diff --git a/.gitignore b/.gitignore index 1de20116f8..e9d08ab34d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ libvgpu.so vendor license vgpuvalidator +/quotacheck _output/ coverage.out .DS_Store diff --git a/hack/tools/quotacheck/main.go b/hack/tools/quotacheck/main.go new file mode 100644 index 0000000000..71f1b6ae9c --- /dev/null +++ b/hack/tools/quotacheck/main.go @@ -0,0 +1,791 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// quotacheck verifies that every device backend's Fit() implementation +// re-checks ResourceQuota before admitting a pod. device.Devices.Fit() is +// implemented separately by every vendor backend under pkg/device//, +// and the admission-vs-schedule TOCTOU race is only closed if Fit() re-reads +// namespace ResourceQuota usage via device.QuotaManager.FitQuota (directly, +// or through a local wrapper such as the fitQuota() helper used by nvidia +// and cambricon) *and* lets the result reject the candidate device: +// +// if !fitQuota(pod, tmpDevs, allocated, pod.Namespace, dev.ID, memreq, coresreq) { +// reason[common.ResourceQuotaNotFit]++ +// continue +// } +// +// Calling FitQuota and discarding what it returns leaves the race wide open, +// so reaching the call is necessary but not sufficient: the value has to +// reach a branch condition or a return. +// +// Usage: go run ./hack/tools/quotacheck/ [-allow vendor1,vendor2,...] [path ...] +// +// With no path arguments, it discovers every pkg/device//device.go +// file (skipping the shared pkg/device/common package) and, for each one, +// checks the package's Fit() method, following calls into other functions +// defined in the same package. Path arguments, if given, are +// pkg/device//device.go paths to check instead of the default set. +// +// -allow lists vendor directory names that are still permitted to fail the +// check, for enabling this gate before every backend has been fixed (see +// #2829). A vendor that fails and isn't listed, or that passes while still +// listed (a stale entry left behind after its fix landed), fails the run. +// +// # Known limits +// +// quotacheck parses each package with go/ast alone, without type information, +// which bounds what it can prove: +// +// - Calls are resolved to same-package declarations only, so a backend that +// factors its re-check into a shared package would be reported as a +// violation. +// - It checks that the result reaches a branch or a return, not that the +// branch rejects the device, and not that the call dominates the point +// where the device is accepted. Reviewers still own that. +// +// See isTargetCall for how a FitQuota call is told apart from an unrelated +// method that happens to share the name. +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "path/filepath" + "strings" +) + +// fitMethodName is the device.Devices interface method every backend must +// implement (pkg/device/devices.go). +const fitMethodName = "Fit" + +// targetMethod is the shared re-check helper backends must reach from Fit(). +// It is always called as a method on the cache the device package owns, e.g. +// device.GetLocalCache().FitQuota(...). +const targetMethod = "FitQuota" + +// devicePkgPath is the package that owns QuotaManager.FitQuota. A FitQuota +// call only counts when its receiver chain roots in this package's import, +// so an unrelated method that happens to be named FitQuota cannot satisfy +// the check. +const devicePkgPath = "github.com/Project-HAMi/HAMi/pkg/device" + +// fitParamCount and fitResultCount are the parameter and result counts of +// device.Devices.Fit (pkg/device/devices.go): +// +// Fit(devices []*DeviceUsage, request ContainerDeviceRequest, pod *corev1.Pod, +// nodeInfo *NodeInfo, allocated *PodDevices) (bool, map[string]ContainerDevices, string) +// +// They identify the interface method among any same-named helpers in the +// package. If the interface signature changes, quotacheck reports that +// explicitly rather than silently finding no Fit() to check. +const ( + fitParamCount = 5 + fitResultCount = 3 +) + +// skipDirs are pkg/device subdirectories that do not implement a vendor +// backend and so have no Fit() method to check. +var skipDirs = map[string]bool{ + "common": true, +} + +// allowFlag lists vendor directory names (pkg/device/) that are +// currently permitted to fail the check. It exists so this CI gate can be +// enabled before every backend has been fixed: each backend's fix PR (see +// #2829) removes its own entry. A vendor left in the list after its Fit() +// is fixed is reported as a stale entry, so the allowlist can't silently +// mask a regression once a backend is compliant. +var allowFlag = flag.String("allow", "", "comma-separated list of vendor directory names allowed to fail the check") + +func main() { + flag.Parse() + + root, err := findRepoRoot() + if err != nil { + fmt.Fprintf(os.Stderr, "quotacheck: %v\n", err) + os.Exit(1) + } + + paths := flag.Args() + if len(paths) == 0 { + paths, err = defaultDeviceFiles(root) + if err != nil { + fmt.Fprintf(os.Stderr, "quotacheck: %v\n", err) + os.Exit(1) + } + } + + allowed := parseAllowList(*allowFlag) + + exitCode, err := run(paths, allowed, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "quotacheck: %v\n", err) + os.Exit(1) + } + if exitCode != 0 { + os.Exit(exitCode) + } +} + +// run checks each of paths and reports one line per path to out: silence +// for a compliant, unlisted vendor; an allowed-failure or stale-entry note +// for a listed vendor; or the check's violation message for a non-compliant, +// unlisted vendor. It returns a non-zero exit code if any path still needs +// attention. +func run(paths []string, allowed map[string]bool, out io.Writer) (int, error) { + exitCode := 0 + for _, path := range paths { + violations, err := checkDeviceFile(path) + if err != nil { + return 0, err + } + + vendor := filepath.Base(filepath.Dir(path)) + switch { + case len(violations) > 0 && allowed[vendor]: + fmt.Fprintf(out, "%s: allowed failure (see hack/verify-quota.sh); remove from -allow once fixed\n", path) + case len(violations) > 0: + for _, v := range violations { + fmt.Fprintln(out, v) + } + exitCode = 1 + case allowed[vendor]: + fmt.Fprintf(out, "%s: passes the check but is still listed in -allow; remove its stale entry\n", path) + exitCode = 1 + } + } + return exitCode, nil +} + +// parseAllowList splits a comma-separated -allow value into a lookup set, +// ignoring blank entries. +func parseAllowList(s string) map[string]bool { + allowed := make(map[string]bool) + for v := range strings.SplitSeq(s, ",") { + v = strings.TrimSpace(v) + if v != "" { + allowed[v] = true + } + } + return allowed +} + +// defaultDeviceFiles returns every pkg/device//device.go path under +// root, skipping vendor directories with no device.go and non-backend +// directories listed in skipDirs. +func defaultDeviceFiles(root string) ([]string, error) { + deviceDir := filepath.Join(root, "pkg", "device") + entries, err := os.ReadDir(deviceDir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", deviceDir, err) + } + + var paths []string + for _, e := range entries { + if !e.IsDir() || skipDirs[e.Name()] { + continue + } + p := filepath.Join(deviceDir, e.Name(), "device.go") + if _, err := os.Stat(p); err != nil { + continue + } + paths = append(paths, p) + } + return paths, nil +} + +// checkDeviceFile checks the Fit() method of the package containing path, +// returning a violation message if Fit() cannot reach a call to FitQuota +// through its package's call graph, or reaches one whose result is discarded +// instead of deciding whether the candidate device is accepted. +func checkDeviceFile(path string) ([]string, error) { + fset := token.NewFileSet() + pkgFiles, err := parsePackage(fset, filepath.Dir(path)) + if err != nil { + return nil, err + } + + fit, problem := findFitMethod(pkgFiles) + if problem != "" { + return []string{fmt.Sprintf("%s: %s", path, problem)}, nil + } + + idx := newDeclIndex(pkgFiles) + + // Collect the calls in Fit()'s own body that reach the re-check, either + // directly or through a local wrapper. Restricting this to Fit()'s body + // is what makes the result check below meaningful: the value has to be + // acted on where the candidate device is chosen. + quotaCalls := map[*ast.CallExpr]bool{} + inspectInvokedCalls(fit.Body, func(call *ast.CallExpr) { + if idx.callReachesTarget(fit, call) { + quotaCalls[call] = true + } + }) + + pos := fset.Position(fit.Pos()) + location := fmt.Sprintf("%s:%d", pos.Filename, pos.Line) + + if len(quotaCalls) == 0 { + return []string{fmt.Sprintf( + "%s: %s() does not call %s() to re-check ResourceQuota before admitting the pod; "+ + "the re-check is recognised as a call on the cache the device package owns "+ + "(e.g. device.GetLocalCache().%s(...)), directly or through a local helper", + location, fitMethodName, targetMethod, targetMethod)}, nil + } + + if !resultGatesAdmission(fit.Body, quotaCalls) { + return []string{fmt.Sprintf( + "%s: %s() calls %s() but discards its result; the returned value must reject the "+ + "candidate device (e.g. `if !fitQuota(...) { continue }`), otherwise the re-check has no effect", + location, fitMethodName, targetMethod)}, nil + } + + return nil, nil +} + +// findFitMethod returns the package's device.Devices Fit implementation: the +// method named fitMethodName whose signature matches the interface. Looking +// across every file of the package (not just device.go) keeps the check +// working if a backend moves Fit() into another file, and matching on the +// signature keeps an unrelated helper that happens to be named Fit from +// shadowing the real one. +// +// The second result is a non-empty message when no single implementation can +// be identified. +func findFitMethod(files []*ast.File) (*ast.FuncDecl, string) { + var named, candidates []*ast.FuncDecl + for _, f := range files { + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || fn.Name.Name != fitMethodName { + continue + } + named = append(named, fn) + if fieldCount(fn.Type.Params) == fitParamCount && fieldCount(fn.Type.Results) == fitResultCount { + candidates = append(candidates, fn) + } + } + } + + switch { + case len(candidates) == 1: + return candidates[0], "" + case len(candidates) > 1: + return nil, fmt.Sprintf("found %d methods named %s() matching device.Devices; cannot tell which one implements the interface", + len(candidates), fitMethodName) + case len(named) > 0: + return nil, fmt.Sprintf("found %d method(s) named %s() but none take %d parameters and return %d values; "+ + "if device.Devices.%s changed, update fitParamCount/fitResultCount in quotacheck", + len(named), fitMethodName, fitParamCount, fitResultCount, fitMethodName) + default: + return nil, fmt.Sprintf("no %s() method found", fitMethodName) + } +} + +// fieldCount returns the number of parameters or results in a signature, +// counting each name in a grouped field (e.g. `a, b int` is two) and an +// unnamed field as one. +func fieldCount(fl *ast.FieldList) int { + if fl == nil { + return 0 + } + n := 0 + for _, f := range fl.List { + if len(f.Names) == 0 { + n++ + continue + } + n += len(f.Names) + } + return n +} + +// parsePackage parses every non-test .go file in dir. +func parsePackage(fset *token.FileSet, dir string) ([]*ast.File, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", dir, err) + } + + var files []*ast.File + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0) + if err != nil { + return nil, fmt.Errorf("parsing %s: %w", name, err) + } + files = append(files, f) + } + return files, nil +} + +// declIndex indexes a package's declarations so a call expression can be +// resolved back to the function or method it invokes. Package-level functions +// and methods are kept apart, and methods are also indexed by receiver type, +// so two same-named methods on different receivers stay distinct instead of +// one overwriting the other. +type declIndex struct { + funcs map[string][]*ast.FuncDecl + methods map[string][]*ast.FuncDecl + byRecv map[string][]*ast.FuncDecl + + // devicePkg is the identifier the device package is imported under in + // the file each declaration lives in, so an aliased import still + // resolves. It is empty for a file that doesn't import it at all. + devicePkg map[*ast.FuncDecl]string + + // deviceVars caches, per function, the local variables assigned from an + // expression rooted in the device package. + deviceVars map[*ast.FuncDecl]map[string]bool +} + +func newDeclIndex(files []*ast.File) *declIndex { + idx := &declIndex{ + funcs: map[string][]*ast.FuncDecl{}, + methods: map[string][]*ast.FuncDecl{}, + byRecv: map[string][]*ast.FuncDecl{}, + devicePkg: map[*ast.FuncDecl]string{}, + deviceVars: map[*ast.FuncDecl]map[string]bool{}, + } + for _, f := range files { + devicePkg := devicePkgIdent(f) + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + idx.devicePkg[fn] = devicePkg + if fn.Recv == nil { + idx.funcs[fn.Name.Name] = append(idx.funcs[fn.Name.Name], fn) + continue + } + idx.methods[fn.Name.Name] = append(idx.methods[fn.Name.Name], fn) + key := receiverType(fn) + "." + fn.Name.Name + idx.byRecv[key] = append(idx.byRecv[key], fn) + } + } + return idx +} + +// devicePkgIdent returns the identifier devicePkgPath is imported under in f +// (its alias, or the package name), or "" if f does not import it. +func devicePkgIdent(f *ast.File) string { + for _, imp := range f.Imports { + if strings.Trim(imp.Path.Value, `"`) != devicePkgPath { + continue + } + if imp.Name != nil { + return imp.Name.Name + } + // Unaliased: the package declares itself as `package device`. + return "device" + } + return "" +} + +// isTargetCall reports whether call is the QuotaManager re-check. +// +// Matching on the method name alone would let any same-named method satisfy +// the gate — a local no-op stub named FitQuota, or an unrelated type's method +// — which is exactly the regression this tool exists to catch. Without type +// information the receiver cannot be resolved to device.QuotaManager, so +// instead the receiver chain has to root in the device package's import: +// +// device.GetLocalCache().FitQuota(...) // root ident is the device import +// cache := device.GetLocalCache() // ...or a local bound to it +// cache.FitQuota(...) +// +// A backend that reaches the re-check some other way is reported as a +// violation rather than silently passing, which is the safe direction: the +// failure is loud and the message says which shape is recognised. +func (idx *declIndex) isTargetCall(from *ast.FuncDecl, call *ast.CallExpr) bool { + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != targetMethod { + return false + } + + devicePkg := idx.devicePkg[from] + if devicePkg == "" { + return false + } + + root := rootIdent(sel.X) + if root == "" { + return false + } + return root == devicePkg || idx.deviceRootedVars(from)[root] +} + +// deviceRootedVars returns the local variables in fn assigned from an +// expression rooted in the device package, so the receiver can be held in a +// variable instead of being called inline. +func (idx *declIndex) deviceRootedVars(fn *ast.FuncDecl) map[string]bool { + if cached, ok := idx.deviceVars[fn]; ok { + return cached + } + + devicePkg := idx.devicePkg[fn] + names := map[string]bool{} + idx.deviceVars[fn] = names + if devicePkg == "" { + return names + } + + record := func(lhs []ast.Expr, rhs []ast.Expr) { + for i, r := range rhs { + if rootIdent(r) != devicePkg || i >= len(lhs) { + continue + } + if id, ok := lhs[i].(*ast.Ident); ok && id.Name != "_" { + names[id.Name] = true + } + } + } + ast.Inspect(fn.Body, func(n ast.Node) bool { + switch v := n.(type) { + case *ast.AssignStmt: + record(v.Lhs, v.Rhs) + case *ast.ValueSpec: + lhs := make([]ast.Expr, len(v.Names)) + for i, name := range v.Names { + lhs[i] = name + } + record(lhs, v.Values) + } + return true + }) + return names +} + +// rootIdent returns the identifier an expression chain starts from, e.g. +// "device" for device.GetLocalCache().FitQuota, or "" if it doesn't start +// from a plain identifier. +func rootIdent(expr ast.Expr) string { + for { + switch e := expr.(type) { + case *ast.Ident: + return e.Name + case *ast.SelectorExpr: + expr = e.X + case *ast.CallExpr: + expr = e.Fun + case *ast.ParenExpr: + expr = e.X + case *ast.StarExpr: + expr = e.X + case *ast.IndexExpr: + expr = e.X + case *ast.IndexListExpr: + expr = e.X + default: + return "" + } + } +} + +// receiverType returns the bare type name a method is declared on, with any +// pointer star and type parameters stripped, or "" if it can't be determined. +func receiverType(fn *ast.FuncDecl) string { + if fn.Recv == nil || len(fn.Recv.List) == 0 { + return "" + } + expr := fn.Recv.List[0].Type + for { + switch t := expr.(type) { + case *ast.StarExpr: + expr = t.X + case *ast.IndexExpr: // generic receiver, e.g. Foo[T] + expr = t.X + case *ast.IndexListExpr: + expr = t.X + case *ast.Ident: + return t.Name + default: + return "" + } + } +} + +// resolve returns the same-package declarations a call made from caller may +// invoke. +// +// A bare call like fitQuota(...) is either a package-level function or a +// method on the caller's own receiver, both of which resolve exactly. A +// selector call like x.helper(...) cannot be resolved without type +// information, so every same-named method in the package is returned. That +// over-approximates rather than dropping the real callee, which is the safe +// direction for a check whose job is to find backends that never reach the +// re-check at all. +func (idx *declIndex) resolve(caller *ast.CallExpr, from *ast.FuncDecl) []*ast.FuncDecl { + switch fun := caller.Fun.(type) { + case *ast.Ident: + out := idx.funcs[fun.Name] + if recv := receiverType(from); recv != "" { + out = append(out, idx.byRecv[recv+"."+fun.Name]...) + } + return out + case *ast.SelectorExpr: + return idx.methods[fun.Sel.Name] + default: + return nil + } +} + +// callReachesTarget reports whether call is the FitQuota re-check itself, or +// a call to a same-package function that reaches it. +func (idx *declIndex) callReachesTarget(from *ast.FuncDecl, call *ast.CallExpr) bool { + if idx.isTargetCall(from, call) { + return true + } + for _, callee := range idx.resolve(call, from) { + if idx.reachesTargetMethod(callee, map[*ast.FuncDecl]bool{}) { + return true + } + } + return false +} + +// reachesTargetMethod reports whether fn's body, or any local function it +// calls (transitively), contains a call to targetMethod. visited breaks +// recursive and mutually recursive call chains. +func (idx *declIndex) reachesTargetMethod(fn *ast.FuncDecl, visited map[*ast.FuncDecl]bool) bool { + if fn == nil || visited[fn] { + return false + } + visited[fn] = true + + found := false + inspectInvokedCalls(fn.Body, func(call *ast.CallExpr) { + if found { + return + } + if idx.isTargetCall(fn, call) { + found = true + return + } + for _, callee := range idx.resolve(call, fn) { + if idx.reachesTargetMethod(callee, visited) { + found = true + return + } + } + }) + return found +} + +// resultGatesAdmission reports whether the value returned by one of quotaCalls +// decides control flow in body, either directly inside a branch condition or a +// return, or through a variable it is assigned to. +// +// This is what separates a re-check that works from one that runs and is +// thrown away: `_ = fitQuota(...)`, a bare `fitQuota(...)` statement, or a +// result passed only to a logger all leave the candidate device admitted. +func resultGatesAdmission(body *ast.BlockStmt, quotaCalls map[*ast.CallExpr]bool) bool { + gating := gatingExprs(body) + + for _, expr := range gating { + if containsCall(expr, quotaCalls) { + return true + } + } + + bound := identsBoundTo(body, quotaCalls) + if len(bound) == 0 { + return false + } + for _, expr := range gating { + if containsIdent(expr, bound) { + return true + } + } + return false +} + +// gatingExprs returns every expression in body whose value decides control +// flow: branch and loop conditions, switch tags and case values, and returned +// values. +func gatingExprs(body *ast.BlockStmt) []ast.Expr { + var out []ast.Expr + ast.Inspect(body, func(n ast.Node) bool { + switch v := n.(type) { + case *ast.IfStmt: + out = append(out, v.Cond) + case *ast.ForStmt: + if v.Cond != nil { + out = append(out, v.Cond) + } + case *ast.SwitchStmt: + if v.Tag != nil { + out = append(out, v.Tag) + } + case *ast.CaseClause: + out = append(out, v.List...) + case *ast.ReturnStmt: + out = append(out, v.Results...) + } + return true + }) + return out +} + +// identsBoundTo returns the names of variables assigned the value of one of +// calls, so that `ok := fitQuota(...)` followed by `if !ok` counts as gating. +// The blank identifier is deliberately excluded: `_ = fitQuota(...)` discards +// the result. +func identsBoundTo(body *ast.BlockStmt, calls map[*ast.CallExpr]bool) map[string]bool { + names := map[string]bool{} + record := func(lhs []ast.Expr) { + for _, e := range lhs { + if id, ok := e.(*ast.Ident); ok && id.Name != "_" { + names[id.Name] = true + } + } + } + ast.Inspect(body, func(n ast.Node) bool { + switch v := n.(type) { + case *ast.AssignStmt: + for _, rhs := range v.Rhs { + if containsCall(rhs, calls) { + record(v.Lhs) + break + } + } + case *ast.ValueSpec: + for _, val := range v.Values { + if containsCall(val, calls) { + for _, id := range v.Names { + if id.Name != "_" { + names[id.Name] = true + } + } + break + } + } + } + return true + }) + return names +} + +// containsCall reports whether expr contains one of calls. +func containsCall(expr ast.Expr, calls map[*ast.CallExpr]bool) bool { + found := false + ast.Inspect(expr, func(n ast.Node) bool { + if call, ok := n.(*ast.CallExpr); ok && calls[call] { + found = true + } + return !found + }) + return found +} + +// containsIdent reports whether expr references one of names. +func containsIdent(expr ast.Expr, names map[string]bool) bool { + found := false + ast.Inspect(expr, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && names[id.Name] { + found = true + } + return !found + }) + return found +} + +// inspectInvokedCalls walks n and reports every call expression that is +// reachable, and runs synchronously, when the enclosing function runs. It +// descends into a function literal only when that literal is the callee of an +// enclosing call expression (an IIFE), so calls inside an uninvoked closure +// are skipped; and it does not treat the callee of a `go` statement as +// synchronous, since Fit() can return before that goroutine runs. +func inspectInvokedCalls(n ast.Node, visit func(*ast.CallExpr)) { + if n == nil { + return + } + ast.Inspect(n, func(node ast.Node) bool { + switch v := node.(type) { + case *ast.FuncLit: + // Do not descend into a closure body here; only an + // immediately-invoked one is walked, via the CallExpr case. + return false + case *ast.GoStmt: + inspectDeferredCall(v.Call, visit) + return false + case *ast.DeferStmt: + // A deferred call runs after Fit()'s body has finished choosing + // a device, so like `go` it cannot reject the candidate. Its + // function value and arguments are evaluated at the `defer` + // statement, so those are still walked. + inspectDeferredCall(v.Call, visit) + return false + case *ast.CallExpr: + visit(v) + if lit, ok := v.Fun.(*ast.FuncLit); ok { + inspectInvokedCalls(lit.Body, visit) + } else { + // Walk the callee expression (e.g. the receiver chain + // of a.b().c()) and the arguments for nested calls. + inspectInvokedCalls(v.Fun, visit) + } + for _, arg := range v.Args { + inspectInvokedCalls(arg, visit) + } + return false + } + return true + }) +} + +// inspectDeferredCall walks the parts of a `go` or `defer` call that Go +// evaluates synchronously at the statement: the function value and the +// arguments. The call itself is deliberately not reported, since it runs +// after Fit() has already chosen a device. +// +// Walking the function value matters for shapes like `go checkedRunner()()`, +// where the outer invocation is asynchronous but checkedRunner() is not. For +// `go cache.FitQuota(...)` it walks only the receiver chain, so the +// asynchronous FitQuota still does not count. +func inspectDeferredCall(call *ast.CallExpr, visit func(*ast.CallExpr)) { + inspectInvokedCalls(call.Fun, visit) + for _, arg := range call.Args { + inspectInvokedCalls(arg, visit) + } +} + +func findRepoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("go.mod not found") + } + dir = parent + } +} diff --git a/hack/tools/quotacheck/main_test.go b/hack/tools/quotacheck/main_test.go new file mode 100644 index 0000000000..43500f4fc1 --- /dev/null +++ b/hack/tools/quotacheck/main_test.go @@ -0,0 +1,550 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// fitSignature mirrors device.Devices.Fit so fixtures are selected by +// findFitMethod the same way a real backend's method is. +const fitSignature = "func (d *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, " + + "pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) " + + "(bool, map[string]device.ContainerDevices, string) {\n" + +// fixtureHeader opens a fixture file, importing the device package the same +// way a real backend does so the re-check resolves. +const fixtureHeader = "package fixture\n\nimport \"github.com/Project-HAMi/HAMi/pkg/device\"\n\n" + +// fitFile returns a parseable fixture file whose Fit() method has the given +// body, followed by any extra package-level declarations. +func fitFile(body string, decls ...string) string { + return fixtureHeader + "type Devices struct{}\n\n" + fitSignature + body + "}\n" + + strings.Join(decls, "\n") +} + +// gatedOn wraps a quota expression in the rejection branch real backends use. +func gatedOn(expr string) string { + return "\tif !" + expr + " {\n\t\treturn false, nil, \"quota\"\n\t}\n\treturn true, nil, \"\"\n" +} + +// wrapper is a local fitQuota() helper that reaches the re-check, like the one +// nvidia and cambricon use. +const wrapper = ` +func fitQuota(ns string, memreq, coresreq int64) bool { + return device.GetLocalCache().FitQuota(ns, memreq, 1, coresreq, "dev") +} +` + +func writeFile(t *testing.T, dir, name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("writing %s: %v", path, err) + } + return path +} + +// checkFixture writes content as device.go in a fresh temp dir and checks it. +func checkFixture(t *testing.T, content string) []string { + t.Helper() + path := writeFile(t, t.TempDir(), "device.go", content) + violations, err := checkDeviceFile(path) + if err != nil { + t.Fatalf("checkDeviceFile: %v", err) + } + return violations +} + +// assertViolation fails unless there is exactly one violation containing want. +func assertViolation(t *testing.T, violations []string, want string) { + t.Helper() + if len(violations) != 1 { + t.Fatalf("expected exactly one violation, got %v", violations) + } + if !strings.Contains(violations[0], want) { + t.Errorf("violation = %q, want it to mention %q", violations[0], want) + } +} + +func TestCheckDeviceFile_DirectCall(t *testing.T) { + got := checkFixture(t, fitFile(gatedOn(`device.GetLocalCache().FitQuota("ns", 0, 1, 0, "dev")`))) + if len(got) != 0 { + t.Errorf("expected no violations for a gated direct FitQuota call, got %v", got) + } +} + +func TestCheckDeviceFile_LocalWrapper(t *testing.T) { + got := checkFixture(t, fitFile(gatedOn(`fitQuota("ns", 0, 0)`), wrapper)) + if len(got) != 0 { + t.Errorf("expected no violations when Fit() reaches FitQuota via a local wrapper, got %v", got) + } +} + +func TestCheckDeviceFile_WrapperInOtherFile(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "device.go", fitFile(gatedOn(`fitQuota("ns", 0, 0)`))) + writeFile(t, dir, "quota.go", fixtureHeader+wrapper) + + violations, err := checkDeviceFile(path) + if err != nil { + t.Fatalf("checkDeviceFile: %v", err) + } + if len(violations) != 0 { + t.Errorf("expected no violations when the wrapper lives in another file of the same package, got %v", violations) + } +} + +// TestCheckDeviceFile_FitInOtherFile guards against the checker only looking +// at device.go: a backend that moves Fit() into another file of the same +// package is still compliant and must not be reported as missing Fit(). +func TestCheckDeviceFile_FitInOtherFile(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "device.go", fixtureHeader+"type Devices struct{}\n"+wrapper) + writeFile(t, dir, "fit.go", "package fixture\n\n"+fitSignature+gatedOn(`fitQuota("ns", 0, 0)`)+"}\n") + + violations, err := checkDeviceFile(path) + if err != nil { + t.Fatalf("checkDeviceFile: %v", err) + } + if len(violations) != 0 { + t.Errorf("expected no violations when Fit() is declared in another file of the package, got %v", violations) + } +} + +func TestCheckDeviceFile_MissingCheck(t *testing.T) { + got := checkFixture(t, fitFile("\treturn true, nil, \"\"\n")) + assertViolation(t, got, "does not call") +} + +func TestCheckDeviceFile_UnrelatedCallsDoNotCount(t *testing.T) { + got := checkFixture(t, fitFile(gatedOn("otherHelper()"), "\nfunc otherHelper() bool {\n\treturn true\n}\n")) + assertViolation(t, got, "does not call") +} + +// TestCheckDeviceFile_DiscardedResult is the case a pure reachability check +// misses: FitQuota runs, but nothing acts on what it returns, so the candidate +// device is admitted regardless and the race stays open. +func TestCheckDeviceFile_DiscardedResult(t *testing.T) { + cases := map[string]string{ + "assigned to the blank identifier": "\t_ = fitQuota(\"ns\", 0, 0)\n\treturn true, nil, \"\"\n", + "called as a bare statement": "\tfitQuota(\"ns\", 0, 0)\n\treturn true, nil, \"\"\n", + "passed only to a logger": "\tklog.V(3).InfoS(\"quota\", \"fits\", fitQuota(\"ns\", 0, 0))\n\treturn true, nil, \"\"\n", + } + + for name, body := range cases { + t.Run(name, func(t *testing.T) { + assertViolation(t, checkFixture(t, fitFile(body, wrapper)), "discards its result") + }) + } +} + +func TestCheckDeviceFile_ResultCheckedViaVariable(t *testing.T) { + cases := map[string]string{ + "assigned then branched on": "\tok := fitQuota(\"ns\", 0, 0)\n\tif !ok {\n\t\treturn false, nil, \"quota\"\n\t}\n\treturn true, nil, \"\"\n", + "bound in the if init": "\tif ok := fitQuota(\"ns\", 0, 0); !ok {\n\t\treturn false, nil, \"quota\"\n\t}\n\treturn true, nil, \"\"\n", + "returned directly": "\treturn fitQuota(\"ns\", 0, 0), nil, \"\"\n", + "declared with var": "\tvar ok = fitQuota(\"ns\", 0, 0)\n\tif !ok {\n\t\treturn false, nil, \"quota\"\n\t}\n\treturn true, nil, \"\"\n", + } + + for name, body := range cases { + t.Run(name, func(t *testing.T) { + if got := checkFixture(t, fitFile(body, wrapper)); len(got) != 0 { + t.Errorf("expected no violations when the result gates admission, got %v", got) + } + }) + } +} + +func TestCheckDeviceFile_UninvokedClosureDoesNotCount(t *testing.T) { + body := "\tcheck := func() bool {\n\t\treturn device.GetLocalCache().FitQuota(\"ns\", 0, 1, 0, \"dev\")\n\t}\n\t_ = check\n\treturn true, nil, \"\"\n" + assertViolation(t, checkFixture(t, fitFile(body)), "does not call") +} + +func TestCheckDeviceFile_InvokedClosureCounts(t *testing.T) { + body := gatedOn("func() bool {\n\t\treturn device.GetLocalCache().FitQuota(\"ns\", 0, 1, 0, \"dev\")\n\t}()") + if got := checkFixture(t, fitFile(body)); len(got) != 0 { + t.Errorf("expected no violations when Fit() invokes a closure that calls FitQuota, got %v", got) + } +} + +func TestCheckDeviceFile_AsyncCallDoesNotCount(t *testing.T) { + cases := map[string]string{ + "go statement calling FitQuota directly": "\tgo device.GetLocalCache().FitQuota(\"ns\", 0, 1, 0, \"dev\")\n\treturn true, nil, \"\"\n", + "go statement launching a closure": "\tgo func() {\n\t\tdevice.GetLocalCache().FitQuota(\"ns\", 0, 1, 0, \"dev\")\n\t}()\n\treturn true, nil, \"\"\n", + } + + for name, body := range cases { + t.Run(name, func(t *testing.T) { + assertViolation(t, checkFixture(t, fitFile(body)), "does not call") + }) + } +} + +// TestCheckDeviceFile_AsyncCallArgsAreSeen documents the two rules meeting: +// a `go` statement's arguments are evaluated synchronously, so the call is +// found, but handing its result to a logger still doesn't gate admission. +func TestCheckDeviceFile_AsyncCallArgsAreSeen(t *testing.T) { + body := "\tgo record(\"checked\", fitQuota(\"ns\", 0, 0))\n\treturn true, nil, \"\"\n" + decls := wrapper + "\nfunc record(msg string, ok bool) {}\n" + assertViolation(t, checkFixture(t, fitFile(body, decls)), "discards its result") +} + +// TestCheckDeviceFile_UnrelatedFitQuotaIgnored guards the receiver +// resolution: a same-named method on an unrelated type must not satisfy the +// gate, or a backend could pass by calling a local no-op stub named FitQuota +// while never reaching device.QuotaManager's re-check. +func TestCheckDeviceFile_UnrelatedFitQuotaIgnored(t *testing.T) { + decls := ` +type quotaStub struct{} + +func (q *quotaStub) FitQuota(ns string, mem, factor, core int64, devType string) bool { + return true +} + +func (d *Devices) stub() *quotaStub { + return "aStub{} +} +` + got := checkFixture(t, fitFile(gatedOn(`d.stub().FitQuota("ns", 0, 1, 0, "dev")`), decls)) + assertViolation(t, got, "does not call") +} + +func TestCheckDeviceFile_AliasedDeviceImport(t *testing.T) { + content := "package fixture\n\nimport dev \"github.com/Project-HAMi/HAMi/pkg/device\"\n\ntype Devices struct{}\n\n" + + fitSignature + gatedOn(`dev.GetLocalCache().FitQuota("ns", 0, 1, 0, "dev")`) + "}\n" + + if got := checkFixture(t, content); len(got) != 0 { + t.Errorf("expected no violations when the device package is imported under an alias, got %v", got) + } +} + +// TestCheckDeviceFile_CacheHeldInVariable covers the re-check being called on +// a local bound to the device package's cache rather than inline. +func TestCheckDeviceFile_CacheHeldInVariable(t *testing.T) { + body := "\tcache := device.GetLocalCache()\n" + + "\tif !cache.FitQuota(\"ns\", 0, 1, 0, \"dev\") {\n\t\treturn false, nil, \"quota\"\n\t}\n\treturn true, nil, \"\"\n" + + if got := checkFixture(t, fitFile(body)); len(got) != 0 { + t.Errorf("expected no violations when the cache is held in a local variable, got %v", got) + } +} + +// TestCheckDeviceFile_DeferredCallDoesNotCount covers the same reasoning as +// the `go` cases: a deferred re-check runs after Fit() has chosen a device. +func TestCheckDeviceFile_DeferredCallDoesNotCount(t *testing.T) { + body := "\tdefer device.GetLocalCache().FitQuota(\"ns\", 0, 1, 0, \"dev\")\n\treturn true, nil, \"\"\n" + assertViolation(t, checkFixture(t, fitFile(body)), "does not call") +} + +// TestCheckDeviceFile_GoStatementFunctionValueIsInspected covers the half of +// a `go` statement that is synchronous: in `go checkedRunner()()` the outer +// invocation is asynchronous, but checkedRunner() is evaluated at the `go` +// statement, so the re-check it reaches must still be seen. It is reported +// for discarding the result, not for never calling FitQuota. +func TestCheckDeviceFile_GoStatementFunctionValueIsInspected(t *testing.T) { + decls := wrapper + ` +func checkedRunner() func() { + fits := fitQuota("ns", 0, 0) + return func() { + _ = fits + } +} +` + body := "\tgo checkedRunner()()\n\treturn true, nil, \"\"\n" + assertViolation(t, checkFixture(t, fitFile(body, decls)), "discards its result") +} + +// TestCheckDeviceFile_ShadowingFitIgnored guards findFitMethod's signature +// match: an unrelated method named Fit must not stand in for the interface +// implementation, or a backend could pass the gate on a helper's check while +// its real Fit() admits pods unguarded. +func TestCheckDeviceFile_ShadowingFitIgnored(t *testing.T) { + shadow := "\ntype cache struct{}\n\nfunc (c *cache) Fit() bool {\n\treturn device.GetLocalCache().FitQuota(\"ns\", 0, 1, 0, \"dev\")\n}\n" + // The shadowing helper is declared first, so a first-match lookup would + // pick it and wrongly report the package as compliant. + content := fixtureHeader + "type Devices struct{}\n" + shadow + "\n" + fitSignature + "\treturn true, nil, \"\"\n}\n" + assertViolation(t, checkFixture(t, content), "does not call") +} + +func TestCheckDeviceFile_FitSignatureChanged(t *testing.T) { + content := "package fixture\n\ntype Devices struct{}\n\nfunc (d *Devices) Fit() bool {\n\treturn true\n}\n" + assertViolation(t, checkFixture(t, content), "update fitParamCount") +} + +// TestCheckDeviceFile_SameNamedMethodsOnDifferentReceivers guards call-graph +// resolution: indexing helpers by name alone lets one receiver's method +// overwrite another's, which can drop the real wrapper and report a compliant +// backend as broken. +func TestCheckDeviceFile_SameNamedMethodsOnDifferentReceivers(t *testing.T) { + decls := ` +type other struct{} + +func (o *other) check() bool { + return true +} + +func (d *Devices) check() bool { + return device.GetLocalCache().FitQuota("ns", 0, 1, 0, "dev") +} +` + if got := checkFixture(t, fitFile(gatedOn("d.check()"), decls)); len(got) != 0 { + t.Errorf("expected no violations when the real wrapper shares its name with another receiver's method, got %v", got) + } +} + +// TestCheckDeviceFile_RecursiveHelperTerminates guards the visited set against +// a mutually recursive call chain hanging the check. +func TestCheckDeviceFile_RecursiveHelperTerminates(t *testing.T) { + decls := ` +func ping() bool { + return pong() +} + +func pong() bool { + return ping() +} +` + assertViolation(t, checkFixture(t, fitFile(gatedOn("ping()"), decls)), "does not call") +} + +func TestCheckDeviceFile_NoFitMethod(t *testing.T) { + content := "package fixture\n\nfunc NotFit() bool {\n\treturn true\n}\n" + assertViolation(t, checkFixture(t, content), "no Fit() method found") +} + +func TestCheckDeviceFile_UnparseableFileErrors(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "device.go", "package fixture\n\nfunc broken( {\n") + + if _, err := checkDeviceFile(path); err == nil { + t.Fatal("expected an error for an unparseable file, got nil") + } +} + +func TestDefaultDeviceFiles(t *testing.T) { + root := t.TempDir() + deviceDir := filepath.Join(root, "pkg", "device") + + mustMkdirAll(t, filepath.Join(deviceDir, "vendora")) + writeFile(t, filepath.Join(deviceDir, "vendora"), "device.go", "package vendora\n") + + mustMkdirAll(t, filepath.Join(deviceDir, "vendorb")) + writeFile(t, filepath.Join(deviceDir, "vendorb"), "device.go", "package vendorb\n") + + // common has no device.go and must be skipped explicitly. + mustMkdirAll(t, filepath.Join(deviceDir, "common")) + writeFile(t, filepath.Join(deviceDir, "common"), "device.go", "package common\n") + + // vendorc has no device.go and must be skipped by absence. + mustMkdirAll(t, filepath.Join(deviceDir, "vendorc")) + + paths, err := defaultDeviceFiles(root) + if err != nil { + t.Fatalf("defaultDeviceFiles: %v", err) + } + + want := map[string]bool{ + filepath.Join(deviceDir, "vendora", "device.go"): true, + filepath.Join(deviceDir, "vendorb", "device.go"): true, + } + if len(paths) != len(want) { + t.Fatalf("defaultDeviceFiles returned %v, want keys of %v", paths, want) + } + for _, p := range paths { + if !want[p] { + t.Errorf("unexpected path %s", p) + } + } +} + +func mustMkdirAll(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } +} + +// TestRealBackends runs quotacheck against the actual pkg/device backends in +// this repository. nvidia and cambricon already re-check ResourceQuota in +// Fit() (see #2536) and must pass; the backends tracked by #2829 as still +// missing the check must be flagged, mirroring this issue's acceptance +// criteria. +// +// It iterates the backends quotacheck discovers rather than a fixed list, so +// a newly added backend fails here until it is classified, instead of being +// silently unverified while hack/verify-quota.sh fails in CI. +func TestRealBackends(t *testing.T) { + root, err := findRepoRoot() + if err != nil { + t.Fatalf("findRepoRoot: %v", err) + } + + compliant := map[string]bool{ + "cambricon": true, + "nvidia": true, + } + // Keep in sync with ALLOWED_VENDORS in hack/verify-quota.sh. + nonCompliant := map[string]bool{ + "amd": true, + "ascend": true, + "awsneuron": true, + "biren": true, + "enflame": true, + "hygon": true, + "iluvatar": true, + "kunlun": true, + "metax": true, + "mthreads": true, + "vastai": true, + } + + paths, err := defaultDeviceFiles(root) + if err != nil { + t.Fatalf("defaultDeviceFiles: %v", err) + } + if len(paths) == 0 { + t.Fatal("no device backends discovered under pkg/device") + } + + seen := map[string]bool{} + for _, path := range paths { + vendor := filepath.Base(filepath.Dir(path)) + seen[vendor] = true + + violations, err := checkDeviceFile(path) + if err != nil { + t.Fatalf("checkDeviceFile(%s): %v", vendor, err) + } + + switch { + case compliant[vendor]: + if len(violations) != 0 { + t.Errorf("%s: expected no violations, got %v", vendor, violations) + } + case nonCompliant[vendor]: + if len(violations) == 0 { + t.Errorf("%s: now re-checks ResourceQuota; move it to the compliant set here and drop it from ALLOWED_VENDORS in hack/verify-quota.sh", vendor) + } + default: + t.Errorf("%s: backend is classified in neither the compliant nor the non-compliant set; add it here, and to ALLOWED_VENDORS in hack/verify-quota.sh if its Fit() does not re-check ResourceQuota yet", vendor) + } + } + + for _, set := range []map[string]bool{compliant, nonCompliant} { + for vendor := range set { + if !seen[vendor] { + t.Errorf("%s: listed here but no longer discovered under pkg/device; remove its stale entry", vendor) + } + } + } +} + +func TestParseAllowList(t *testing.T) { + got := parseAllowList(" foo, bar ,,baz") + want := map[string]bool{"foo": true, "bar": true, "baz": true} + if !reflect.DeepEqual(got, want) { + t.Errorf("parseAllowList(...) = %v, want %v", got, want) + } + + if got := parseAllowList(""); len(got) != 0 { + t.Errorf("parseAllowList(\"\") = %v, want empty", got) + } +} + +// newRunFixture writes a device.go for a "vendora" backend under a fresh +// temp root and returns its path. +func newRunFixture(t *testing.T, compliant bool) string { + t.Helper() + vendorDir := filepath.Join(t.TempDir(), "vendora") + mustMkdirAll(t, vendorDir) + + body := "\treturn true, nil, \"\"\n" + if compliant { + body = gatedOn(`device.GetLocalCache().FitQuota("ns", 0, 1, 0, "dev")`) + } + return writeFile(t, vendorDir, "device.go", fitFile(body)) +} + +func TestRun_UnlistedCompliant(t *testing.T) { + path := newRunFixture(t, true) + + var buf bytes.Buffer + code, err := run([]string{path}, nil, &buf) + if err != nil { + t.Fatalf("run: %v", err) + } + if code != 0 { + t.Errorf("exit code = %d, want 0", code) + } + if buf.Len() != 0 { + t.Errorf("expected no output for a compliant, unlisted vendor, got %q", buf.String()) + } +} + +func TestRun_UnlistedNonCompliant(t *testing.T) { + path := newRunFixture(t, false) + + var buf bytes.Buffer + code, err := run([]string{path}, nil, &buf) + if err != nil { + t.Fatalf("run: %v", err) + } + if code != 1 { + t.Errorf("exit code = %d, want 1", code) + } + if !strings.Contains(buf.String(), "does not call") { + t.Errorf("expected a violation message, got %q", buf.String()) + } +} + +func TestRun_ListedNonCompliant_Allowed(t *testing.T) { + path := newRunFixture(t, false) + + var buf bytes.Buffer + code, err := run([]string{path}, map[string]bool{"vendora": true}, &buf) + if err != nil { + t.Fatalf("run: %v", err) + } + if code != 0 { + t.Errorf("exit code = %d, want 0 for an allowed failure", code) + } + if !strings.Contains(buf.String(), "allowed failure") { + t.Errorf("expected an allowed-failure note, got %q", buf.String()) + } +} + +func TestRun_ListedCompliant_StaleEntry(t *testing.T) { + path := newRunFixture(t, true) + + var buf bytes.Buffer + code, err := run([]string{path}, map[string]bool{"vendora": true}, &buf) + if err != nil { + t.Fatalf("run: %v", err) + } + if code != 1 { + t.Errorf("exit code = %d, want 1 for a stale allowlist entry", code) + } + if !strings.Contains(buf.String(), "stale entry") { + t.Errorf("expected a stale-entry note, got %q", buf.String()) + } +} diff --git a/hack/verify-all.sh b/hack/verify-all.sh index 72ee895630..2a432438d4 100755 --- a/hack/verify-all.sh +++ b/hack/verify-all.sh @@ -33,3 +33,5 @@ bash "$REPO_ROOT/hack/verify-license.sh" bash "$REPO_ROOT/hack/verify-import-aliases.sh" bash "$REPO_ROOT/hack/verify-rbac.sh" + +bash "$REPO_ROOT/hack/verify-quota.sh" diff --git a/hack/verify-quota.sh b/hack/verify-quota.sh new file mode 100755 index 0000000000..f964671f8d --- /dev/null +++ b/hack/verify-quota.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Copyright 2024 The HAMi Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script verifies that every pkg/device//device.go backend +# re-checks namespace ResourceQuota usage in its Fit() implementation, to +# guard against the check silently regressing for a vendor that has it, or +# never landing for a vendor that doesn't. +# +# It runs the quotacheck tool, which uses Go AST analysis to confirm each +# backend's Fit() method reaches a call to the shared +# device.QuotaManager.FitQuota re-check, and that what the re-check returns +# actually rejects the candidate device rather than being discarded. +# +# ALLOWED_VENDORS lists backends tracked by #2829 that don't call the +# re-check yet; each gets its own fix PR, and removes itself from this list +# when it lands. quotacheck fails if a listed vendor starts passing (a +# stale entry) or an unlisted vendor fails (a new or regressed backend). +# +# 11 of the 13 backends under pkg/device are listed. The two that are not, +# nvidia and cambricon, already re-check (#2536). Keep this list in sync with +# the nonCompliant set in hack/tools/quotacheck/main_test.go, which asserts +# the same split against the real backends. +ALLOWED_VENDORS="amd,ascend,awsneuron,biren,enflame,hygon,iluvatar,kunlun,metax,mthreads,vastai" + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. + +cd "${REPO_ROOT}" + +echo "Running ResourceQuota re-check verification..." +go run ./hack/tools/quotacheck/ -allow "${ALLOWED_VENDORS}"