// Copyright 2023 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package python import ( "fmt" "log" "os" "path/filepath" "strings" "github.com/bazelbuild/bazel-gazelle/config" "github.com/bazelbuild/bazel-gazelle/label" "github.com/bazelbuild/bazel-gazelle/repo" "github.com/bazelbuild/bazel-gazelle/resolve" "github.com/bazelbuild/bazel-gazelle/rule" bzl "github.com/bazelbuild/buildtools/build" "github.com/emirpasic/gods/sets/treeset" godsutils "github.com/emirpasic/gods/utils" "github.com/bazelbuild/rules_python/gazelle/pythonconfig" ) const languageName = "py" const ( // resolvedDepsKey is the attribute key used to pass dependencies that don't // need to be resolved by the dependency resolver in the Resolver step. resolvedDepsKey = "_gazelle_python_resolved_deps" ) // Resolver satisfies the resolve.Resolver interface. It resolves dependencies // in rules generated by this extension. type Resolver struct{} // Name returns the name of the language. This is the prefix of the kinds of // rules generated. E.g. py_library and py_binary. func (*Resolver) Name() string { return languageName } // Imports returns a list of ImportSpecs that can be used to import the rule // r. This is used to populate RuleIndex. // // If nil is returned, the rule will not be indexed. If any non-nil slice is // returned, including an empty slice, the rule will be indexed. func (py *Resolver) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { cfgs := c.Exts[languageName].(pythonconfig.Configs) cfg := cfgs[f.Pkg] srcs := r.AttrStrings("srcs") provides := make([]resolve.ImportSpec, 0, len(srcs)+1) for _, src := range srcs { ext := filepath.Ext(src) if ext != ".py" { continue } if cfg.PerFileGeneration() && len(srcs) > 1 && src == pyLibraryEntrypointFilename { // Do not provide import spec from __init__.py when it is being included as // part of another module. continue } pythonProjectRoot := cfg.PythonProjectRoot() provide := importSpecFromSrc(pythonProjectRoot, f.Pkg, src) provides = append(provides, provide) } if len(provides) == 0 { return nil } return provides } // importSpecFromSrc determines the ImportSpec based on the target that contains the src so that // the target can be indexed for import statements that match the calculated src relative to the its // Python project root. func importSpecFromSrc(pythonProjectRoot, bzlPkg, src string) resolve.ImportSpec { pythonPkgDir := filepath.Join(bzlPkg, filepath.Dir(src)) relPythonPkgDir, err := filepath.Rel(pythonProjectRoot, pythonPkgDir) if err != nil { panic(fmt.Errorf("unexpected failure: %v", err)) } if relPythonPkgDir == "." { relPythonPkgDir = "" } pythonPkg := strings.ReplaceAll(relPythonPkgDir, "/", ".") filename := filepath.Base(src) if filename == pyLibraryEntrypointFilename { if pythonPkg != "" { return resolve.ImportSpec{ Lang: languageName, Imp: pythonPkg, } } } moduleName := strings.TrimSuffix(filename, ".py") var imp string if pythonPkg == "" { imp = moduleName } else { imp = fmt.Sprintf("%s.%s", pythonPkg, moduleName) } return resolve.ImportSpec{ Lang: languageName, Imp: imp, } } // Embeds returns a list of labels of rules that the given rule embeds. If // a rule is embedded by another importable rule of the same language, only // the embedding rule will be indexed. The embedding rule will inherit // the imports of the embedded rule. func (py *Resolver) Embeds(r *rule.Rule, from label.Label) []label.Label { // TODO(f0rmiga): implement. return make([]label.Label, 0) } // Resolve translates imported libraries for a given rule into Bazel // dependencies. Information about imported libraries is returned for each // rule generated by language.GenerateRules in // language.GenerateResult.Imports. Resolve generates a "deps" attribute (or // the appropriate language-specific equivalent) for each import according to // language-specific rules and heuristics. func (py *Resolver) Resolve( c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, modulesRaw interface{}, from label.Label, ) { // TODO(f0rmiga): may need to be defensive here once this Gazelle extension // join with the main Gazelle binary with other rules. It may conflict with // other generators that generate py_* targets. deps := treeset.NewWith(godsutils.StringComparator) if modulesRaw != nil { cfgs := c.Exts[languageName].(pythonconfig.Configs) cfg := cfgs[from.Pkg] pythonProjectRoot := cfg.PythonProjectRoot() modules := modulesRaw.(*treeset.Set) it := modules.Iterator() explainDependency := os.Getenv("EXPLAIN_DEPENDENCY") hasFatalError := false MODULES_LOOP: for it.Next() { mod := it.Value().(module) moduleParts := strings.Split(mod.Name, ".") possibleModules := []string{mod.Name} for len(moduleParts) > 1 { // Iterate back through the possible imports until // a match is found. // For example, "from foo.bar import baz" where baz is a module, we should try `foo.bar.baz` first, then // `foo.bar`, then `foo`. // In the first case, the import could be file `baz.py` in the directory `foo/bar`. // Or, the import could be variable `baz` in file `foo/bar.py`. // The import could also be from a standard module, e.g. `six.moves`, where // the dependency is actually `six`. moduleParts = moduleParts[:len(moduleParts)-1] possibleModules = append(possibleModules, strings.Join(moduleParts, ".")) } errs := []error{} POSSIBLE_MODULE_LOOP: for _, moduleName := range possibleModules { imp := resolve.ImportSpec{Lang: languageName, Imp: moduleName} if override, ok := resolve.FindRuleWithOverride(c, imp, languageName); ok { if override.Repo == "" { override.Repo = from.Repo } if !override.Equal(from) { if override.Repo == from.Repo { override.Repo = "" } dep := override.Rel(from.Repo, from.Pkg).String() deps.Add(dep) if explainDependency == dep { log.Printf("Explaining dependency (%s): "+ "in the target %q, the file %q imports %q at line %d, "+ "which resolves using the \"gazelle:resolve\" directive.\n", explainDependency, from.String(), mod.Filepath, moduleName, mod.LineNumber) } continue MODULES_LOOP } } else { if dep, ok := cfg.FindThirdPartyDependency(moduleName); ok { deps.Add(dep) if explainDependency == dep { log.Printf("Explaining dependency (%s): "+ "in the target %q, the file %q imports %q at line %d, "+ "which resolves from the third-party module %q from the wheel %q.\n", explainDependency, from.String(), mod.Filepath, moduleName, mod.LineNumber, mod.Name, dep) } continue MODULES_LOOP } else { matches := ix.FindRulesByImportWithConfig(c, imp, languageName) if len(matches) == 0 { // Check if the imported module is part of the standard library. if isStdModule(module{Name: moduleName}) { continue MODULES_LOOP } else if cfg.ValidateImportStatements() { err := fmt.Errorf( "%[1]q, line %[2]d: %[3]q is an invalid dependency: possible solutions:\n"+ "\t1. Add it as a dependency in the requirements.txt file.\n"+ "\t2. Use the '# gazelle:resolve py %[3]s TARGET_LABEL' BUILD file directive to resolve to a known dependency.\n"+ "\t3. Ignore it with a comment '# gazelle:ignore %[3]s' in the Python file.\n", mod.Filepath, mod.LineNumber, moduleName, ) errs = append(errs, err) continue POSSIBLE_MODULE_LOOP } } filteredMatches := make([]resolve.FindResult, 0, len(matches)) for _, match := range matches { if match.IsSelfImport(from) { // Prevent from adding itself as a dependency. continue MODULES_LOOP } filteredMatches = append(filteredMatches, match) } if len(filteredMatches) == 0 { continue POSSIBLE_MODULE_LOOP } if len(filteredMatches) > 1 { sameRootMatches := make([]resolve.FindResult, 0, len(filteredMatches)) for _, match := range filteredMatches { if strings.HasPrefix(match.Label.Pkg, pythonProjectRoot) { sameRootMatches = append(sameRootMatches, match) } } if len(sameRootMatches) != 1 { err := fmt.Errorf( "%[1]q, line %[2]d: multiple targets (%[3]s) may be imported with %[4]q: possible solutions:\n"+ "\t1. Disambiguate the above multiple targets by removing duplicate srcs entries.\n"+ "\t2. Use the '# gazelle:resolve py %[4]s TARGET_LABEL' BUILD file directive to resolve to one of the above targets.\n", mod.Filepath, mod.LineNumber, targetListFromResults(filteredMatches), moduleName) errs = append(errs, err) continue POSSIBLE_MODULE_LOOP } filteredMatches = sameRootMatches } matchLabel := filteredMatches[0].Label.Rel(from.Repo, from.Pkg) dep := matchLabel.String() deps.Add(dep) if explainDependency == dep { log.Printf("Explaining dependency (%s): "+ "in the target %q, the file %q imports %q at line %d, "+ "which resolves from the first-party indexed labels.\n", explainDependency, from.String(), mod.Filepath, moduleName, mod.LineNumber) } continue MODULES_LOOP } } } // End possible modules loop. if len(errs) > 0 { // If, after trying all possible modules, we still haven't found anything, error out. joinedErrs := "" for _, err := range errs { joinedErrs = fmt.Sprintf("%s%s\n", joinedErrs, err) } log.Printf("ERROR: failed to validate dependencies for target %q:\n\n%v", from.String(), joinedErrs) hasFatalError = true } } if hasFatalError { os.Exit(1) } } resolvedDeps := r.PrivateAttr(resolvedDepsKey).(*treeset.Set) if !resolvedDeps.Empty() { it := resolvedDeps.Iterator() for it.Next() { deps.Add(it.Value()) } } if !deps.Empty() { r.SetAttr("deps", convertDependencySetToExpr(deps)) } } // targetListFromResults returns a string with the human-readable list of // targets contained in the given results. func targetListFromResults(results []resolve.FindResult) string { list := make([]string, len(results)) for i, result := range results { list[i] = result.Label.String() } return strings.Join(list, ", ") } // convertDependencySetToExpr converts the given set of dependencies to an // expression to be used in the deps attribute. func convertDependencySetToExpr(set *treeset.Set) bzl.Expr { deps := make([]bzl.Expr, set.Size()) it := set.Iterator() for it.Next() { dep := it.Value().(string) deps[it.Index()] = &bzl.StringExpr{Value: dep} } return &bzl.ListExpr{List: deps} }