diff --git a/build_defs/cgo.build_defs b/build_defs/cgo.build_defs index 1676732b..802483ed 100644 --- a/build_defs/cgo.build_defs +++ b/build_defs/cgo.build_defs @@ -74,18 +74,8 @@ def cgo_library(name:str, srcs:list=[], resources:list=None, go_srcs:list=[], c_ 'c': [subdir2 + src.replace('.go', '.cgo2.c') for src in file_srcs] + [subdir2 + '_cgo_export.c'], 'h': [subdir2 + '_cgo_export.h'], }, - cmd = ' && '.join([ - (f'OUT_DIR="$TMP_DIR/{subdir}"') if subdir else 'OUT_DIR="$TMP_DIR"', - 'mkdir -p "$OUT_DIR"', - 'cd $PKG_DIR/' + subdir, - f'$TOOL tool cgo -objdir "$OUT_DIR" -importpath {import_path} -- {compiler_flags_cmd} *.go', - # Remove the .o file which BSD sed gets upset about in the next command - 'rm -f "$OUT_DIR"/_cgo_.o "$OUT_DIR"/_cgo_main.c', - # cgo leaves absolute paths in these files which we must get rid of :( - 'find "$OUT_DIR" -type f -maxdepth 1 | xargs sed -i -e "s|"$TMP_DIR"/||g"', - 'cd "$TMP_DIR"', - f'ls {subdir2}*.c {subdir2}*.go', - ]), + cmd = _cgo_cmd(subdir, subdir2, import_path, compiler_flags_cmd, " ".join([f"'{flag}'" for flag in linker_flags])), + deps = deps, tools = [CONFIG.GO.GO_TOOL], post_build = post_build if file_srcs != srcs else None, requires = ['go', 'cc_hdrs'], @@ -99,7 +89,7 @@ def cgo_library(name:str, srcs:list=[], resources:list=None, go_srcs:list=[], c_ compiler_flags = compiler_flags + [ '-Wno-error', '-Wno-unused-parameter', # Generated code doesn't compile clean - ], + ] + linker_flags, pkg_config_libs = pkg_config, test_only = test_only, deps = deps, @@ -143,3 +133,28 @@ def cgo_library(name:str, srcs:list=[], resources:list=None, go_srcs:list=[], c_ deps = deps, exported_deps=[import_config], ) + +def _cgo_cmd(subdir, subdir2, import_path, compiler_flags_cmd, linker_flags=None): + linker_flags = linker_flags or "" + return ' && '.join([ + f'export CGO_LDFLAGS="{linker_flags}"', + f'OUT_DIR="$TMP_DIR/{subdir}"' if subdir else 'OUT_DIR="$TMP_DIR"', + 'mkdir -p "$OUT_DIR"', + 'cd $PKG_DIR/' + subdir, + f'$TOOL tool cgo -objdir "$OUT_DIR" -importpath {import_path} -- {compiler_flags_cmd} *.go', + # Remove the .o file which BSD sed gets upset about in the next command + 'rm -f "$OUT_DIR"/_cgo_.o "$OUT_DIR"/_cgo_main.c', + # cgo leaves absolute paths in these files which we must get rid of :( + 'find "$OUT_DIR" -maxdepth 1 -type f | xargs sed -i -e "s|"$TMP_DIR"/||g"', + 'cd "$TMP_DIR"', + f'ls {subdir2}*.c {subdir2}*.go', + ]) + +def _collect_cgo_linker_flags(subdir, subdir2, import_path, compiler_flags_cmd): + """Returns a pre-build function to apply transitive linker flags to a go_binary rule.""" + def collect_linker_flags(name): + ldflags, _ = _get_ldflags_and_pkgconfig(name) + if ldflags: + set_command(name, _cgo_cmd(subdir, subdir2, import_path, compiler_flags_cmd)) + return collect_linker_flags + diff --git a/build_defs/go.build_defs b/build_defs/go.build_defs index 70c50e71..179b11f0 100644 --- a/build_defs/go.build_defs +++ b/build_defs/go.build_defs @@ -1184,18 +1184,21 @@ def go_repo(module: str, version:str='', download:str=None, name:str=None, insta else: modFileArg = "" + src_root = f"pkg/{CONFIG.OS}_{CONFIG.ARCH}/{module}" + requirements = " ".join(requirements) repo = build_rule( name = name, tag = "repo" if install else None, srcs = srcs, - cmd = f"rm -rf $SRCS_DOWNLOAD/.plzconfig && find $SRCS_DOWNLOAD -name BUILD -delete && $TOOL generate {modFileArg} --module {module} --version '{version}' --src_root=$SRCS_DOWNLOAD {install_args} {requirements} && mv $SRCS_DOWNLOAD $OUT", + cmd = f"rm -rf $SRCS_DOWNLOAD/.plzconfig && find $SRCS_DOWNLOAD -name BUILD -delete && mkdir -p $SRC_ROOT && mv $SRCS_DOWNLOAD/* $SRC_ROOT && $TOOL generate {modFileArg} --module {module} --version '{version}' --src_root=$SRC_ROOT {install_args} {requirements} && mv $SRC_ROOT $OUT", outs = [subrepo_name], tools = [CONFIG.GO.PLEASE_GO_TOOL], env= { "GOOS": CONFIG.OS, "GOARCH": CONFIG.ARCH, "CGO_ENABLED": CONFIG.GO.CGO_ENABLED, + "SRC_ROOT": src_root, }, deps = [CONFIG.GO.MOD_FILE] if CONFIG.GO.MOD_FILE else None, _subrepo = True, diff --git a/tools/please_go/generate/generate.go b/tools/please_go/generate/generate.go index 5ee42c5a..346e9cdd 100644 --- a/tools/please_go/generate/generate.go +++ b/tools/please_go/generate/generate.go @@ -26,9 +26,18 @@ type Generate struct { knownImportTargets map[string]string // cache these so we don't end up looping over all the modules for every import thirdPartyFolder string install []string + // Any paths included via -I or -L in this module that we should generate filegroups for + localIncludePaths map[string]struct{} } func New(srcRoot, thirdPartyFolder, modFile, module, version string, buildFileNames, moduleDeps, install []string) *Generate { + // Ensure the srcRoot is absolute otherwise build.Import behaves badly + wd, err := os.Getwd() + if err != nil { + panic(err) + } + srcRoot = filepath.Join(wd, srcRoot) + moduleArg := module if version != "" { moduleArg += "@" + version @@ -44,6 +53,7 @@ func New(srcRoot, thirdPartyFolder, modFile, module, version string, buildFileNa install: install, moduleName: module, moduleArg: moduleArg, + localIncludePaths: map[string]struct{}{}, } } @@ -59,9 +69,61 @@ func (g *Generate) Generate() error { if err := g.generateAll(g.srcRoot); err != nil { return err } + if err := g.generateIncludeFilegroups(); err != nil { + return err + } return g.writeInstallFilegroup() } +func (g *Generate) generateIncludeFilegroups() error { + for pkg := range g.localIncludePaths { + if err := g.generateIncludeFilegroup(pkg); err != nil { + return err + } + } + return nil +} + +func (g *Generate) generateIncludeFilegroup(pkg string) error { + var srcs []string + var exportedDeps []string + + entries, err := os.ReadDir(filepath.Join(g.srcRoot, pkg)) + if err != nil { + return err + } + + for _, entry := range entries { + if !entry.IsDir() { + if g.isBuildFile(entry.Name()) { + continue + } + srcs = append(srcs, entry.Name()) + continue + } + + if err := g.generateIncludeFilegroup(filepath.Join(pkg, entry.Name())); err != nil { + return err + } + + exportedDeps = append(exportedDeps, buildTarget("inc", filepath.Join(pkg, entry.Name()), "")) + } + + file, err := parseOrCreateBuildFile(filepath.Join(g.srcRoot, pkg), g.buildFileNames) + if err != nil { + return err + } + + rule := NewRule("filegroup", "inc") + rule.SetAttr("srcs", NewStringList(srcs)) + rule.SetAttr("exported_deps", NewStringList(exportedDeps)) + rule.SetAttr("visibility", NewStringList([]string{"PUBLIC"})) + + file.Stmt = append(file.Stmt, rule.Call) + + return saveBuildFile(file) +} + func (g *Generate) installTargets() []string { var targets []string @@ -193,8 +255,23 @@ func (g *Generate) pkgDir(target string) string { } func (g *Generate) importDir(target string) (*build.Package, error) { - dir := filepath.Join(os.Getenv("TMP_DIR"), g.pkgDir(target)) - return g.buildContext.ImportDir(dir, 0) + i := "./" + trimPath(target, g.moduleName) + if i == "./." { + i = "." + } + pkg, err := g.buildContext.Import("./"+i, g.srcRoot, 0) + if err != nil { + return nil, err + } + + // We import with an absolute source dir which means some -I and -L args will include the current tmp dir. We should + // replace those with an env var that will be expanded correctly at build time. + replaceTmpDirWithEnvVar(pkg.CgoCFLAGS) + replaceTmpDirWithEnvVar(pkg.CgoCPPFLAGS) + replaceTmpDirWithEnvVar(pkg.CgoCXXFLAGS) + replaceTmpDirWithEnvVar(pkg.CgoLDFLAGS) + + return pkg, nil } func (g *Generate) generate(dir string) error { @@ -211,6 +288,21 @@ func (g *Generate) generate(dir string) error { return g.createBuildFile(dir, lib) } +func (g *Generate) getIncludeDepsFromCFlags(pkg *build.Package) []string { + goRootPath := `"$TMP_DIR"/pkg/` + g.buildContext.GOOS + "_" + g.buildContext.GOARCH + "/" + + var ret []string + for _, flag := range append(pkg.CgoCFLAGS, pkg.CgoLDFLAGS...) { + if strings.HasPrefix(flag, "-I"+goRootPath) { + ret = append(ret, g.includeTarget(strings.TrimPrefix(flag, "-I"+goRootPath))) + } + if strings.HasPrefix(flag, "-L"+goRootPath) { + ret = append(ret, g.includeTarget(strings.TrimPrefix(flag, "-L"+goRootPath))) + } + } + return ret +} + func (g *Generate) matchesInstall(dir string) bool { for _, i := range g.install { i := filepath.Join(g.srcRoot, i) @@ -354,12 +446,77 @@ func (g *Generate) ruleForPackage(pkg *build.Package, dir string) *Rule { pkgConfigs: pkg.CgoPkgConfig, asmFiles: pkg.SFiles, hdrs: pkg.HFiles, - deps: g.depTargets(pkg.Imports), + deps: append(g.getIncludeDepsFromCFlags(pkg), g.depTargets(pkg.Imports)...), embedPatterns: pkg.EmbedPatterns, isCMD: pkg.IsCommand(), } } +// replaceTmpDirWithEnvVar replaces the current wd with the env var $TMP_DIR. This helps in cases where ${SRC_DIR} has +// been resolved to the absolute path based on the current build directory. To make this portable, it needs to use the +// env var which will be expanded at build time. +func replaceTmpDirWithEnvVar(flags []string) { + tmpDir, err := os.Getwd() + if err != nil { + panic(err) + } + for i := range flags { + flags[i] = strings.ReplaceAll(flags[i], tmpDir, `"$TMP_DIR"`) + } +} + +func (g *Generate) testRule(pkg *build.Package, prodRule *Rule) *Rule { + // The name of the target should match the dir it's in, or the basename of the module if it's in the repo root. + name := filepath.Base(pkg.Dir) + if strings.HasSuffix(pkg.Dir, g.srcRoot) || name == "" { + name = filepath.Base(g.moduleName) + } + + if len(pkg.TestGoFiles) == 0 { + return nil + } + deps := g.depTargets(pkg.TestImports) + + if prodRule != nil { + deps = append(deps, ":"+prodRule.name) + } + // TODO(jpoole): handle external tests + return &Rule{ + name: name, + kind: "go_test", + srcs: pkg.TestGoFiles, + deps: deps, + embedPatterns: pkg.TestEmbedPatterns, + } +} + +func (g *Generate) includeTarget(path string) string { + module := "" + for _, mod := range append(g.moduleDeps, g.moduleName) { + if strings.HasPrefix(path, mod) { + if len(module) < len(mod) { + module = mod + } + } + } + + if module == "" { + // If we can't find this import, we can return nothing and the build rule will fail at build time reporting a + // sensible error. It may also be an import from the go SDK which is fine. + return "" + } + + subrepoName := g.subrepoName(module) + packageName := strings.TrimPrefix(path, module) + packageName = strings.TrimPrefix(packageName, "/") + + target := buildTarget("inc", packageName, subrepoName) + if module == g.moduleName { + g.localIncludePaths[packageName] = struct{}{} + } + return target +} + func (g *Generate) depTarget(importPath string) string { if target, ok := g.knownImportTargets[importPath]; ok { return target