compiler_wrapper: automatic sync
This CL automatically brings toolchain-utils' compiler_wrapper/
directory in sync with chromiumos-overlay's. Please see
go/crostc-repo/+/main:sync_compiler_wrapper_within_cros.sh for more info
on this process.
BUG=None
TEST=None
Change-Id: I81b8f6c5c407b414f647f525fffd6fd7bcc446e6
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/5359381
Commit-Queue: George Burgess <[email protected]>
Tested-by: mobiletc-prebuild Role Account <[email protected]>
Auto-Submit: mobiletc-prebuild Role Account <[email protected]>
Reviewed-by: George Burgess <[email protected]>
diff --git a/compiler_wrapper/README.md b/compiler_wrapper/README.md
index 7a05c81..38ff8a6 100644
--- a/compiler_wrapper/README.md
+++ b/compiler_wrapper/README.md
@@ -2,12 +2,9 @@
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
+### What
Toolchain utils compiler wrapper sources.
-Build the wrapper:
-./build.py --config=<config name> --use_ccache=<bool> \
- --use_llvm_next=<bool> --output_file=<file>
-
Please note that there's a regular syncing operation between
`chromiumos-overlay/sys-devel/llvm/files/compiler_wrapper` and
`toolchain-utils/compiler_wrapper`. This sync is one way (from
@@ -15,3 +12,33 @@
toolchain keep up-to-date with our wrapper easily, as they're a downstream
consumer of it. For this reason, **please be sure to land all actual changes in
chromeos-overlay**.
+
+### Build + Run Tests
+1. Install the wrapper locally in chroot (builds as well)
+```
+(chroot) ./install_compiler_wrapper.sh
+```
+
+#### Running a manual test
+Test a manual build command with `-print-cmdline`
+```
+(chroot) x86_64-cros-linux-gnu-clang++ -o test_exec -f<some_flag_to_add>='some_value' -print-cmdline test.cc
+```
+- `test.cc` doesn't actually have to exist.
+- The command above will output the additional build flags that are added in by the wrapper.
+
+#### Testing your changes
+1. Add tests to your wrapper changes
+1. Run all the tests via:
+```
+go test -vet=all
+```
+
+### Build Only
+This is handy if you just want to test that the build works.
+
+Build the wrapper:
+```
+./build.py --config=<config name> --use_ccache=<bool> \
+ --use_llvm_next=<bool> --output_file=<file>
+ ```
diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go
index 22109e3..7f2c8d1 100644
--- a/compiler_wrapper/compiler_wrapper.go
+++ b/compiler_wrapper/compiler_wrapper.go
@@ -168,8 +168,7 @@
return 0, newErrorwithSourceLocf("unsupported compiler: %s", mainBuilder.target.compiler)
}
} else {
- _, tidyFlags, tidyMode := processClangTidyFlags(mainBuilder)
- cSrcFile, iwyuFlags, iwyuMode := processIWYUFlags(mainBuilder)
+ cSrcFile, tidyFlags, tidyMode := processClangTidyFlags(mainBuilder)
crashArtifactsDir := detectCrashArtifactsDir(env, cfg)
if mainBuilder.target.compilerType == clangType {
err := prepareClangCommand(crashArtifactsDir, mainBuilder)
@@ -200,19 +199,6 @@
}
}
- if iwyuMode != iwyuModeNone {
- if iwyuMode == iwyuModeError {
- panic("Unknown IWYU mode")
- }
-
- allowCCache = false
- clangCmdWithoutRemoteBuildAndCCache := mainBuilder.build()
- err := runIWYU(env, clangCmdWithoutRemoteBuildAndCCache, cSrcFile, iwyuFlags)
- if err != nil {
- return 0, err
- }
- }
-
if remoteBuildUsed, err = processRemoteBuildAndCCacheFlags(allowCCache, mainBuilder); err != nil {
return 0, err
}
@@ -350,9 +336,9 @@
}
}
-func hasFlag(flag string, flagList ...string) bool {
- for _, flagVal := range flagList {
- if strings.Contains(flagVal, flag) {
+func hasUserArg(argName string, builder *commandBuilder) bool {
+ for _, argValue := range builder.args {
+ if strings.Contains(argValue.value, argName) && argValue.fromUser {
return true
}
}
@@ -367,8 +353,7 @@
var crashDiagFlagName = "-fcrash-diagnostics-dir"
if crashArtifactsDir != "" &&
- !hasFlag(crashDiagFlagName, builder.cfg.clangFlags...) &&
- !hasFlag(crashDiagFlagName, builder.cfg.clangPostFlags...) {
+ !hasUserArg(crashDiagFlagName, builder) {
builder.addPreUserArgs(crashDiagFlagName + "=" + crashArtifactsDir)
}
diff --git a/compiler_wrapper/compiler_wrapper_test.go b/compiler_wrapper/compiler_wrapper_test.go
index 2cace6e..79edab6 100644
--- a/compiler_wrapper/compiler_wrapper_test.go
+++ b/compiler_wrapper/compiler_wrapper_test.go
@@ -253,36 +253,6 @@
}
}
-// If "crash-diagnostics-dir" flag is already provided, only use that flag and don't add a dupe
-func TestCrashDiagPreFlag(t *testing.T) {
- withTestContext(t, func(ctx *testContext) {
- ctx.cfg.clangFlags = []string{"-fcrash-diagnostics-dir=/build/something/foo"}
- ctx.env = []string{
- "CROS_ARTIFACTS_TMP_DIR=/tmp/foo",
- }
- cmd := ctx.must(callCompiler(ctx, ctx.cfg,
- ctx.newCommand(clangX86_64, mainCc)))
- if err := verifyArgCount(cmd, 1, "-fcrash-diagnostics-dir=/build/something/foo"); err != nil {
- t.Error(err)
- }
- })
-}
-
-// If "crash-diagnostics-dir" flag is already provided, only use that flag and don't add a dupe
-func TestCrashDiagPostFlag(t *testing.T) {
- withTestContext(t, func(ctx *testContext) {
- ctx.cfg.clangPostFlags = []string{"-fcrash-diagnostics-dir=/build/something/foo"}
- ctx.env = []string{
- "CROS_ARTIFACTS_TMP_DIR=/tmp/foo",
- }
- cmd := ctx.must(callCompiler(ctx, ctx.cfg,
- ctx.newCommand(clangX86_64, mainCc)))
- if err := verifyArgCount(cmd, 1, "-fcrash-diagnostics-dir=/build/something/foo"); err != nil {
- t.Error(err)
- }
- })
-}
-
// If "crash-diagnostics-dir" flag is not provided, add one in
func TestCrashDiagDefault(t *testing.T) {
withTestContext(t, func(ctx *testContext) {
@@ -291,8 +261,31 @@
}
cmd := ctx.must(callCompiler(ctx, ctx.cfg,
ctx.newCommand(clangX86_64, mainCc)))
+
+ // Verify that we added the default flag
if err := verifyArgCount(cmd, 1, "-fcrash-diagnostics-dir=/tmp/foo/toolchain/clang_crash_diagnostics"); err != nil {
t.Error(err)
}
})
}
+
+// If "crash-diagnostics-dir" flag is already provided by the user, only use that flag and don't add a dupe
+func TestCrashDiagUserFlag(t *testing.T) {
+ withTestContext(t, func(ctx *testContext) {
+ ctx.env = []string{
+ "CROS_ARTIFACTS_TMP_DIR=/tmp/foo",
+ }
+ cmd := ctx.must(callCompiler(ctx, ctx.cfg,
+ ctx.newCommand(clangX86_64, mainCc, "-fcrash-diagnostics-dir=/build/something/foozz")))
+
+ // Verify that user flag is not removed
+ if err := verifyArgCount(cmd, 1, "-fcrash-diagnostics-dir=/build/something/foozz"); err != nil {
+ t.Error(err)
+ }
+
+ // Verify that we did not add the default flag
+ if err := verifyArgCount(cmd, 0, "-fcrash-diagnostics-dir=/tmp/foo/toolchain/clang_crash_diagnostics"); err != nil {
+ t.Error(err)
+ }
+ })
+}
diff --git a/compiler_wrapper/iwyu_flag.go b/compiler_wrapper/iwyu_flag.go
deleted file mode 100644
index 5788d8c..0000000
--- a/compiler_wrapper/iwyu_flag.go
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright 2022 The ChromiumOS Authors
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package main
-
-import (
- "bufio"
- "bytes"
- "fmt"
- "os"
- "path/filepath"
- "strings"
-)
-
-type useIWYUMode int
-
-const (
- iwyuModeNone useIWYUMode = iota
- iwyuModeAll
- iwyuModeError
-)
-
-var srcFileSuffixes = []string{
- ".c",
- ".cc",
- ".cpp",
- ".C",
- ".cxx",
- ".c++",
-}
-
-func findWithIWYUFlag(args []builderArg) (string, []builderArg) {
- for i := range args {
- if args[i].value == "--with-iwyu" {
- args = append(args[:i], args[i+1:]...)
- return "1", args
- }
- }
- return "", args
-}
-
-func processIWYUFlags(builder *commandBuilder) (cSrcFile string, iwyuFlags []string, mode useIWYUMode) {
- builder.transformArgs(func(arg builderArg) string {
- const prefix = "-iwyu-flag="
- if !strings.HasPrefix(arg.value, prefix) {
- return arg.value
- }
-
- iwyuFlags = append(iwyuFlags, arg.value[len(prefix):])
- return ""
- })
-
- cSrcFile = ""
- lastArg := ""
- for _, arg := range builder.args {
- if lastArg != "-o" {
- for _, suffix := range srcFileSuffixes {
- if strings.HasSuffix(arg.value, suffix) {
- cSrcFile = arg.value
- break
- }
- }
- }
- lastArg = arg.value
- }
-
- if cSrcFile == "" {
- return "", iwyuFlags, iwyuModeNone
- }
-
- withIWYU, _ := builder.env.getenv("WITH_IWYU")
- if withIWYU == "" {
- withIWYU, builder.args = findWithIWYUFlag(builder.args)
- if withIWYU == "" {
- return cSrcFile, iwyuFlags, iwyuModeNone
- }
- }
-
- if withIWYU != "1" {
- return cSrcFile, iwyuFlags, iwyuModeError
- }
-
- return cSrcFile, iwyuFlags, iwyuModeAll
-}
-
-func calcIWYUInvocation(env env, clangCmd *command, cSrcFile string, iwyuFlags ...string) (*command, error) {
- resourceDir, err := getClangResourceDir(env, clangCmd.Path)
- if err != nil {
- return nil, err
- }
-
- iwyuPath := filepath.Join(filepath.Dir(clangCmd.Path), "include-what-you-use")
- args := append([]string{}, iwyuFlags...)
- args = append(args, "-resource-dir="+resourceDir)
- args = append(args, clangCmd.Args...)
-
- for i := 0; i < len(args); i++ {
- for j := 0; j < len(srcFileSuffixes); j++ {
- if strings.HasSuffix(args[i], srcFileSuffixes[j]) {
- args = append(args[:i], args[i+1:]...)
- break
- }
- }
- }
- args = append(args, cSrcFile)
-
- return &command{
- Path: iwyuPath,
- Args: args,
- EnvUpdates: clangCmd.EnvUpdates,
- }, nil
-}
-
-func runIWYU(env env, clangCmd *command, cSrcFile string, extraIWYUFlags []string) error {
- extraIWYUFlags = append(extraIWYUFlags, "-Xiwyu", "--mapping_file=/usr/share/include-what-you-use/libcxx.imp", "-Xiwyu", "--no_fwd_decls")
- iwyuCmd, err := calcIWYUInvocation(env, clangCmd, cSrcFile, extraIWYUFlags...)
- if err != nil {
- return fmt.Errorf("calculating include-what-you-use invocation: %v", err)
- }
-
- // Note: We pass nil as stdin as we checked before that the compiler
- // was invoked with a source file argument.
- var stderr bytes.Buffer
- stderrWriter := bufio.NewWriter(&stderr)
- exitCode, err := wrapSubprocessErrorWithSourceLoc(iwyuCmd,
- env.run(iwyuCmd, nil, nil, stderrWriter))
- stderrMessage := stderr.String()
- fmt.Fprintln(env.stderr(), stderrMessage)
-
- if err == nil && exitCode != 0 {
- // Note: We continue on purpose when include-what-you-use fails
- // to maintain compatibility with the previous wrapper.
- fmt.Fprintln(env.stderr(), "include-what-you-use failed")
- }
-
- iwyuDir := filepath.Join(getCompilerArtifactsDir(env), "linting-output", "iwyu")
- if err := os.MkdirAll(iwyuDir, 0777); err != nil {
- return fmt.Errorf("creating fixes directory at %q: %v", iwyuDir, err)
- }
-
- f, err := os.CreateTemp(iwyuDir, "*.out")
- if err != nil {
- return fmt.Errorf("making output file for iwyu: %v", err)
- }
- writer := bufio.NewWriter(f)
- if _, err := writer.WriteString(stderrMessage); err != nil {
- return fmt.Errorf("writing output file for iwyu: %v", err)
- }
- if err := writer.Flush(); err != nil {
- return fmt.Errorf("flushing output file buffer for iwyu: %v", err)
- }
- if err := f.Close(); err != nil {
- return fmt.Errorf("finalizing output file for iwyu: %v", err)
- }
-
- return err
-}
diff --git a/compiler_wrapper/iwyu_flag_test.go b/compiler_wrapper/iwyu_flag_test.go
deleted file mode 100644
index e2db3b4..0000000
--- a/compiler_wrapper/iwyu_flag_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright 2022 The ChromiumOS Authors
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package main
-
-import (
- "errors"
- "io"
- "strings"
- "testing"
-)
-
-func TestIWYUArgOrder(t *testing.T) {
- withIWYUTestContext(t, func(ctx *testContext) {
- ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
- if ctx.cmdCount == 2 {
- if err := verifyArgOrder(cmd, "-checks=.*", mainCc, "--", "-resource-dir=.*", mainCc, "--some_arg"); err != nil {
- return err
- }
- }
- return nil
- }
- ctx.must(callCompiler(ctx, ctx.cfg,
- ctx.newCommand(clangX86_64, mainCc, "--some_arg")))
- if ctx.cmdCount < 2 {
- t.Error("expected multiple calls.")
- }
- })
-}
-
-func TestIgnoreNonZeroExitCodeFromIWYU(t *testing.T) {
- withIWYUTestContext(t, func(ctx *testContext) {
- ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
- if ctx.cmdCount == 2 {
- return newExitCodeError(23)
- }
- return nil
- }
- ctx.must(callCompiler(ctx, ctx.cfg,
- ctx.newCommand(clangX86_64, mainCc)))
- stderr := ctx.stderrString()
- if err := verifyNonInternalError(stderr, "include-what-you-use failed"); err != nil {
- t.Error(err)
- }
- })
-}
-
-func TestReportGeneralErrorsFromIWYU(t *testing.T) {
- withIWYUTestContext(t, func(ctx *testContext) {
- ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
- if ctx.cmdCount > 1 {
- return errors.New("someerror")
- }
- return nil
- }
- stderr := ctx.mustFail(callCompiler(ctx, ctx.cfg,
- ctx.newCommand(clangX86_64, mainCc)))
- if err := verifyInternalError(stderr); err != nil {
- t.Fatal(err)
- }
- if !strings.Contains(stderr, "someerror") {
- t.Errorf("unexpected error. Got: %s", stderr)
- }
- })
-}
-
-func TestUseIWYUBasedOnFileExtension(t *testing.T) {
- withIWYUTestContext(t, func(ctx *testContext) {
- testData := []struct {
- args []string
- iwyu bool
- }{
- {[]string{"main.cc"}, true},
- {[]string{"main.cc"}, true},
- {[]string{"main.C"}, true},
- {[]string{"main.cxx"}, true},
- {[]string{"main.c++"}, true},
- {[]string{"main.xy"}, false},
- {[]string{"-o", "main.cc"}, false},
- {[]string{}, false},
- }
- for _, tt := range testData {
- ctx.cmdCount = 0
- ctx.must(callCompiler(ctx, ctx.cfg,
- ctx.newCommand(clangX86_64, tt.args...)))
- if ctx.cmdCount == 2 && !tt.iwyu {
- t.Errorf("expected a call to iwyu but got none for args %s", tt.args)
- }
- if ctx.cmdCount == 1 && tt.iwyu {
- t.Errorf("expected no call to iwyu but got one for args %s", tt.args)
- }
- }
- })
-}
-
-func TestIWYUFiltersIWYUFlags(t *testing.T) {
- withIWYUTestContext(t, func(ctx *testContext) {
- addedFlag := "--some_iwyu_flag=flag"
- ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
- switch ctx.cmdCount {
- case 1:
- if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
- t.Error(err)
- } else if err := verifyArgCount(cmd, 0, addedFlag); err != nil {
- t.Error(err)
- }
- return nil
- case 2:
- if err := verifyPath(cmd, "usr/bin/include-what-you-use"); err != nil {
- t.Error(err)
- } else if verifyArgCount(cmd, 1, addedFlag); err != nil {
- t.Error(err)
- }
- return nil
- default:
- return nil
- }
- }
- cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc, "-iwyu-flag="+addedFlag)))
- if ctx.cmdCount < 2 {
- t.Errorf("expected multiple calls.")
- }
- if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
- t.Error(err)
- }
- })
-}
-
-func withIWYUTestContext(t *testing.T, work func(ctx *testContext)) {
- withTestContext(t, func(ctx *testContext) {
- artifactDir := t.TempDir()
- ctx.env = []string{"WITH_IWYU=1", "CROS_ARTIFACTS_TMP_DIR=" + artifactDir}
- work(ctx)
- })
-}
