Add some tests for singlethreaded usage of the Finder.
Also change the existing tests to use exactly 2 threads.
Also update some tests to also same number of threads as will
be used by the finder.
Bug: 62455338
Test: m -j # which runs unit tests
Change-Id: I2b9d39680ecddd6d28c951df982ac51c077d2777
diff --git a/finder/finder.go b/finder/finder.go
index c2df928..ffda155 100644
--- a/finder/finder.go
+++ b/finder/finder.go
@@ -18,6 +18,7 @@
"bufio"
"bytes"
"encoding/json"
+ "errors"
"fmt"
"io"
"os"
@@ -30,7 +31,6 @@
"time"
"android/soong/fs"
- "errors"
)
// This file provides a Finder struct that can quickly search for files satisfying
@@ -159,11 +159,17 @@
nodes pathMap
}
+var defaultNumThreads = runtime.NumCPU() * 2
+
// New creates a new Finder for use
func New(cacheParams CacheParams, filesystem fs.FileSystem,
logger Logger, dbPath string) (f *Finder, err error) {
+ return newImpl(cacheParams, filesystem, logger, dbPath, defaultNumThreads)
+}
- numThreads := runtime.NumCPU() * 2
+// newImpl is like New but accepts more params
+func newImpl(cacheParams CacheParams, filesystem fs.FileSystem,
+ logger Logger, dbPath string, numThreads int) (f *Finder, err error) {
numDbLoadingThreads := numThreads
numSearchingThreads := numThreads
diff --git a/finder/finder_test.go b/finder/finder_test.go
index 8d1bbd7..1522c68 100644
--- a/finder/finder_test.go
+++ b/finder/finder_test.go
@@ -35,14 +35,18 @@
}
func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Finder {
- f, err := newFinderAndErr(t, filesystem, cacheParams)
+ return newFinderWithNumThreads(t, filesystem, cacheParams, 2)
+}
+
+func newFinderWithNumThreads(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) *Finder {
+ f, err := newFinderAndErr(t, filesystem, cacheParams, numThreads)
if err != nil {
fatal(t, err.Error())
}
return f
}
-func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) (*Finder, error) {
+func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams, numThreads int) (*Finder, error) {
cachePath := "/finder/finder-db"
cacheDir := filepath.Dir(cachePath)
filesystem.MkDirs(cacheDir)
@@ -51,7 +55,7 @@
}
logger := log.New(ioutil.Discard, "", 0)
- f, err := New(cacheParams, filesystem, logger, cachePath)
+ f, err := newImpl(cacheParams, filesystem, logger, cachePath, numThreads)
return f, err
}
@@ -64,11 +68,13 @@
}
func finderAndErrorWithSameParams(t *testing.T, original *Finder) (*Finder, error) {
- f, err := New(
+ f, err := newImpl(
original.cacheMetadata.Config.CacheParams,
original.filesystem,
original.logger,
- original.DbPath)
+ original.DbPath,
+ original.numDbLoadingThreads,
+ )
return f, err
}
@@ -234,6 +240,21 @@
assertSameResponse(t, foundPaths, absoluteMatches)
}
+// testAgainstSeveralThreadcounts runs the given test for each threadcount that we care to test
+func testAgainstSeveralThreadcounts(t *testing.T, tester func(t *testing.T, numThreads int)) {
+ // test singlethreaded, multithreaded, and also using the same number of threads as
+ // will be used on the current system
+ threadCounts := []int{1, 2, defaultNumThreads}
+ for _, numThreads := range threadCounts {
+ testName := fmt.Sprintf("%v threads", numThreads)
+ // store numThreads in a new variable to prevent numThreads from changing in each loop
+ localNumThreads := numThreads
+ t.Run(testName, func(t *testing.T) {
+ tester(t, localNumThreads)
+ })
+ }
+}
+
// end of utils, start of individual tests
func TestSingleFile(t *testing.T) {
@@ -285,24 +306,30 @@
}
func TestFilesystemRoot(t *testing.T) {
- filesystem := newFs()
- root := "/"
- createdPath := "/findme.txt"
- create(t, createdPath, filesystem)
- finder := newFinder(
- t,
- filesystem,
- CacheParams{
- RootDirs: []string{root},
- IncludeFiles: []string{"findme.txt", "skipme.txt"},
- },
- )
- defer finder.Shutdown()
+ testWithNumThreads := func(t *testing.T, numThreads int) {
+ filesystem := newFs()
+ root := "/"
+ createdPath := "/findme.txt"
+ create(t, createdPath, filesystem)
- foundPaths := finder.FindNamedAt(root, "findme.txt")
+ finder := newFinderWithNumThreads(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{root},
+ IncludeFiles: []string{"findme.txt", "skipme.txt"},
+ },
+ numThreads,
+ )
+ defer finder.Shutdown()
- assertSameResponse(t, foundPaths, []string{createdPath})
+ foundPaths := finder.FindNamedAt(root, "findme.txt")
+
+ assertSameResponse(t, foundPaths, []string{createdPath})
+ }
+
+ testAgainstSeveralThreadcounts(t, testWithNumThreads)
}
func TestNonexistentDir(t *testing.T) {
@@ -316,6 +343,7 @@
RootDirs: []string{"/tmp/IDontExist"},
IncludeFiles: []string{"findme.txt", "skipme.txt"},
},
+ 1,
)
if err == nil {
fatal(t, "Did not fail when given a nonexistent root directory")
@@ -380,6 +408,8 @@
"/tmp/include/findme.txt"})
}
+// TestRootDir tests that the value of RootDirs is used
+// tests of the filesystem root are in TestFilesystemRoot
func TestRootDir(t *testing.T) {
filesystem := newFs()
create(t, "/tmp/a/findme.txt", filesystem)
@@ -548,48 +578,54 @@
}
func TestConcurrentFindSameDirectory(t *testing.T) {
- filesystem := newFs()
- // create a bunch of files and directories
- paths := []string{}
- for i := 0; i < 10; i++ {
- parentDir := fmt.Sprintf("/tmp/%v", i)
- for j := 0; j < 10; j++ {
- filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
- paths = append(paths, filePath)
+ testWithNumThreads := func(t *testing.T, numThreads int) {
+ filesystem := newFs()
+
+ // create a bunch of files and directories
+ paths := []string{}
+ for i := 0; i < 10; i++ {
+ parentDir := fmt.Sprintf("/tmp/%v", i)
+ for j := 0; j < 10; j++ {
+ filePath := filepath.Join(parentDir, fmt.Sprintf("%v/findme.txt", j))
+ paths = append(paths, filePath)
+ }
+ }
+ sort.Strings(paths)
+ for _, path := range paths {
+ create(t, path, filesystem)
+ }
+
+ // set up a finder
+ finder := newFinderWithNumThreads(
+ t,
+ filesystem,
+ CacheParams{
+ RootDirs: []string{"/tmp"},
+ IncludeFiles: []string{"findme.txt"},
+ },
+ numThreads,
+ )
+ defer finder.Shutdown()
+
+ numTests := 20
+ results := make(chan []string, numTests)
+ // make several parallel calls to the finder
+ for i := 0; i < numTests; i++ {
+ go func() {
+ foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
+ results <- foundPaths
+ }()
+ }
+
+ // check that each response was correct
+ for i := 0; i < numTests; i++ {
+ foundPaths := <-results
+ assertSameResponse(t, foundPaths, paths)
}
}
- sort.Strings(paths)
- for _, path := range paths {
- create(t, path, filesystem)
- }
- // set up a finder
- finder := newFinder(
- t,
- filesystem,
- CacheParams{
- RootDirs: []string{"/tmp"},
- IncludeFiles: []string{"findme.txt"},
- },
- )
- defer finder.Shutdown()
-
- numTests := 20
- results := make(chan []string, numTests)
- // make several parallel calls to the finder
- for i := 0; i < numTests; i++ {
- go func() {
- foundPaths := finder.FindNamedAt("/tmp", "findme.txt")
- results <- foundPaths
- }()
- }
-
- // check that each response was correct
- for i := 0; i < numTests; i++ {
- foundPaths := <-results
- assertSameResponse(t, foundPaths, paths)
- }
+ testAgainstSeveralThreadcounts(t, testWithNumThreads)
}
func TestConcurrentFindDifferentDirectories(t *testing.T) {
