refactor sh builtins
add -use_shell_builtins flag to disable the feature
(when some builtin is broken)
diff --git a/expr.go b/expr.go
index 6af75f1..6af8d6e 100644
--- a/expr.go
+++ b/expr.go
@@ -584,3 +584,32 @@
// TODO(ukai): per functype?
traceEvent.end(te)
}
+
+type matchVarref struct{}
+
+func (m matchVarref) String() string { return "$(match-any)" }
+func (m matchVarref) Eval(w io.Writer, ev *Evaluator) { panic("not implemented") }
+func (m matchVarref) Serialize() SerializableVar { panic("not implemented") }
+func (m matchVarref) Dump(w io.Writer) { panic("not implemented") }
+
+func matchExpr(expr, pat Expr) ([]Value, bool) {
+ if len(expr) != len(pat) {
+ return nil, false
+ }
+ var mv matchVarref
+ var matches []Value
+ for i := range expr {
+ if pat[i] == mv {
+ switch expr[i].(type) {
+ case paramref, varref:
+ matches = append(matches, expr[i])
+ continue
+ }
+ return nil, false
+ }
+ if expr[i] != pat[i] {
+ return nil, false
+ }
+ }
+ return matches, true
+}
diff --git a/func.go b/func.go
index 954a1d5..3ac9404 100644
--- a/func.go
+++ b/func.go
@@ -763,290 +763,19 @@
if !ok {
return f
}
- // hack for android
- if v, ok := matchAndroidRot13(expr); ok {
- return &funcShellAndroidRot13{
- funcShell: f,
- v: v,
+ if useShellBuiltins {
+ // hack for android
+ for _, sb := range shBuiltins {
+ if v, ok := matchExpr(expr, sb.pattern); ok {
+ Logf("shell compact apply %s for %s", sb.name, expr)
+ return sb.compact(f, v)
+ }
}
+ Logf("shell compact no match: %s", expr)
}
- if dir, ok := matchAndroidFindFileInDir(expr); ok {
- androidFindCache.init(nil)
- return &funcShellAndroidFindFileInDir{
- funcShell: f,
- dir: dir,
- }
- }
- if chdir, roots, ok := matchAndroidFindJavaInDir(expr); ok {
- androidFindCache.init(nil)
- return &funcShellAndroidFindJavaInDir{
- funcShell: f,
- chdir: chdir,
- roots: roots,
- }
- }
- if dir, ok := matchAndroidFindJavaResourceFileGroup(expr); ok {
- androidFindCache.init(nil)
- return &funcShellAndroidFindJavaResourceFileGroup{
- funcShell: f,
- dir: dir,
- }
- }
- Logf("shell compact no match: %s", expr)
return f
}
-// pattern in repo/android/build/core/definisions.mk
-// rot13
-// echo $(1) | tr 'a-zA-Z' 'n-za-mN-ZA-M'
-func matchAndroidRot13(expr Expr) (Value, bool) {
- // literal: "echo "
- // paramref: 1
- // literal: " | tr 'a-zA-Z' 'n-za-mN-ZA-M'"
- if len(expr) != 3 {
- return nil, false
- }
- if expr[0] != literal("echo ") {
- return nil, false
- }
- if expr[1] != paramref(1) {
- return nil, false
- }
- if expr[2] != literal(" | tr 'a-zA-Z' 'n-za-mN-ZA-M'") {
- return nil, false
- }
- return expr[1], true
-}
-
-type funcShellAndroidRot13 struct {
- *funcShell
- v Value
-}
-
-func rot13(buf []byte) {
- for i, b := range buf {
- // tr 'a-zA-Z' 'n-za-mN-ZA-M'
- if b >= 'a' && b <= 'z' {
- b += 'n' - 'a'
- if b > 'z' {
- b -= 'z' - 'a' + 1
- }
- } else if b >= 'A' && b <= 'Z' {
- b += 'N' - 'A'
- if b > 'Z' {
- b -= 'Z' - 'A' + 1
- }
- }
- buf[i] = b
- }
-}
-
-func (f *funcShellAndroidRot13) Eval(w io.Writer, ev *Evaluator) {
- abuf := newBuf()
- fargs := ev.args(abuf, f.v)
- rot13(fargs[0])
- w.Write(fargs[0])
- freeBuf(abuf)
-}
-
-// pattern in repo/android/build/core/definitions.mk
-// find-subdir-assets
-// if [ -d $1 ] ; then cd $1 ; find ./ -not -name '.*' -and -type f -and -not -type l ; fi
-func matchAndroidFindFileInDir(expr Expr) (Value, bool) {
- // literal: "if [ -d "
- // paramref: 1
- // literal: " ] ; then cd "
- // paramref: 1
- // literal: " ; find ./ -not -name '.*' -and -type f -and -not -type l ; fi"
- if len(expr) != 5 {
- return nil, false
- }
- if expr[0] != literal("if [ -d ") {
- return nil, false
- }
- if expr[1] != paramref(1) {
- return nil, false
- }
- if expr[2] != literal(" ] ; then cd ") {
- return nil, false
- }
- if expr[3] != paramref(1) {
- return nil, false
- }
- if expr[4] != literal(" ; find ./ -not -name '.*' -and -type f -and -not -type l ; fi") {
- return nil, false
- }
- return paramref(1), true
-}
-
-type funcShellAndroidFindFileInDir struct {
- *funcShell
- dir Value
-}
-
-func (f *funcShellAndroidFindFileInDir) Eval(w io.Writer, ev *Evaluator) {
- abuf := newBuf()
- fargs := ev.args(abuf, f.dir)
- dir := string(trimSpaceBytes(fargs[0]))
- freeBuf(abuf)
- Logf("shellAndroidFindFileInDir %s => %s", f.dir.String(), dir)
- if strings.Contains(dir, "..") {
- Logf("shellAndroidFindFileInDir contains ..: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- if !androidFindCache.ready() {
- Logf("shellAndroidFindFileInDir androidFindCache is not ready: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- sw := ssvWriter{w: w}
- androidFindCache.findInDir(&sw, dir)
-}
-
-// pattern in repo/android/build/core/definitions.mk
-// all-java-files-under
-// cd ${LOCAL_PATH} ; find -L $1 -name "*.java" -and -not -name ".*"
-func matchAndroidFindJavaInDir(expr Expr) (Value, Value, bool) {
- // literal: "cd "
- // varref: xxx
- // literal: " ; find -L "
- // paramref: 1
- // literal: " -name "*.java" -and -not -name ".*"
- if len(expr) != 5 {
- return nil, nil, false
- }
- if expr[0] != literal("cd ") {
- return nil, nil, false
- }
- if _, ok := expr[1].(varref); !ok {
- return nil, nil, false
- }
- if expr[2] != literal(" ; find -L ") {
- return nil, nil, false
- }
- if expr[3] != paramref(1) {
- return nil, nil, false
- }
- if expr[4] != literal(` -name "*.java" -and -not -name ".*"`) {
- return nil, nil, false
- }
- return expr[1], paramref(1), true
-}
-
-type funcShellAndroidFindJavaInDir struct {
- *funcShell
- chdir Value
- roots Value
-}
-
-func (f *funcShellAndroidFindJavaInDir) Eval(w io.Writer, ev *Evaluator) {
- abuf := newBuf()
- fargs := ev.args(abuf, f.chdir, f.roots)
- chdir := string(trimSpaceBytes(fargs[0]))
- var roots []string
- hasDotDot := false
- ws := newWordScanner(fargs[1])
- for ws.Scan() {
- root := string(ws.Bytes())
- if strings.Contains(root, "..") {
- hasDotDot = true
- }
- roots = append(roots, string(ws.Bytes()))
- }
- freeBuf(abuf)
- Logf("shellAndroidFindJavaInDir %s,%s => %s,%s", f.chdir.String(), f.roots.String(), chdir, roots)
- if strings.Contains(chdir, "..") || hasDotDot {
- Logf("shellAndroidFindJavaInDir contains ..: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- if !androidFindCache.ready() {
- Logf("shellAndroidFindJavaInDir androidFindCache is not ready: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- buf := newBuf()
- sw := ssvWriter{w: buf}
- for _, root := range roots {
- if !androidFindCache.findJavaInDir(&sw, chdir, root) {
- freeBuf(buf)
- Logf("shellAndroidFindJavaInDir androidFindCache couldn't handle: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- }
- w.Write(buf.Bytes())
- freeBuf(buf)
-}
-
-// pattern: in repo/android/build/core/base_rules.mk
-// java_resource_file_groups+= ...
-// cd ${TOP_DIR}${LOCAL_PATH}/${dir} && find . -type d -a -name ".svn" -prune \
-// -o -type f -a \! -name "*.java" -a \! -name "package.html" -a \! \
-// -name "overview.html" -a \! -name ".*.swp" -a \! -name ".DS_Store" \
-// -a \! -name "*~" -print )
-func matchAndroidFindJavaResourceFileGroup(expr Expr) (Value, bool) {
- // literal: "cd "
- // varref: TOP_DIR
- // varref: LOCAL_PATH
- // literal: "/"
- // varref: dir
- // literal: " && find . -type d -a name ".svn" -prune -o .."
- if len(expr) != 6 {
- return nil, false
- }
- if expr[0] != literal("cd ") {
- return nil, false
- }
- if _, ok := expr[1].(varref); !ok {
- return nil, false
- }
- if _, ok := expr[2].(varref); !ok {
- return nil, false
- }
- if expr[3] != literal("/") {
- return nil, false
- }
- if _, ok := expr[4].(varref); !ok {
- return nil, false
- }
- if expr[5] != literal(` && find . -type d -a -name ".svn" -prune -o -type f -a \! -name "*.java" -a \! -name "package.html" -a \! -name "overview.html" -a \! -name ".*.swp" -a \! -name ".DS_Store" -a \! -name "*~" -print `) {
- Logf("shell compact mismatch: expr[5]=%q", expr[5])
- return nil, false
- }
- return expr[1:5], true
-}
-
-type funcShellAndroidFindJavaResourceFileGroup struct {
- *funcShell
- dir Value
-}
-
-func (f *funcShellAndroidFindJavaResourceFileGroup) Eval(w io.Writer, ev *Evaluator) {
- abuf := newBuf()
- fargs := ev.args(abuf, f.dir)
- dir := string(trimSpaceBytes(fargs[0]))
- freeBuf(abuf)
- Logf("shellAndroidFindJavaResourceFileGroup %s => %s", f.dir.String(), dir)
- if strings.Contains(dir, "..") {
- Logf("shellAndroidFindJavaResourceFileGroup contains ..: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- if !androidFindCache.ready() {
- Logf("shellAndroidFindJavaResourceFileGroup androidFindCache is not ready: call original shell")
- f.funcShell.Eval(w, ev)
- return
- }
- sw := ssvWriter{w: w}
- androidFindCache.findJavaResourceFileGroup(&sw, dir)
-}
-
-// TODO(ukai): pattern:
-//
-// echo $1 | tr 'a-zA-Z' 'n-za-mN-ZA-M'
-
// https://www.gnu.org/software/make/manual/html_node/Call-Function.html#Call-Function
type funcCall struct{ fclosure }
diff --git a/main.go b/main.go
index f414a0b..b6ccf98 100644
--- a/main.go
+++ b/main.go
@@ -53,6 +53,7 @@
useFindCache bool
findCachePrunes string
useWildcardCache bool
+ useShellBuiltins bool
generateNinja bool
ignoreOptionalInclude string
gomaDir string
@@ -99,6 +100,7 @@
flag.StringVar(&findCachePrunes, "find_cache_prunes", "",
"space separated prune directories for find cache.")
flag.BoolVar(&useWildcardCache, "use_wildcard_cache", true, "Use wildcard cache.")
+ flag.BoolVar(&useShellBuiltins, "use_shell_builtins", true, "Use shell builtins")
flag.BoolVar(&generateNinja, "ninja", false, "Generate build.ninja.")
flag.StringVar(&ignoreOptionalInclude, "ignore_optional_include", "", "If specified, skip reading -include directives start with the specified path.")
flag.StringVar(&gomaDir, "goma_dir", "", "If specified, use goma to build C/C++ files.")
diff --git a/shellutil.go b/shellutil.go
new file mode 100644
index 0000000..0c0cd17
--- /dev/null
+++ b/shellutil.go
@@ -0,0 +1,222 @@
+package main
+
+import (
+ "io"
+ "strings"
+)
+
+var shBuiltins = []struct {
+ name string
+ pattern Expr
+ compact func(*funcShell, []Value) Value
+}{
+ {
+ name: "android:rot13",
+ // in repo/android/build/core/definisions.mk
+ // echo $(1) | tr 'a-zA-Z' 'n-za-mN-ZA-M'
+ pattern: Expr{
+ literal("echo "),
+ matchVarref{},
+ literal(" | tr 'a-zA-Z' 'n-za-mN-ZA-M'"),
+ },
+ compact: func(sh *funcShell, matches []Value) Value {
+ return &funcShellAndroidRot13{
+ funcShell: sh,
+ v: matches[0],
+ }
+ },
+ },
+ {
+ name: "android:find-subdir-assets",
+ // in repo/android/build/core/definitions.mk
+ // if [ -d $1 ] ; then cd $1 ; find ./ -not -name '.*' -and -type f -and -not -type l ; fi
+ pattern: Expr{
+ literal("if [ -d "),
+ matchVarref{},
+ literal(" ] ; then cd "),
+ matchVarref{},
+ literal(" ; find ./ -not -name '.*' -and -type f -and -not -type l ; fi"),
+ },
+ compact: func(sh *funcShell, v []Value) Value {
+ if v[0] != v[1] {
+ return sh
+ }
+ androidFindCache.init(nil)
+ return &funcShellAndroidFindFileInDir{
+ funcShell: sh,
+ dir: v[0],
+ }
+ },
+ },
+ {
+ name: "android:all-java-files-under",
+ // in repo/android/build/core/definitions.mk
+ // cd ${LOCAL_PATH} ; find -L $1 -name "*.java" -and -not -name ".*"
+ pattern: Expr{
+ literal("cd "),
+ matchVarref{},
+ literal(" ; find -L "),
+ matchVarref{},
+ literal(` -name "*.java" -and -not -name ".*"`),
+ },
+ compact: func(sh *funcShell, v []Value) Value {
+ androidFindCache.init(nil)
+ return &funcShellAndroidFindJavaInDir{
+ funcShell: sh,
+ chdir: v[0],
+ roots: v[1],
+ }
+ },
+ },
+ {
+ name: "android:java_resource_file_groups",
+ // in repo/android/build/core/base_rules.mk
+ // cd ${TOP_DIR}${LOCAL_PATH}/${dir} && find . -type d -a \
+ // -name ".svn" -prune -o -type f -a \! -name "*.java" \
+ // -a \! -name "package.html" -a \! -name "overview.html" \
+ // -a \! -name ".*.swp" -a \! -name ".DS_Store" \
+ // -a \! -name "*~" -print )
+ pattern: Expr{
+ literal("cd "),
+ matchVarref{},
+ matchVarref{},
+ literal("/"),
+ matchVarref{},
+ literal(` && find . -type d -a -name ".svn" -prune -o -type f -a \! -name "*.java" -a \! -name "package.html" -a \! -name "overview.html" -a \! -name ".*.swp" -a \! -name ".DS_Store" -a \! -name "*~" -print `),
+ },
+ compact: func(sh *funcShell, v []Value) Value {
+ androidFindCache.init(nil)
+ return &funcShellAndroidFindJavaResourceFileGroup{
+ funcShell: sh,
+ dir: Expr(v),
+ }
+ },
+ },
+}
+
+type funcShellAndroidRot13 struct {
+ *funcShell
+ v Value
+}
+
+func rot13(buf []byte) {
+ for i, b := range buf {
+ // tr 'a-zA-Z' 'n-za-mN-ZA-M'
+ if b >= 'a' && b <= 'z' {
+ b += 'n' - 'a'
+ if b > 'z' {
+ b -= 'z' - 'a' + 1
+ }
+ } else if b >= 'A' && b <= 'Z' {
+ b += 'N' - 'A'
+ if b > 'Z' {
+ b -= 'Z' - 'A' + 1
+ }
+ }
+ buf[i] = b
+ }
+}
+
+func (f *funcShellAndroidRot13) Eval(w io.Writer, ev *Evaluator) {
+ abuf := newBuf()
+ fargs := ev.args(abuf, f.v)
+ rot13(fargs[0])
+ w.Write(fargs[0])
+ freeBuf(abuf)
+}
+
+type funcShellAndroidFindFileInDir struct {
+ *funcShell
+ dir Value
+}
+
+func (f *funcShellAndroidFindFileInDir) Eval(w io.Writer, ev *Evaluator) {
+ abuf := newBuf()
+ fargs := ev.args(abuf, f.dir)
+ dir := string(trimSpaceBytes(fargs[0]))
+ freeBuf(abuf)
+ Logf("shellAndroidFindFileInDir %s => %s", f.dir.String(), dir)
+ if strings.Contains(dir, "..") {
+ Logf("shellAndroidFindFileInDir contains ..: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ if !androidFindCache.ready() {
+ Logf("shellAndroidFindFileInDir androidFindCache is not ready: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ sw := ssvWriter{w: w}
+ androidFindCache.findInDir(&sw, dir)
+}
+
+type funcShellAndroidFindJavaInDir struct {
+ *funcShell
+ chdir Value
+ roots Value
+}
+
+func (f *funcShellAndroidFindJavaInDir) Eval(w io.Writer, ev *Evaluator) {
+ abuf := newBuf()
+ fargs := ev.args(abuf, f.chdir, f.roots)
+ chdir := string(trimSpaceBytes(fargs[0]))
+ var roots []string
+ hasDotDot := false
+ ws := newWordScanner(fargs[1])
+ for ws.Scan() {
+ root := string(ws.Bytes())
+ if strings.Contains(root, "..") {
+ hasDotDot = true
+ }
+ roots = append(roots, string(ws.Bytes()))
+ }
+ freeBuf(abuf)
+ Logf("shellAndroidFindJavaInDir %s,%s => %s,%s", f.chdir.String(), f.roots.String(), chdir, roots)
+ if strings.Contains(chdir, "..") || hasDotDot {
+ Logf("shellAndroidFindJavaInDir contains ..: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ if !androidFindCache.ready() {
+ Logf("shellAndroidFindJavaInDir androidFindCache is not ready: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ buf := newBuf()
+ sw := ssvWriter{w: buf}
+ for _, root := range roots {
+ if !androidFindCache.findJavaInDir(&sw, chdir, root) {
+ freeBuf(buf)
+ Logf("shellAndroidFindJavaInDir androidFindCache couldn't handle: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ }
+ w.Write(buf.Bytes())
+ freeBuf(buf)
+}
+
+type funcShellAndroidFindJavaResourceFileGroup struct {
+ *funcShell
+ dir Value
+}
+
+func (f *funcShellAndroidFindJavaResourceFileGroup) Eval(w io.Writer, ev *Evaluator) {
+ abuf := newBuf()
+ fargs := ev.args(abuf, f.dir)
+ dir := string(trimSpaceBytes(fargs[0]))
+ freeBuf(abuf)
+ Logf("shellAndroidFindJavaResourceFileGroup %s => %s", f.dir.String(), dir)
+ if strings.Contains(dir, "..") {
+ Logf("shellAndroidFindJavaResourceFileGroup contains ..: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ if !androidFindCache.ready() {
+ Logf("shellAndroidFindJavaResourceFileGroup androidFindCache is not ready: call original shell")
+ f.funcShell.Eval(w, ev)
+ return
+ }
+ sw := ssvWriter{w: w}
+ androidFindCache.findJavaResourceFileGroup(&sw, dir)
+}
diff --git a/func_test.go b/shellutil_test.go
similarity index 100%
rename from func_test.go
rename to shellutil_test.go
