import_scrypt.sh

#!/bin/bash
#
# Copyright (C) 2009 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# This script imports new versions of scrypt (http://www.tarsnap.com/scrypt/) into the
# Android source tree.  To run, (1) fetch the appropriate tarball from the scrypt repository,
# (2) check the gpg/pgp signature, and then (3) run:
#   ./import_scrypt.sh import scrypt-*.tar.gz
#
# IMPORTANT: See README.android for additional details.

# turn on exit on error as well as a warning when it happens
set -e
set -x
trap  "echo WARNING: Exiting on non-zero subprocess exit code" ERR;

# Ensure consistent sorting order / tool output.
export LANG=C
export LC_ALL=C

export DIRNAME=$(dirname $0)

function die() {
  declare -r message=$1

  echo $message
  exit 1
}

function usage() {
  declare -r message=$1

  if [ ! "$message" = "" ]; then
    echo $message
  fi
  echo "Usage:"
  echo "  ./import_scrypt.sh import </path/to/scrypt-*.tar.gz>"
  echo "  ./import_scrypt.sh regenerate <patch/*.patch>"
  echo "  ./import_scrypt.sh generate <patch/*.patch> </path/to/scrypt-*.tar.gz>"
  exit 1
}

function main() {
  if [ ! -d patches ]; then
    die "scrypt patch directory patches/ not found"
  fi

  if [ ! -f scrypt.version ]; then
    die "scrypt.version not found"
  fi

  source $DIRNAME/scrypt.version
  if [ "$SCRYPT_VERSION" == "" ]; then
    die "Invalid scrypt.version; see README.android for more information"
  fi

  SCRYPT_DIR=scrypt-$SCRYPT_VERSION
  SCRYPT_DIR_ORIG=$SCRYPT_DIR.orig

  if [ ! -f scrypt.config ]; then
    die "scrypt.config not found"
  fi

  source $DIRNAME/scrypt.config
  if [ "$CONFIGURE_ARGS" == "" -o "$UNNEEDED_SOURCES" == "" -o "$NEEDED_SOURCES" == "" ]; then
    die "Invalid scrypt.config; see README.android for more information"
  fi

  declare -r command=$1
  shift || usage "No command specified. Try import, regenerate, or generate."
  if [ "$command" = "import" ]; then
    declare -r tar=$1
    shift || usage "No tar file specified."
    import $tar
  elif [ "$command" = "regenerate" ]; then
    declare -r patch=$1
    shift || usage "No patch file specified."
    [ -d $SCRYPT_DIR ] || usage "$SCRYPT_DIR not found, did you mean to use generate?"
    [ -d $SCRYPT_DIR_ORIG ] || usage "$SCRYPT_DIR_ORIG not found, did you mean to use generate?"
    regenerate $patch
  elif [ "$command" = "generate" ]; then
    declare -r patch=$1
    shift || usage "No patch file specified."
    declare -r tar=$1
    shift || usage "No tar file specified."
    generate $patch $tar
  else
    usage "Unknown command specified $command. Try import, regenerate, or generate."
  fi
}

# Compute the name of an assembly source file generated by one of the
# gen_asm_xxxx() functions below. The logic is the following:
# - if "$2" is not empty, output it directly
# - otherwise, change the file extension of $1 from .pl to .S and output
#   it.
# Usage: default_asm_file "$1" "$2"
#     or default_asm_file "$@"
#
# $1: generator path (perl script)
# $2: optional output file name.
function default_asm_file () {
  if [ "$2" ]; then
    echo "$2"
  else
    echo "${1%%.pl}.S"
  fi
}

# Generate an ARM assembly file.
# $1: generator (perl script)
# $2: [optional] output file name
function gen_asm_arm () {
  local OUT
  OUT=$(default_asm_file "$@")
  perl "$1" > "$OUT"
}

function gen_asm_mips () {
  local OUT
  OUT=$(default_asm_file "$@")
  # The perl scripts expect to run the target compiler as $CC to determine
  # the endianess of the target. Setting CC to true is a hack that forces the scripts
  # to generate little endian output
  CC=true perl "$1" o32 > "$OUT"
}

function gen_asm_x86 () {
  local OUT
  OUT=$(default_asm_file "$@")
  perl "$1" elf -fPIC > "$OUT"
}

function gen_asm_x86_64 () {
  local OUT
  OUT=$(default_asm_file "$@")
  perl "$1" elf "$OUT" > "$OUT"
}


# Filter all items in a list that match a given pattern.
# $1: space-separated list
# $2: egrep pattern.
# Out: items in $1 that match $2
function filter_by_egrep() {
  declare -r pattern=$1
  shift
  echo "$@" | tr ' ' '\n' | grep -e "$pattern" | tr '\n' ' '
}

# Sort and remove duplicates in a space-separated list
# $1: space-separated list
# Out: new space-separated list
function uniq_sort () {
  echo "$@" | tr ' ' '\n' | sort -u | tr '\n' ' ' | \
    sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
}

function print_autogenerated_header() {
  echo "// Auto-generated - DO NOT EDIT!"
  echo "// To regenerate, edit scrypt.config, then run:"
  echo "//     ./import_scrypt.sh import /path/to/scrypt-$SCRYPT_VERSION.tar.gz"
  echo "//"
}

# Return the value of a computed variable name.
# E.g.:
#   FOO=foo
#   BAR=bar
#   echo $(var_value FOO_$BAR)   -> prints the value of ${FOO_bar}
# $1: Variable name
# Out: variable value
var_value() {
  # Note: don't use 'echo' here, because it's sensitive to values
  #       that begin with an underscore (e.g. "-n")
  eval printf \"%s\\n\" \$$1
}

# Same as var_value, but returns sorted output without duplicates.
# $1: Variable name
# Out: variable value (if space-separated list, sorted with no duplicates)
var_sorted_value() {
  uniq_sort $(var_value $1)
}

# $1: Property name
# $2: Indent
# $3: Values
print_list_in_bp() {
  declare -r indent="$1"
  shift
  declare -r name="$1"
  shift

  if [ -n "$*" ]; then
    echo "${indent}${name}: ["
    for value in "$@"; do
      echo "${indent}    \"${value}\","
    done
    echo "${indent}],"
  fi
}

# Print an architecture-specific section
# $1: Arch
# $2: Arch variant
print_arch_section_in_bp() {
  declare -r arch="$1"
  declare -r variant="$2"

  declare -r srcs="$(var_sorted_value SCRYPT_SOURCES_${arch})"
  declare -r exclude_srcs="$(var_sorted_value SCRYPT_SOURCES_EXCLUDES_${arch})"

  declare -r srcs_variant="$(var_sorted_value SCRYPT_SOURCES_${arch}_${variant})"
  declare -r exclude_srcs_variant="$(var_sorted_value SCRYPT_SOURCES_EXCLUDES_${arch}_${variant})"

  if [ -n "${srcs}${exclude_srcs}${srcs_variant}${exclude_srcs_variant}" ]; then
    echo "        ${arch}: {"

    print_list_in_bp "            " "srcs" ${srcs}
    print_list_in_bp "            " "exclude_srcs" ${exclude_srcs}

    if [ -n "${src_variant}${exclude_srcs_variant}" ]; then
      echo "            ${variant}: {"
      print_list_in_bp "                " "srcs" ${srcs_variant}
      print_list_in_bp "                " "exclude_srcs" ${exclude_srcs_variant}
      echo "            },"
    fi

    echo "        },"
  fi
}

# Generate sources.bp from scrypt.config
#
# $1: Target file name.  (e.g. sources.bp)
function generate_android_bp() {
  declare -r output="$1"

  # Extract cflags from upstream `./configure` script
  cd $SCRYPT_DIR

  ./configure $CONFIGURE_ARGS

  declare -r tmpfile=$(mktemp)
  (grep -e -D Makefile | grep -v CONFIGURE_ARGS= | grep -v OPTIONS=) > $tmpfile

  declare -r ac_cflags=$(filter_by_egrep "^-D" $(grep -e "^CFLAG=" $tmpfile))
  declare -r ac_depflags=$(filter_by_egrep "^-D" $(grep -e "^DEPFLAG=" $tmpfile))
  rm -f $tmpfile

  cd ..

  # Generate sources.bp
  echo "Generating $(basename $output)"
  (
    print_autogenerated_header

    cat <<__EOF__

//
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

cc_defaults {
    name: "libscrypt_sources",

__EOF__

    print_list_in_bp "    " "cflags" \
      $(var_sorted_value SCRYPT_CFLAGS) ${ac_cflags} ${ac_depflags}
    echo

    print_list_in_bp "    " "export_include_dirs" \
      $(var_sorted_value SCRYPT_EXPORT_C_INCLUDE_DIRS)
    echo

    print_list_in_bp "    " "local_include_dirs" \
      $(var_sorted_value SCRYPT_INCLUDES)
    echo

    print_list_in_bp "    " "srcs" $(var_sorted_value SCRYPT_SOURCES)
    echo

    echo "    arch: {"

    print_arch_section_in_bp "arm" "neon"
    print_arch_section_in_bp "arm64"
    print_arch_section_in_bp "x86"
    print_arch_section_in_bp "x86_64"
    print_arch_section_in_bp "mips"
    print_arch_section_in_bp "mips64"

    cat <<__EOF__
    },
}
__EOF__
  ) > "$output"
}

function import() {
  declare -r SCRYPT_SOURCE=$1

  untar $SCRYPT_SOURCE readonly
  applypatches $SCRYPT_DIR

  cd $SCRYPT_DIR/..

  touch MODULE_LICENSE_BSD_LIKE

  generate_android_bp "sources.bp"

  # Prune unnecessary sources
  prune

  NEEDED_SOURCES="$NEEDED_SOURCES"
  for i in $NEEDED_SOURCES; do
    echo "Updating $i"
    rm -r $i
    mv $SCRYPT_DIR/$i .
  done

  cleantar
}

function regenerate() {
  declare -r patch=$1

  generatepatch $patch
}

function generate() {
  declare -r patch=$1
  declare -r SCRYPT_SOURCE=$2

  untar $SCRYPT_SOURCE
  applypatches $SCRYPT_DIR_ORIG $patch
  prune

  for i in $NEEDED_SOURCES; do
    echo "Restoring $i"
    rm -rf $SCRYPT_DIR/$i
    cp -rf $i $SCRYPT_DIR/$i
  done

  generatepatch $patch
  cleantar
}

# Find all files in a sub-directory that are encoded in ISO-8859
# $1: Directory.
# Out: list of files in $1 that are encoded as ISO-8859.
function find_iso8859_files() {
  find $1 -type f -print0 | xargs -0 file | fgrep "ISO-8859" | cut -d: -f1
}

# Convert all ISO-8859 files in a given subdirectory to UTF-8
# $1: Directory name
function convert_iso8859_to_utf8() {
  declare -r iso_files=$(find_iso8859_files "$1")
  for iso_file in $iso_files; do
    iconv --from-code iso-8859-1 --to-code utf-8 $iso_file > $iso_file.tmp
    rm -f $iso_file
    mv $iso_file.tmp $iso_file
  done
}

function untar() {
  declare -r SCRYPT_SOURCE=$1
  declare -r readonly=$2

  # Remove old source
  cleantar

  # Process new source
  tar -zxf $SCRYPT_SOURCE
  convert_iso8859_to_utf8 $SCRYPT_DIR
  cp -rfP $SCRYPT_DIR $SCRYPT_DIR_ORIG
  if [ ! -z $readonly ]; then
    find $SCRYPT_DIR_ORIG -type f -print0 | xargs -0 chmod a-w
  fi
}

function prune() {
  echo "Removing $UNNEEDED_SOURCES"
  (cd $SCRYPT_DIR_ORIG && rm -rf $UNNEEDED_SOURCES)
  (cd $SCRYPT_DIR      && rm -r  $UNNEEDED_SOURCES)
}

function cleantar() {
  rm -rf $SCRYPT_DIR_ORIG
  rm -rf $SCRYPT_DIR
}

function applypatches () {
  declare -r dir=$1
  declare -r skip_patch=$2

  cd $dir

  # Apply appropriate patches
  for i in $SCRYPT_PATCHES; do
    if [ ! "$skip_patch" = "patches/$i" ]; then
      echo "Applying patch $i"
      patch -p1 --merge < ../patches/$i || die "Could not apply patches/$i. Fix source and run: $0 regenerate patches/$i"
    else
      echo "Skiping patch $i"
    fi

  done

  # Cleanup patch output
  find . \( -type f -o -type l \) -name "*.orig" -print0 | xargs -0 rm -f

  cd ..
}

function generatepatch() {
  declare -r patch=$1

  # Cleanup stray files before generating patch
  find $SCRYPT_DIR -type f -name "*.orig" -print0 | xargs -0 rm -f
  find $SCRYPT_DIR -type f -name "*~" -print0 | xargs -0 rm -f

  declare -r variable_name=SCRYPT_PATCHES_`basename $patch .patch | sed s/-/_/`_SOURCES
  # http://tldp.org/LDP/abs/html/ivr.html
  eval declare -r sources=\$$variable_name
  rm -f $patch
  touch $patch
  for i in $sources; do
    LC_ALL=C TZ=UTC0 diff -aupN $SCRYPT_DIR_ORIG/$i $SCRYPT_DIR/$i >> $patch && die "ERROR: No diff for patch $path in file $i"
  done
  echo "Generated patch $patch"
  echo "NOTE To make sure there are not unwanted changes from conflicting patches, be sure to review the generated patch."
}

main $@