benchmarks/bionic_benchmarks.cpp

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <err.h>
#include <getopt.h>
#include <inttypes.h>
#include <math.h>
#include <sys/resource.h>

#include <map>
#include <mutex>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include <android-base/file.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <benchmark/benchmark.h>
#include <tinyxml2.h>
#include "util.h"

#define _STR(x) #x
#define STRINGFY(x) _STR(x)

static const std::vector<int> kCommonSizes{
  8,
  16,
  32,
  64,
  512,
  1 * KB,
  8 * KB,
  16 * KB,
  32 * KB,
  64 * KB,
  128 * KB,
};

static const std::vector<int> kSmallSizes{
  // Increment by 1
  1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
  // Increment by 8
  24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144,
  // Increment by 16
  160, 176, 192, 208, 224, 240, 256,
};

static const std::vector<int> kMediumSizes{
  512,
  1 * KB,
  8 * KB,
  16 * KB,
  32 * KB,
  64 * KB,
  128 * KB,
};

static const std::vector<int> kLargeSizes{
  256 * KB,
  512 * KB,
  1024 * KB,
  2048 * KB,
};

static std::map<std::string, const std::vector<int> &> kSizes{
  { "SMALL",  kSmallSizes },
  { "MEDIUM", kMediumSizes },
  { "LARGE",  kLargeSizes },
};

std::map<std::string, std::pair<benchmark_func_t, std::string>> g_str_to_func;

std::mutex g_map_lock;

static struct option g_long_options[] =
{
  {"bionic_cpu", required_argument, nullptr, 'c'},
  {"bionic_xml", required_argument, nullptr, 'x'},
  {"bionic_iterations", required_argument, nullptr, 'i'},
  {"bionic_extra", required_argument, nullptr, 'a'},
  {"help", no_argument, nullptr, 'h'},
  {nullptr, 0, nullptr, 0},
};

typedef std::vector<std::vector<int64_t>> args_vector_t;

void Usage() {
  printf("Usage:\n");
  printf("bionic_benchmarks [--bionic_cpu=<cpu_to_isolate>]\n");
  printf("                  [--bionic_xml=<path_to_xml>]\n");
  printf("                  [--bionic_iterations=<num_iter>]\n");
  printf("                  [--bionic_extra=\"<fn_name> <arg1> <arg 2> ...\"]\n");
  printf("                  [<Google benchmark flags>]\n");
  printf("Google benchmark flags:\n");

  int fake_argc = 2;
  char argv0[] = "bionic_benchmarks";
  char argv1[] = "--help";
  char* fake_argv[3] {argv0, argv1, nullptr};
  benchmark::Initialize(&fake_argc, fake_argv);
  exit(1);
}

// This function removes any bionic benchmarks command line arguments by checking them
// against g_long_options. It fills new_argv with the filtered args.
void SanitizeOpts(int argc, char** argv, std::vector<char*>* new_argv) {
  // TO THOSE ADDING OPTIONS: This currently doesn't support optional arguments.
  (*new_argv)[0] = argv[0];
  for (int i = 1; i < argc; ++i) {
    char* optarg = argv[i];
    size_t opt_idx = 0;

    // Iterate through g_long_options until either we hit the end or we have a match.
    for (opt_idx = 0; g_long_options[opt_idx].name &&
                      strncmp(g_long_options[opt_idx].name, optarg + 2,
                              strlen(g_long_options[opt_idx].name)); ++opt_idx) {
    }

    if (!g_long_options[opt_idx].name) {
      new_argv->push_back(optarg);
    } else {
      if (g_long_options[opt_idx].has_arg == required_argument) {
        // If the arg was passed in with an =, it spans one char *.
        // Otherwise, we skip a spot for the argument.
        if (!strchr(optarg, '=')) {
          i++;
        }
      }
    }
  }
  new_argv->push_back(nullptr);
}

bench_opts_t ParseOpts(int argc, char** argv) {
  bench_opts_t opts;
  int opt;
  int option_index = 0;

  // To make this parser handle the benchmark options silently:
  extern int opterr;
  opterr = 0;

  while ((opt = getopt_long(argc, argv, "c:x:i:a:h", g_long_options, &option_index)) != -1) {
    if (opt == -1) {
      break;
    }
    switch (opt) {
      case 'c':
        if (*optarg) {
          char* check_null;
          opts.cpu_to_lock = strtol(optarg, &check_null, 10);
          if (*check_null) {
            errx(1, "ERROR: Args %s is not a valid integer.", optarg);
          }
        } else {
          printf("ERROR: no argument specified for bionic_cpu\n");
          Usage();
        }
        break;
      case 'x':
        if (*optarg) {
          opts.xmlpath = optarg;
        } else {
          printf("ERROR: no argument specified for bionic_xml\n");
          Usage();
        }
        break;
      case 'a':
        if (*optarg) {
          opts.extra_benchmarks.push_back(optarg);
        } else {
          printf("ERROR: no argument specified for bionic_extra\n");
          Usage();
        }
        break;
      case 'i':
        if (*optarg){
          char* check_null;
          opts.num_iterations = strtol(optarg, &check_null, 10);
          if (*check_null != '\0' or opts.num_iterations < 0) {
            errx(1, "ERROR: Args %s is not a valid number of iterations.", optarg);
          }
        } else {
          printf("ERROR: no argument specified for bionic_iterations\n");
          Usage();
        }
        break;
      case 'h':
        Usage();
        break;
      case '?':
        break;
      default:
        exit(1);
    }
  }
  return opts;
}

// This is a wrapper for every function call for per-benchmark cpu pinning.
void LockAndRun(benchmark::State& state, benchmark_func_t func_to_bench, int cpu_to_lock) {
  if (cpu_to_lock >= 0) LockToCPU(cpu_to_lock);

  // To avoid having to link against Google benchmarks in libutil,
  // benchmarks are kept without parameter information, necessitating this cast.
  reinterpret_cast<void(*) (benchmark::State&)>(func_to_bench)(state);
}

static constexpr char kOnebufManualStr[] = "AT_ONEBUF_MANUAL_ALIGN_";
static constexpr char kTwobufManualStr[] = "AT_TWOBUF_MANUAL_ALIGN1_";

static bool ParseOnebufManualStr(std::string& arg, args_vector_t* to_populate) {
  // The format of this is:
  //   AT_ONEBUF_MANUAL_ALIGN_XX_SIZE_YY
  // Where:
  //   XX is the alignment
  //   YY is the size
  // The YY size can be either a number or a string representing the pre-defined
  // sets of values:
  //   SMALL (for values between 1 and 256)
  //   MEDIUM (for values between 512 and 128KB)
  //   LARGE (for values between 256KB and 2048KB)
  int64_t align;
  int64_t size;
  char sizes[32] = { 0 };
  int ret;

  ret = sscanf(arg.c_str(), "AT_ONEBUF_MANUAL_ALIGN_%" SCNd64 "_SIZE_%" SCNd64,
               &align, &size);
  if (ret == 1) {
    ret = sscanf(arg.c_str(), "AT_ONEBUF_MANUAL_ALIGN_%" SCNd64 "_SIZE_"
                              "%" STRINGFY(sizeof(sizes)-1) "s", &align, sizes);
  }
  if (ret != 2) {
    return false;
  }

  // Verify the alignment is powers of 2.
  if (align != 0 && (align & (align - 1)) != 0) {
    return false;
  }

  auto sit = kSizes.find(sizes);
  if (sit == kSizes.cend()) {
    to_populate->push_back({size, align});
  } else {
    for (auto ssize : sit->second) {
      to_populate->push_back({ssize, align});
    }
  }
  return true;
}

static bool ParseTwobufManualStr(std::string& arg, args_vector_t* to_populate) {
  // The format of this is:
  //   AT_TWOBUF_MANUAL_ALIGN1_XX_ALIGN2_YY_SIZE_ZZ
  // Where:
  //   XX is the alignment of the first argument
  //   YY is the alignment of the second argument
  //   ZZ is the size
  // The ZZ size can be either a number or a string representing the pre-defined
  // sets of values:
  //   SMALL (for values between 1 and 256)
  //   MEDIUM (for values between 512 and 128KB)
  //   LARGE (for values between 256KB and 2048KB)
  int64_t align1;
  int64_t align2;
  int64_t size;
  char sizes[32] = { 0 };
  int ret;

  ret = sscanf(arg.c_str(), "AT_TWOBUF_MANUAL_ALIGN1_%" SCNd64 "_ALIGN2_%" SCNd64 "_SIZE_%" SCNd64,
                            &align1, &align2, &size);
  if (ret == 2) {
    ret = sscanf(arg.c_str(), "AT_TWOBUF_MANUAL_ALIGN1_%" SCNd64 "_ALIGN2_%" SCNd64 "_SIZE_"
                               "%" STRINGFY(sizeof(sizes)-1) "s",
                               &align1, &align2, sizes);
  }
  if (ret != 3) {
    return false;
  }

  // Verify the alignments are powers of 2.
  if ((align1 != 0 && (align1 & (align1 - 1)) != 0)
      || (align2 != 0 && (align2 & (align2 - 1)) != 0)) {
    return false;
  }

  auto sit = kSizes.find(sizes);
  if (sit == kSizes.cend()) {
    to_populate->push_back({size, align1, align2});
  } else {
    for (auto ssize : sit->second) {
      to_populate->push_back({ssize, align1, align2});
    }
  }
  return true;
}

args_vector_t* ResolveArgs(args_vector_t* to_populate, std::string args,
                           std::map<std::string, args_vector_t>& args_shorthand) {
  // args is either a space-separated list of ints, a macro name, or
  // special free form macro.
  // To ease formatting in XML files, args is left and right trimmed.
  if (args_shorthand.count(args)) {
    return &args_shorthand[args];
  }
  // Check for free form macro.
  if (android::base::StartsWith(args, kOnebufManualStr)) {
    if (!ParseOnebufManualStr(args, to_populate)) {
      errx(1, "ERROR: Bad format of macro %s, should be AT_ONEBUF_MANUAL_ALIGN_XX_SIZE_YY",
           args.c_str());
    }
    return to_populate;
  } else if (android::base::StartsWith(args, kTwobufManualStr)) {
    if (!ParseTwobufManualStr(args, to_populate)) {
      errx(1,
           "ERROR: Bad format of macro %s, should be AT_TWOBUF_MANUAL_ALIGN1_XX_ALIGNE2_YY_SIZE_ZZ",
           args.c_str());
    }
    return to_populate;
  }

  std::string trimmed_args = android::base::Trim(args);
  if (!trimmed_args.empty()) {
    std::stringstream sstream(trimmed_args);
    std::string argstr;
    while (sstream >> argstr) {
      char* check_null;
      int converted = static_cast<int>(strtol(argstr.c_str(), &check_null, 10));
      if (*check_null == '\0') {
        to_populate->emplace_back(std::vector<int64_t>{converted});
        continue;
      } else if (*check_null == '/') {
        // The only supported format with a / is \d+(/\d+)\s*. Example 8/8/8 or 16/23.
        std::vector<int64_t> test_args{converted};
        while (true) {
          converted = static_cast<int>(strtol(check_null + 1, &check_null, 10));
          test_args.push_back(converted);
          if (*check_null == '\0') {
            to_populate->emplace_back(std::move(test_args));
            break;
          } else if (*check_null != '/') {
            errx(1, "ERROR: Args str %s contains an invalid macro or int.", args.c_str());
          }
        }
      } else {
        errx(1, "ERROR: Args str %s contains an invalid macro or int.", args.c_str());
      }
    }
  } else {
    // No arguments, only the base benchmark.
    to_populate->emplace_back(std::vector<int64_t>{});
  }
  return to_populate;
}

void RegisterGoogleBenchmarks(bench_opts_t primary_opts, bench_opts_t secondary_opts,
                              const std::string& fn_name, args_vector_t* run_args) {
  if (!g_str_to_func.contains(fn_name)) {
    errx(1, "ERROR: No benchmark for function %s", fn_name.c_str());
  }
  long iterations_to_use = primary_opts.num_iterations ? primary_opts.num_iterations :
                                                         secondary_opts.num_iterations;
  int cpu_to_use = -1;
  if (primary_opts.cpu_to_lock >= 0) {
    cpu_to_use = primary_opts.cpu_to_lock;

  } else if (secondary_opts.cpu_to_lock >= 0) {
    cpu_to_use = secondary_opts.cpu_to_lock;
  }

  benchmark_func_t benchmark_function = g_str_to_func.at(fn_name).first;
  for (const std::vector<int64_t>& args : (*run_args)) {
    auto registration = benchmark::RegisterBenchmark(fn_name.c_str(), LockAndRun,
                                                     benchmark_function,
                                                     cpu_to_use)->Args(args);
    if (iterations_to_use > 0) {
      registration->Iterations(iterations_to_use);
    }
  }
}

void RegisterCliBenchmarks(bench_opts_t cmdline_opts,
                           std::map<std::string, args_vector_t>& args_shorthand) {
  // Register any of the extra benchmarks that were specified in the options.
  args_vector_t arg_vector;
  args_vector_t* run_args = &arg_vector;
  for (const std::string& extra_fn : cmdline_opts.extra_benchmarks) {
    android::base::Trim(extra_fn);
    size_t first_space_pos = extra_fn.find(' ');
    std::string fn_name = extra_fn.substr(0, first_space_pos);
    std::string cmd_args;
    if (first_space_pos != std::string::npos) {
      cmd_args = extra_fn.substr(extra_fn.find(' ') + 1);
    } else {
      cmd_args = "";
    }
    run_args = ResolveArgs(run_args, cmd_args, args_shorthand);
    RegisterGoogleBenchmarks(bench_opts_t(), cmdline_opts, fn_name, run_args);

    run_args = &arg_vector;
    arg_vector.clear();
  }
}

int RegisterXmlBenchmarks(bench_opts_t cmdline_opts,
                          std::map<std::string, args_vector_t>& args_shorthand) {
  // Structure of the XML file:
  // - Element "fn"           Function to benchmark.
  // - - Element "iterations" Number of iterations to run. Leaving this blank uses
  //                          Google benchmarks' convergence heuristics.
  // - - Element "cpu"        CPU to isolate to, if any.
  // - - Element "args"       Whitespace-separated list of per-function integer arguments, or
  //                          one of the macros defined in util.h.
  tinyxml2::XMLDocument doc;
  if (doc.LoadFile(cmdline_opts.xmlpath.c_str()) != tinyxml2::XML_SUCCESS) {
    doc.PrintError();
    return doc.ErrorID();
  }

  // Read and register the functions.
  tinyxml2::XMLNode* fn = doc.FirstChildElement("fn");
  while (fn) {
    if (fn == fn->ToComment()) {
      // Skip comments.
      fn = fn->NextSibling();
      continue;
    }

    auto fn_elem = fn->FirstChildElement("name");
    if (!fn_elem) {
      errx(1, "ERROR: Malformed XML entry: missing name element.");
    }
    std::string fn_name = fn_elem->GetText();
    if (fn_name.empty()) {
      errx(1, "ERROR: Malformed XML entry: error parsing name text.");
    }
    auto* xml_args = fn->FirstChildElement("args");
    args_vector_t arg_vector;
    args_vector_t* run_args = ResolveArgs(&arg_vector,
                                          xml_args ? android::base::Trim(xml_args->GetText()) : "",
                                          args_shorthand);

    // XML values for CPU and iterations take precedence over those passed in via CLI.
    bench_opts_t xml_opts{};
    auto* num_iterations_elem = fn->FirstChildElement("iterations");
    if (num_iterations_elem) {
      int temp;
      num_iterations_elem->QueryIntText(&temp);
      xml_opts.num_iterations = temp;
    }
    auto* cpu_to_lock_elem = fn->FirstChildElement("cpu");
    if (cpu_to_lock_elem) {
      int temp;
      cpu_to_lock_elem->QueryIntText(&temp);
      xml_opts.cpu_to_lock = temp;
    }

    RegisterGoogleBenchmarks(xml_opts, cmdline_opts, fn_name, run_args);

    fn = fn->NextSibling();
  }
  return 0;
}

static void SetArgs(const std::vector<int>& sizes, args_vector_t* args) {
  for (int size : sizes) {
    args->push_back({size});
  }
}

static void SetArgs(const std::vector<int>& sizes, int align, args_vector_t* args) {
  for (int size : sizes) {
    args->push_back({size, align});
  }
}


static void SetArgs(const std::vector<int>& sizes, int align1, int align2, args_vector_t* args) {
  for (int size : sizes) {
    args->push_back({size, align1, align2});
  }
}

static args_vector_t GetArgs(const std::vector<int>& sizes) {
  args_vector_t args;
  SetArgs(sizes, &args);
  return args;
}

static args_vector_t GetArgs(const std::vector<int>& sizes, int align) {
  args_vector_t args;
  SetArgs(sizes, align, &args);
  return args;
}

static args_vector_t GetArgs(const std::vector<int>& sizes, int align1, int align2) {
  args_vector_t args;
  SetArgs(sizes, align1, align2, &args);
  return args;
}

std::map<std::string, args_vector_t> GetShorthand() {
  std::vector<int> all_sizes(kSmallSizes);
  all_sizes.insert(all_sizes.end(), kMediumSizes.begin(), kMediumSizes.end());
  all_sizes.insert(all_sizes.end(), kLargeSizes.begin(), kLargeSizes.end());

  int page_sz = getpagesize();
  std::vector<int> sub_page_sizes = {page_sz / 2, page_sz / 4, page_sz / 8};
  std::vector<int> multi_page_sizes = {page_sz,      page_sz * 2,  page_sz * 3,  page_sz * 10,
                                       page_sz * 25, page_sz * 50, page_sz * 75, page_sz * 100};
  std::vector<int> all_page_sizes(sub_page_sizes);
  all_page_sizes.insert(all_page_sizes.end(), multi_page_sizes.begin(), multi_page_sizes.end());

  std::map<std::string, args_vector_t> args_shorthand{
      {"AT_COMMON_SIZES", GetArgs(kCommonSizes)},
      {"AT_SMALL_SIZES", GetArgs(kSmallSizes)},
      {"AT_MEDIUM_SIZES", GetArgs(kMediumSizes)},
      {"AT_LARGE_SIZES", GetArgs(kLargeSizes)},
      {"AT_ALL_SIZES", GetArgs(all_sizes)},
      {"AT_SUB_PAGE_SIZES", GetArgs(sub_page_sizes)},
      {"AT_MULTI_PAGE_SIZES", GetArgs(multi_page_sizes)},
      {"AT_ALL_PAGE_SIZES", GetArgs(all_page_sizes)},

      {"AT_ALIGNED_ONEBUF", GetArgs(kCommonSizes, 0)},
      {"AT_ALIGNED_ONEBUF_SMALL", GetArgs(kSmallSizes, 0)},
      {"AT_ALIGNED_ONEBUF_MEDIUM", GetArgs(kMediumSizes, 0)},
      {"AT_ALIGNED_ONEBUF_LARGE", GetArgs(kLargeSizes, 0)},
      {"AT_ALIGNED_ONEBUF_ALL", GetArgs(all_sizes, 0)},

      {"AT_ALIGNED_TWOBUF", GetArgs(kCommonSizes, 0, 0)},
      {"AT_ALIGNED_TWOBUF_SMALL", GetArgs(kSmallSizes, 0, 0)},
      {"AT_ALIGNED_TWOBUF_MEDIUM", GetArgs(kMediumSizes, 0, 0)},
      {"AT_ALIGNED_TWOBUF_LARGE", GetArgs(kLargeSizes, 0, 0)},
      {"AT_ALIGNED_TWOBUF_ALL", GetArgs(all_sizes, 0, 0)},

      // Do not exceed 512. that is about the largest number of properties
      // that can be created with the current property area size.
      {"NUM_PROPS", args_vector_t{{1}, {4}, {16}, {64}, {128}, {256}, {512}}},

      {"MATH_COMMON", args_vector_t{{0}, {1}, {2}, {3}}},
      {"MATH_SINCOS_COMMON", args_vector_t{{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}},
  };

  args_vector_t args_onebuf;
  args_vector_t args_twobuf;
  for (int size : all_sizes) {
    args_onebuf.push_back({size, 0});
    args_twobuf.push_back({size, 0, 0});
    // Skip alignments on zero sizes.
    if (size == 0) {
      continue;
    }
    for (int align1 = 1; align1 <= 32; align1 <<= 1) {
      args_onebuf.push_back({size, align1});
      for (int align2 = 1; align2 <= 32; align2 <<= 1) {
        args_twobuf.push_back({size, align1, align2});
      }
    }
  }
  args_shorthand.emplace("AT_MANY_ALIGNED_ONEBUF", args_onebuf);
  args_shorthand.emplace("AT_MANY_ALIGNED_TWOBUF", args_twobuf);

  return args_shorthand;
}

static bool FileExists(const std::string& file) {
  struct stat st;
  return stat(file.c_str(), &st) != -1 && S_ISREG(st.st_mode);
}

void RegisterAllBenchmarks(const bench_opts_t& opts,
                           std::map<std::string, args_vector_t>& args_shorthand) {
  for (auto& entry : g_str_to_func) {
    auto& function_info = entry.second;
    args_vector_t arg_vector;
    args_vector_t* run_args = ResolveArgs(&arg_vector, function_info.second,
                                          args_shorthand);
    RegisterGoogleBenchmarks(bench_opts_t(), opts, entry.first, run_args);
  }
}

int main(int argc, char** argv) {
  std::map<std::string, args_vector_t> args_shorthand = GetShorthand();
  bench_opts_t opts = ParseOpts(argc, argv);
  std::vector<char*> new_argv(argc);
  SanitizeOpts(argc, argv, &new_argv);

  if (opts.xmlpath.empty()) {
    // Don't add the default xml file if a user is specifying the tests to run.
    if (opts.extra_benchmarks.empty()) {
      RegisterAllBenchmarks(opts, args_shorthand);
    }
  } else if (!FileExists(opts.xmlpath)) {
    // See if this is a file in the suites directory.
    std::string file(android::base::GetExecutableDirectory() + "/suites/" + opts.xmlpath);
    if (opts.xmlpath[0] == '/' || !FileExists(file)) {
      printf("Cannot find xml file %s: does not exist or is not a file.\n", opts.xmlpath.c_str());
      return 1;
    }
    opts.xmlpath = file;
  }

  if (!opts.xmlpath.empty()) {
    if (int err = RegisterXmlBenchmarks(opts, args_shorthand)) {
      return err;
    }
  }
  RegisterCliBenchmarks(opts, args_shorthand);

  // Set the thread priority to the maximum.
  if (setpriority(PRIO_PROCESS, 0, -20)) {
    perror("Failed to raise priority of process. Are you root?\n");
  }

  int new_argc = new_argv.size();
  benchmark::Initialize(&new_argc, new_argv.data());
  benchmark::RunSpecifiedBenchmarks();
}