run.py - toolchain/benchmark - Git at Google

 #!/usr/bin/env python2
 #
 # Copyright 2017 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
 # pylint: disable=cros-logging-import

 # This is the script to run specified benchmark with different toolchain
 # settings. It includes the process of building benchmark locally and running
 # benchmark on DUT.

 """Main script to run the benchmark suite from building to testing."""
 from __future__ import print_function

 import argparse
 import config
 import ConfigParser
 import logging
 import os
 import subprocess
 import sys

 logging.basicConfig(level=logging.INFO)

 def _parse_arguments(argv):
     parser = argparse.ArgumentParser(description='Build and run specific '
                                      'benchamrk')
     parser.add_argument(
         '-b',
         '--bench',
         action='append',
         default=[],
         help='Select which benchmark to run')

     # Only one of compiler directory and llvm prebuilts version can be indicated
     # at the beginning, so set -c and -l into a exclusive group.
     group = parser.add_mutually_exclusive_group()

     # The toolchain setting arguments has action of 'append', so that users
     # could compare performance with several toolchain settings together.
     group.add_argument(
         '-c',
         '--compiler_dir',
         metavar='DIR',
         action='append',
         default=[],
         help='Specify path to the compiler\'s bin directory. '
         'You shall give several paths, each with a -c, to '
         'compare performance differences in '
         'each compiler.')

     parser.add_argument(
         '-o',
         '--build_os',
         action='append',
         default=[],
         help='Specify the host OS to build the benchmark.')

     group.add_argument(
         '-l',
         '--llvm_prebuilts_version',
         action='append',
         default=[],
         help='Specify the version of prebuilt LLVM. When '
         'specific prebuilt version of LLVM already '
         'exists, no need to pass the path to compiler '
         'directory.')

     parser.add_argument(
         '-f',
         '--cflags',
         action='append',
         default=[],
         help='Specify the cflags options for the toolchain. '
         'Be sure to quote all the cflags with quotation '
         'mark("") or use equal(=).')
     parser.add_argument(
         '--ldflags',
         action='append',
         default=[],
         help='Specify linker flags for the toolchain.')

     parser.add_argument(
         '-i',
         '--iterations',
         type=int,
         default=1,
         help='Specify how many iterations does the test '
         'take.')

     # Arguments -s and -r are for connecting to DUT.
     parser.add_argument(
         '-s',
         '--serials',
         help='Comma separate list of device serials under '
         'test.')

     parser.add_argument(
         '-r',
         '--remote',
         default='localhost',
         help='hostname[:port] if the ADB device is connected '
         'to a remote machine. Ensure this workstation '
         'is configured for passwordless ssh access as '
         'users "root" or "adb"')

     # Arguments -frequency and -m are for device settings
     parser.add_argument(
         '--frequency',
         type=int,
         default=979200,
         help='Specify the CPU frequency of the device. The '
         'unit is KHZ. The available value is defined in'
         'cpufreq/scaling_available_frequency file in '
         'device\'s each core directory. '
         'The default value is 979200, which shows a '
         'balance in noise and performance. Lower '
         'frequency will slow down the performance but '
         'reduce noise.')

     parser.add_argument(
         '-m',
         '--mode',
         default='little',
         help='User can specify whether \'little\' or \'big\' '
         'mode to use. The default one is little mode. '
         'The little mode runs on a single core of '
         'Cortex-A53, while big mode runs on single core '
         'of Cortex-A57.')

     # Configure file for benchmark test
     parser.add_argument(
         '-t',
         '--test',
         help='Specify the test settings with configuration '
         'file.')

     # Whether to keep old json result or not
     parser.add_argument(
         '-k',
         '--keep',
         default='False',
         help='User can specify whether to keep the old json '
         'results from last run. This can be useful if you '
         'want to compare performance differences in two or '
         'more different runs. Default is False(off).')

     return parser.parse_args(argv)


 # Clear old log files in bench suite directory
 def clear_logs():
     logging.info('Removing old logfiles...')
     for f in ['build_log', 'device_log', 'test_log']:
         logfile = os.path.join(config.bench_suite_dir, f)
         try:
             os.remove(logfile)
         except OSError:
             logging.info('No logfile %s need to be removed. Ignored.', f)
     logging.info('Old logfiles been removed.')


 # Clear old json files in bench suite directory
 def clear_results():
     logging.info('Clearing old json results...')
     for bench in config.bench_list:
         result = os.path.join(config.bench_suite_dir, bench + '.json')
         try:
             os.remove(result)
         except OSError:
             logging.info('no %s json file need to be removed. Ignored.', bench)
     logging.info('Old json results been removed.')


 # Use subprocess.check_call to run other script, and put logs to files
 def check_call_with_log(cmd, log_file):
     log_file = os.path.join(config.bench_suite_dir, log_file)
     with open(log_file, 'a') as logfile:
         log_header = 'Log for command: %s\n' % (cmd)
         logfile.write(log_header)
         try:
             subprocess.check_call(cmd, stdout=logfile)
         except subprocess.CalledProcessError:
             logging.error('Error running %s, please check %s for more info.',
                           cmd, log_file)
             raise
     logging.info('Logs for %s are written to %s.', cmd, log_file)


 def set_device(serials, remote, frequency):
     setting_cmd = [
         os.path.join(
             os.path.join(config.android_home, config.autotest_dir),
             'site_utils/set_device.py')
     ]
     setting_cmd.append('-r=' + remote)
     setting_cmd.append('-q=' + str(frequency))

     # Deal with serials.
     # If there is no serails specified, try to run test on the only device.
     # If specified, split the serials into a list and run test on each device.
     if serials:
         for serial in serials.split(','):
             setting_cmd.append('-s=' + serial)
             check_call_with_log(setting_cmd, 'device_log')
             setting_cmd.pop()
     else:
         check_call_with_log(setting_cmd, 'device_log')

     logging.info('CPU mode and frequency set successfully!')


 def log_ambiguous_args():
     logging.error('The count of arguments does not match!')
     raise ValueError('The count of arguments does not match.')


 # Check if the count of building arguments are log_ambiguous or not.  The
 # number of -c/-l, -f, and -os should be either all 0s or all the same.
 def check_count(compiler, llvm_version, build_os, cflags, ldflags):
     # Count will be set to 0 if no compiler or llvm_version specified.
     # Otherwise, one of these two args length should be 0 and count will be
     # the other one.
     count = max(len(compiler), len(llvm_version))

     # Check if number of cflags is 0 or the same with before.
     if len(cflags) != 0:
         if count != 0 and len(cflags) != count:
             log_ambiguous_args()
         count = len(cflags)

     if len(ldflags) != 0:
         if count != 0 and len(ldflags) != count:
             log_ambiguous_args()
         count = len(ldflags)

     if len(build_os) != 0:
         if count != 0 and len(build_os) != count:
             log_ambiguous_args()
         count = len(build_os)

     # If no settings are passed, only run default once.
     return max(1, count)


 # Build benchmark binary with toolchain settings
 def build_bench(setting_no, bench, compiler, llvm_version, build_os, cflags,
                 ldflags):
     # Build benchmark locally
     build_cmd = ['./build_bench.py', '-b=' + bench]
     if compiler:
         build_cmd.append('-c=' + compiler[setting_no])
     if llvm_version:
         build_cmd.append('-l=' + llvm_version[setting_no])
     if build_os:
         build_cmd.append('-o=' + build_os[setting_no])
     if cflags:
         build_cmd.append('-f=' + cflags[setting_no])
     if ldflags:
         build_cmd.append('--ldflags=' + ldflags[setting_no])

     logging.info('Building benchmark for toolchain setting No.%d...',
                  setting_no)
     logging.info('Command: %s', build_cmd)

     try:
         subprocess.check_call(build_cmd)
     except:
         logging.error('Error while building benchmark!')
         raise


 def run_and_collect_result(test_cmd, setting_no, i, bench, serial='default'):

     # Run autotest script for benchmark on DUT
     check_call_with_log(test_cmd, 'test_log')

     logging.info('Benchmark with setting No.%d, iter.%d finished testing on '
                  'device %s.', setting_no, i, serial)

     # Rename results from the bench_result generated in autotest
     bench_result = os.path.join(config.bench_suite_dir, 'bench_result')
     if not os.path.exists(bench_result):
         logging.error('No result found at %s, '
                       'please check test_log for details.', bench_result)
         raise OSError('Result file %s not found.' % bench_result)

     new_bench_result = 'bench_result_%s_%s_%d_%d' % (bench, serial,
                                                      setting_no, i)
     new_bench_result_path = os.path.join(config.bench_suite_dir,
                                          new_bench_result)
     try:
         os.rename(bench_result, new_bench_result_path)
     except OSError:
         logging.error('Error while renaming raw result %s to %s',
                       bench_result, new_bench_result_path)
         raise

     logging.info('Benchmark result saved at %s.', new_bench_result_path)


 def test_bench(bench, setting_no, iterations, serials, remote, mode):
     logging.info('Start running benchmark on device...')

     # Run benchmark and tests on DUT
     for i in xrange(iterations):
         logging.info('Iteration No.%d:', i)
         test_cmd = [
             os.path.join(
                 os.path.join(config.android_home, config.autotest_dir),
                 'site_utils/test_bench.py')
         ]
         test_cmd.append('-b=' + bench)
         test_cmd.append('-r=' + remote)
         test_cmd.append('-m=' + mode)

         # Deal with serials. If there is no serails specified, try to run test
         # on the only device. If specified, split the serials into a list and
         # run test on each device.
         if serials:
             for serial in serials.split(','):
                 test_cmd.append('-s=' + serial)

                 run_and_collect_result(test_cmd, setting_no, i, bench, serial)
                 test_cmd.pop()
         else:
             run_and_collect_result(test_cmd, setting_no, i, bench)


 def gen_json(bench, setting_no, iterations, serials):
     bench_result = os.path.join(config.bench_suite_dir, 'bench_result')

     logging.info('Generating JSON file for Crosperf...')

     if not serials:
         serials = 'default'

     for serial in serials.split(','):

         # Platform will be used as device lunch combo instead
         #experiment = '_'.join([serial, str(setting_no)])
         experiment = config.product_combo

         # Input format: bench_result_{bench}_{serial}_{setting_no}_
         input_file = '_'.join([bench_result, bench,
                                serial, str(setting_no), ''])
         gen_json_cmd = [
             './gen_json.py', '--input=' + input_file,
             '--output=%s.json' % os.path.join(config.bench_suite_dir, bench),
             '--bench=' + bench, '--platform=' + experiment,
             '--iterations=' + str(iterations)
         ]

         logging.info('Command: %s', gen_json_cmd)
         if subprocess.call(gen_json_cmd):
             logging.error('Error while generating JSON file, please check raw'
                           ' data of the results at %s.', input_file)


 def gen_crosperf(infile, outfile):
     # Set environment variable for crosperf
     os.environ['PYTHONPATH'] = os.path.dirname(config.toolchain_utils)

     logging.info('Generating Crosperf Report...')
     crosperf_cmd = [
         os.path.join(config.toolchain_utils, 'generate_report.py'),
         '-i=' + infile, '-o=' + outfile, '-f'
     ]

     # Run crosperf generate_report.py
     logging.info('Command: %s', crosperf_cmd)
     subprocess.call(crosperf_cmd)

     logging.info('Report generated successfully!')
     logging.info('Report Location: ' + outfile + '.html at bench'
                  'suite directory.')


 def main(argv):
     # Set environment variable for the local loacation of benchmark suite.
     # This is for collecting testing results to benchmark suite directory.
     os.environ['BENCH_SUITE_DIR'] = config.bench_suite_dir

     # Set Android type, used for the difference part between aosp and internal.
     os.environ['ANDROID_TYPE'] = config.android_type

     # Set ANDROID_HOME for both building and testing.
     os.environ['ANDROID_HOME'] = config.android_home

     # Set environment variable for architecture, this will be used in
     # autotest.
     os.environ['PRODUCT'] = config.product

     arguments = _parse_arguments(argv)

     bench_list = arguments.bench
     if not bench_list:
         bench_list = config.bench_list

     compiler = arguments.compiler_dir
     build_os = arguments.build_os
     llvm_version = arguments.llvm_prebuilts_version
     cflags = arguments.cflags
     ldflags = arguments.ldflags
     iterations = arguments.iterations
     serials = arguments.serials
     remote = arguments.remote
     frequency = arguments.frequency
     mode = arguments.mode
     keep = arguments.keep

     # Clear old logs every time before run script
     clear_logs()

     if keep == 'False':
         clear_results()

     # Set test mode and frequency of CPU on the DUT
     set_device(serials, remote, frequency)

     test = arguments.test
     # if test configuration file has been given, use the build settings
     # in the configuration file and run the test.
     if test:
         test_config = ConfigParser.ConfigParser(allow_no_value=True)
         if not test_config.read(test):
             logging.error('Error while reading from building '
                           'configuration file %s.', test)
             raise RuntimeError('Error while reading configuration file %s.'
                                % test)

         for setting_no, section in enumerate(test_config.sections()):
             bench = test_config.get(section, 'bench')
             compiler = [test_config.get(section, 'compiler')]
             build_os = [test_config.get(section, 'build_os')]
             llvm_version = [test_config.get(section, 'llvm_version')]
             cflags = [test_config.get(section, 'cflags')]
             ldflags = [test_config.get(section, 'ldflags')]

             # Set iterations from test_config file, if not exist, use the one
             # from command line.
             it = test_config.get(section, 'iterations')
             if not it:
                 it = iterations
             it = int(it)

             # Build benchmark for each single test configuration
             build_bench(0, bench, compiler, llvm_version,
                         build_os, cflags, ldflags)

             test_bench(bench, setting_no, it, serials, remote, mode)

             gen_json(bench, setting_no, it, serials)

         for bench in config.bench_list:
             infile = os.path.join(config.bench_suite_dir, bench + '.json')
             if os.path.exists(infile):
                 outfile = os.path.join(config.bench_suite_dir,
                                        bench + '_report')
                 gen_crosperf(infile, outfile)

         # Stop script if there is only config file provided
         return 0

     # If no configuration file specified, continue running.
     # Check if the count of the setting arguments are log_ambiguous.
     setting_count = check_count(compiler, llvm_version, build_os,
                                 cflags, ldflags)

     for bench in bench_list:
         logging.info('Start building and running benchmark: [%s]', bench)
         # Run script for each toolchain settings
         for setting_no in xrange(setting_count):
             build_bench(setting_no, bench, compiler, llvm_version,
                         build_os, cflags, ldflags)

             # Run autotest script for benchmark test on device
             test_bench(bench, setting_no, iterations, serials, remote, mode)

             gen_json(bench, setting_no, iterations, serials)

         infile = os.path.join(config.bench_suite_dir, bench + '.json')
         outfile = os.path.join(config.bench_suite_dir, bench + '_report')
         gen_crosperf(infile, outfile)


 if __name__ == '__main__':
     main(sys.argv[1:])
	#!/usr/bin/env python2
	#
	# Copyright 2017 The Chromium OS Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	#
	# pylint: disable=cros-logging-import

	# This is the script to run specified benchmark with different toolchain
	# settings. It includes the process of building benchmark locally and running
	# benchmark on DUT.

	"""Main script to run the benchmark suite from building to testing."""
	from __future__ import print_function

	import argparse
	import config
	import ConfigParser
	import logging
	import os
	import subprocess
	import sys

	logging.basicConfig(level=logging.INFO)

	def _parse_arguments(argv):
	parser = argparse.ArgumentParser(description='Build and run specific '
	'benchamrk')
	parser.add_argument(
	'-b',
	'--bench',
	action='append',
	default=[],
	help='Select which benchmark to run')

	# Only one of compiler directory and llvm prebuilts version can be indicated
	# at the beginning, so set -c and -l into a exclusive group.
	group = parser.add_mutually_exclusive_group()

	# The toolchain setting arguments has action of 'append', so that users
	# could compare performance with several toolchain settings together.
	group.add_argument(
	'-c',
	'--compiler_dir',
	metavar='DIR',
	action='append',
	default=[],
	help='Specify path to the compiler\'s bin directory. '
	'You shall give several paths, each with a -c, to '
	'compare performance differences in '
	'each compiler.')

	parser.add_argument(
	'-o',
	'--build_os',
	action='append',
	default=[],
	help='Specify the host OS to build the benchmark.')

	group.add_argument(
	'-l',
	'--llvm_prebuilts_version',
	action='append',
	default=[],
	help='Specify the version of prebuilt LLVM. When '
	'specific prebuilt version of LLVM already '
	'exists, no need to pass the path to compiler '
	'directory.')

	parser.add_argument(
	'-f',
	'--cflags',
	action='append',
	default=[],
	help='Specify the cflags options for the toolchain. '
	'Be sure to quote all the cflags with quotation '
	'mark("") or use equal(=).')
	parser.add_argument(
	'--ldflags',
	action='append',
	default=[],
	help='Specify linker flags for the toolchain.')

	parser.add_argument(
	'-i',
	'--iterations',
	type=int,
	default=1,
	help='Specify how many iterations does the test '
	'take.')

	# Arguments -s and -r are for connecting to DUT.
	parser.add_argument(
	'-s',
	'--serials',
	help='Comma separate list of device serials under '
	'test.')

	parser.add_argument(
	'-r',
	'--remote',
	default='localhost',
	help='hostname[:port] if the ADB device is connected '
	'to a remote machine. Ensure this workstation '
	'is configured for passwordless ssh access as '
	'users "root" or "adb"')

	# Arguments -frequency and -m are for device settings
	parser.add_argument(
	'--frequency',
	type=int,
	default=979200,
	help='Specify the CPU frequency of the device. The '
	'unit is KHZ. The available value is defined in'
	'cpufreq/scaling_available_frequency file in '
	'device\'s each core directory. '
	'The default value is 979200, which shows a '
	'balance in noise and performance. Lower '
	'frequency will slow down the performance but '
	'reduce noise.')

	parser.add_argument(
	'-m',
	'--mode',
	default='little',
	help='User can specify whether \'little\' or \'big\' '
	'mode to use. The default one is little mode. '
	'The little mode runs on a single core of '
	'Cortex-A53, while big mode runs on single core '
	'of Cortex-A57.')

	# Configure file for benchmark test
	parser.add_argument(
	'-t',
	'--test',
	help='Specify the test settings with configuration '
	'file.')

	# Whether to keep old json result or not
	parser.add_argument(
	'-k',
	'--keep',
	default='False',
	help='User can specify whether to keep the old json '
	'results from last run. This can be useful if you '
	'want to compare performance differences in two or '
	'more different runs. Default is False(off).')

	return parser.parse_args(argv)


	# Clear old log files in bench suite directory
	def clear_logs():
	logging.info('Removing old logfiles...')
	for f in ['build_log', 'device_log', 'test_log']:
	logfile = os.path.join(config.bench_suite_dir, f)
	try:
	os.remove(logfile)
	except OSError:
	logging.info('No logfile %s need to be removed. Ignored.', f)
	logging.info('Old logfiles been removed.')


	# Clear old json files in bench suite directory
	def clear_results():
	logging.info('Clearing old json results...')
	for bench in config.bench_list:
	result = os.path.join(config.bench_suite_dir, bench + '.json')
	try:
	os.remove(result)
	except OSError:
	logging.info('no %s json file need to be removed. Ignored.', bench)
	logging.info('Old json results been removed.')


	# Use subprocess.check_call to run other script, and put logs to files
	def check_call_with_log(cmd, log_file):
	log_file = os.path.join(config.bench_suite_dir, log_file)
	with open(log_file, 'a') as logfile:
	log_header = 'Log for command: %s\n' % (cmd)
	logfile.write(log_header)
	try:
	subprocess.check_call(cmd, stdout=logfile)
	except subprocess.CalledProcessError:
	logging.error('Error running %s, please check %s for more info.',
	cmd, log_file)
	raise
	logging.info('Logs for %s are written to %s.', cmd, log_file)


	def set_device(serials, remote, frequency):
	setting_cmd = [
	os.path.join(
	os.path.join(config.android_home, config.autotest_dir),
	'site_utils/set_device.py')
	]
	setting_cmd.append('-r=' + remote)
	setting_cmd.append('-q=' + str(frequency))

	# Deal with serials.
	# If there is no serails specified, try to run test on the only device.
	# If specified, split the serials into a list and run test on each device.
	if serials:
	for serial in serials.split(','):
	setting_cmd.append('-s=' + serial)
	check_call_with_log(setting_cmd, 'device_log')
	setting_cmd.pop()
	else:
	check_call_with_log(setting_cmd, 'device_log')

	logging.info('CPU mode and frequency set successfully!')


	def log_ambiguous_args():
	logging.error('The count of arguments does not match!')
	raise ValueError('The count of arguments does not match.')


	# Check if the count of building arguments are log_ambiguous or not. The
	# number of -c/-l, -f, and -os should be either all 0s or all the same.
	def check_count(compiler, llvm_version, build_os, cflags, ldflags):
	# Count will be set to 0 if no compiler or llvm_version specified.
	# Otherwise, one of these two args length should be 0 and count will be
	# the other one.
	count = max(len(compiler), len(llvm_version))

	# Check if number of cflags is 0 or the same with before.
	if len(cflags) != 0:
	if count != 0 and len(cflags) != count:
	log_ambiguous_args()
	count = len(cflags)

	if len(ldflags) != 0:
	if count != 0 and len(ldflags) != count:
	log_ambiguous_args()
	count = len(ldflags)

	if len(build_os) != 0:
	if count != 0 and len(build_os) != count:
	log_ambiguous_args()
	count = len(build_os)

	# If no settings are passed, only run default once.
	return max(1, count)


	# Build benchmark binary with toolchain settings
	def build_bench(setting_no, bench, compiler, llvm_version, build_os, cflags,
	ldflags):
	# Build benchmark locally
	build_cmd = ['./build_bench.py', '-b=' + bench]
	if compiler:
	build_cmd.append('-c=' + compiler[setting_no])
	if llvm_version:
	build_cmd.append('-l=' + llvm_version[setting_no])
	if build_os:
	build_cmd.append('-o=' + build_os[setting_no])
	if cflags:
	build_cmd.append('-f=' + cflags[setting_no])
	if ldflags:
	build_cmd.append('--ldflags=' + ldflags[setting_no])

	logging.info('Building benchmark for toolchain setting No.%d...',
	setting_no)
	logging.info('Command: %s', build_cmd)

	try:
	subprocess.check_call(build_cmd)
	except:
	logging.error('Error while building benchmark!')
	raise


	def run_and_collect_result(test_cmd, setting_no, i, bench, serial='default'):

	# Run autotest script for benchmark on DUT
	check_call_with_log(test_cmd, 'test_log')

	logging.info('Benchmark with setting No.%d, iter.%d finished testing on '
	'device %s.', setting_no, i, serial)

	# Rename results from the bench_result generated in autotest
	bench_result = os.path.join(config.bench_suite_dir, 'bench_result')
	if not os.path.exists(bench_result):
	logging.error('No result found at %s, '
	'please check test_log for details.', bench_result)
	raise OSError('Result file %s not found.' % bench_result)

	new_bench_result = 'bench_result_%s_%s_%d_%d' % (bench, serial,
	setting_no, i)
	new_bench_result_path = os.path.join(config.bench_suite_dir,
	new_bench_result)
	try:
	os.rename(bench_result, new_bench_result_path)
	except OSError:
	logging.error('Error while renaming raw result %s to %s',
	bench_result, new_bench_result_path)
	raise

	logging.info('Benchmark result saved at %s.', new_bench_result_path)


	def test_bench(bench, setting_no, iterations, serials, remote, mode):
	logging.info('Start running benchmark on device...')

	# Run benchmark and tests on DUT
	for i in xrange(iterations):
	logging.info('Iteration No.%d:', i)
	test_cmd = [
	os.path.join(
	os.path.join(config.android_home, config.autotest_dir),
	'site_utils/test_bench.py')
	]
	test_cmd.append('-b=' + bench)
	test_cmd.append('-r=' + remote)
	test_cmd.append('-m=' + mode)

	# Deal with serials. If there is no serails specified, try to run test
	# on the only device. If specified, split the serials into a list and
	# run test on each device.
	if serials:
	for serial in serials.split(','):
	test_cmd.append('-s=' + serial)

	run_and_collect_result(test_cmd, setting_no, i, bench, serial)
	test_cmd.pop()
	else:
	run_and_collect_result(test_cmd, setting_no, i, bench)


	def gen_json(bench, setting_no, iterations, serials):
	bench_result = os.path.join(config.bench_suite_dir, 'bench_result')

	logging.info('Generating JSON file for Crosperf...')

	if not serials:
	serials = 'default'

	for serial in serials.split(','):

	# Platform will be used as device lunch combo instead
	#experiment = '_'.join([serial, str(setting_no)])
	experiment = config.product_combo

	# Input format: bench_result_{bench}_{serial}_{setting_no}_
	input_file = '_'.join([bench_result, bench,
	serial, str(setting_no), ''])
	gen_json_cmd = [
	'./gen_json.py', '--input=' + input_file,
	'--output=%s.json' % os.path.join(config.bench_suite_dir, bench),
	'--bench=' + bench, '--platform=' + experiment,
	'--iterations=' + str(iterations)
	]

	logging.info('Command: %s', gen_json_cmd)
	if subprocess.call(gen_json_cmd):
	logging.error('Error while generating JSON file, please check raw'
	' data of the results at %s.', input_file)


	def gen_crosperf(infile, outfile):
	# Set environment variable for crosperf
	os.environ['PYTHONPATH'] = os.path.dirname(config.toolchain_utils)

	logging.info('Generating Crosperf Report...')
	crosperf_cmd = [
	os.path.join(config.toolchain_utils, 'generate_report.py'),
	'-i=' + infile, '-o=' + outfile, '-f'
	]

	# Run crosperf generate_report.py
	logging.info('Command: %s', crosperf_cmd)
	subprocess.call(crosperf_cmd)

	logging.info('Report generated successfully!')
	logging.info('Report Location: ' + outfile + '.html at bench'
	'suite directory.')


	def main(argv):
	# Set environment variable for the local loacation of benchmark suite.
	# This is for collecting testing results to benchmark suite directory.
	os.environ['BENCH_SUITE_DIR'] = config.bench_suite_dir

	# Set Android type, used for the difference part between aosp and internal.
	os.environ['ANDROID_TYPE'] = config.android_type

	# Set ANDROID_HOME for both building and testing.
	os.environ['ANDROID_HOME'] = config.android_home

	# Set environment variable for architecture, this will be used in
	# autotest.
	os.environ['PRODUCT'] = config.product

	arguments = _parse_arguments(argv)

	bench_list = arguments.bench
	if not bench_list:
	bench_list = config.bench_list

	compiler = arguments.compiler_dir
	build_os = arguments.build_os
	llvm_version = arguments.llvm_prebuilts_version
	cflags = arguments.cflags
	ldflags = arguments.ldflags
	iterations = arguments.iterations
	serials = arguments.serials
	remote = arguments.remote
	frequency = arguments.frequency
	mode = arguments.mode
	keep = arguments.keep

	# Clear old logs every time before run script
	clear_logs()

	if keep == 'False':
	clear_results()

	# Set test mode and frequency of CPU on the DUT
	set_device(serials, remote, frequency)

	test = arguments.test
	# if test configuration file has been given, use the build settings
	# in the configuration file and run the test.
	if test:
	test_config = ConfigParser.ConfigParser(allow_no_value=True)
	if not test_config.read(test):
	logging.error('Error while reading from building '
	'configuration file %s.', test)
	raise RuntimeError('Error while reading configuration file %s.'
	% test)

	for setting_no, section in enumerate(test_config.sections()):
	bench = test_config.get(section, 'bench')
	compiler = [test_config.get(section, 'compiler')]
	build_os = [test_config.get(section, 'build_os')]
	llvm_version = [test_config.get(section, 'llvm_version')]
	cflags = [test_config.get(section, 'cflags')]
	ldflags = [test_config.get(section, 'ldflags')]

	# Set iterations from test_config file, if not exist, use the one
	# from command line.
	it = test_config.get(section, 'iterations')
	if not it:
	it = iterations
	it = int(it)

	# Build benchmark for each single test configuration
	build_bench(0, bench, compiler, llvm_version,
	build_os, cflags, ldflags)

	test_bench(bench, setting_no, it, serials, remote, mode)

	gen_json(bench, setting_no, it, serials)

	for bench in config.bench_list:
	infile = os.path.join(config.bench_suite_dir, bench + '.json')
	if os.path.exists(infile):
	outfile = os.path.join(config.bench_suite_dir,
	bench + '_report')
	gen_crosperf(infile, outfile)

	# Stop script if there is only config file provided
	return 0

	# If no configuration file specified, continue running.
	# Check if the count of the setting arguments are log_ambiguous.
	setting_count = check_count(compiler, llvm_version, build_os,
	cflags, ldflags)

	for bench in bench_list:
	logging.info('Start building and running benchmark: [%s]', bench)
	# Run script for each toolchain settings
	for setting_no in xrange(setting_count):
	build_bench(setting_no, bench, compiler, llvm_version,
	build_os, cflags, ldflags)

	# Run autotest script for benchmark test on device
	test_bench(bench, setting_no, iterations, serials, remote, mode)

	gen_json(bench, setting_no, iterations, serials)

	infile = os.path.join(config.bench_suite_dir, bench + '.json')
	outfile = os.path.join(config.bench_suite_dir, bench + '_report')
	gen_crosperf(infile, outfile)


	if __name__ == '__main__':
	main(sys.argv[1:])