Merge remote branch 'cros/upstream' into master
Merged to upstream autotest @4749~@5215.
The entire change list description is too big to enlist here. Please refer to upstream (http://autotest.kernel.org/browser) for more details.
BUG=
TEST=emerged both x86 and arm build.
Tested emerged x86 build bvt against a chromeos device.
Review URL: http://codereview.chromium.org/6246035
Change-Id: I8455f2135c87c321c6efc232e2869dc8f675395e
diff --git a/utils/check_patch.py b/utils/check_patch.py
index 576b97d..78af6b9 100755
--- a/utils/check_patch.py
+++ b/utils/check_patch.py
@@ -20,7 +20,7 @@
@author: Lucas Meneghel Rodrigues <[email protected]>
"""
-import os, stat, logging, sys, optparse
+import os, stat, logging, sys, optparse, time
import common
from autotest_lib.client.common_lib import utils, error, logging_config
from autotest_lib.client.common_lib import logging_manager
@@ -32,6 +32,20 @@
verbose=verbose)
+def ask(question, auto=False):
+ """
+ Raw input with a prompt that emulates logging.
+
+ @param question: Question to be asked
+ @param auto: Whether to return "y" instead of asking the question
+ """
+ if auto:
+ logging.info("%s (y/n) y" % question)
+ return "y"
+ return raw_input("%s INFO | %s (y/n) " %
+ (time.strftime("%H:%M:%S", time.localtime()), question))
+
+
class VCS(object):
"""
Abstraction layer to the version control system.
@@ -104,6 +118,7 @@
"""
def __init__(self):
logging.debug("Subversion VCS backend initialized.")
+ self.ignored_extension_list = ['.orig', '.bak']
def get_unknown_files(self):
@@ -112,7 +127,9 @@
for line in status.split("\n"):
status_flag = line[0]
if line and status_flag == "?":
- unknown_files.append(line[1:].strip())
+ for extension in self.ignored_extension_list:
+ if not line.endswith(extension):
+ unknown_files.append(line[1:].strip())
return unknown_files
@@ -181,13 +198,16 @@
Picks up a given file and performs various checks, looking after problems
and eventually suggesting solutions.
"""
- def __init__(self, path):
+ def __init__(self, path, confirm=False):
"""
Class constructor, sets the path attribute.
@param path: Path to the file that will be checked.
+ @param confirm: Whether to answer yes to all questions asked without
+ prompting the user.
"""
self.path = path
+ self.confirm = confirm
self.basename = os.path.basename(self.path)
if self.basename.endswith('.py'):
self.is_python = True
@@ -204,7 +224,7 @@
self.first_line = checked_file.readline()
checked_file.close()
self.corrective_actions = []
- self.indentation_exceptions = ['cli/job_unittest.py']
+ self.indentation_exceptions = ['job_unittest.py']
def _check_indent(self):
@@ -226,8 +246,6 @@
reindent_results = reindent_raw.split(" ")[-1].strip(".")
if reindent_results == "changed":
if self.basename not in self.indentation_exceptions:
- logging.error("Possible indentation and spacing issues on "
- "file %s" % self.path)
self.corrective_actions.append("reindent.py -v %s" % self.path)
@@ -242,8 +260,7 @@
c_cmd = 'run_pylint.py %s' % self.path
rc = utils.system(c_cmd, ignore_status=True)
if rc != 0:
- logging.error("Possible syntax problems on file %s", self.path)
- logging.error("You might want to rerun '%s'", c_cmd)
+ logging.error("Syntax issues found during '%s'", c_cmd)
def _check_unittest(self):
@@ -260,9 +277,8 @@
unittest_cmd = 'python %s' % unittest_path
rc = utils.system(unittest_cmd, ignore_status=True)
if rc != 0:
- logging.error("Problems during unit test execution "
- "for file %s", self.path)
- logging.error("You might want to rerun '%s'", unittest_cmd)
+ logging.error("Unittest issues found during '%s'",
+ unittest_cmd)
def _check_permissions(self):
@@ -273,14 +289,10 @@
"""
if self.first_line.startswith("#!"):
if not self.is_executable:
- logging.info("File %s seems to require execution "
- "permissions. ", self.path)
- self.corrective_actions.append("chmod +x %s" % self.path)
+ self.corrective_actions.append("svn propset svn:executable ON %s" % self.path)
else:
if self.is_executable:
- logging.info("File %s does not seem to require execution "
- "permissions. ", self.path)
- self.corrective_actions.append("chmod -x %s" % self.path)
+ self.corrective_actions.append("svn propdel svn:executable %s" % self.path)
def report(self):
@@ -294,10 +306,9 @@
self._check_code()
self._check_unittest()
if self.corrective_actions:
- logging.info("The following corrective actions are suggested:")
for action in self.corrective_actions:
- logging.info(action)
- answer = raw_input("Would you like to apply it? (y/n) ")
+ answer = ask("Would you like to execute %s?" % action,
+ auto=self.confirm)
if answer == "y":
rc = utils.system(action, ignore_status=True)
if rc != 0:
@@ -305,7 +316,8 @@
class PatchChecker(object):
- def __init__(self, patch=None, patchwork_id=None):
+ def __init__(self, patch=None, patchwork_id=None, confirm=False):
+ self.confirm = confirm
self.base_dir = os.getcwd()
if patch:
self.patch = os.path.abspath(patch)
@@ -322,7 +334,7 @@
if changed_files_before:
logging.error("Repository has changed files prior to patch "
"application. ")
- answer = raw_input("Would you like to revert them? (y/n) ")
+ answer = ask("Would you like to revert them?", auto=self.confirm)
if answer == "n":
logging.error("Not safe to proceed without reverting files.")
sys.exit(1)
@@ -370,20 +382,20 @@
for untracked_file in add_to_vcs:
logging.info(untracked_file)
logging.info("Might need to be added to VCS")
- logging.info("Would you like to add them to VCS ? (y/n/abort) ")
- answer = raw_input()
+ answer = ask("Would you like to add them to VCS ?")
if answer == "y":
for untracked_file in add_to_vcs:
self.vcs.add_untracked_file(untracked_file)
modified_files_after.append(untracked_file)
elif answer == "n":
pass
- elif answer == "abort":
- sys.exit(1)
for modified_file in modified_files_after:
- file_checker = FileChecker(modified_file)
- file_checker.report()
+ # Additional safety check, new commits might introduce
+ # new directories
+ if os.path.isfile(modified_file):
+ file_checker = FileChecker(modified_file)
+ file_checker.report()
def check(self):
@@ -399,20 +411,37 @@
help='id of a given patchwork patch')
parser.add_option('--verbose', dest="debug", action='store_true',
help='include debug messages in console output')
+ parser.add_option('-f', '--full-check', dest="full_check",
+ action='store_true',
+ help='check the full tree for corrective actions')
+ parser.add_option('-y', '--yes', dest="confirm",
+ action='store_true',
+ help='Answer yes to all questions')
options, args = parser.parse_args()
local_patch = options.local_patch
id = options.id
debug = options.debug
+ full_check = options.full_check
+ confirm = options.confirm
logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
- if local_patch:
- patch_checker = PatchChecker(patch=local_patch)
- elif id:
- patch_checker = PatchChecker(patchwork_id=id)
+ ignore_file_list = ['common.py']
+ if full_check:
+ for root, dirs, files in os.walk('.'):
+ if not '.svn' in root:
+ for file in files:
+ if file not in ignore_file_list:
+ path = os.path.join(root, file)
+ file_checker = FileChecker(path, confirm=confirm)
+ file_checker.report()
else:
- logging.error('No patch or patchwork id specified. Aborting.')
- sys.exit(1)
-
- patch_checker.check()
+ if local_patch:
+ patch_checker = PatchChecker(patch=local_patch, confirm=confirm)
+ elif id:
+ patch_checker = PatchChecker(patchwork_id=id, confirm=confirm)
+ else:
+ logging.error('No patch or patchwork id specified. Aborting.')
+ sys.exit(1)
+ patch_checker.check()
diff --git a/utils/external_packages.py b/utils/external_packages.py
old mode 100644
new mode 100755
index d505eef..bcfd15d
--- a/utils/external_packages.py
+++ b/utils/external_packages.py
@@ -613,7 +613,8 @@
class ParamikoPackage(ExternalPackage):
version = '1.7.5'
local_filename = 'paramiko-%s.tar.gz' % version
- urls = ('http://www.lag.net/paramiko/download/' + local_filename,)
+ urls = ('http://www.lag.net/paramiko/download/' + local_filename,
+ 'ftp://mirrors.kernel.org/gentoo/distfiles/' + local_filename,)
hex_sum = '592be7a08290070b71da63a8e6f28a803399e5c5'
diff --git a/utils/run_pylint.py b/utils/run_pylint.py
index 0bf5f95..3c3e225 100755
--- a/utils/run_pylint.py
+++ b/utils/run_pylint.py
@@ -6,10 +6,13 @@
# do a basic check to see if pylint is even installed
try:
import pylint
+ from pylint.__pkginfo__ import version as pylint_version
except ImportError:
print "Unable to import pylint, it may need to be installed"
sys.exit(1)
+major, minor, release = pylint_version.split('.')
+pylint_version = float("%s.%s" % (major, minor))
pylintrc_path = os.path.expanduser('~/.pylintrc')
if not os.path.exists(pylintrc_path):
open(pylintrc_path, 'w').close()
@@ -54,10 +57,13 @@
# * common_lib.enum.Enum objects
# * DB model objects (scheduler models are the worst, but Django models also
# generate some errors)
-pylint_base_opts = ['--disable-msg-cat=warning,refactor,convention',
- '--disable-msg=E1101,E1103',
- '--reports=no',
- '--include-ids=y']
+if pylint_version >= 0.21:
+ pylint_base_opts = ['--disable=W,R,C,E1101,E1103']
+else:
+ pylint_base_opts = ['--disable-msg-cat=warning,refactor,convention',
+ '--disable-msg=E1101,E1103']
+pylint_base_opts += ['--reports=no',
+ '--include-ids=y']
file_list = sys.argv[1:]
if '--' in file_list:
diff --git a/utils/test_importer.py b/utils/test_importer.py
index 00b912c..a7be755 100755
--- a/utils/test_importer.py
+++ b/utils/test_importer.py
@@ -242,6 +242,12 @@
_log_or_execute(repr(new_test), new_test.save)
add_label_dependencies(new_test)
+ # save TestParameter
+ for para_name in data.test_parameters:
+ test_parameter = models.TestParameter.objects.get_or_create(
+ test=new_test, name=para_name)[0]
+ test_parameter.save()
+
def _set_attributes_clean(test, data):
"""
diff --git a/utils/unittest_suite.py b/utils/unittest_suite.py
index 629eba6..78a9602 100755
--- a/utils/unittest_suite.py
+++ b/utils/unittest_suite.py
@@ -16,6 +16,8 @@
parser.add_option("--skip-tests", dest="skip_tests", default=[],
help="A space separated list of tests to skip")
+parser.set_defaults(module_list=None)
+
REQUIRES_DJANGO = set((
'monitor_db_unittest.py',
@@ -64,6 +66,11 @@
'logging_manager_test.py',
))
+# This particular KVM autotest test is not a unittest
+SKIP = set((
+ 'guest_test.py',
+ ))
+
LONG_TESTS = (REQUIRES_DJANGO |
REQUIRES_MYSQLDB |
REQUIRES_GWT |
@@ -103,16 +110,10 @@
raise TestFailure(msg)
-def find_and_run_tests(start, options):
- """
- Find and run Python unittest suites below the given directory. Only look
- in subdirectories of start that are actual importable Python modules.
-
- @param start: The absolute directory to look for tests under.
- @param options: optparse options.
- """
+def scan_for_modules(start, options):
modules = []
- skip_tests = set()
+
+ skip_tests = SKIP
if options.skip_tests:
skip_tests.update(options.skip_tests.split())
@@ -142,6 +143,22 @@
modules.append(['autotest_lib'] + names)
if options.debug:
print 'testing', path_no_py
+ return modules
+
+def find_and_run_tests(start, options):
+ """
+ Find and run Python unittest suites below the given directory. Only look
+ in subdirectories of start that are actual importable Python modules.
+
+ @param start: The absolute directory to look for tests under.
+ @param options: optparse options.
+ """
+ if options.module_list:
+ modules = []
+ for m in options.module_list:
+ modules.append(m.split('.'))
+ else:
+ modules = scan_for_modules(start, options)
if options.debug:
print 'Number of test modules found:', len(modules)
@@ -170,9 +187,7 @@
def main():
options, args = parser.parse_args()
if args:
- parser.error('Unexpected argument(s): %s' % args)
- parser.print_help()
- sys.exit(1)
+ options.module_list = args
# Strip the arguments off the command line, so that the unit tests do not
# see them.
