Attached is the CLI code tarball. It is documented at http://test.kernel.org/autotest/CLIHowTo
From: [email protected]
git-svn-id: http://test.kernel.org/svn/autotest/trunk@1950 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/cli/topic_common.py b/cli/topic_common.py
new file mode 100755
index 0000000..4d94a0e
--- /dev/null
+++ b/cli/topic_common.py
@@ -0,0 +1,552 @@
+#
+# Copyright 2008 Google Inc. All Rights Reserved.
+#
+"""
+This module contains the generic CLI object
+
+High Level Design:
+
+The atest class contains attributes & method generic to all the CLI
+operations.
+
+The class inheritance is shown here using the command
+'atest host create ...' as an example:
+
+atest <-- host <-- host_create <-- site_host_create
+
+Note: The site_<topic>.py and its classes are only needed if you need
+to override the common <topic>.py methods with your site specific ones.
+
+
+High Level Algorithm:
+
+1. atest figures out the topic and action from the 2 first arguments
+ on the command line and imports the <topic> (or site_<topic>)
+ module.
+
+1. Init
+ The main atest module creates a <topic>_<action> object. The
+ __init__() function is used to setup the parser options, if this
+ <action> has some specific options to add to its <topic>.
+
+ If it exists, the child __init__() method must call its parent
+ class __init__() before adding its own parser arguments.
+
+2. Parsing
+ If the child wants to validate the parsing (e.g. make sure that
+ there are hosts in the arguments), or if it wants to check the
+ options it added in its __init__(), it should implement a parse()
+ method.
+
+ The child parser must call its parent parser and gets back the
+ options dictionary and the rest of the command line arguments
+ (leftover). Each level gets to see all the options, but the
+ leftovers can be deleted as they can be consumed by only one
+ object.
+
+3. Execution
+ This execute() method is specific to the child and should use the
+ self.execute_rpc() to send commands to the Autotest Front-End. It
+ should return results.
+
+4. Output
+ The child output() method is called with the execute() resutls as a
+ parameter. This is child-specific, but should leverage the
+ atest.print_*() methods.
+"""
+
+import os, sys, pwd, optparse, re, textwrap, urllib2, getpass, socket
+from autotest_lib.cli import rpc
+from autotest_lib.frontend.afe.json_rpc import proxy
+
+
+# Maps the AFE keys to printable names.
+KEYS_TO_NAMES_EN = {'hostname': 'Host',
+ 'platform': 'Platform',
+ 'status': 'Status',
+ 'locked': 'Locked',
+ 'locked_by': 'Locked by',
+ 'labels': 'Labels',
+ 'description': 'Description',
+ 'hosts': 'Hosts',
+ 'users': 'Users',
+ 'id': 'Id',
+ 'name': 'Name',
+ 'invalid': 'Valid',
+ 'login': 'Login',
+ 'access_level': 'Access Level',
+ 'job_id': 'Job Id',
+ 'job_owner': 'Job Owner',
+ 'job_name': 'Job Name',
+ 'test_type': 'Test Type',
+ 'test_class': 'Test Class',
+ 'path': 'Path',
+ 'owner': 'Owner',
+ 'status_counts': 'Status Counts',
+ 'hosts_status': 'Host Status',
+ 'priority': 'Priority',
+ 'control_type': 'Control Type',
+ 'created_on': 'Created On',
+ 'synch_type': 'Synch Type',
+ 'control_file': 'Control File',
+ }
+
+# In the failure, tag that will replace the item.
+FAIL_TAG = '<XYZ>'
+
+# Global socket timeout
+DEFAULT_SOCKET_TIMEOUT = 5
+# For list commands, can take longer
+LIST_SOCKET_TIMEOUT = 30
+# For uploading kernels, can take much, much longer
+UPLOAD_SOCKET_TIMEOUT = 60*30
+
+
+# Convertion functions to be called for printing,
+# e.g. to print True/False for booleans.
+def __convert_platform(field):
+ if not field:
+ # Can be None
+ return ""
+ elif type(field) == int:
+ # Can be 0/1 for False/True
+ return str(bool(field))
+ else:
+ # Can be a platform name
+ return field
+
+
+KEYS_CONVERT = {'locked': lambda flag: str(bool(flag)),
+ 'invalid': lambda flag: str(bool(not flag)),
+ 'platform': __convert_platform,
+ 'labels': lambda labels: ', '.join(labels)}
+
+class CliError(Exception):
+ pass
+
+
+class atest(object):
+ """Common class for generic processing
+ Should only be instantiated by itself for usage
+ references, otherwise, the <topic> objects should
+ be used."""
+ msg_topic = "[acl|host|job|label|user]"
+ usage_action = "[action]"
+ msg_items = ''
+
+ def invalid_arg(self, header, follow_up=''):
+ twrap = textwrap.TextWrapper(initial_indent=' ',
+ subsequent_indent=' ')
+ rest = twrap.fill(follow_up)
+
+ if self.kill_on_failure:
+ self.invalid_syntax(header + rest)
+ else:
+ print >> sys.stderr, header + rest
+
+
+ def invalid_syntax(self, msg):
+ print
+ print >> sys.stderr, msg
+ print
+ print "usage:",
+ print self._get_usage()
+ print
+ sys.exit(1)
+
+
+ def generic_error(self, msg):
+ print >> sys.stderr, msg
+ sys.exit(1)
+
+
+ def failure(self, full_error, item=None, what_failed=''):
+ """If kill_on_failure, print this error and die,
+ otherwise, queue the error and accumulate all the items
+ that triggered the same error."""
+
+ if self.debug:
+ errmsg = str(full_error)
+ else:
+ errmsg = str(full_error).split('Traceback')[0].rstrip('\n')
+
+ if self.kill_on_failure:
+ print >> sys.stderr, "%s\n %s" % (what_failed, errmsg)
+ sys.exit(1)
+
+ # Build a dictionary with the 'what_failed' as keys. The
+ # values are dictionaries with the errmsg as keys and a set
+ # of items as values.
+ # self.failed =
+ # {'Operation delete_host_failed': {'AclAccessViolation:
+ # set('host0', 'host1')}}
+ # Try to gather all the same error messages together,
+ # even if they contain the 'item'
+ if item and item in errmsg:
+ errmsg = errmsg.replace(item, FAIL_TAG)
+ if self.failed.has_key(what_failed):
+ self.failed[what_failed].setdefault(errmsg, set()).add(item)
+ else:
+ self.failed[what_failed] = {errmsg: set([item])}
+
+
+ def show_all_failures(self):
+ if not self.failed:
+ return 0
+ for what_failed in self.failed.keys():
+ print >> sys.stderr, what_failed + ':'
+ for (errmsg, items) in self.failed[what_failed].iteritems():
+ if len(items) == 0:
+ print >> sys.stderr, errmsg
+ elif items == set(['']):
+ print >> sys.stderr, ' ' + errmsg
+ elif len(items) == 1:
+ # Restore the only item
+ if FAIL_TAG in errmsg:
+ errmsg = errmsg.replace(FAIL_TAG, items.pop())
+ else:
+ errmsg = '%s (%s)' % (errmsg, items.pop())
+ print >> sys.stderr, ' ' + errmsg
+ else:
+ print >> sys.stderr, ' ' + errmsg + ' with <XYZ> in:'
+ twrap = textwrap.TextWrapper(initial_indent=' ',
+ subsequent_indent=' ')
+ items = list(items)
+ items.sort()
+ print >> sys.stderr, twrap.fill(', '.join(items))
+ return 1
+
+
+ def __init__(self):
+ """Setup the parser common options"""
+ # Initialized for unit tests.
+ self.afe = None
+ self.failed = {}
+ self.data = {}
+ self.debug = False
+ self.kill_on_failure = False
+ self.web_server = ''
+ self.verbose = False
+
+ self.parser = optparse.OptionParser(self._get_usage())
+ self.parser.add_option('-g', '--debug',
+ help='Print debugging information',
+ action='store_true', default=False)
+ self.parser.add_option('--kill-on-failure',
+ help='Stop at the first failure',
+ action='store_true', default=False)
+ self.parser.add_option('--parse',
+ help='Print the output using colon '
+ 'separated key=value fields',
+ action='store_true', default=False)
+ self.parser.add_option('-v', '--verbose',
+ action='store_true', default=False)
+ self.parser.add_option('-w', '--web',
+ help='Specify the autotest server '
+ 'to talk to',
+ action='store', type='string',
+ dest='web_server', default=None)
+
+ # Shorten the TCP timeout.
+ socket.setdefaulttimeout(DEFAULT_SOCKET_TIMEOUT)
+
+
+ def _file_list(self, options, opt_file='', opt_list='', add_on=[]):
+ """Returns a list containing the unique items from the
+ options.<opt_list>, from the file options.<opt_file>,
+ and from the space separated add_on strings.
+ The opt_list can be space or comma separated list.
+ Used for host, acls, labels... arguments"""
+ # Start with the add_on
+ result = set()
+ [result.add(item)
+ for items in add_on
+ for item in items.split(',')]
+
+ # Process the opt_list, if any
+ try:
+ args = getattr(options, opt_list)
+ [result.add(arg) for arg in re.split(r'[\s,]', args)]
+ except (AttributeError, TypeError):
+ pass
+
+ # Process the file list, if any and not empty
+ # The file can contain space and/or comma separated items
+ try:
+ flist = getattr(options, opt_file)
+ file_content = []
+ for line in open(flist).readlines():
+ if line == '\n':
+ continue
+ file_content += re.split(r'[\s,]',
+ line.rstrip('\n'))
+ if len(file_content) == 0:
+ self.invalid_syntax("Empty file %s" % flist)
+ result = result.union(file_content)
+ except (AttributeError, TypeError):
+ pass
+ except IOError:
+ self.invalid_syntax("Could not open file %s" % flist)
+
+ return list(result)
+
+
+ def _get_usage(self):
+ return "atest %s %s [options] %s" % (self.msg_topic.lower(),
+ self.usage_action,
+ self.msg_items)
+
+
+ def parse_with_flist(self, flists, req_items):
+ """Flists is a list of tuples containing:
+ (attribute, opt_fname, opt_list, use_leftover)
+
+ self.<atttribute> will be populated with a set
+ containing the lines of the file named
+ options.<opt_fname> and the options.<opt_list> values
+ and the leftover from the parsing if use_leftover is
+ True. There should only be one use_leftover set to
+ True in the list.
+ Also check if the req_items is not empty after parsing."""
+ (options, leftover) = atest.parse(self)
+ if leftover == None:
+ leftover = []
+
+ for (attribute, opt_fname, opt_list, use_leftover) in flists:
+ if use_leftover:
+ add_on = leftover
+ leftover = []
+ else:
+ add_on = []
+
+ setattr(self, attribute,
+ self._file_list(options,
+ opt_file=opt_fname,
+ opt_list=opt_list,
+ add_on=add_on))
+
+ if (req_items and not getattr(self, req_items, None)):
+ self.invalid_syntax('%s %s requires at least one %s' %
+ (self.msg_topic,
+ self.usage_action,
+ self.msg_topic))
+
+ return (options, leftover)
+
+
+ def parse(self):
+ """Parse all the arguments.
+
+ It consumes what the common object needs to know, and
+ let the children look at all the options. We could
+ remove the options that we have used, but there is no
+ harm in leaving them, and the children may need them
+ in the future.
+
+ Must be called from its children parse()"""
+ (options, leftover) = self.parser.parse_args()
+ # Handle our own options setup in __init__()
+ self.debug = options.debug
+ self.kill_on_failure = options.kill_on_failure
+
+ if options.parse:
+ suffix = '_parse'
+ else:
+ suffix = '_std'
+ for func in ['print_fields', 'print_table',
+ 'print_by_ids']:
+ setattr(self, func, getattr(self, func + suffix))
+
+ self.verbose = options.verbose
+ self.web_server = options.web_server
+ self.afe = rpc.afe_comm(self.web_server)
+
+ return (options, leftover)
+
+
+ def check_and_create_items(self, op_get, op_create,
+ items, **data_create):
+ """Create the items if they don't exist already"""
+ for item in items:
+ ret = self.execute_rpc(op_get, name=item)
+
+ if len(ret) == 0:
+ try:
+ data_create['name'] = item
+ self.execute_rpc(op_create, **data_create)
+ except CliError:
+ continue
+
+
+ def execute_rpc(self, op, item='', **data):
+ retry = 2
+ while retry:
+ try:
+ return self.afe.run(op, **data)
+ except urllib2.URLError, err:
+ if 'timed out' not in err.reason:
+ self.invalid_syntax('Invalid server name %s: %s' %
+ (self.afe.web_server, err))
+ if self.debug:
+ print 'retrying: %r %d' % (data, retry)
+ retry -= 1
+ if retry == 0:
+ if item:
+ myerr = '%s timed out for %s' % (op, item)
+ else:
+ myerr = '%s timed out' % op
+ self.failure(myerr, item=item,
+ what_failed=("Timed-out contacting "
+ "the Autotest server"))
+ raise CliError("Timed-out contacting the Autotest server")
+ except Exception, full_error:
+ # There are various exceptions throwns by JSON,
+ # urllib & httplib, so catch them all.
+ self.failure(full_error, item=item,
+ what_failed='Operation %s failed' % op)
+ raise CliError(str(full_error))
+
+
+ # There is no output() method in the atest object (yet?)
+ # but here are some helper functions to be used by its
+ # children
+ def print_wrapped(self, msg, values):
+ if len(values) == 0:
+ return
+ elif len(values) == 1:
+ print msg + ': '
+ elif len(values) > 1:
+ if msg.endswith('s'):
+ print msg + ': '
+ else:
+ print msg + 's: '
+
+ values.sort()
+ twrap = textwrap.TextWrapper(initial_indent='\t',
+ subsequent_indent='\t')
+ print twrap.fill(', '.join(values))
+
+
+ def __conv_value(self, type, value):
+ return KEYS_CONVERT.get(type, str)(value)
+
+
+ def print_fields_std(self, items, keys, title=None):
+ """Print the keys in each item, one on each line"""
+ if not items:
+ print "No results"
+ return
+ if title:
+ print title
+ for item in items:
+ for key in keys:
+ print '%s: %s' % (KEYS_TO_NAMES_EN[key],
+ self.__conv_value(key,
+ item[key]))
+
+
+ def print_fields_parse(self, items, keys, title=None):
+ """Print the keys in each item as comma
+ separated name=value"""
+ if not items:
+ print "No results"
+ return
+ for item in items:
+ values = ['%s=%s' % (KEYS_TO_NAMES_EN[key],
+ self.__conv_value(key,
+ item[key]))
+ for key in keys
+ if self.__conv_value(key,
+ item[key]) != '']
+ print ':'.join(values)
+
+
+ def __find_justified_fmt(self, items, keys):
+ """Find the max length for each field."""
+ lens = {}
+ # Don't justify the last field, otherwise we have blank
+ # lines when the max is overlaps but the current values
+ # are smaller
+ if not items:
+ print "No results"
+ return
+ for key in keys[:-1]:
+ lens[key] = max(len(self.__conv_value(key,
+ item[key]))
+ for item in items)
+ lens[key] = max(lens[key], len(KEYS_TO_NAMES_EN[key]))
+ lens[keys[-1]] = 0
+
+ return ' '.join(["%%-%ds" % lens[key] for key in keys])
+
+
+ def print_table_std(self, items, keys_header, sublist_keys={}):
+ """Print a mix of header and lists in a user readable
+ format
+ The headers are justified, the sublist_keys are wrapped."""
+ if not items:
+ print "No results"
+ return
+ fmt = self.__find_justified_fmt(items, keys_header)
+ header = tuple(KEYS_TO_NAMES_EN[key] for key in keys_header)
+ print fmt % header
+ for item in items:
+ values = tuple(self.__conv_value(key, item[key])
+ for key in keys_header)
+ print fmt % values
+ if self.verbose and sublist_keys:
+ for key in sublist_keys:
+ self.print_wrapped(KEYS_TO_NAMES_EN[key],
+ item[key])
+ print '\n'
+
+
+ def print_table_parse(self, items, keys_header, sublist_keys=[]):
+ """Print a mix of header and lists in a user readable
+ format"""
+ if not items:
+ print "No results"
+ return
+ for item in items:
+ values = ['%s=%s' % (KEYS_TO_NAMES_EN[key],
+ self.__conv_value(key, item[key]))
+ for key in keys_header
+ if self.__conv_value(key,
+ item[key]) != '']
+
+ if self.verbose:
+ [values.append('%s=%s'% (KEYS_TO_NAMES_EN[key],
+ ','.join(item[key])))
+ for key in sublist_keys
+ if len(item[key])]
+
+ print ':'.join(values)
+
+
+ def print_by_ids_std(self, items, title=None, line_before=False):
+ """Prints ID & names of items in a user readable form"""
+ if not items:
+ return
+ if line_before:
+ print
+ if title:
+ print title + ':'
+ self.print_table_std(items, keys_header=['id', 'name'])
+
+
+ def print_by_ids_parse(self, items, title=None, line_before=False):
+ """Prints ID & names of items in a parseable format"""
+ if not items:
+ print "No results"
+ return
+ if title:
+ print title + '=',
+ values = []
+ for item in items:
+ values += ['%s=%s' % (KEYS_TO_NAMES_EN[key],
+ self.__conv_value(key,
+ item[key]))
+ for key in ['id', 'name']
+ if self.__conv_value(key,
+ item[key]) != '']
+ print ':'.join(values)
