| #!/usr/bin/python |
| # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """ |
| Automatically update the afe_stable_versions table. |
| |
| This command updates the stable repair version for selected boards |
| in the lab. For each board, if the version that Omaha is serving |
| on the Beta channel for the board is more recent than the current |
| stable version in the AFE database, then the AFE is updated to use |
| the version on Omaha. |
| |
| The upgrade process is applied to every "managed board" in the test |
| lab. Generally, a managed board is a board with both spare and |
| critical scheduling pools. |
| |
| See `autotest_lib.site_utils.lab_inventory` for the full definition |
| of "managed board". |
| |
| The command supports a `--dry-run` option that reports changes that |
| would be made, without making the actual RPC calls to change the |
| database. |
| |
| """ |
| |
| import argparse |
| import json |
| import subprocess |
| import sys |
| |
| import common |
| from autotest_lib.client.common_lib import utils |
| from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
| from autotest_lib.site_utils import lab_inventory |
| |
| |
| # _OMAHA_STATUS - URI of a file in GoogleStorage with a JSON object |
| # summarizing all versions currently being served by Omaha. |
| # |
| # The principle data is in an array named 'omaha_data'. Each entry |
| # in the array contains information relevant to one image being |
| # served by Omaha, including the following information: |
| # * The board name of the product, as known to Omaha. |
| # * The channel associated with the image. |
| # * The Chrome and Chrome OS version strings for the image |
| # being served. |
| # |
| _OMAHA_STATUS = 'gs://chromeos-build-release-console/omaha_status.json' |
| |
| |
| # _BUILD_METADATA_PATTERN - Format string for the URI of a file in |
| # GoogleStorage with a JSON object that contains metadata about |
| # a given build. The metadata includes the version of firmware |
| # bundled with the build. |
| # |
| _BUILD_METADATA_PATTERN = 'gs://chromeos-image-archive/%s/metadata.json' |
| |
| |
| # _DEFAULT_BOARD - The distinguished board name used to identify a |
| # stable version mapping that is used for any board without an explicit |
| # mapping of its own. |
| # |
| # _DEFAULT_VERSION_TAG - A string used to signify that there is no |
| # mapping for a board, in other words, the board is mapped to the |
| # default version. |
| # |
| _DEFAULT_BOARD = 'DEFAULT' |
| _DEFAULT_VERSION_TAG = '(default)' |
| |
| |
| # _FIRMWARE_UPGRADE_BLACKLIST - a set of boards that are exempt from |
| # automatic stable firmware version assignment. This blacklist is |
| # here out of an abundance of caution, on the general principle of "if |
| # it ain't broke, don't fix it." Specifically, these are old, legacy |
| # boards and: |
| # * They're working fine with whatever firmware they have in the lab |
| # right now. Moreover, because of their age, we can expect that |
| # they will never get any new firmware updates in future. |
| # * Servo support is spotty or missing, so there's no certainty |
| # that DUTs bricked by a firmware update can be repaired. |
| # * Because of their age, they are somewhere between hard and |
| # impossible to replace. In some cases, they are also already |
| # in short supply. |
| # |
| # N.B. HARDCODED BOARD NAMES ARE EVIL!!! This blacklist uses hardcoded |
| # names because it's meant to define a list of legacies that will shrivel |
| # and die over time. |
| # |
| # DO NOT ADD TO THIS LIST. If there's a new use case that requires |
| # extending the blacklist concept, you should find a maintainable |
| # solution that deletes this code. |
| # |
| # TODO(jrbarnette): When any board is past EOL, and removed from the |
| # lab, it can be removed from the blacklist. When all the boards are |
| # past EOL, the blacklist should be removed. |
| |
| _FIRMWARE_UPGRADE_BLACKLIST = set([ |
| 'butterfly', |
| 'daisy', |
| 'daisy_skate', |
| 'daisy_spring', |
| 'lumpy', |
| 'parrot', |
| 'parrot_ivb', |
| 'peach_pi', |
| 'peach_pit', |
| 'stout', |
| 'stumpy', |
| 'x86-alex', |
| 'x86-mario', |
| 'x86-zgb', |
| ]) |
| |
| |
| def _get_by_key_path(dictdict, key_path): |
| """ |
| Traverse a sequence of keys in a dict of dicts. |
| |
| The `dictdict` parameter is a dict of nested dict values, and |
| `key_path` a list of keys. |
| |
| A single-element key path returns `dictdict[key_path[0]]`, a |
| two-element path returns `dictdict[key_path[0]][key_path[1]]`, and |
| so forth. If any key in the path is not found, return `None`. |
| |
| @param dictdict A dictionary of nested dictionaries. |
| @param key_path The sequence of keys to look up in `dictdict`. |
| @return The value found by successive dictionary lookups, or `None`. |
| """ |
| value = dictdict |
| for key in key_path: |
| value = value.get(key) |
| if value is None: |
| break |
| return value |
| |
| |
| def _get_model_firmware_versions(metadata_json, board): |
| """ |
| Get the firmware version for each model for a unibuild board. |
| |
| @param metadata_json The metadata_json dict parsed from the metadata.json |
| file generated by the build. |
| @param board The board name of the unibuild. |
| @return If no models found for a board, return {board: None}; elase, return |
| a dict mapping from model name to its upgrade firmware version. |
| """ |
| model_firmware_versions = {} |
| key_path = ['board-metadata', board, 'models'] |
| model_versions = _get_by_key_path(metadata_json, key_path) |
| |
| if model_versions is not None: |
| for model, fw_versions in model_versions.iteritems(): |
| fw_version = (fw_versions.get('main-readwrite-firmware-version') or |
| fw_versions.get('main-readonly-firmware-version')) |
| model_firmware_versions[model] = fw_version |
| else: |
| model_firmware_versions[board] = None |
| |
| return model_firmware_versions |
| |
| |
| def get_firmware_versions(version_map, board, cros_version): |
| """ |
| Get the firmware versions for a given board and CrOS version. |
| |
| Typically, CrOS builds bundle firmware that is installed at update |
| time. The returned firmware version value will be `None` if the build isn't |
| found in storage, if there is no firmware found for the build, or if the |
| board is blacklisted from firmware updates in the test lab. |
| |
| @param version_map An AFE cros version map object; used to |
| locate the build in storage. |
| @param board The board for the firmware version to be |
| determined. |
| @param cros_version The CrOS version bundling the firmware. |
| @return A dict mapping from board to firmware version string for |
| non-unibuild board, or a dict mapping from models to firmware |
| versions for a unibuild board (see return type of |
| _get_model_firmware_versions) |
| """ |
| if board in _FIRMWARE_UPGRADE_BLACKLIST: |
| return {board: None} |
| try: |
| image_path = version_map.format_image_name(board, cros_version) |
| uri = _BUILD_METADATA_PATTERN % image_path |
| metadata_json = _read_gs_json_data(uri) |
| unibuild = bool(_get_by_key_path(metadata_json, ['unibuild'])) |
| if unibuild: |
| return _get_model_firmware_versions(metadata_json, board) |
| else: |
| key_path = ['board-metadata', board, 'main-firmware-version'] |
| fw_version = _get_by_key_path(metadata_json, key_path) |
| return {board: fw_version} |
| except Exception as e: |
| # TODO(jrbarnette): If we get here, it likely means that |
| # the repair build for our board doesn't exist. That can |
| # happen if a board doesn't release on the Beta channel for |
| # at least 6 months. |
| # |
| # We can't allow this error to propogate up the call chain |
| # because that will kill assigning versions to all the other |
| # boards that are still OK, so for now we ignore it. We |
| # really should do better. |
| print ('Failed to get firmware version for board %s: %s.' % (board, e)) |
| return {board: None} |
| |
| |
| class _VersionUpdater(object): |
| """ |
| Class to report and apply version changes. |
| |
| This class is responsible for the low-level logic of applying |
| version upgrades and reporting them as command output. |
| |
| This class exists to solve two problems: |
| 1. To distinguish "normal" vs. "dry-run" modes. Each mode has a |
| subclass; methods that perform actual AFE updates are |
| implemented for the normal mode subclass only. |
| 2. To provide hooks for unit tests. The unit tests override both |
| the reporting and modification behaviors, in order to test the |
| higher level logic that decides what changes are needed. |
| |
| Methods meant merely to report changes to command output have names |
| starting with "report" or "_report". Methods that are meant to |
| change the AFE in normal mode have names starting with "_do" |
| """ |
| |
| def __init__(self, afe): |
| image_types = [afe.CROS_IMAGE_TYPE, afe.FIRMWARE_IMAGE_TYPE] |
| self._version_maps = { |
| image_type: afe.get_stable_version_map(image_type) |
| for image_type in image_types |
| } |
| self._cros_map = self._version_maps[afe.CROS_IMAGE_TYPE] |
| self._selected_map = None |
| |
| def select_version_map(self, image_type): |
| """ |
| Select an AFE version map object based on `image_type`. |
| |
| This creates and remembers an AFE version mapper object to be |
| used for making changes in normal mode. |
| |
| @param image_type Image type parameter for the version mapper |
| object. |
| """ |
| self._selected_map = self._version_maps[image_type] |
| return self._selected_map |
| |
| def announce(self): |
| """Announce the start of processing to the user.""" |
| pass |
| |
| def report(self, message): |
| """ |
| Report a pre-formatted message for the user. |
| |
| The message is printed to stdout, followed by a newline. |
| |
| @param message The message to be provided to the user. |
| """ |
| print message |
| |
| def report_default_changed(self, old_default, new_default): |
| """ |
| Report that the default version mapping is changing. |
| |
| This merely reports a text description of the pending change |
| without executing it. |
| |
| @param old_default The original default version. |
| @param new_default The new default version to be applied. |
| """ |
| self.report('Default %s -> %s' % (old_default, new_default)) |
| |
| def _report_board_changed(self, board, old_version, new_version): |
| """ |
| Report a change in one board's assigned version mapping. |
| |
| This merely reports a text description of the pending change |
| without executing it. |
| |
| @param board The board with the changing version. |
| @param old_version The original version mapped to the board. |
| @param new_version The new version to be applied to the board. |
| """ |
| template = ' %-22s %s -> %s' |
| self.report(template % (board, old_version, new_version)) |
| |
| def report_board_unchanged(self, board, old_version): |
| """ |
| Report that a board's version mapping is unchanged. |
| |
| This reports that a board has a non-default mapping that will be |
| unchanged. |
| |
| @param board The board that is not changing. |
| @param old_version The board's version mapping. |
| """ |
| self._report_board_changed(board, '(no change)', old_version) |
| |
| def _do_set_mapping(self, board, new_version): |
| """ |
| Change one board's assigned version mapping. |
| |
| @param board The board with the changing version. |
| @param new_version The new version to be applied to the board. |
| """ |
| pass |
| |
| def _do_delete_mapping(self, board): |
| """ |
| Delete one board's assigned version mapping. |
| |
| @param board The board with the version to be deleted. |
| """ |
| pass |
| |
| def set_mapping(self, board, old_version, new_version): |
| """ |
| Change and report a board version mapping. |
| |
| @param board The board with the changing version. |
| @param old_version The original version mapped to the board. |
| @param new_version The new version to be applied to the board. |
| """ |
| self._report_board_changed(board, old_version, new_version) |
| self._do_set_mapping(board, new_version) |
| |
| def upgrade_default(self, new_default): |
| """ |
| Apply a default version change. |
| |
| @param new_default The new default version to be applied. |
| """ |
| self._do_set_mapping(_DEFAULT_BOARD, new_default) |
| |
| def delete_mapping(self, board, old_version): |
| """ |
| Delete a board version mapping, and report the change. |
| |
| @param board The board with the version to be deleted. |
| @param old_version The board's verson prior to deletion. |
| """ |
| assert board != _DEFAULT_BOARD |
| self._report_board_changed(board, |
| old_version, |
| _DEFAULT_VERSION_TAG) |
| self._do_delete_mapping(board) |
| |
| |
| class _DryRunUpdater(_VersionUpdater): |
| """Code for handling --dry-run execution.""" |
| |
| def announce(self): |
| self.report('Dry run: no changes will be made.') |
| |
| |
| class _NormalModeUpdater(_VersionUpdater): |
| """Code for handling normal execution.""" |
| |
| def _do_set_mapping(self, board, new_version): |
| self._selected_map.set_version(board, new_version) |
| |
| def _do_delete_mapping(self, board): |
| self._selected_map.delete_version(board) |
| |
| |
| def _read_gs_json_data(gs_uri): |
| """ |
| Read and parse a JSON file from googlestorage. |
| |
| This is a wrapper around `gsutil cat` for the specified URI. |
| The standard output of the command is parsed as JSON, and the |
| resulting object returned. |
| |
| @return A JSON object parsed from `gs_uri`. |
| """ |
| with open('/dev/null', 'w') as ignore_errors: |
| sp = subprocess.Popen(['gsutil', 'cat', gs_uri], |
| stdout=subprocess.PIPE, |
| stderr=ignore_errors) |
| try: |
| json_object = json.load(sp.stdout) |
| finally: |
| sp.stdout.close() |
| sp.wait() |
| return json_object |
| |
| |
| def _make_omaha_versions(omaha_status): |
| """ |
| Convert parsed omaha versions data to a versions mapping. |
| |
| Returns a dictionary mapping board names to the currently preferred |
| version for the Beta channel as served by Omaha. The mappings are |
| provided by settings in the JSON object `omaha_status`. |
| |
| The board names are the names as known to Omaha: If the board name |
| in the AFE contains '_', the corresponding Omaha name uses '-' |
| instead. The boards mapped may include boards not in the list of |
| managed boards in the lab. |
| |
| @return A dictionary mapping Omaha boards to Beta versions. |
| """ |
| def _entry_valid(json_entry): |
| return json_entry['channel'] == 'beta' |
| |
| def _get_omaha_data(json_entry): |
| board = json_entry['board']['public_codename'] |
| milestone = json_entry['milestone'] |
| build = json_entry['chrome_os_version'] |
| version = 'R%d-%s' % (milestone, build) |
| return (board, version) |
| |
| return dict(_get_omaha_data(e) for e in omaha_status['omaha_data'] |
| if _entry_valid(e)) |
| |
| |
| def _get_upgrade_versions(cros_versions, omaha_versions, boards): |
| """ |
| Get the new stable versions to which we should update. |
| |
| The new versions are returned as a tuple of a dictionary mapping |
| board names to versions, plus a new default board setting. The |
| new default is determined as the most commonly used version |
| across the given boards. |
| |
| The new dictionary will have a mapping for every board in `boards`. |
| That mapping will be taken from `cros_versions`, unless the board has |
| a mapping in `omaha_versions` _and_ the omaha version is more recent |
| than the AFE version. |
| |
| @param cros_versions The current board->version mappings in the |
| AFE. |
| @param omaha_versions The current board->version mappings from |
| Omaha for the Beta channel. |
| @param boards Set of boards to be upgraded. |
| @return Tuple of (mapping, default) where mapping is a dictionary |
| mapping boards to versions, and default is a version string. |
| """ |
| upgrade_versions = {} |
| version_counts = {} |
| afe_default = cros_versions[_DEFAULT_BOARD] |
| for board in boards: |
| version = cros_versions.get(board, afe_default) |
| omaha_version = omaha_versions.get(board.replace('_', '-')) |
| if (omaha_version is not None and |
| utils.compare_versions(version, omaha_version) < 0): |
| version = omaha_version |
| upgrade_versions[board] = version |
| version_counts.setdefault(version, 0) |
| version_counts[version] += 1 |
| return (upgrade_versions, |
| max(version_counts.items(), key=lambda x: x[1])[0]) |
| |
| |
| def _get_firmware_upgrades(cros_version_map, cros_versions): |
| """ |
| Get the new firmware versions to which we should update. |
| |
| @param cros_version_map An instance of frontend._CrosVersionMap. |
| @param cros_versions Current board->cros version mappings in the |
| AFE. |
| @return A dictionary mapping boards/models to firmware upgrade versions. |
| If the build is unibuild, the key is a model name; else, the key |
| is a board name. |
| """ |
| firmware_upgrades = {} |
| for board, version in cros_versions.iteritems(): |
| firmware_upgrades.update(get_firmware_versions( |
| cros_version_map, board, version)) |
| |
| return firmware_upgrades |
| |
| |
| def _apply_cros_upgrades(updater, old_versions, new_versions, |
| new_default): |
| """ |
| Change CrOS stable version mappings in the AFE. |
| |
| The input `old_versions` dictionary represents the content of the |
| `afe_stable_versions` database table; it contains mappings for a |
| default version, plus exceptions for boards with non-default |
| mappings. |
| |
| The `new_versions` dictionary contains a mapping for every board, |
| including boards that will be mapped to the new default version. |
| |
| This function applies the AFE changes necessary to produce the new |
| AFE mappings indicated by `new_versions` and `new_default`. The |
| changes are ordered so that at any moment, every board is mapped |
| either according to the old or the new mapping. |
| |
| @param updater Instance of _VersionUpdater responsible for |
| making the actual database changes. |
| @param old_versions The current board->version mappings in the |
| AFE. |
| @param new_versions New board->version mappings obtained by |
| applying Beta channel upgrades from Omaha. |
| @param new_default The new default build for the AFE. |
| """ |
| old_default = old_versions[_DEFAULT_BOARD] |
| if old_default != new_default: |
| updater.report_default_changed(old_default, new_default) |
| updater.report('Applying stable version changes:') |
| default_count = 0 |
| for board, new_build in new_versions.items(): |
| if new_build == new_default: |
| default_count += 1 |
| elif board in old_versions and new_build == old_versions[board]: |
| updater.report_board_unchanged(board, new_build) |
| else: |
| old_build = old_versions.get(board) |
| if old_build is None: |
| old_build = _DEFAULT_VERSION_TAG |
| updater.set_mapping(board, old_build, new_build) |
| if old_default != new_default: |
| updater.upgrade_default(new_default) |
| for board, new_build in new_versions.items(): |
| if new_build == new_default and board in old_versions: |
| updater.delete_mapping(board, old_versions[board]) |
| updater.report('%d boards now use the default mapping' % |
| default_count) |
| |
| |
| def _apply_firmware_upgrades(updater, old_versions, new_versions): |
| """ |
| Change firmware version mappings in the AFE. |
| |
| The input `old_versions` dictionary represents the content of the |
| firmware mappings in the `afe_stable_versions` database table. |
| There is no default version; missing boards simply have no current |
| version. |
| |
| This function applies the AFE changes necessary to produce the new |
| AFE mappings indicated by `new_versions`. |
| |
| TODO(jrbarnette) This function ought to remove any mapping not found |
| in `new_versions`. However, in theory, that's only needed to |
| account for boards that are removed from the lab, and that hasn't |
| happened yet. |
| |
| @param updater Instance of _VersionUpdater responsible for |
| making the actual database changes. |
| @param old_versions The current board->version mappings in the |
| AFE. |
| @param new_versions New board->version mappings obtained by |
| applying Beta channel upgrades from Omaha. |
| """ |
| unchanged = 0 |
| no_version = 0 |
| for board, new_firmware in new_versions.items(): |
| if new_firmware is None: |
| no_version += 1 |
| elif board not in old_versions: |
| updater.set_mapping(board, '(nothing)', new_firmware) |
| else: |
| old_firmware = old_versions[board] |
| if new_firmware != old_firmware: |
| updater.set_mapping(board, old_firmware, new_firmware) |
| else: |
| unchanged += 1 |
| updater.report('%d boards have no firmware mapping' % no_version) |
| updater.report('%d boards are unchanged' % unchanged) |
| |
| |
| def _parse_command_line(argv): |
| """ |
| Parse the command line arguments. |
| |
| Create an argument parser for this command's syntax, parse the |
| command line, and return the result of the ArgumentParser |
| parse_args() method. |
| |
| @param argv Standard command line argument vector; argv[0] is |
| assumed to be the command name. |
| @return Result returned by ArgumentParser.parse_args(). |
| |
| """ |
| parser = argparse.ArgumentParser( |
| prog=argv[0], |
| description='Update the stable repair version for all ' |
| 'boards') |
| parser.add_argument('-n', '--dry-run', dest='updater_mode', |
| action='store_const', const=_DryRunUpdater, |
| help='print changes without executing them') |
| parser.add_argument('extra_boards', nargs='*', metavar='BOARD', |
| help='Names of additional boards to be updated.') |
| arguments = parser.parse_args(argv[1:]) |
| if not arguments.updater_mode: |
| arguments.updater_mode = _NormalModeUpdater |
| return arguments |
| |
| |
| def main(argv): |
| """ |
| Standard main routine. |
| |
| @param argv Command line arguments including `sys.argv[0]`. |
| """ |
| arguments = _parse_command_line(argv) |
| afe = frontend_wrappers.RetryingAFE(server=None) |
| updater = arguments.updater_mode(afe) |
| updater.announce() |
| boards = (set(arguments.extra_boards) | |
| lab_inventory.get_managed_boards(afe)) |
| |
| cros_version_map = updater.select_version_map(afe.CROS_IMAGE_TYPE) |
| cros_versions = cros_version_map.get_all_versions() |
| omaha_versions = _make_omaha_versions( |
| _read_gs_json_data(_OMAHA_STATUS)) |
| upgrade_versions, new_default = ( |
| _get_upgrade_versions(cros_versions, omaha_versions, boards)) |
| _apply_cros_upgrades(updater, cros_versions, |
| upgrade_versions, new_default) |
| |
| updater.report('\nApplying firmware updates:') |
| firmware_version_map = updater.select_version_map(afe.FIRMWARE_IMAGE_TYPE) |
| fw_versions = firmware_version_map.get_all_versions() |
| firmware_upgrades = _get_firmware_upgrades( |
| cros_version_map, upgrade_versions) |
| _apply_firmware_upgrades(updater, fw_versions, firmware_upgrades) |
| |
| |
| if __name__ == '__main__': |
| main(sys.argv) |
