Update payload library + command-line tool
An initial implementation of a Python module for parsing, checking and
applying a Chrome OS update payload. Comes with a command-line tool
(paycheck.py) for applying such operations on payload files, and a test
script (test_paycheck.sh) for ensuring that the library and tool are
working correctly.
Since update_payload is introduced as a package, we're moving some
previously merged utilities into the package's directory.
(Unit testing for this code will be uploaded on a separate CL; see
chromium-os:39663)
BUG=chromium-os:34911,chromium-os:33607,chromium-os:7597
TEST=test_paycheck.sh successful on MP-signed payloads
CQ-DEPEND=I5746a1d80e822a575f0d96f94d0b4e765fc64507
Change-Id: I77123a1fffbb2059c239b7145c6922968fdffb6a
Reviewed-on: https://gerrit.chromium.org/gerrit/43041
Reviewed-by: Gilad Arnold <[email protected]>
Tested-by: Gilad Arnold <[email protected]>
Reviewed-by: Chris Sosa <[email protected]>
Reviewed-by: Jay Srinivasan <[email protected]>
Reviewed-by: Don Garrett <[email protected]>
Commit-Queue: Gilad Arnold <[email protected]>
diff --git a/scripts/paycheck.py b/scripts/paycheck.py
new file mode 100755
index 0000000..77af744
--- /dev/null
+++ b/scripts/paycheck.py
@@ -0,0 +1,160 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Command-line tool for checking and applying Chrome OS update payloads."""
+
+import optparse
+import os
+import sys
+
+# pylint: disable=F0401
+lib_dir = os.path.join(os.path.dirname(__file__), 'lib')
+if os.path.exists(lib_dir) and os.path.isdir(lib_dir):
+ sys.path.insert(1, lib_dir)
+import update_payload
+
+
+_TYPE_FULL = 'full'
+_TYPE_DELTA = 'delta'
+
+
+def ParseArguments(parser, argv):
+ """Parse and validate command-line arguments.
+
+ Args:
+ parser: the command-line parser
+ Returns:
+ A tuple (options, payload, extra_args), where `options' are the options
+ returned by the parser, `payload' is the name of the payload file
+ (mandatory argument) and `extra_args' are any additional command-line
+ arguments.
+
+ """
+ options, args = parser.parse_args(argv)
+
+ # Validate a value given to --type, if any.
+ if options.assert_type not in (None, _TYPE_FULL, _TYPE_DELTA):
+ parser.error('invalid argument to --type: %s' % options.assert_type)
+
+ # There are several options that imply --check.
+ options.check = (options.check or options.report or options.assert_type or
+ options.block_size or options.allow_unhashed or
+ options.key or options.meta_sig)
+
+ # Check number of arguments, enforce payload type accordingly.
+ if len(args) == 3:
+ if options.assert_type == _TYPE_DELTA:
+ parser.error('%s payload requires source partition arguments' %
+ _TYPE_DELTA)
+ options.assert_type = _TYPE_FULL
+ elif len(args) == 5:
+ if options.assert_type == _TYPE_FULL:
+ parser.error('%s payload does not accept source partition arguments' %
+ _TYPE_FULL)
+ options.assert_type = _TYPE_DELTA
+ elif len(args) != 1:
+ parser.error('unexpected number of arguments')
+
+ return options, args[0], args[1:]
+
+
+def main(argv):
+ parser = optparse.OptionParser(
+ usage='Usage: %prog [OPTION...] PAYLOAD [DST_KERN DST_ROOT '
+ '[SRC_KERN SRC_ROOT]]',
+ description='Applies a Chrome OS update PAYLOAD to SRC_KERN and '
+ 'SRC_ROOT emitting DST_KERN and DST_ROOT, respectively. '
+ 'SRC_KERN and SRC_ROOT need only be provided for delta '
+ 'payloads. If no partitions are provided, only verifies '
+ 'payload integrity.')
+
+ check_opts = optparse.OptionGroup(parser, 'Payload checking')
+ check_opts.add_option('-c', '--check', action='store_true', default=False,
+ help='check payload integrity')
+ check_opts.add_option('-r', '--report', metavar='FILE',
+ help="dump payload report (`-' for stdout)")
+ check_opts.add_option('-t', '--type', metavar='TYPE', dest='assert_type',
+ help="assert that payload is either `%s' or `%s'" %
+ (_TYPE_FULL, _TYPE_DELTA))
+ check_opts.add_option('-z', '--block-size', metavar='NUM', default=0,
+ type='int',
+ help='assert a non-default (4096) payload block size')
+ check_opts.add_option('-u', '--allow-unhashed', action='store_true',
+ default=False, help='allow unhashed operations')
+ check_opts.add_option('-k', '--key', metavar='FILE',
+ help='public key to be used for signature verification')
+ check_opts.add_option('-m', '--meta-sig', metavar='FILE',
+ help='verify metadata against its signature')
+ parser.add_option_group(check_opts)
+
+ trace_opts = optparse.OptionGroup(parser, 'Block tracing')
+ trace_opts.add_option('-b', '--root-block', metavar='BLOCK', type='int',
+ help='trace the origin for a rootfs block')
+ trace_opts.add_option('-B', '--kern-block', metavar='BLOCK', type='int',
+ help='trace the origin for a kernel block')
+ trace_opts.add_option('-s', '--skip', metavar='NUM', default='0', type='int',
+ help='skip first NUM occurrences of traced block')
+ parser.add_option_group(trace_opts)
+
+ # Parse and validate arguments.
+ options, payload_file_name, extra_args = ParseArguments(parser, argv[1:])
+
+ with open(payload_file_name) as payload_file:
+ payload = update_payload.Payload(payload_file)
+ try:
+ # Initialize payload.
+ payload.Init()
+
+ # Perform payload integrity checks.
+ if options.check:
+ report_file = None
+ do_close_report_file = False
+ try:
+ if options.report:
+ if options.report == '-':
+ report_file = sys.stdout
+ else:
+ report_file = open(options.report, 'w')
+ do_close_report_file = True
+
+ payload.Check(
+ pubkey_file_name=options.key,
+ metadata_sig_file=open(options.meta_sig)
+ if options.meta_sig else None,
+ report_out_file=report_file,
+ assert_type=options.assert_type,
+ block_size=int(options.block_size),
+ allow_unhashed=options.allow_unhashed)
+ finally:
+ if do_close_report_file:
+ report_file.close()
+
+ # Trace blocks.
+ if options.root_block is not None:
+ payload.TraceBlock(options.root_block, options.skip, sys.stdout, False)
+ if options.kern_block is not None:
+ payload.TraceBlock(options.kern_block, options.skip, sys.stdout, True)
+
+ # Apply payload.
+ if extra_args:
+ if options.assert_type == _TYPE_FULL:
+ payload.Apply(extra_args[0], extra_args[1])
+ elif options.assert_type == _TYPE_DELTA:
+ payload.Apply(extra_args[0], extra_args[1],
+ src_kernel_part=extra_args[2],
+ src_rootfs_part=extra_args[3])
+ else:
+ assert False, 'cannot get here'
+
+ except update_payload.PayloadError, e:
+ sys.stderr.write('Error: %s\n' % e)
+ return 1
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/scripts/test_paycheck.sh b/scripts/test_paycheck.sh
new file mode 100755
index 0000000..d6b9310
--- /dev/null
+++ b/scripts/test_paycheck.sh
@@ -0,0 +1,168 @@
+#!/bin/bash
+#
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# A test script for paycheck.py and the update_payload.py library.
+#
+# This script requires three payload files, along with a metadata signature for
+# each, and a public key for verifying signatures. Payload include:
+#
+# - A full payload for release X (old_full_payload)
+#
+# - A full payload for release Y (new_full_payload), where Y > X
+#
+# - A delta payload from X to Y (delta_payload)
+#
+# The test performs the following:
+#
+# - It verifies each payload against its metadata signature, also asserting the
+# payload type. Another artifact is a human-readable payload report, which
+# is output to stdout to be inspected by the user.
+#
+# - It performs a random block trace on the delta payload (both kernel and
+# rootfs blocks), dumping the traces to stdout for the user to inspect.
+#
+# - It applies old_full_payload to yield old kernel (old_kern.part) and rootfs
+# (old_root.part) partitions.
+#
+# - It applies delta_payload to old_{kern,root}.part to yield new kernel
+# (new_delta_kern.part) and rootfs (new_delta_root.part) partitions.
+#
+# - It applies new_full_payload to yield reference new kernel
+# (new_full_kern.part) and rootfs (new_full_root.part) partitions.
+#
+# - It compares new_{delta,full}_kern.part and new_{delta,full}_root.part to
+# ensure that they are binary identical.
+#
+# If all steps have completed successfully we know with high certainty that
+# paycheck.py (and hence update_payload.py) correctly parses both full and
+# delta payloads, and applies them to yield the expected result. We also know
+# that tracing works, to the extent it does not crash. Manual inspection of
+# payload reports and block traces will improve this our confidence and are
+# strongly encouraged. Finally, each paycheck.py execution is timed.
+
+
+OLD_KERN_PART=old_kern.part
+OLD_ROOT_PART=old_root.part
+NEW_DELTA_KERN_PART=new_delta_kern.part
+NEW_DELTA_ROOT_PART=new_delta_root.part
+NEW_FULL_KERN_PART=new_full_kern.part
+NEW_FULL_ROOT_PART=new_full_root.part
+
+# Stop on errors, unset variables.
+set -e
+set -u
+
+log() {
+ echo "$@" >&2
+}
+
+die() {
+ log "$@"
+ exit 1
+}
+
+usage_and_exit() {
+ cat >&2 <<EOF
+Usage: ${0##*/} pubkey old_full_payload old_full_metasig \\
+ delta_payload delta_metasig new_full_payload new_full_metasig
+EOF
+ exit
+}
+
+check_payload() {
+ payload_file=$1
+ metasig_file=$2
+ payload_type=$3
+
+ time ${paycheck} -r - -k ${pubkey_file} -m ${metasig_file} \
+ -t ${payload_type} ${payload_file}
+}
+
+trace_kern_block() {
+ payload_file=$1
+ block=$2
+ time ${paycheck} -B ${block} ${payload_file}
+}
+
+trace_root_block() {
+ payload_file=$1
+ block=$2
+ time ${paycheck} -b ${block} ${payload_file}
+}
+
+apply_full_payload() {
+ payload_file=$1
+ dst_kern_part=$2
+ dst_root_part=$3
+
+ time ${paycheck} ${payload_file} ${dst_kern_part} ${dst_root_part}
+}
+
+apply_delta_payload() {
+ payload_file=$1
+ dst_kern_part=$2
+ dst_root_part=$3
+ src_kern_part=$4
+ src_root_part=$5
+
+ time ${paycheck} ${payload_file} ${dst_kern_part} ${dst_root_part} \
+ ${src_kern_part} ${src_root_part}
+}
+
+main() {
+ # Read command-line arguments.
+ if [ $# == 1 ] && [ "$1" == "-h" ]; then
+ usage_and_exit
+ elif [ $# != 7 ]; then
+ die "Error: unexpected number of arguments"
+ fi
+ pubkey_file="$1"
+ old_full_payload="$2"
+ old_full_metasig="$3"
+ delta_payload="$4"
+ delta_metasig="$5"
+ new_full_payload="$6"
+ new_full_metasig="$7"
+
+ # Find paycheck.py
+ paycheck=${0%/*}/paycheck.py
+ if [ -z "${paycheck}" ] || [ ! -x ${paycheck} ]; then
+ die "cannot find paycheck.py or file is not executable"
+ fi
+
+ log "Checking payloads..."
+ check_payload "${old_full_payload}" "${old_full_metasig}" full
+ check_payload "${new_full_payload}" "${new_full_metasig}" full
+ check_payload "${delta_payload}" "${delta_metasig}" delta
+ log "Done"
+
+ # Pick a random block between 0-1024
+ block=$((RANDOM * 1024 / 32767))
+ log "Tracing a random block (${block}) in full/delta payloads..."
+ trace_kern_block "${new_full_payload}" ${block}
+ trace_root_block "${new_full_payload}" ${block}
+ trace_kern_block "${delta_payload}" ${block}
+ trace_root_block "${delta_payload}" ${block}
+ log "Done"
+
+ log "Apply old full payload..."
+ apply_full_payload "${old_full_payload}" "${OLD_KERN_PART}" "${OLD_ROOT_PART}"
+ log "Done"
+ log "Apply delta payload to old partitions..."
+ time ./paycheck.py "${delta_payload}" "${NEW_DELTA_KERN_PART}" \
+ "${NEW_DELTA_ROOT_PART}" "${OLD_KERN_PART}" "${OLD_ROOT_PART}"
+ log "Done"
+ log "Apply new full payload..."
+ time ./paycheck.py "${new_full_payload}" "${NEW_FULL_KERN_PART}" \
+ "${NEW_FULL_ROOT_PART}"
+ log "Done"
+ log "Comparing results of delta and new full updates..."
+ diff "${NEW_FULL_KERN_PART}" "${NEW_DELTA_KERN_PART}"
+ diff "${NEW_FULL_ROOT_PART}" "${NEW_DELTA_ROOT_PART}"
+ log "Done"
+}
+
+main "$@"
diff --git a/scripts/update_payload/__init__.py b/scripts/update_payload/__init__.py
new file mode 100644
index 0000000..e437f40
--- /dev/null
+++ b/scripts/update_payload/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Library for processing, verifying and applying Chrome OS update payloads."""
+
+# Just raise the interface classes to the root namespace.
+# pylint: disable=W0401
+from error import PayloadError
+from payload import Payload
diff --git a/scripts/update_payload/applier.py b/scripts/update_payload/applier.py
new file mode 100644
index 0000000..6780e9a
--- /dev/null
+++ b/scripts/update_payload/applier.py
@@ -0,0 +1,382 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Applying a Chrome OS update payload.
+
+This module is used internally by the main Payload class for applying an update
+payload. The interface for invoking the applier is as follows:
+
+ applier = PayloadApplier(payload)
+ applier.Run(...)
+
+"""
+
+import array
+import bz2
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+import common
+from error import PayloadError
+
+
+#
+# Helper functions.
+#
+def _VerifySha256(file_obj, expected_hash, name, max_length=-1):
+ """Verifies the SHA256 hash of a file.
+
+ Args:
+ file_obj: file object to read
+ expected_hash: the hash digest we expect to be getting
+ name: name string of this hash, for error reporting
+ max_length: maximum length of data to read/hash (optional)
+ Raises:
+ PayloadError if file hash fails to verify.
+
+ """
+ # pylint: disable=E1101
+ hasher = hashlib.sha256()
+ block_length = 1024 * 1024
+ if max_length < 0:
+ max_length = sys.maxint
+
+ while max_length != 0:
+ read_length = min(max_length, block_length)
+ data = file_obj.read(read_length)
+ if not data:
+ break
+ max_length -= len(data)
+ hasher.update(data)
+
+ actual_hash = hasher.digest()
+ if actual_hash != expected_hash:
+ raise PayloadError('%s hash (%s) not as expected (%s)' %
+ (name, actual_hash.encode('hex'),
+ expected_hash.encode('hex')))
+
+
+def _ReadExtents(file_obj, extents, block_size, max_length=-1):
+ """Reads data from file as defined by extent sequence.
+
+ This tries to be efficient by not copying data as it is read in chunks.
+
+ Args:
+ file_obj: file object
+ extents: sequence of block extents (offset and length)
+ block_size: size of each block
+ max_length: maximum length to read (optional)
+ Returns:
+ A character array containing the concatenated read data.
+
+ """
+ data = array.array('c')
+ for ex in extents:
+ if max_length == 0:
+ break
+ file_obj.seek(ex.start_block * block_size)
+ read_length = ex.num_blocks * block_size
+ if max_length > 0:
+ read_length = min(max_length, read_length)
+ max_length -= read_length
+ data.fromfile(file_obj, read_length)
+ return data
+
+
+def _WriteExtents(file_obj, data, extents, block_size, base_name):
+ """Write data to file as defined by extent sequence.
+
+ This tries to be efficient by not copy data as it is written in chunks.
+
+ Args:
+ file_obj: file object
+ data: data to write
+ extents: sequence of block extents (offset and length)
+ block_size: size of each block
+ base_name: name string of extent block for error reporting
+ Raises:
+ PayloadError when things don't add up.
+
+ """
+ data_offset = 0
+ data_length = len(data)
+ for ex, ex_name in common.ExtentIter(extents, base_name):
+ if data_offset == data_length:
+ raise PayloadError('%s: more write extents than data' % ex_name)
+ write_length = min(data_length - data_offset, ex.num_blocks * block_size)
+ file_obj.seek(ex.start_block * block_size)
+ data_view = buffer(data, data_offset, write_length)
+ file_obj.write(data_view)
+ data_offset += write_length
+
+ if data_offset < data_length:
+ raise PayloadError('%s: more data than write extents' % base_name)
+
+
+#
+# Payload application.
+#
+class PayloadApplier(object):
+ """Applying an update payload.
+
+ This is a short-lived object whose purpose is to isolate the logic used for
+ applying an update payload.
+
+ """
+
+ def __init__(self, payload):
+ assert payload.is_init, 'uninitialized update payload'
+ self.payload = payload
+ self.block_size = payload.manifest.block_size
+
+ def _ApplyReplaceOperation(self, op, op_name, out_data, part_file, part_size):
+ """Applies a REPLACE{,_BZ} operation.
+
+ Args:
+ op: the operation object
+ op_name: name string for error reporting
+ out_data: the data to be written
+ part_file: the partition file object
+ part_size: the size of the partition
+ Raises:
+ PayloadError if something goes wrong.
+
+ """
+ block_size = self.block_size
+ data_length = len(out_data)
+
+ # Decompress data if needed.
+ if op.type == common.OpType.REPLACE_BZ:
+ out_data = bz2.decompress(out_data)
+ data_length = len(out_data)
+
+ # Write data to blocks specified in dst extents.
+ data_start = 0
+ for ex, ex_name in common.ExtentIter(op.dst_extents,
+ '%s.dst_extents' % op_name):
+ start_block = ex.start_block
+ num_blocks = ex.num_blocks
+ count = num_blocks * block_size
+
+ # Make sure it's not a fake (signature) operation.
+ if start_block != common.PSEUDO_EXTENT_MARKER:
+ data_end = data_start + count
+
+ # Make sure we're not running past partition boundary.
+ if (start_block + num_blocks) * block_size > part_size:
+ raise PayloadError(
+ '%s: extent (%s) exceeds partition size (%d)' %
+ (ex_name, common.FormatExtent(ex, block_size),
+ part_size))
+
+ # Make sure that we have enough data to write.
+ if data_end >= data_length + block_size:
+ raise PayloadError(
+ '%s: more dst blocks than data (even with padding)')
+
+ # Pad with zeros if necessary.
+ if data_end > data_length:
+ padding = data_end - data_length
+ out_data += '\0' * padding
+
+ self.payload.payload_file.seek(start_block * block_size)
+ part_file.seek(start_block * block_size)
+ part_file.write(out_data[data_start:data_end])
+
+ data_start += count
+
+ # Make sure we wrote all data.
+ if data_start < data_length:
+ raise PayloadError('%s: wrote fewer bytes (%d) than expected (%d)' %
+ (op_name, data_start, data_length))
+
+ def _ApplyMoveOperation(self, op, op_name, part_file):
+ """Applies a MOVE operation.
+
+ Args:
+ op: the operation object
+ op_name: name string for error reporting
+ part_file: the partition file object
+ Raises:
+ PayloadError if something goes wrong.
+
+ """
+ block_size = self.block_size
+
+ # Gather input raw data from src extents.
+ in_data = _ReadExtents(part_file, op.src_extents, block_size)
+
+ # Dump extracted data to dst extents.
+ _WriteExtents(part_file, in_data, op.dst_extents, block_size,
+ '%s.dst_extents' % op_name)
+
+ def _ApplyBsdiffOperation(self, op, op_name, patch_data, part_file):
+ """Applies a BSDIFF operation.
+
+ Args:
+ op: the operation object
+ op_name: name string for error reporting
+ patch_data: the binary patch content
+ part_file: the partition file object
+ Raises:
+ PayloadError if something goes wrong.
+
+ """
+ block_size = self.block_size
+
+ # Gather input raw data and write to a temp file.
+ in_data = _ReadExtents(part_file, op.src_extents, block_size,
+ max_length=op.src_length)
+ with tempfile.NamedTemporaryFile(delete=False) as in_file:
+ in_file_name = in_file.name
+ in_file.write(in_data)
+
+ # Dump patch data to file.
+ with tempfile.NamedTemporaryFile(delete=False) as patch_file:
+ patch_file_name = patch_file.name
+ patch_file.write(patch_data)
+
+ # Allocate tepmorary output file.
+ with tempfile.NamedTemporaryFile(delete=False) as out_file:
+ out_file_name = out_file.name
+
+ # Invoke bspatch.
+ bspatch_cmd = ['bspatch', in_file_name, out_file_name, patch_file_name]
+ subprocess.check_call(bspatch_cmd)
+
+ # Read output.
+ with open(out_file_name, 'rb') as out_file:
+ out_data = out_file.read()
+ if len(out_data) != op.dst_length:
+ raise PayloadError(
+ '%s: actual patched data length (%d) not as expected (%d)' %
+ (op_name, len(out_data), op.dst_length))
+
+ # Write output back to partition, with padding.
+ unaligned_out_len = len(out_data) % block_size
+ if unaligned_out_len:
+ out_data += '\0' * (block_size - unaligned_out_len)
+ _WriteExtents(part_file, out_data, op.dst_extents, block_size,
+ '%s.dst_extents' % op_name)
+
+ # Delete all temporary files.
+ os.remove(in_file_name)
+ os.remove(out_file_name)
+ os.remove(patch_file_name)
+
+ def _ApplyOperations(self, operations, base_name, part_file, part_size):
+ """Applies a sequence of update operations to a partition.
+
+ This assumes an in-place update semantics, namely all reads are performed
+ first, then the data is processed and written back to the same file.
+
+ Args:
+ operations: the sequence of operations
+ base_name: the name of the operation sequence
+ part_file: the partition file object, open for reading/writing
+ part_size: the partition size
+ Raises:
+ PayloadError if anything goes wrong while processing the payload.
+
+ """
+ for op, op_name in common.OperationIter(operations, base_name):
+ # Read data blob.
+ data = self.payload.ReadDataBlob(op.data_offset, op.data_length)
+
+ if op.type in (common.OpType.REPLACE, common.OpType.REPLACE_BZ):
+ self._ApplyReplaceOperation(op, op_name, data, part_file, part_size)
+ elif op.type == common.OpType.MOVE:
+ self._ApplyMoveOperation(op, op_name, part_file)
+ elif op.type == common.OpType.BSDIFF:
+ self._ApplyBsdiffOperation(op, op_name, data, part_file)
+ else:
+ raise PayloadError('%s: unknown operation type (%d)' %
+ (op_name, op.type))
+
+ def _ApplyToPartition(self, operations, part_name, base_name,
+ dst_part_file_name, dst_part_info,
+ src_part_file_name=None, src_part_info=None):
+ """Applies an update to a partition.
+
+ Args:
+ operations: the sequence of update operations to apply
+ part_name: the name of the partition, for error reporting
+ base_name: the name of the operation sequence
+ dst_part_file_name: file name to write partition data to
+ dst_part_info: size and expected hash of dest partition
+ src_part_file_name: file name of source partition (optional)
+ src_part_info: size and expected hash of source partition (optional)
+ Raises:
+ PayloadError if anything goes wrong with the update.
+
+ """
+ # Do we have a source partition?
+ if src_part_file_name:
+ # Verify the source partition.
+ with open(src_part_file_name, 'rb') as src_part_file:
+ _VerifySha256(src_part_file, src_part_info.hash, part_name)
+
+ # Copy the src partition to the dst one.
+ shutil.copyfile(src_part_file_name, dst_part_file_name)
+ else:
+ # Preallocate the dst partition file.
+ subprocess.check_call(
+ ['fallocate', '-l', str(dst_part_info.size), dst_part_file_name])
+
+ # Apply operations.
+ with open(dst_part_file_name, 'r+b') as dst_part_file:
+ self._ApplyOperations(operations, base_name, dst_part_file,
+ dst_part_info.size)
+
+ # Verify the resulting partition.
+ with open(dst_part_file_name, 'rb') as dst_part_file:
+ _VerifySha256(dst_part_file, dst_part_info.hash, part_name)
+
+ def Run(self, dst_kernel_part, dst_rootfs_part, src_kernel_part=None,
+ src_rootfs_part=None):
+ """Applier entry point, invoking all update operations.
+
+ Args:
+ dst_kernel_part: name of dest kernel partition file
+ dst_rootfs_part: name of dest rootfs partition file
+ src_kernel_part: name of source kernel partition file (optional)
+ src_rootfs_part: name of source rootfs partition file (optional)
+ Raises:
+ PayloadError if payload application failed.
+
+ """
+ self.payload.ResetFile()
+
+ # Make sure the arguments are sane and match the payload.
+ if not (dst_kernel_part and dst_rootfs_part):
+ raise PayloadError('missing dst {kernel,rootfs} partitions')
+
+ if not (src_kernel_part or src_rootfs_part):
+ if not self.payload.IsFull():
+ raise PayloadError('trying to apply a non-full update without src '
+ '{kernel,rootfs} partitions')
+ elif src_kernel_part and src_rootfs_part:
+ if not self.payload.IsDelta():
+ raise PayloadError('trying to apply a non-delta update onto src '
+ '{kernel,rootfs} partitions')
+ else:
+ raise PayloadError('not all src partitions provided')
+
+ # Apply update to rootfs.
+ self._ApplyToPartition(
+ self.payload.manifest.install_operations, 'rootfs',
+ 'install_operations', dst_rootfs_part,
+ self.payload.manifest.new_rootfs_info, src_rootfs_part,
+ self.payload.manifest.old_rootfs_info)
+
+ # Apply update to kernel update.
+ self._ApplyToPartition(
+ self.payload.manifest.kernel_install_operations, 'kernel',
+ 'kernel_install_operations', dst_kernel_part,
+ self.payload.manifest.new_kernel_info, src_kernel_part,
+ self.payload.manifest.old_kernel_info)
diff --git a/scripts/update_payload/block_tracer.py b/scripts/update_payload/block_tracer.py
new file mode 100644
index 0000000..e7a9d27
--- /dev/null
+++ b/scripts/update_payload/block_tracer.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tracing block data source through a Chrome OS update payload.
+
+This module is used internally by the main Payload class for tracing block
+content through an update payload. This is a useful feature in debugging
+payload applying functionality in this package. The interface for invoking the
+tracer is as follows:
+
+ tracer = PayloadBlockTracer(payload)
+ tracer.Run(...)
+
+"""
+
+import common
+
+
+#
+# Payload block tracing.
+#
+class PayloadBlockTracer(object):
+ """Tracing the origin of block data through update instructions.
+
+ This is a short-lived object whose purpose is to isolate the logic used for
+ tracing the origin of destination partition blocks.
+
+ """
+
+ def __init__(self, payload):
+ assert payload.is_init, 'uninitialized update payload'
+ self.payload = payload
+
+ @staticmethod
+ def _TraceBlock(block, skip, trace_out_file, operations, base_name):
+ """Trace the origin of a given block through a sequence of operations.
+
+ This method tries to map the given dest block to the corresponding source
+ block from which its content originates in the course of an update. It
+ further tries to trace transitive origins through MOVE operations. It is
+ rather efficient, doing the actual tracing by means of a single reverse
+ sweep through the operation sequence. It dumps a log of operations and
+ source blocks responsible for the data in the given dest block to the
+ provided output file.
+
+ Args:
+ block: the block number to trace
+ skip: number of initial transitive origins to ignore
+ trace_out_file: a file object to dump the trace to
+ operations: the sequence of operations
+ base_name: name of the operation sequence
+
+ """
+ # Traverse operations backwards.
+ for op, op_name in common.OperationIter(operations, base_name,
+ reverse=True):
+ total_block_offset = 0
+ found = False
+
+ # Is the traced block mentioned in the dest extents?
+ for dst_ex, dst_ex_name in common.ExtentIter(op.dst_extents,
+ op_name + '.dst_extents'):
+ if (block >= dst_ex.start_block
+ and block < dst_ex.start_block + dst_ex.num_blocks):
+ if skip:
+ skip -= 1
+ else:
+ total_block_offset += block - dst_ex.start_block
+ trace_out_file.write(
+ '%s: found %s (total block offset: %d)\n' %
+ (dst_ex_name, common.FormatExtent(dst_ex), total_block_offset))
+ found = True
+ break
+
+ total_block_offset += dst_ex.num_blocks
+
+ if found:
+ # Don't trace further, unless it's a MOVE.
+ if op.type != common.OpType.MOVE:
+ break
+
+ # For MOVE, find corresponding source block and keep tracing.
+ for src_ex, src_ex_name in common.ExtentIter(op.src_extents,
+ op_name + '.src_extents'):
+ if total_block_offset < src_ex.num_blocks:
+ block = src_ex.start_block + total_block_offset
+ trace_out_file.write(
+ '%s: mapped to %s (%d)\n' %
+ (src_ex_name, common.FormatExtent(src_ex), block))
+ break
+
+ total_block_offset -= src_ex.num_blocks
+
+ def Run(self, block, skip, trace_out_file, is_kernel):
+ """Block tracer entry point, invoking the actual search.
+
+ Args:
+ block: the block number whose origin to trace
+ skip: the number of first origin mappings to skip
+ trace_out_file: file object to dump the trace to
+ is_kernel: trace through kernel (True) or rootfs (False) operations
+
+ """
+ if is_kernel:
+ self._TraceBlock(block, skip, trace_out_file,
+ self.payload.manifest.kernel_install_operations,
+ 'kernel_install_operations')
+ else:
+ self._TraceBlock(block, skip, trace_out_file,
+ self.payload.manifest.install_operations,
+ 'install_operations')
diff --git a/scripts/update_payload/checker.py b/scripts/update_payload/checker.py
new file mode 100644
index 0000000..b2920a0
--- /dev/null
+++ b/scripts/update_payload/checker.py
@@ -0,0 +1,1068 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Verifying the integrity of a Chrome OS update payload.
+
+This module is used internally by the main Payload class for verifying the
+integrity of an update payload. The interface for invoking the checks is as
+follows:
+
+ checker = PayloadChecker(payload)
+ checker.Run(...)
+
+"""
+
+import array
+import base64
+import hashlib
+import subprocess
+
+import common
+from error import PayloadError
+import format_utils
+import histogram
+import update_metadata_pb2
+
+
+#
+# Constants / helper functions.
+#
+_SIG_ASN1_HEADER = (
+ '\x30\x31\x30\x0d\x06\x09\x60\x86'
+ '\x48\x01\x65\x03\x04\x02\x01\x05'
+ '\x00\x04\x20'
+)
+
+_TYPE_FULL = 'full'
+_TYPE_DELTA = 'delta'
+
+_DEFAULT_BLOCK_SIZE = 4096
+
+
+#
+# Helper functions.
+#
+def _IsPowerOfTwo(val):
+ """Returns True iff val is a power of two."""
+ return val > 0 and (val & (val - 1)) == 0
+
+
+def _AddFormat(format_func, value):
+ """Adds a custom formatted representation to ordinary string representation.
+
+ Args:
+ format_func: a value formatter
+ value: value to be formatted and returned
+ Returns:
+ A string 'x (y)' where x = str(value) and y = format_func(value).
+
+ """
+ return '%s (%s)' % (value, format_func(value))
+
+
+def _AddHumanReadableSize(size):
+ """Adds a human readable representation to a byte size value."""
+ return _AddFormat(format_utils.BytesToHumanReadable, size)
+
+
+#
+# Payload report generator.
+#
+class _PayloadReport(object):
+ """A payload report generator.
+
+ A report is essentially a sequence of nodes, which represent data points. It
+ is initialized to have a "global", untitled section. A node may be a
+ sub-report itself.
+
+ """
+
+ # Report nodes: field, sub-report, section.
+ class Node(object):
+ """A report node interface."""
+
+ @staticmethod
+ def _Indent(indent, line):
+ """Indents a line by a given indentation amount.
+
+ Args:
+ indent: the indentation amount
+ line: the line content (string)
+ Returns:
+ The properly indented line (string).
+
+ """
+ return '%*s%s' % (indent, '', line)
+
+ def GenerateLines(self, base_indent, sub_indent, curr_section):
+ """Generates the report lines for this node.
+
+ Args:
+ base_indent: base indentation for each line
+ sub_indent: additional indentation for sub-nodes
+ curr_section: the current report section object
+ Returns:
+ A pair consisting of a list of properly indented report lines and a new
+ current section object.
+
+ """
+ raise NotImplementedError()
+
+ class FieldNode(Node):
+ """A field report node, representing a (name, value) pair."""
+
+ def __init__(self, name, value, linebreak, indent):
+ super(_PayloadReport.FieldNode, self).__init__()
+ self.name = name
+ self.value = value
+ self.linebreak = linebreak
+ self.indent = indent
+
+ def GenerateLines(self, base_indent, sub_indent, curr_section):
+ """Generates a properly formatted 'name : value' entry."""
+ report_output = ''
+ if self.name:
+ report_output += self.name.ljust(curr_section.max_field_name_len) + ' :'
+ value_lines = str(self.value).splitlines()
+ if self.linebreak and self.name:
+ report_output += '\n' + '\n'.join(
+ ['%*s%s' % (self.indent, '', line) for line in value_lines])
+ else:
+ if self.name:
+ report_output += ' '
+ report_output += '%*s' % (self.indent, '')
+ cont_line_indent = len(report_output)
+ indented_value_lines = [value_lines[0]]
+ indented_value_lines.extend(['%*s%s' % (cont_line_indent, '', line)
+ for line in value_lines[1:]])
+ report_output += '\n'.join(indented_value_lines)
+
+ report_lines = [self._Indent(base_indent, line + '\n')
+ for line in report_output.split('\n')]
+ return report_lines, curr_section
+
+ class SubReportNode(Node):
+ """A sub-report node, representing a nested report."""
+
+ def __init__(self, title, report):
+ super(_PayloadReport.SubReportNode, self).__init__()
+ self.title = title
+ self.report = report
+
+ def GenerateLines(self, base_indent, sub_indent, curr_section):
+ """Recurse with indentation."""
+ report_lines = [self._Indent(base_indent, self.title + ' =>\n')]
+ report_lines.extend(self.report.GenerateLines(base_indent + sub_indent,
+ sub_indent))
+ return report_lines, curr_section
+
+ class SectionNode(Node):
+ """A section header node."""
+
+ def __init__(self, title=None):
+ super(_PayloadReport.SectionNode, self).__init__()
+ self.title = title
+ self.max_field_name_len = 0
+
+ def GenerateLines(self, base_indent, sub_indent, curr_section):
+ """Dump a title line, return self as the (new) current section."""
+ report_lines = []
+ if self.title:
+ report_lines.append(self._Indent(base_indent,
+ '=== %s ===\n' % self.title))
+ return report_lines, self
+
+ def __init__(self):
+ self.report = []
+ self.last_section = self.global_section = self.SectionNode()
+ self.is_finalized = False
+
+ def GenerateLines(self, base_indent, sub_indent):
+ """Generates the lines in the report, properly indented.
+
+ Args:
+ base_indent: the indentation used for root-level report lines
+ sub_indent: the indentation offset used for sub-reports
+ Returns:
+ A list of indented report lines.
+
+ """
+ report_lines = []
+ curr_section = self.global_section
+ for node in self.report:
+ node_report_lines, curr_section = node.GenerateLines(
+ base_indent, sub_indent, curr_section)
+ report_lines.extend(node_report_lines)
+
+ return report_lines
+
+ def Dump(self, out_file, base_indent=0, sub_indent=2):
+ """Dumps the report to a file.
+
+ Args:
+ out_file: file object to output the content to
+ base_indent: base indentation for report lines
+ sub_indent: added indentation for sub-reports
+
+ """
+
+ report_lines = self.GenerateLines(base_indent, sub_indent)
+ if report_lines and not self.is_finalized:
+ report_lines.append('(incomplete report)\n')
+
+ for line in report_lines:
+ out_file.write(line)
+
+ def AddField(self, name, value, linebreak=False, indent=0):
+ """Adds a field/value pair to the payload report.
+
+ Args:
+ name: the field's name
+ value: the field's value
+ linebreak: whether the value should be printed on a new line
+ indent: amount of extra indent for each line of the value
+
+ """
+ assert not self.is_finalized
+ if name and self.last_section.max_field_name_len < len(name):
+ self.last_section.max_field_name_len = len(name)
+ self.report.append(self.FieldNode(name, value, linebreak, indent))
+
+ def AddSubReport(self, title):
+ """Adds and returns a sub-report with a title."""
+ assert not self.is_finalized
+ sub_report = self.SubReportNode(title, type(self)())
+ self.report.append(sub_report)
+ return sub_report.report
+
+ def AddSection(self, title):
+ """Adds a new section title."""
+ assert not self.is_finalized
+ self.last_section = self.SectionNode(title)
+ self.report.append(self.last_section)
+
+ def Finalize(self):
+ """Seals the report, marking it as complete."""
+ self.is_finalized = True
+
+
+#
+# Payload verification.
+#
+class PayloadChecker(object):
+ """Checking the integrity of an update payload.
+
+ This is a short-lived object whose purpose is to isolate the logic used for
+ verifying the integrity of an update payload.
+
+ """
+
+ def __init__(self, payload):
+ assert payload.is_init, 'uninitialized update payload'
+ self.payload = payload
+
+ # Reset state; these will be assigned when the manifest is checked.
+ self.block_size = _DEFAULT_BLOCK_SIZE
+ self.sigs_offset = 0
+ self.sigs_size = 0
+ self.old_rootfs_size = 0
+ self.old_kernel_size = 0
+ self.new_rootfs_size = 0
+ self.new_kernel_size = 0
+ self.payload_type = None
+
+ @staticmethod
+ def _CheckElem(msg, name, report, is_mandatory, is_submsg, convert=str,
+ msg_name=None, linebreak=False, indent=0):
+ """Adds an element from a protobuf message to the payload report.
+
+ Checks to see whether a message contains a given element, and if so adds
+ the element value to the provided report. A missing mandatory element
+ causes an exception to be raised.
+
+ Args:
+ msg: the message containing the element
+ name: the name of the element
+ report: a report object to add the element name/value to
+ is_mandatory: whether or not this element must be present
+ is_submsg: whether this element is itself a message
+ convert: a function for converting the element value for reporting
+ msg_name: the name of the message object (for error reporting)
+ linebreak: whether the value report should induce a line break
+ indent: amount of indent used for reporting the value
+ Returns:
+ A pair consisting of the element value and the generated sub-report for
+ it (if the element is a sub-message, None otherwise). If the element is
+ missing, returns (None, None).
+ Raises:
+ PayloadError if a mandatory element is missing.
+
+ """
+ if not msg.HasField(name):
+ if is_mandatory:
+ raise PayloadError("%smissing mandatory %s '%s'" %
+ (msg_name + ' ' if msg_name else '',
+ 'sub-message' if is_submsg else 'field',
+ name))
+ return (None, None)
+
+ value = getattr(msg, name)
+ if is_submsg:
+ return (value, report and report.AddSubReport(name))
+ else:
+ if report:
+ report.AddField(name, convert(value), linebreak=linebreak,
+ indent=indent)
+ return (value, None)
+
+ @staticmethod
+ def _CheckMandatoryField(msg, field_name, report, msg_name, convert=str,
+ linebreak=False, indent=0):
+ """Adds a mandatory field; returning first component from _CheckElem."""
+ return PayloadChecker._CheckElem(msg, field_name, report, True, False,
+ convert=convert, msg_name=msg_name,
+ linebreak=linebreak, indent=indent)[0]
+
+ @staticmethod
+ def _CheckOptionalField(msg, field_name, report, convert=str,
+ linebreak=False, indent=0):
+ """Adds an optional field; returning first component from _CheckElem."""
+ return PayloadChecker._CheckElem(msg, field_name, report, False, False,
+ convert=convert, linebreak=linebreak,
+ indent=indent)[0]
+
+ @staticmethod
+ def _CheckMandatorySubMsg(msg, submsg_name, report, msg_name):
+ """Adds a mandatory sub-message; wrapper for _CheckElem."""
+ return PayloadChecker._CheckElem(msg, submsg_name, report, True, True,
+ msg_name)
+
+ @staticmethod
+ def _CheckOptionalSubMsg(msg, submsg_name, report):
+ """Adds an optional sub-message; wrapper for _CheckElem."""
+ return PayloadChecker._CheckElem(msg, submsg_name, report, False, True)
+
+ @staticmethod
+ def _CheckPresentIff(val1, val2, name1, name2, obj_name):
+ """Checks that val1 is None iff val2 is None.
+
+ Args:
+ val1: first value to be compared
+ val2: second value to be compared
+ name1: name of object holding the first value
+ name2: name of object holding the second value
+ obj_name: name of the object containing these values
+ Raises:
+ PayloadError if assertion does not hold.
+
+ """
+ if None in (val1, val2) and val1 is not val2:
+ present, missing = (name1, name2) if val2 is None else (name2, name1)
+ raise PayloadError("'%s' present without '%s'%s" %
+ (present, missing,
+ ' in ' + obj_name if obj_name else ''))
+
+ @staticmethod
+ def _Run(cmd, send_data=None):
+ """Runs a subprocess, returns its output.
+
+ Args:
+ cmd: list of command-line argument for invoking the subprocess
+ send_data: data to feed to the process via its stdin
+ Returns:
+ A tuple containing the stdout and stderr output of the process.
+
+ """
+ run_process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ return run_process.communicate(input=send_data)
+
+ @staticmethod
+ def _CheckSha256Signature(sig_data, pubkey_file_name, actual_hash, sig_name):
+ """Verifies an actual hash against a signed one.
+
+ Args:
+ sig_data: the raw signature data
+ pubkey_file_name: public key used for verifying signature
+ actual_hash: the actual hash digest
+ sig_name: signature name for error reporting
+ Raises:
+ PayloadError if signature could not be verified.
+
+ """
+ if len(sig_data) != 256:
+ raise PayloadError('%s: signature size (%d) not as expected (256)' %
+ (sig_name, len(sig_data)))
+ signed_data, _ = PayloadChecker._Run(
+ ['openssl', 'rsautl', '-verify', '-pubin', '-inkey', pubkey_file_name],
+ send_data=sig_data)
+
+ if len(signed_data) != len(_SIG_ASN1_HEADER) + 32:
+ raise PayloadError('%s: unexpected signed data length (%d)' %
+ (sig_name, len(signed_data)))
+
+ if not signed_data.startswith(_SIG_ASN1_HEADER):
+ raise PayloadError('%s: not containing standard ASN.1 prefix' % sig_name)
+
+ signed_hash = signed_data[len(_SIG_ASN1_HEADER):]
+ if signed_hash != actual_hash:
+ raise PayloadError('%s: signed hash (%s) different from actual (%s)' %
+ (sig_name, signed_hash.encode('hex'),
+ actual_hash.encode('hex')))
+
+ @staticmethod
+ def _CheckBlocksFitLength(length, num_blocks, block_size, length_name,
+ block_name=None):
+ """Checks that a given length fits given block space.
+
+ This ensures that the number of blocks allocated is appropriate for the
+ length of the data residing in these blocks.
+
+ Args:
+ length: the actual length of the data
+ num_blocks: the number of blocks allocated for it
+ block_size: the size of each block in bytes
+ length_name: name of length (used for error reporting)
+ block_name: name of block (used for error reporting)
+ Raises:
+ PayloadError if the aforementioned invariant is not satisfied.
+
+ """
+ # Check: length <= num_blocks * block_size.
+ if not length <= num_blocks * block_size:
+ raise PayloadError(
+ '%s (%d) > num %sblocks (%d) * block_size (%d)' %
+ (length_name, length, block_name or '', num_blocks, block_size))
+
+ # Check: length > (num_blocks - 1) * block_size.
+ if not length > (num_blocks - 1) * block_size:
+ raise PayloadError(
+ '%s (%d) <= (num %sblocks - 1 (%d)) * block_size (%d)' %
+ (length_name, length, block_name or '', num_blocks - 1, block_size))
+
+ def _CheckManifest(self, report):
+ """Checks the payload manifest.
+
+ Args:
+ report: a report object to add to
+ Returns:
+ A tuple consisting of the partition block size used during the update
+ (integer), the signatures block offset and size.
+ Raises:
+ PayloadError if any of the checks fail.
+
+ """
+ manifest = self.payload.manifest
+ report.AddSection('manifest')
+
+ # Check: block_size must exist and match the expected value.
+ actual_block_size = self._CheckMandatoryField(manifest, 'block_size',
+ report, 'manifest')
+ if actual_block_size != self.block_size:
+ raise PayloadError('block_size (%d) not as expected (%d)' %
+ (actual_block_size, self.block_size))
+
+ # Check: signatures_offset <==> signatures_size.
+ self.sigs_offset = self._CheckOptionalField(manifest, 'signatures_offset',
+ report)
+ self.sigs_size = self._CheckOptionalField(manifest, 'signatures_size',
+ report)
+ self._CheckPresentIff(self.sigs_offset, self.sigs_size,
+ 'signatures_offset', 'signatures_size', 'manifest')
+
+ # Check: old_kernel_info <==> old_rootfs_info.
+ oki_msg, oki_report = self._CheckOptionalSubMsg(manifest,
+ 'old_kernel_info', report)
+ ori_msg, ori_report = self._CheckOptionalSubMsg(manifest,
+ 'old_rootfs_info', report)
+ self._CheckPresentIff(oki_msg, ori_msg, 'old_kernel_info',
+ 'old_rootfs_info', 'manifest')
+ if oki_msg: # equivalently, ori_msg
+ # Assert/mark delta payload.
+ if self.payload_type == _TYPE_FULL:
+ raise PayloadError(
+ 'apparent full payload contains old_{kernel,rootfs}_info')
+ self.payload_type = _TYPE_DELTA
+
+ # Check: {size, hash} present in old_{kernel,rootfs}_info.
+ self.old_kernel_size = self._CheckMandatoryField(
+ oki_msg, 'size', oki_report, 'old_kernel_info')
+ self._CheckMandatoryField(oki_msg, 'hash', oki_report, 'old_kernel_info',
+ convert=common.FormatSha256)
+ self.old_rootfs_size = self._CheckMandatoryField(
+ ori_msg, 'size', ori_report, 'old_rootfs_info')
+ self._CheckMandatoryField(ori_msg, 'hash', ori_report, 'old_rootfs_info',
+ convert=common.FormatSha256)
+ else:
+ # Assert/mark full payload.
+ if self.payload_type == _TYPE_DELTA:
+ raise PayloadError(
+ 'apparent delta payload missing old_{kernel,rootfs}_info')
+ self.payload_type = _TYPE_FULL
+
+ # Check: new_kernel_info present; contains {size, hash}.
+ nki_msg, nki_report = self._CheckMandatorySubMsg(
+ manifest, 'new_kernel_info', report, 'manifest')
+ self.new_kernel_size = self._CheckMandatoryField(
+ nki_msg, 'size', nki_report, 'new_kernel_info')
+ self._CheckMandatoryField(nki_msg, 'hash', nki_report, 'new_kernel_info',
+ convert=common.FormatSha256)
+
+ # Check: new_rootfs_info present; contains {size, hash}.
+ nri_msg, nri_report = self._CheckMandatorySubMsg(
+ manifest, 'new_rootfs_info', report, 'manifest')
+ self.new_rootfs_size = self._CheckMandatoryField(
+ nri_msg, 'size', nri_report, 'new_rootfs_info')
+ self._CheckMandatoryField(nri_msg, 'hash', nri_report, 'new_rootfs_info',
+ convert=common.FormatSha256)
+
+ # Check: payload must contain at least one operation.
+ if not(len(manifest.install_operations) or
+ len(manifest.kernel_install_operations)):
+ raise PayloadError('payload contains no operations')
+
+ def _CheckLength(self, length, total_blocks, op_name, length_name):
+ """Checks whether a length matches the space designated in extents.
+
+ Args:
+ length: the total length of the data
+ total_blocks: the total number of blocks in extents
+ op_name: operation name (for error reporting)
+ length_name: length name (for error reporting)
+ Raises:
+ PayloadError is there a problem with the length.
+
+ """
+ # Check: length is non-zero.
+ if length == 0:
+ raise PayloadError('%s: %s is zero' % (op_name, length_name))
+
+ # Check that length matches number of blocks.
+ self._CheckBlocksFitLength(length, total_blocks, self.block_size,
+ '%s: %s' % (op_name, length_name))
+
+ def _CheckExtents(self, extents, part_size, block_counters, name,
+ allow_pseudo=False, allow_signature=False):
+ """Checks a sequence of extents.
+
+ Args:
+ extents: the sequence of extents to check
+ part_size: the total size of the partition to which the extents apply
+ block_counters: an array of counters corresponding to the number of blocks
+ name: the name of the extent block
+ allow_pseudo: whether or not pseudo block numbers are allowed
+ allow_signature: whether or not the extents are used for a signature
+ Returns:
+ The total number of blocks in the extents.
+ Raises:
+ PayloadError if any of the entailed checks fails.
+
+ """
+ total_num_blocks = 0
+ num_extents = 0
+ for ex, ex_name in common.ExtentIter(extents, name):
+ num_extents += 1
+
+ # Check: mandatory fields.
+ start_block = PayloadChecker._CheckMandatoryField(ex, 'start_block',
+ None, ex_name)
+ num_blocks = PayloadChecker._CheckMandatoryField(ex, 'num_blocks', None,
+ ex_name)
+ end_block = start_block + num_blocks
+
+ # Check: num_blocks > 0.
+ if num_blocks == 0:
+ raise PayloadError('%s: extent length is zero' % ex_name)
+
+ if start_block != common.PSEUDO_EXTENT_MARKER:
+ # Check: make sure we're within the partition limit.
+ if part_size and (end_block - 1) * self.block_size > part_size:
+ raise PayloadError(
+ '%s: extent (%s) exceeds partition size (%d)' %
+ (ex_name, common.FormatExtent(ex, self.block_size), part_size))
+
+ # Record block usage.
+ for i in range(start_block, end_block):
+ block_counters[i] += 1
+ elif not (allow_pseudo or
+ (allow_signature and
+ (num_extents == len(extents) and num_blocks == 1))):
+ raise PayloadError('%s: unexpected pseudo-extent' % ex_name)
+
+ total_num_blocks += num_blocks
+
+ return total_num_blocks
+
+ def _CheckReplaceOperation(self, op, data_length, total_dst_blocks, op_name):
+ """Specific checks for REPLACE/REPLACE_BZ operations.
+
+ Args:
+ op: the operation object from the manifest
+ data_length: the length of the data blob associated with the operation
+ total_dst_blocks: total number of blocks in dst_extents
+ op_name: operation name for error reporting
+ Raises:
+ PayloadError if any check fails.
+
+ """
+ if op.src_extents:
+ raise PayloadError('%s: contains src_extents' % op_name)
+
+ if op.type == common.OpType.REPLACE:
+ PayloadChecker._CheckBlocksFitLength(data_length, total_dst_blocks,
+ self.block_size,
+ op_name + '.data_length', 'dst')
+ else:
+ # Check: data_length must be smaller than the alotted dst blocks.
+ if data_length >= total_dst_blocks * self.block_size:
+ raise PayloadError(
+ '%s: data_length (%d) must be less than allotted dst block '
+ 'space (%d * %d)' %
+ (op_name, data_length, total_dst_blocks, self.block_size))
+
+ def _CheckMoveOperation(self, op, data_offset, total_src_blocks,
+ total_dst_blocks, op_name):
+ """Specific checks for MOVE operations.
+
+ Args:
+ op: the operation object from the manifest
+ data_offset: the offset of a data blob for the operation
+ total_src_blocks: total number of blocks in src_extents
+ total_dst_blocks: total number of blocks in dst_extents
+ op_name: operation name for error reporting
+ Raises:
+ PayloadError if any check fails.
+
+ """
+ # Check: no data_{offset,length}.
+ if data_offset is not None:
+ raise PayloadError('%s: contains data_{offset,length}' % op_name)
+
+ # Check: total src blocks == total dst blocks.
+ if total_src_blocks != total_dst_blocks:
+ raise PayloadError(
+ '%s: total src blocks (%d) != total dst blocks (%d)' %
+ (op_name, total_src_blocks, total_dst_blocks))
+
+ # Check: for all i, i-th src block index != i-th dst block index.
+ i = 0
+ src_extent_iter = iter(op.src_extents)
+ dst_extent_iter = iter(op.dst_extents)
+ src_extent = dst_extent = None
+ src_idx = src_num = dst_idx = dst_num = 0
+ while i < total_src_blocks:
+ # Get the next source extent, if needed.
+ if not src_extent:
+ try:
+ src_extent = src_extent_iter.next()
+ except StopIteration:
+ raise PayloadError('%s: ran out of src extents (%d/%d)' %
+ (op_name, i, total_src_blocks))
+ src_idx = src_extent.start_block
+ src_num = src_extent.num_blocks
+
+ # Get the next dest extent, if needed.
+ if not dst_extent:
+ try:
+ dst_extent = dst_extent_iter.next()
+ except StopIteration:
+ raise PayloadError('%s: ran out of dst extents (%d/%d)' %
+ (op_name, i, total_dst_blocks))
+ dst_idx = dst_extent.start_block
+ dst_num = dst_extent.num_blocks
+
+ if src_idx == dst_idx:
+ raise PayloadError('%s: src/dst blocks %d are the same (%d)' %
+ (op_name, i, src_idx))
+
+ advance = min(src_num, dst_num)
+ i += advance
+
+ src_idx += advance
+ src_num -= advance
+ if src_num == 0:
+ src_extent = None
+
+ dst_idx += advance
+ dst_num -= advance
+ if dst_num == 0:
+ dst_extent = None
+
+ def _CheckBsdiffOperation(self, data_length, total_dst_blocks, op_name):
+ """Specific checks for BSDIFF operations.
+
+ Args:
+ data_length: the length of the data blob associated with the operation
+ total_dst_blocks: total number of blocks in dst_extents
+ op_name: operation name for error reporting
+ Raises:
+ PayloadError if any check fails.
+
+ """
+ # Check: data_length is strictly smaller than the alotted dst blocks.
+ if data_length >= total_dst_blocks * self.block_size:
+ raise PayloadError(
+ '%s: data_length (%d) must be smaller than num dst blocks (%d) * '
+ 'block_size (%d)' %
+ (op_name, data_length, total_dst_blocks, self.block_size))
+
+ def _CheckOperation(self, op, op_name, is_last, old_block_counters,
+ new_block_counters, old_part_size, new_part_size,
+ prev_data_offset, allow_signature, allow_unhashed,
+ blob_hash_counts):
+ """Checks a single update operation.
+
+ Args:
+ op: the operation object
+ op_name: operation name string for error reporting
+ is_last: whether this is the last operation in the sequence
+ old_block_counters: arrays of block read counters
+ new_block_counters: arrays of block write counters
+ old_part_size: the source partition size in bytes
+ new_part_size: the target partition size in bytes
+ prev_data_offset: offset of last used data bytes
+ allow_signature: whether this may be a signature operation
+ allow_unhashed: allow operations with unhashed data blobs
+ blob_hash_counts: counters for hashed/unhashed blobs
+ Returns:
+ The amount of data blob associated with the operation.
+ Raises:
+ PayloadError if any check has failed.
+
+ """
+ # Check extents.
+ total_src_blocks = self._CheckExtents(
+ op.src_extents, old_part_size, old_block_counters,
+ op_name + '.src_extents', allow_pseudo=True)
+ allow_signature_in_extents = (allow_signature and is_last and
+ op.type == common.OpType.REPLACE)
+ total_dst_blocks = self._CheckExtents(
+ op.dst_extents, new_part_size, new_block_counters,
+ op_name + '.dst_extents', allow_signature=allow_signature_in_extents)
+
+ # Check: data_offset present <==> data_length present.
+ data_offset = self._CheckOptionalField(op, 'data_offset', None)
+ data_length = self._CheckOptionalField(op, 'data_length', None)
+ self._CheckPresentIff(data_offset, data_length, 'data_offset',
+ 'data_length', op_name)
+
+ # Check: at least one dst_extent.
+ if not op.dst_extents:
+ raise PayloadError('%s: dst_extents is empty' % op_name)
+
+ # Check {src,dst}_length, if present.
+ if op.HasField('src_length'):
+ self._CheckLength(op.src_length, total_src_blocks, op_name, 'src_length')
+ if op.HasField('dst_length'):
+ self._CheckLength(op.dst_length, total_dst_blocks, op_name, 'dst_length')
+
+ if op.HasField('data_sha256_hash'):
+ blob_hash_counts['hashed'] += 1
+
+ # Check: operation carries data.
+ if data_offset is None:
+ raise PayloadError(
+ '%s: data_sha256_hash present but no data_{offset,length}' %
+ op_name)
+
+ # Check: hash verifies correctly.
+ # pylint: disable=E1101
+ actual_hash = hashlib.sha256(self.payload.ReadDataBlob(data_offset,
+ data_length))
+ if op.data_sha256_hash != actual_hash.digest():
+ raise PayloadError(
+ '%s: data_sha256_hash (%s) does not match actual hash (%s)' %
+ (op_name, op.data_sha256_hash.encode('hex'),
+ actual_hash.hexdigest()))
+ elif data_offset is not None:
+ if allow_signature_in_extents:
+ blob_hash_counts['signature'] += 1
+ elif allow_unhashed:
+ blob_hash_counts['unhashed'] += 1
+ else:
+ raise PayloadError('%s: unhashed operation not allowed' % op_name)
+
+ if data_offset is not None:
+ # Check: contiguous use of data section.
+ if data_offset != prev_data_offset:
+ raise PayloadError(
+ '%s: data offset (%d) not matching amount used so far (%d)' %
+ (op_name, data_offset, prev_data_offset))
+
+ # Type-specific checks.
+ if op.type in (common.OpType.REPLACE, common.OpType.REPLACE_BZ):
+ self._CheckReplaceOperation(op, data_length, total_dst_blocks, op_name)
+ elif self.payload_type == _TYPE_FULL:
+ raise PayloadError('%s: non-REPLACE operation in a full payload' %
+ op_name)
+ elif op.type == common.OpType.MOVE:
+ self._CheckMoveOperation(op, data_offset, total_src_blocks,
+ total_dst_blocks, op_name)
+ elif op.type == common.OpType.BSDIFF:
+ self._CheckBsdiffOperation(data_length, total_dst_blocks, op_name)
+ else:
+ assert False, 'cannot get here'
+
+ return data_length if data_length is not None else 0
+
+ def _AllocBlockCounterss(self, part_size):
+ """Returns a freshly initialized array of block counters.
+
+ Args:
+ part_size: the size of the partition
+ Returns:
+ An array of unsigned char elements initialized to zero, one for each of
+ the blocks necessary for containing the partition.
+
+ """
+ num_blocks = (part_size + self.block_size - 1) / self.block_size
+ return array.array('B', [0] * num_blocks)
+
+ def _CheckOperations(self, operations, report, base_name, old_part_size,
+ new_part_size, prev_data_offset, allow_unhashed,
+ allow_signature):
+ """Checks a sequence of update operations.
+
+ Args:
+ operations: the sequence of operations to check
+ report: the report object to add to
+ base_name: the name of the operation block
+ old_part_size: the old partition size in bytes
+ new_part_size: the new partition size in bytes
+ prev_data_offset: offset of last used data bytes
+ allow_unhashed: allow operations with unhashed data blobs
+ allow_signature: whether this sequence may contain signature operations
+ Returns:
+ A pair consisting of the number of operations and the total data blob
+ size used.
+ Raises:
+ PayloadError if any of the checks fails.
+
+ """
+ # The total size of data blobs used by operations scanned thus far.
+ total_data_used = 0
+ # Counts of specific operation types.
+ op_counts = {
+ common.OpType.REPLACE: 0,
+ common.OpType.REPLACE_BZ: 0,
+ common.OpType.MOVE: 0,
+ common.OpType.BSDIFF: 0,
+ }
+ # Total blob sizes for each operation type.
+ op_blob_totals = {
+ common.OpType.REPLACE: 0,
+ common.OpType.REPLACE_BZ: 0,
+ # MOVE operations don't have blobs
+ common.OpType.BSDIFF: 0,
+ }
+ # Counts of hashed vs unhashed operations.
+ blob_hash_counts = {
+ 'hashed': 0,
+ 'unhashed': 0,
+ }
+ if allow_signature:
+ blob_hash_counts['signature'] = 0
+
+ # Allocate old and new block counters.
+ old_block_counters = (self._AllocBlockCounterss(old_part_size)
+ if old_part_size else None)
+ new_block_counters = self._AllocBlockCounterss(new_part_size)
+
+ # Process and verify each operation.
+ op_num = 0
+ for op, op_name in common.OperationIter(operations, base_name):
+ op_num += 1
+
+ # Check: type is valid.
+ if op.type not in op_counts.keys():
+ raise PayloadError('%s: invalid type (%d)' % (op_name, op.type))
+ op_counts[op.type] += 1
+
+ is_last = op_num == len(operations)
+ curr_data_used = self._CheckOperation(
+ op, op_name, is_last, old_block_counters, new_block_counters,
+ old_part_size, new_part_size, prev_data_offset + total_data_used,
+ allow_signature, allow_unhashed, blob_hash_counts)
+ if curr_data_used:
+ op_blob_totals[op.type] += curr_data_used
+ total_data_used += curr_data_used
+
+ # Report totals and breakdown statistics.
+ report.AddField('total operations', op_num)
+ report.AddField(
+ None,
+ histogram.Histogram.FromCountDict(op_counts,
+ key_names=common.OpType.NAMES),
+ indent=1)
+ report.AddField('total blobs', sum(blob_hash_counts.values()))
+ report.AddField(None,
+ histogram.Histogram.FromCountDict(blob_hash_counts),
+ indent=1)
+ report.AddField('total blob size', _AddHumanReadableSize(total_data_used))
+ report.AddField(
+ None,
+ histogram.Histogram.FromCountDict(op_blob_totals,
+ formatter=_AddHumanReadableSize,
+ key_names=common.OpType.NAMES),
+ indent=1)
+
+ # Report read/write histograms.
+ if old_block_counters:
+ report.AddField('block read hist',
+ histogram.Histogram.FromKeyList(old_block_counters),
+ linebreak=True, indent=1)
+
+ new_write_hist = histogram.Histogram.FromKeyList(new_block_counters)
+ # Check: full update must write each dst block once.
+ if self.payload_type == _TYPE_FULL and new_write_hist.GetKeys() != [1]:
+ raise PayloadError(
+ '%s: not all blocks written exactly once during full update' %
+ base_name)
+
+ report.AddField('block write hist', new_write_hist, linebreak=True,
+ indent=1)
+
+ return total_data_used
+
+ def _CheckSignatures(self, report, pubkey_file_name):
+ """Checks a payload's signature block."""
+ sigs_raw = self.payload.ReadDataBlob(self.sigs_offset, self.sigs_size)
+ sigs = update_metadata_pb2.Signatures()
+ sigs.ParseFromString(sigs_raw)
+ report.AddSection('signatures')
+
+ # Check: at least one signature present.
+ # pylint: disable=E1101
+ if not sigs.signatures:
+ raise PayloadError('signature block is empty')
+
+ # Check: signatures_{offset,size} must match the last (fake) operation.
+ last_ops_section = (self.payload.manifest.kernel_install_operations or
+ self.payload.manifest.install_operations)
+ fake_sig_op = last_ops_section[-1]
+ if not (self.sigs_offset == fake_sig_op.data_offset and
+ self.sigs_size == fake_sig_op.data_length):
+ raise PayloadError(
+ 'signatures_{offset,size} (%d+%d) does not match last operation '
+ '(%d+%d)' %
+ (self.sigs_offset, self.sigs_size, fake_sig_op.data_offset,
+ fake_sig_op.data_length))
+
+ # Compute the checksum of all data up to signature blob.
+ # TODO(garnold) we're re-reading the whole data section into a string
+ # just to compute the checksum; instead, we could do it incrementally as
+ # we read the blobs one-by-one, under the assumption that we're reading
+ # them in order (which currently holds). This should be reconsidered.
+ payload_hasher = self.payload.manifest_hasher.copy()
+ common.Read(self.payload.payload_file, self.sigs_offset,
+ offset=self.payload.data_offset, hasher=payload_hasher)
+
+ for sig, sig_name in common.SignatureIter(sigs.signatures, 'signatures'):
+ sig_report = report.AddSubReport(sig_name)
+
+ # Check: signature contains mandatory fields.
+ self._CheckMandatoryField(sig, 'version', sig_report, sig_name)
+ self._CheckMandatoryField(sig, 'data', None, sig_name)
+ sig_report.AddField('data len', len(sig.data))
+
+ # Check: signatures pertains to actual payload hash.
+ if sig.version == 1:
+ self._CheckSha256Signature(sig.data, pubkey_file_name,
+ payload_hasher.digest(), sig_name)
+ else:
+ raise PayloadError('unknown signature version (%d)' % sig.version)
+
+ def Run(self, pubkey_file_name=None, metadata_sig_file=None,
+ report_out_file=None, assert_type=None, block_size=0,
+ allow_unhashed=False):
+ """Checker entry point, invoking all checks.
+
+ Args:
+ pubkey_file_name: public key used for signature verification
+ metadata_sig_file: metadata signature, if verification is desired
+ report_out_file: file object to dump the report to
+ assert_type: assert that payload is either 'full' or 'delta' (optional)
+ block_size: expected filesystem / payload block size
+ allow_unhashed: allow operations with unhashed data blobs
+ Raises:
+ PayloadError if payload verification failed.
+
+ """
+ report = _PayloadReport()
+
+ if assert_type not in (None, _TYPE_FULL, _TYPE_DELTA):
+ raise PayloadError("invalid assert_type value (`%s')" % assert_type)
+ self.payload_type = assert_type
+
+ if block_size:
+ self.block_size = block_size
+ if not _IsPowerOfTwo(self.block_size):
+ raise PayloadError('expected block (%d) size is not a power of two' %
+ self.block_size)
+
+ # Get payload file size.
+ self.payload.payload_file.seek(0, 2)
+ payload_file_size = self.payload.payload_file.tell()
+ self.payload.ResetFile()
+
+ try:
+ # Check metadata signature (if provided).
+ if metadata_sig_file:
+ if not pubkey_file_name:
+ raise PayloadError(
+ 'no public key provided, cannot verify metadata signature')
+ metadata_sig = base64.b64decode(metadata_sig_file.read())
+ self._CheckSha256Signature(metadata_sig, pubkey_file_name,
+ self.payload.manifest_hasher.digest(),
+ 'metadata signature')
+
+ # Part 1: check the file header.
+ report.AddSection('header')
+ # Check: payload version is valid.
+ if self.payload.header.version != 1:
+ raise PayloadError('unknown payload version (%d)' %
+ self.payload.header.version)
+ report.AddField('version', self.payload.header.version)
+ report.AddField('manifest len', self.payload.header.manifest_len)
+
+ # Part 2: check the manifest.
+ self._CheckManifest(report)
+ assert self.payload_type, 'payload type should be known by now'
+
+ # Part 3: examine rootfs operations.
+ report.AddSection('rootfs operations')
+ total_blob_size = self._CheckOperations(
+ self.payload.manifest.install_operations, report,
+ 'install_operations', self.old_rootfs_size,
+ self.new_rootfs_size, 0, allow_unhashed, False)
+
+ # Part 4: examine kernel operations.
+ report.AddSection('kernel operations')
+ total_blob_size += self._CheckOperations(
+ self.payload.manifest.kernel_install_operations, report,
+ 'kernel_install_operations', self.old_kernel_size,
+ self.new_kernel_size, total_blob_size, allow_unhashed, True)
+
+ # Check: operations data reach the end of the payload file.
+ used_payload_size = self.payload.data_offset + total_blob_size
+ if used_payload_size != payload_file_size:
+ raise PayloadError(
+ 'used payload size (%d) different from actual file size (%d)' %
+ (used_payload_size, payload_file_size))
+
+ # Part 5: handle payload signatures message.
+ if self.sigs_size:
+ if not pubkey_file_name:
+ raise PayloadError(
+ 'no public key provided, cannot verify payload signature')
+ self._CheckSignatures(report, pubkey_file_name)
+
+ # Part 6: summary.
+ report.AddSection('summary')
+ report.AddField('update type', self.payload_type)
+
+ report.Finalize()
+ finally:
+ if report_out_file:
+ report.Dump(report_out_file)
diff --git a/scripts/update_payload/common.py b/scripts/update_payload/common.py
new file mode 100644
index 0000000..1650991
--- /dev/null
+++ b/scripts/update_payload/common.py
@@ -0,0 +1,141 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Utilities for update payload processing."""
+
+import ctypes
+import textwrap
+
+from error import PayloadError
+import update_metadata_pb2
+
+
+#
+# Constants.
+#
+PSEUDO_EXTENT_MARKER = ctypes.c_uint64(-1).value
+
+
+#
+# Payload operation types.
+#
+class OpType(object):
+ """Container for operation type constants."""
+ _CLASS = update_metadata_pb2.DeltaArchiveManifest.InstallOperation
+ # pylint: disable=E1101
+ REPLACE = _CLASS.REPLACE
+ REPLACE_BZ = _CLASS.REPLACE_BZ
+ MOVE = _CLASS.MOVE
+ BSDIFF = _CLASS.BSDIFF
+ NAMES = {
+ REPLACE: 'REPLACE',
+ REPLACE_BZ: 'REPLACE_BZ',
+ MOVE: 'MOVE',
+ BSDIFF: 'BSDIFF',
+ }
+
+ def __init__(self):
+ pass
+
+
+#
+# Checker and hashed reading of data.
+#
+def Read(file_obj, length, offset=None, hasher=None):
+ """Reads binary data from a file.
+
+ Args:
+ file_obj: an open file object
+ length: the length of the data to read
+ offset: an offset to seek to prior to reading; this is an absolute offset
+ from either the beginning (non-negative) or end (negative) of the
+ file. (optional)
+ hasher: a hashing object to pass the read data through (optional)
+ Returns:
+ A string containing the read data.
+ Raises:
+ PayloadError if a read error occurred or not enough data was read.
+
+ """
+ if offset is not None:
+ if offset >= 0:
+ file_obj.seek(offset)
+ else:
+ file_obj.seek(offset, 2)
+
+ try:
+ data = file_obj.read(length)
+ except IOError, e:
+ raise PayloadError('error reading from file (%s): %s' % (file_obj.name, e))
+
+ if len(data) != length:
+ raise PayloadError(
+ 'reading from file (%s) too short (%d instead of %d bytes)' %
+ (file_obj.name, len(data), length))
+
+ if hasher:
+ hasher.update(data)
+
+ return data
+
+
+#
+# Formatting functions.
+#
+def FormatExtent(ex, block_size=0):
+ end_block = ex.start_block + ex.num_blocks
+ if block_size:
+ return '%d->%d * %d' % (ex.start_block, end_block, block_size)
+ else:
+ return '%d->%d' % (ex.start_block, end_block)
+
+
+def FormatSha256(digest):
+ """Returns a canonical string representation of a SHA256 digest."""
+ return '\n'.join(textwrap.wrap(digest.encode('hex'), 32))
+
+
+#
+# Useful iterators.
+#
+def _ObjNameIter(items, base_name, reverse=False, name_format_func=None):
+ """A generic (item, name) tuple iterators.
+
+ Args:
+ items: the sequence of objects to iterate on
+ base_name: the base name for all objects
+ reverse: whether iteration should be in reverse order
+ name_format_func: a function to apply to the name string
+ Yields:
+ An iterator whose i-th invocation returns (items[i], name), where name ==
+ base_name + '[i]' (with a formatting function optionally applied to it).
+
+ """
+ idx, inc = (len(items), -1) if reverse else (1, 1)
+ for item in items:
+ item_name = '%s[%d]' % (base_name, idx)
+ if name_format_func:
+ item_name = name_format_func(item, item_name)
+ yield (item, item_name)
+ idx += inc
+
+
+def _OperationNameFormatter(op, op_name):
+ return '%s(%s)' % (op_name, OpType.NAMES.get(op.type, '?'))
+
+
+def OperationIter(operations, base_name, reverse=False):
+ """An (item, name) iterator for update operations."""
+ return _ObjNameIter(operations, base_name, reverse=reverse,
+ name_format_func=_OperationNameFormatter)
+
+
+def ExtentIter(extents, base_name, reverse=False):
+ """An (item, name) iterator for operation extents."""
+ return _ObjNameIter(extents, base_name, reverse=reverse)
+
+
+def SignatureIter(sigs, base_name, reverse=False):
+ """An (item, name) iterator for signatures."""
+ return _ObjNameIter(sigs, base_name, reverse=reverse)
diff --git a/scripts/update_payload/error.py b/scripts/update_payload/error.py
new file mode 100644
index 0000000..8b9cadd
--- /dev/null
+++ b/scripts/update_payload/error.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Payload handling errors."""
+
+
+class PayloadError(Exception):
+ """An update payload general processing error."""
diff --git a/scripts/update_payload/format_utils.py b/scripts/update_payload/format_utils.py
new file mode 100644
index 0000000..2c82f32
--- /dev/null
+++ b/scripts/update_payload/format_utils.py
@@ -0,0 +1,93 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Various formatting functions."""
+
+
+def NumToPercent(num, total, min_precision=1, max_precision=5):
+ """Returns the percentage (string) of |num| out of |total|.
+
+ If the percentage includes a fraction, it will be computed down to the least
+ precision that yields a non-zero and ranging between |min_precision| and
+ |max_precision|. Values are always rounded down. All arithmetic operations
+ are integer built-ins. Examples (using default precision):
+
+ (1, 1) => 100%
+ (3, 10) => 30%
+ (3, 9) => 33.3%
+ (3, 900) => 0.3%
+ (3, 9000000) => 0.00003%
+ (3, 900000000) => 0%
+ (5, 2) => 250%
+
+ Args:
+ num: the value of the part
+ total: the value of the whole
+ min_precision: minimum precision for fractional percentage
+ max_precision: maximum precision for fractional percentage
+ Returns:
+ Percentage string.
+
+ """
+ percent = 0
+ precision = min(min_precision, max_precision)
+ factor = 10 ** precision
+ while precision <= max_precision:
+ percent = num * 100 * factor / total
+ if percent:
+ break
+ factor *= 10
+ precision += 1
+
+ whole, frac = divmod(percent, factor)
+ while frac and not frac % 10:
+ frac /= 10
+ precision -= 1
+
+ return '%d%s%%' % (whole, '.%0*d' % (precision, frac) if frac else '')
+
+
+def BytesToHumanReadable(size, precision=1, decimal=False):
+ """Returns a human readable representation of a given |size|.
+
+ The returned string includes unit notations in either binary (KiB, MiB, etc)
+ or decimal (kB, MB, etc), based on the value of |decimal|. The chosen unit is
+ the largest that yields a whole (or mixed) number. It may contain up to
+ |precision| fractional digits. Values are always rounded down. Largest unit
+ is an exabyte. All arithmetic operations are integer built-ins. Examples
+ (using default precision and binary units):
+
+ 4096 => 4 KiB
+ 5000 => 4.8 KiB
+ 500000 => 488.2 KiB
+ 5000000 => 4.7 MiB
+
+ Args:
+ size: the size in bytes
+ precision: the number of digits past the decimal point
+ decimal: whether to compute/present decimal or binary units
+ Returns:
+ Readable size string, or None if no conversion is applicable (i.e. size is
+ less than the smallest unit).
+
+ """
+ constants = (
+ (('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB'), 1024),
+ (('kB', 'MB', 'GB', 'TB', 'PB', 'EB'), 1000)
+ )
+ suffixes, base = constants[decimal]
+ exp, magnitude = 0, 1
+ while exp < len(suffixes):
+ next_magnitude = magnitude * base
+ if size < next_magnitude:
+ break
+ exp += 1
+ magnitude = next_magnitude
+
+ if exp != 0:
+ whole = size / magnitude
+ frac = (size % magnitude) * (10 ** precision) / magnitude
+ while frac and not frac % 10:
+ frac /= 10
+ return '%d%s %s' % (whole, '.%d' % frac if frac else '', suffixes[exp - 1])
diff --git a/scripts/update_payload/format_utils_unittest.py b/scripts/update_payload/format_utils_unittest.py
new file mode 100755
index 0000000..8c5ba8e
--- /dev/null
+++ b/scripts/update_payload/format_utils_unittest.py
@@ -0,0 +1,76 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for format_utils.py."""
+
+import unittest
+
+import format_utils
+
+
+class NumToPercentTest(unittest.TestCase):
+ def testHundredPercent(self):
+ self.assertEqual(format_utils.NumToPercent(1, 1), '100%')
+
+ def testOverHundredPercent(self):
+ self.assertEqual(format_utils.NumToPercent(5, 2), '250%')
+
+ def testWholePercent(self):
+ self.assertEqual(format_utils.NumToPercent(3, 10), '30%')
+
+ def testDefaultMinPrecision(self):
+ self.assertEqual(format_utils.NumToPercent(3, 9), '33.3%')
+ self.assertEqual(format_utils.NumToPercent(3, 900), '0.3%')
+
+ def testDefaultMaxPrecision(self):
+ self.assertEqual(format_utils.NumToPercent(3, 9000000), '0.00003%')
+ self.assertEqual(format_utils.NumToPercent(3, 90000000), '0%')
+
+ def testCustomMinPrecision(self):
+ self.assertEqual(format_utils.NumToPercent(3, 9, min_precision=3),
+ '33.333%')
+ self.assertEqual(format_utils.NumToPercent(3, 9, min_precision=0),
+ '33%')
+
+ def testCustomMaxPrecision(self):
+ self.assertEqual(format_utils.NumToPercent(3, 900, max_precision=1),
+ '0.3%')
+ self.assertEqual(format_utils.NumToPercent(3, 9000, max_precision=1),
+ '0%')
+
+
+class BytesToHumanReadableTest(unittest.TestCase):
+ def testBaseTwo(self):
+ self.assertEqual(format_utils.BytesToHumanReadable(0x1000), '4 KiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(0x400000), '4 MiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(0x100000000), '4 GiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(0x40000000000), '4 TiB')
+
+ def testDecimal(self):
+ self.assertEqual(format_utils.BytesToHumanReadable(5000, decimal=True),
+ '5 kB')
+ self.assertEqual(format_utils.BytesToHumanReadable(5000000, decimal=True),
+ '5 MB')
+ self.assertEqual(format_utils.BytesToHumanReadable(5000000000,
+ decimal=True),
+ '5 GB')
+
+ def testDefaultPrecision(self):
+ self.assertEqual(format_utils.BytesToHumanReadable(5000), '4.8 KiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(500000), '488.2 KiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(5000000), '4.7 MiB')
+
+ def testCustomPrecision(self):
+ self.assertEqual(format_utils.BytesToHumanReadable(5000, precision=3),
+ '4.882 KiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(500000, precision=0),
+ '488 KiB')
+ self.assertEqual(format_utils.BytesToHumanReadable(5000000, precision=5),
+ '4.76837 MiB')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/scripts/update_payload/histogram.py b/scripts/update_payload/histogram.py
new file mode 100644
index 0000000..a5ddac4
--- /dev/null
+++ b/scripts/update_payload/histogram.py
@@ -0,0 +1,115 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Histogram generation tools."""
+
+from collections import defaultdict
+
+import format_utils
+
+
+class Histogram(object):
+ """A histogram generating object.
+
+ This object serves the sole purpose of formatting (key, val) pairs as an
+ ASCII histogram, including bars and percentage markers, and taking care of
+ label alignment, scaling, etc. In addition to the standard __init__
+ interface, two static methods are provided for conveniently converting data
+ in different formats into a histogram. Histogram generation is exported via
+ its __str__ method, and looks as follows:
+
+ Yes |################ | 5 (83.3%)
+ No |### | 1 (16.6%)
+
+ TODO(garnold) we may want to add actual methods for adding data or tweaking
+ the output layout and formatting. For now, though, this is fine.
+
+ """
+
+ def __init__(self, data, scale=20, formatter=None):
+ """Initialize a histogram object.
+
+ Args:
+ data: list of (key, count) pairs constituting the histogram
+ scale: number of characters used to indicate 100%
+ formatter: function used for formatting raw histogram values
+
+ """
+ self.data = data
+ self.scale = scale
+ self.formatter = formatter or str
+ self.max_key_len = max([len(str(key)) for key, count in self.data])
+ self.total = sum([count for key, count in self.data])
+
+ @staticmethod
+ def FromCountDict(count_dict, scale=20, formatter=None, key_names=None):
+ """Takes a dictionary of counts and returns a histogram object.
+
+ This simply converts a mapping from names to counts into a list of (key,
+ count) pairs, optionally translating keys into name strings, then
+ generating and returning a histogram for them. This is a useful convenience
+ call for clients that update a dictionary of counters as they (say) scan a
+ data stream.
+
+ Args:
+ count_dict: dictionary mapping keys to occurrence counts
+ scale: number of characters used to indicate 100%
+ formatter: function used for formatting raw histogram values
+ key_names: dictionary mapping keys to name strings
+ Returns:
+ A histogram object based on the given data.
+
+ """
+ namer = None
+ if key_names:
+ namer = lambda key: key_names[key]
+ else:
+ namer = lambda key: key
+
+ hist = [(namer(key), count) for key, count in count_dict.items()]
+ return Histogram(hist, scale, formatter)
+
+ @staticmethod
+ def FromKeyList(key_list, scale=20, formatter=None, key_names=None):
+ """Takes a list of (possibly recurring) keys and returns a histogram object.
+
+ This converts the list into a dictionary of counters, then uses
+ FromCountDict() to generate the actual histogram. For example:
+
+ ['a', 'a', 'b', 'a', 'b'] --> {'a': 3, 'b': 2} --> ...
+
+ Args:
+ key_list: list of (possibly recurring) keys
+ scale: number of characters used to indicate 100%
+ formatter: function used for formatting raw histogram values
+ key_names: dictionary mapping keys to name strings
+ Returns:
+ A histogram object based on the given data.
+
+ """
+ count_dict = defaultdict(int) # Unset items default to zero
+ for key in key_list:
+ count_dict[key] += 1
+ return Histogram.FromCountDict(count_dict, scale, formatter, key_names)
+
+ def __str__(self):
+ hist_lines = []
+ hist_bar = '|'
+ for key, count in self.data:
+ if self.total:
+ bar_len = count * self.scale / self.total
+ hist_bar = '|%s|' % ('#' * bar_len).ljust(self.scale)
+
+ line = '%s %s %s (%s)' % (
+ str(key).ljust(self.max_key_len),
+ hist_bar,
+ self.formatter(count),
+ format_utils.NumToPercent(count, self.total))
+ hist_lines.append(line)
+
+ return '\n'.join(hist_lines)
+
+ def GetKeys(self):
+ """Returns the keys of the histogram."""
+ return [key for key, _ in self.data]
diff --git a/scripts/update_payload/histogram_unittest.py b/scripts/update_payload/histogram_unittest.py
new file mode 100755
index 0000000..421ff20
--- /dev/null
+++ b/scripts/update_payload/histogram_unittest.py
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for histogram.py."""
+
+import unittest
+
+import format_utils
+import histogram
+
+
+class HistogramTest(unittest.TestCase):
+
+ @staticmethod
+ def AddHumanReadableSize(size):
+ fmt = format_utils.BytesToHumanReadable(size)
+ return '%s (%s)' % (size, fmt) if fmt else str(size)
+
+ def CompareToExpectedDefault(self, actual_str):
+ expected_str = (
+ 'Yes |################ | 5 (83.3%)\n'
+ 'No |### | 1 (16.6%)'
+ )
+ self.assertEqual(actual_str, expected_str)
+
+ def testExampleHistogram(self):
+ self.CompareToExpectedDefault(str(histogram.Histogram(
+ [('Yes', 5), ('No', 1)])))
+
+ def testFromCountDict(self):
+ self.CompareToExpectedDefault(str(histogram.Histogram.FromCountDict(
+ {'Yes': 5, 'No': 1})))
+
+ def testFromKeyList(self):
+ self.CompareToExpectedDefault(str(histogram.Histogram.FromKeyList(
+ ['Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes'])))
+
+ def testCustomScale(self):
+ expected_str = (
+ 'Yes |#### | 5 (83.3%)\n'
+ 'No | | 1 (16.6%)'
+ )
+ actual_str = str(histogram.Histogram([('Yes', 5), ('No', 1)], scale=5))
+ self.assertEqual(actual_str, expected_str)
+
+ def testCustomFormatter(self):
+ expected_str = (
+ 'Yes |################ | 5000 (4.8 KiB) (83.3%)\n'
+ 'No |### | 1000 (16.6%)'
+ )
+ actual_str = str(histogram.Histogram(
+ [('Yes', 5000), ('No', 1000)], formatter=self.AddHumanReadableSize))
+ self.assertEqual(actual_str, expected_str)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py
new file mode 100644
index 0000000..6dda644
--- /dev/null
+++ b/scripts/update_payload/payload.py
@@ -0,0 +1,257 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tools for reading, verifying and applying Chrome OS update payloads."""
+
+import hashlib
+import struct
+
+import applier
+import block_tracer
+import checker
+import common
+from error import PayloadError
+import update_metadata_pb2
+
+
+#
+# Helper functions.
+#
+def _ReadInt(file_obj, size, is_unsigned, hasher=None):
+ """Read a binary-encoded integer from a file.
+
+ It will do the correct conversion based on the reported size and whether or
+ not a signed number is expected. Assumes a network (big-endian) byte
+ ordering.
+
+ Args:
+ file_obj: a file object
+ size: the integer size in bytes (2, 4 or 8)
+ is_unsigned: whether it is signed or not
+ hasher: an optional hasher to pass the value through
+ Returns:
+ An "unpacked" (Python) integer value.
+ Raises:
+ PayloadError if an read error occurred.
+
+ """
+ # Determine the base conversion format.
+ if size == 2:
+ fmt = 'h'
+ elif size == 4:
+ fmt = 'i'
+ elif size == 8:
+ fmt = 'q'
+ else:
+ raise PayloadError('unsupport numeric field size (%s)' % size)
+
+ # Signed or unsigned?
+ if is_unsigned:
+ fmt = fmt.upper()
+
+ # Our numeric values are in network byte order (big-endian).
+ fmt = '!' + fmt
+
+ return struct.unpack(fmt, common.Read(file_obj, size, hasher=hasher))[0]
+
+
+#
+# Update payload.
+#
+class Payload(object):
+ """Chrome OS update payload processor."""
+
+ class _PayloadHeader(object):
+ """Update payload header struct."""
+
+ def __init__(self, version, manifest_len):
+ self.version = version
+ self.manifest_len = manifest_len
+
+ # Header constants; sizes are in bytes.
+ _MAGIC = 'CrAU'
+ _VERSION_SIZE = 8
+ _MANIFEST_LEN_SIZE = 8
+
+ def __init__(self, payload_file):
+ """Initialize the payload object.
+
+ Args:
+ payload_file: update payload file object open for reading
+
+ """
+ self.payload_file = payload_file
+ self.manifest_hasher = None
+ self.is_init = False
+ self.header = None
+ self.manifest = None
+ self.data_offset = 0
+
+ def _ReadHeader(self):
+ """Reads and returns the payload header.
+
+ Returns:
+ A payload header object.
+ Raises:
+ PayloadError if a read error occurred.
+
+ """
+ # Verify magic
+ magic = common.Read(self.payload_file, len(self._MAGIC),
+ hasher=self.manifest_hasher)
+ if magic != self._MAGIC:
+ raise PayloadError('invalid payload magic: %s' % magic)
+
+ return self._PayloadHeader(
+ _ReadInt(self.payload_file, self._VERSION_SIZE, True,
+ hasher=self.manifest_hasher),
+ _ReadInt(self.payload_file, self._MANIFEST_LEN_SIZE, True,
+ hasher=self.manifest_hasher))
+
+ def _ReadManifest(self):
+ """Reads and returns the payload manifest.
+
+ Returns:
+ A string containing the payload manifest in binary form.
+ Raises:
+ PayloadError if a read error occurred.
+
+ """
+ if not self.header:
+ raise PayloadError('payload header not present')
+
+ return common.Read(self.payload_file, self.header.manifest_len,
+ hasher=self.manifest_hasher)
+
+ def ReadDataBlob(self, offset, length):
+ """Reads and returns a single data blob from the update payload.
+
+ Args:
+ offset: offset to the beginning of the blob from the end of the manifest
+ length: the blob's length
+ Returns:
+ A string containing the raw blob data.
+ Raises:
+ PayloadError if a read error occurred.
+
+ """
+ return common.Read(self.payload_file, length,
+ offset=self.data_offset + offset)
+
+ def Init(self):
+ """Initializes the payload object.
+
+ This is a prerequisite for any other public API call.
+
+ Raises:
+ PayloadError if object already initialized or fails to initialize
+ correctly.
+
+ """
+ if self.is_init:
+ raise PayloadError('payload object already initialized')
+
+ # Initialize hash context.
+ # pylint: disable=E1101
+ self.manifest_hasher = hashlib.sha256()
+
+ # Read the file header.
+ self.header = self._ReadHeader()
+
+ # Read the manifest.
+ manifest_raw = self._ReadManifest()
+ self.manifest = update_metadata_pb2.DeltaArchiveManifest()
+ self.manifest.ParseFromString(manifest_raw)
+
+ # Store data offset.
+ self.data_offset = (len(self._MAGIC) + self._VERSION_SIZE +
+ self._MANIFEST_LEN_SIZE + self.header.manifest_len)
+
+ self.is_init = True
+
+ def _AssertInit(self):
+ """Raises an exception if the object was not initialized."""
+ if not self.is_init:
+ raise PayloadError('payload object not initialized')
+
+ def ResetFile(self):
+ """Resets the offset of the payload file to right past the manifest."""
+ self.payload_file.seek(self.data_offset)
+
+ def IsDelta(self):
+ """Returns True iff the payload appears to be a delta."""
+ self._AssertInit()
+ return (self.manifest.HasField('old_kernel_info') or
+ self.manifest.HasField('old_rootfs_info'))
+
+ def IsFull(self):
+ """Returns True iff the payload appears to be a full."""
+ return not self.IsDelta()
+
+ def Check(self, pubkey_file_name=None, metadata_sig_file=None,
+ report_out_file=None, assert_type=None, block_size=0,
+ allow_unhashed=False):
+ """Checks the payload integrity.
+
+ Args:
+ pubkey_file_name: public key used for signature verification
+ metadata_sig_file: metadata signature, if verification is desired
+ report_out_file: file object to dump the report to
+ assert_type: assert that payload is either 'full' or 'delta'
+ block_size: expected filesystem / payload block size
+ allow_unhashed: allow unhashed operation blobs
+ Raises:
+ PayloadError if payload verification failed.
+
+ """
+ self._AssertInit()
+
+ # Create a short-lived payload checker object and run it.
+ helper = checker.PayloadChecker(self)
+ helper.Run(pubkey_file_name=pubkey_file_name,
+ metadata_sig_file=metadata_sig_file,
+ report_out_file=report_out_file, assert_type=assert_type,
+ block_size=block_size, allow_unhashed=allow_unhashed)
+
+ def Apply(self, dst_kernel_part, dst_rootfs_part, src_kernel_part=None,
+ src_rootfs_part=None):
+ """Applies the update payload.
+
+ Args:
+ dst_kernel_part: name of dest kernel partition file
+ dst_rootfs_part: name of dest rootfs partition file
+ src_kernel_part: name of source kernel partition file (optional)
+ src_rootfs_part: name of source rootfs partition file (optional)
+ Raises:
+ PayloadError if payload application failed.
+
+ """
+ self._AssertInit()
+
+ # Create a short-lived payload applier object and run it.
+ helper = applier.PayloadApplier(self)
+ helper.Run(dst_kernel_part, dst_rootfs_part,
+ src_kernel_part=src_kernel_part,
+ src_rootfs_part=src_rootfs_part)
+
+ def TraceBlock(self, block, skip, trace_out_file, is_kernel):
+ """Traces the origin(s) of a given dest partition block.
+
+ The tracing tries to find origins transitively, when possible (it currently
+ only works for move operations, where the mapping of src/dst is
+ one-to-one). It will dump a list of operations and source blocks
+ responsible for the data in the given dest block.
+
+ Args:
+ block: the block number whose origin to trace
+ skip: the number of first origin mappings to skip
+ trace_out_file: file object to dump the trace to
+ is_kernel: trace through kernel (True) or rootfs (False) operations
+
+ """
+ self._AssertInit()
+
+ # Create a short-lived payload block tracer object and run it.
+ helper = block_tracer.PayloadBlockTracer(self)
+ helper.Run(block, skip, trace_out_file, is_kernel)
