update_payload: Port scripts to python3
Update the update_payload scripts to be compatible with
python2 and python3. Python2 compatibility is needed since
the repo is shared with Android.
BUG=chromium:1011631
TEST=Executed aosp/system/update_engine/scripts/run_unittests and
cros_generate_update_payload
Cq-Depend: chromium:1904837, chromium:1911499
Change-Id: Ie450b80b5f7550051b38d320173ccc0c915f65e7
Reviewed-on: https://chromium-review.googlesource.com/c/aosp/platform/system/update_engine/+/1904310
Commit-Queue: Andrew Lassalle <[email protected]>
Tested-by: Andrew Lassalle <[email protected]>
Reviewed-by: Mike Frysinger <[email protected]>
Reviewed-by: Amin Hassani <[email protected]>
Auto-Submit: Andrew Lassalle <[email protected]>
diff --git a/scripts/blockdiff.py b/scripts/blockdiff.py
index 5793def..95893cf 100755
--- a/scripts/blockdiff.py
+++ b/scripts/blockdiff.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2013 The Android Open Source Project
#
@@ -17,6 +17,7 @@
"""Block diff utility."""
+from __future__ import absolute_import
from __future__ import print_function
# pylint: disable=import-error
@@ -46,7 +47,7 @@
"""
if max_length < 0:
- max_length = sys.maxint
+ max_length = sys.maxsize
diff_list = []
num_blocks = extent_start = extent_length = 0
while max_length or extent_length:
diff --git a/scripts/paycheck.py b/scripts/paycheck.py
index 875b00f..3587750 100755
--- a/scripts/paycheck.py
+++ b/scripts/paycheck.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2013 The Android Open Source Project
#
@@ -17,6 +17,7 @@
"""Command-line tool for checking and applying Chrome OS update payloads."""
+from __future__ import absolute_import
from __future__ import print_function
# pylint: disable=import-error
@@ -26,13 +27,14 @@
import sys
import tempfile
-from update_payload import common
+from six.moves import zip
from update_payload import error
+
lib_dir = os.path.join(os.path.dirname(__file__), 'lib')
if os.path.exists(lib_dir) and os.path.isdir(lib_dir):
sys.path.insert(1, lib_dir)
-import update_payload
+import update_payload # pylint: disable=wrong-import-position
_TYPE_FULL = 'full'
@@ -287,7 +289,7 @@
# files are created as temp files and will be deleted upon close().
for handle in file_handles:
handle.close()
- except error.PayloadError, e:
+ except error.PayloadError as e:
sys.stderr.write('Error: %s\n' % e)
return 1
diff --git a/scripts/payload_info.py b/scripts/payload_info.py
index bb7f8a4..965bb76 100755
--- a/scripts/payload_info.py
+++ b/scripts/payload_info.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 The Android Open Source Project
@@ -18,15 +18,17 @@
"""payload_info: Show information about an update payload."""
+from __future__ import absolute_import
from __future__ import print_function
import argparse
-import itertools
import sys
import textwrap
+from six.moves import range
import update_payload
+
MAJOR_PAYLOAD_VERSION_BRILLO = 2
def DisplayValue(key, value):
@@ -40,12 +42,12 @@
def DisplayHexData(data, indent=0):
"""Print out binary data as a hex values."""
for off in range(0, len(data), 16):
- chunk = data[off:off + 16]
+ chunk = bytearray(data[off:off + 16])
print(' ' * indent +
- ' '.join('%.2x' % ord(c) for c in chunk) +
+ ' '.join('%.2x' % c for c in chunk) +
' ' * (16 - len(chunk)) +
' | ' +
- ''.join(c if 32 <= ord(c) < 127 else '.' for c in chunk))
+ ''.join(chr(c) if 32 <= c < 127 else '.' for c in chunk))
class PayloadCommand(object):
@@ -144,7 +146,7 @@
op_dict = update_payload.common.OpType.NAMES
print('%s:' % name)
- for op, op_count in itertools.izip(operations, itertools.count()):
+ for op_count, op in enumerate(operations):
print(' %d: %s' % (op_count, op_dict[op.type]))
if op.HasField('data_offset'):
print(' Data offset: %s' % op.data_offset)
@@ -178,8 +180,8 @@
last_ext = curr_ext
# Old and new partitions are read once during verification.
- read_blocks += partition.old_partition_info.size / manifest.block_size
- read_blocks += partition.new_partition_info.size / manifest.block_size
+ read_blocks += partition.old_partition_info.size // manifest.block_size
+ read_blocks += partition.new_partition_info.size // manifest.block_size
stats = {'read_blocks': read_blocks,
'written_blocks': written_blocks,
@@ -212,7 +214,7 @@
def main():
parser = argparse.ArgumentParser(
description='Show information about an update payload.')
- parser.add_argument('payload_file', type=file,
+ parser.add_argument('payload_file', type=argparse.FileType('rb'),
help='The update payload file.')
parser.add_argument('--list_ops', default=False, action='store_true',
help='List the install operations and their extents.')
@@ -224,5 +226,6 @@
PayloadCommand(args).Run()
+
if __name__ == '__main__':
sys.exit(main())
diff --git a/scripts/payload_info_unittest.py b/scripts/payload_info_unittest.py
index bf9f60a..07bb679 100755
--- a/scripts/payload_info_unittest.py
+++ b/scripts/payload_info_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2015 The Android Open Source Project
#
@@ -17,14 +17,19 @@
"""Unit testing payload_info.py."""
+# Disable check for function names to avoid errors based on old code
+# pylint: disable-msg=invalid-name
+
+from __future__ import absolute_import
from __future__ import print_function
-import StringIO
import sys
import unittest
from contextlib import contextmanager
+from six.moves import StringIO
+
import mock # pylint: disable=import-error
import payload_info
@@ -32,9 +37,11 @@
from update_payload import update_metadata_pb2
+
class FakePayloadError(Exception):
"""A generic error when using the FakePayload."""
+
class FakeOption(object):
"""Fake options object for testing."""
@@ -42,11 +49,12 @@
self.list_ops = False
self.stats = False
self.signatures = False
- for key, val in kwargs.iteritems():
+ for key, val in kwargs.items():
setattr(self, key, val)
if not hasattr(self, 'payload_file'):
self.payload_file = None
+
class FakeOp(object):
"""Fake manifest operation for testing."""
@@ -54,23 +62,26 @@
self.src_extents = src_extents
self.dst_extents = dst_extents
self.type = op_type
- for key, val in kwargs.iteritems():
+ for key, val in kwargs.items():
setattr(self, key, val)
def HasField(self, field):
return hasattr(self, field)
+
class FakeExtent(object):
"""Fake Extent for testing."""
def __init__(self, start_block, num_blocks):
self.start_block = start_block
self.num_blocks = num_blocks
+
class FakePartitionInfo(object):
"""Fake PartitionInfo for testing."""
def __init__(self, size):
self.size = size
+
class FakePartition(object):
"""Fake PartitionUpdate field for testing."""
@@ -80,6 +91,7 @@
self.old_partition_info = FakePartitionInfo(old_size)
self.new_partition_info = FakePartitionInfo(new_size)
+
class FakeManifest(object):
"""Fake manifest for testing."""
@@ -94,7 +106,7 @@
], 1 * 4096, 3 * 4096),
FakePartition(update_payload.common.KERNEL,
[FakeOp([FakeExtent(1, 1)],
- [FakeExtent(x, x) for x in xrange(20)],
+ [FakeExtent(x, x) for x in range(20)],
update_payload.common.OpType.SOURCE_COPY,
src_length=4096)
], 2 * 4096, 4 * 4096),
@@ -108,6 +120,7 @@
"""Fake HasField method based on the python members."""
return hasattr(self, field_name) and getattr(self, field_name) is not None
+
class FakeHeader(object):
"""Fake payload header for testing."""
@@ -120,6 +133,7 @@
def size(self):
return 24
+
class FakePayload(object):
"""Fake payload for testing."""
@@ -156,7 +170,7 @@
def _AddSignatureToProto(proto, **kwargs):
"""Add a new Signature element to the passed proto."""
new_signature = proto.signatures.add()
- for key, val in kwargs.iteritems():
+ for key, val in kwargs.items():
setattr(new_signature, key, val)
def AddPayloadSignature(self, **kwargs):
@@ -174,6 +188,7 @@
self._header.metadata_signature_len = len(blob)
self._blobs[-len(blob)] = blob
+
class PayloadCommandTest(unittest.TestCase):
"""Test class for our PayloadCommand class."""
@@ -182,7 +197,7 @@
"""A tool for capturing the sys.stdout"""
stdout = sys.stdout
try:
- sys.stdout = StringIO.StringIO()
+ sys.stdout = StringIO()
yield sys.stdout
finally:
sys.stdout = stdout
@@ -196,13 +211,13 @@
with mock.patch.object(update_payload, 'Payload', return_value=payload), \
self.OutputCapturer() as output:
payload_cmd.Run()
- self.assertEquals(output.getvalue(), expected_out)
+ self.assertEqual(output.getvalue(), expected_out)
def testDisplayValue(self):
"""Verify that DisplayValue prints what we expect."""
with self.OutputCapturer() as output:
payload_info.DisplayValue('key', 'value')
- self.assertEquals(output.getvalue(), 'key: value\n')
+ self.assertEqual(output.getvalue(), 'key: value\n')
def testRun(self):
"""Verify that Run parses and displays the payload like we expect."""
@@ -288,9 +303,9 @@
FakeOption(action='show', signatures=True))
payload = FakePayload()
payload.AddPayloadSignature(version=1,
- data='12345678abcdefgh\x00\x01\x02\x03')
- payload.AddPayloadSignature(data='I am a signature so access is yes.')
- payload.AddMetadataSignature(data='\x00\x0a\x0c')
+ data=b'12345678abcdefgh\x00\x01\x02\x03')
+ payload.AddPayloadSignature(data=b'I am a signature so access is yes.')
+ payload.AddMetadataSignature(data=b'\x00\x0a\x0c')
expected_out = """Payload version: 2
Manifest length: 222
Number of partitions: 2
@@ -314,5 +329,6 @@
"""
self.TestCommand(payload_cmd, payload, expected_out)
+
if __name__ == '__main__':
unittest.main()
diff --git a/scripts/update_device.py b/scripts/update_device.py
index 5c19b89..f970bd3 100755
--- a/scripts/update_device.py
+++ b/scripts/update_device.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2017 The Android Open Source Project
#
@@ -17,8 +17,9 @@
"""Send an A/B update to an Android device over adb."""
+from __future__ import absolute_import
+
import argparse
-import BaseHTTPServer
import hashlib
import logging
import os
@@ -29,6 +30,8 @@
import xml.etree.ElementTree
import zipfile
+from six.moves import BaseHTTPServer
+
import update_payload.payload
@@ -41,6 +44,7 @@
# The port on the device that update_engine should connect to.
DEVICE_PORT = 1234
+
def CopyFileObjLength(fsrc, fdst, buffer_size=128 * 1024, copy_length=None):
"""Copy from a file object to another.
@@ -130,7 +134,6 @@
start_range = file_size - int(e)
return start_range, end_range
-
def do_GET(self): # pylint: disable=invalid-name
"""Reply with the requested payload file."""
if self.path != '/payload':
@@ -173,7 +176,6 @@
f.seek(serving_start + start_range)
CopyFileObjLength(f, self.wfile, copy_length=end_range - start_range)
-
def do_POST(self): # pylint: disable=invalid-name
"""Reply with the omaha response xml."""
if self.path != '/update':
@@ -442,5 +444,6 @@
return 0
+
if __name__ == '__main__':
sys.exit(main())
diff --git a/scripts/update_payload/__init__.py b/scripts/update_payload/__init__.py
index 8ee95e2..6e77678 100644
--- a/scripts/update_payload/__init__.py
+++ b/scripts/update_payload/__init__.py
@@ -17,6 +17,8 @@
"""Library for processing, verifying and applying Chrome OS update payloads."""
# Just raise the interface classes to the root namespace.
+from __future__ import absolute_import
+
from update_payload.checker import CHECKS_TO_DISABLE
from update_payload.error import PayloadError
from update_payload.payload import Payload
diff --git a/scripts/update_payload/applier.py b/scripts/update_payload/applier.py
index 511ed49..7830c00 100644
--- a/scripts/update_payload/applier.py
+++ b/scripts/update_payload/applier.py
@@ -24,6 +24,7 @@
"""
+from __future__ import absolute_import
from __future__ import print_function
import array
@@ -70,7 +71,7 @@
"""
hasher = hashlib.sha256()
block_length = 1024 * 1024
- max_length = length if length >= 0 else sys.maxint
+ max_length = length if length >= 0 else sys.maxsize
while max_length > 0:
read_length = min(max_length, block_length)
@@ -108,7 +109,7 @@
"""
data = array.array('c')
if max_length < 0:
- max_length = sys.maxint
+ max_length = sys.maxsize
for ex in extents:
if max_length == 0:
break
@@ -176,7 +177,7 @@
arg = ''
pad_off = pad_len = 0
if data_length < 0:
- data_length = sys.maxint
+ data_length = sys.maxsize
for ex, ex_name in common.ExtentIter(extents, base_name):
if not data_length:
raise PayloadError('%s: more extents than total data length' % ex_name)
@@ -416,7 +417,7 @@
"--dst_extents=%s" % out_extents_arg]
subprocess.check_call(puffpatch_cmd)
else:
- raise PayloadError("Unknown operation %s", op.type)
+ raise PayloadError("Unknown operation %s" % op.type)
# Pad with zeros past the total output length.
if pad_len:
@@ -451,7 +452,7 @@
"--patch_file=%s" % patch_file_name]
subprocess.check_call(puffpatch_cmd)
else:
- raise PayloadError("Unknown operation %s", op.type)
+ raise PayloadError("Unknown operation %s" % op.type)
# Read output.
with open(out_file_name, 'rb') as out_file:
diff --git a/scripts/update_payload/checker.py b/scripts/update_payload/checker.py
index 4558872..4c65516 100644
--- a/scripts/update_payload/checker.py
+++ b/scripts/update_payload/checker.py
@@ -24,6 +24,7 @@
checker.Run(...)
"""
+from __future__ import absolute_import
from __future__ import print_function
import array
@@ -34,13 +35,14 @@
import os
import subprocess
+from six.moves import range
+
from update_payload import common
from update_payload import error
from update_payload import format_utils
from update_payload import histogram
from update_payload import update_metadata_pb2
-
#
# Constants.
#
@@ -71,6 +73,7 @@
6: (_TYPE_DELTA,),
}
+
#
# Helper functions.
#
@@ -647,7 +650,7 @@
'Apparent full payload contains old_{kernel,rootfs}_info.')
self.payload_type = _TYPE_DELTA
- for part, (msg, part_report) in self.old_part_info.iteritems():
+ for part, (msg, part_report) in self.old_part_info.items():
# Check: {size, hash} present in old_{kernel,rootfs}_info.
field = 'old_%s_info' % part
self.old_fs_sizes[part] = self._CheckMandatoryField(msg, 'size',
@@ -668,7 +671,7 @@
self.payload_type = _TYPE_FULL
# Check: new_{kernel,rootfs}_info present; contains {size, hash}.
- for part, (msg, part_report) in self.new_part_info.iteritems():
+ for part, (msg, part_report) in self.new_part_info.items():
field = 'new_%s_info' % part
self.new_fs_sizes[part] = self._CheckMandatoryField(msg, 'size',
part_report, field)
@@ -740,7 +743,7 @@
(ex_name, common.FormatExtent(ex, self.block_size), usable_size))
# Record block usage.
- for i in xrange(start_block, end_block):
+ for i in range(start_block, end_block):
block_counters[i] += 1
total_num_blocks += num_blocks
@@ -759,6 +762,11 @@
Raises:
error.PayloadError if any check fails.
"""
+ # Check: total_dst_blocks is not a floating point.
+ if isinstance(total_dst_blocks, float):
+ raise error.PayloadError('%s: contains invalid data type of '
+ 'total_dst_blocks.' % op_name)
+
# Check: Does not contain src extents.
if op.src_extents:
raise error.PayloadError('%s: contains src_extents.' % op_name)
@@ -975,7 +983,7 @@
def _SizeToNumBlocks(self, size):
"""Returns the number of blocks needed to contain a given byte size."""
- return (size + self.block_size - 1) / self.block_size
+ return (size + self.block_size - 1) // self.block_size
def _AllocBlockCounters(self, total_size):
"""Returns a freshly initialized array of block counters.
@@ -1054,7 +1062,7 @@
op_num += 1
# Check: Type is valid.
- if op.type not in op_counts.keys():
+ if op.type not in op_counts:
raise error.PayloadError('%s: invalid type (%d).' % (op_name, op.type))
op_counts[op.type] += 1
@@ -1127,7 +1135,6 @@
raise error.PayloadError('It seems like the last operation is the '
'signature blob. This is an invalid payload.')
-
# Compute the checksum of all data up to signature blob.
# TODO(garnold) we're re-reading the whole data section into a string
# just to compute the checksum; instead, we could do it incrementally as
diff --git a/scripts/update_payload/checker_unittest.py b/scripts/update_payload/checker_unittest.py
index 4881653..993b785 100755
--- a/scripts/update_payload/checker_unittest.py
+++ b/scripts/update_payload/checker_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2013 The Android Open Source Project
#
@@ -17,30 +17,33 @@
"""Unit testing checker.py."""
-from __future__ import print_function
+# Disable check for function names to avoid errors based on old code
+# pylint: disable-msg=invalid-name
+
+from __future__ import absolute_import
import array
import collections
-import cStringIO
import hashlib
+import io
import itertools
import os
import unittest
-# pylint cannot find mox.
-# pylint: disable=F0401
-import mox
+from six.moves import zip
+
+import mock # pylint: disable=import-error
from update_payload import checker
from update_payload import common
from update_payload import test_utils
from update_payload import update_metadata_pb2
from update_payload.error import PayloadError
-from update_payload.payload import Payload # Avoid name conflicts later.
+from update_payload.payload import Payload # Avoid name conflicts later.
def _OpTypeByName(op_name):
- """Returns the type of an operation from itsname."""
+ """Returns the type of an operation from its name."""
op_name_to_type = {
'REPLACE': common.OpType.REPLACE,
'REPLACE_BZ': common.OpType.REPLACE_BZ,
@@ -63,7 +66,7 @@
if checker_init_dargs is None:
checker_init_dargs = {}
- payload_file = cStringIO.StringIO()
+ payload_file = io.BytesIO()
payload_gen_write_to_file_func(payload_file, **payload_gen_dargs)
payload_file.seek(0)
payload = Payload(payload_file)
@@ -73,7 +76,7 @@
def _GetPayloadCheckerWithData(payload_gen):
"""Returns a payload checker from a given payload generator."""
- payload_file = cStringIO.StringIO()
+ payload_file = io.BytesIO()
payload_gen.WriteToFile(payload_file)
payload_file.seek(0)
payload = Payload(payload_file)
@@ -87,7 +90,7 @@
# pylint: disable=W0212
# Don't bark about missing members of classes you cannot import.
# pylint: disable=E1101
-class PayloadCheckerTest(mox.MoxTestBase):
+class PayloadCheckerTest(unittest.TestCase):
"""Tests the PayloadChecker class.
In addition to ordinary testFoo() methods, which are automatically invoked by
@@ -100,11 +103,42 @@
all such tests is done in AddAllParametricTests().
"""
+ def setUp(self):
+ """setUp function for unittest testcase"""
+ self.mock_checks = []
+
+ def tearDown(self):
+ """tearDown function for unittest testcase"""
+ # Verify that all mock functions were called.
+ for check in self.mock_checks:
+ check.mock_fn.assert_called_once_with(*check.exp_args, **check.exp_kwargs)
+
+ class MockChecksAtTearDown(object):
+ """Mock data storage.
+
+ This class stores the mock functions and its arguments to be checked at a
+ later point.
+ """
+ def __init__(self, mock_fn, *args, **kwargs):
+ self.mock_fn = mock_fn
+ self.exp_args = args
+ self.exp_kwargs = kwargs
+
+ def addPostCheckForMockFunction(self, mock_fn, *args, **kwargs):
+ """Store a mock function and its arguments to self.mock_checks
+
+ Args:
+ mock_fn: mock function object
+ args: expected positional arguments for the mock_fn
+ kwargs: expected named arguments for the mock_fn
+ """
+ self.mock_checks.append(self.MockChecksAtTearDown(mock_fn, *args, **kwargs))
+
def MockPayload(self):
"""Create a mock payload object, complete with a mock manifest."""
- payload = self.mox.CreateMock(Payload)
+ payload = mock.create_autospec(Payload)
payload.is_init = True
- payload.manifest = self.mox.CreateMock(
+ payload.manifest = mock.create_autospec(
update_metadata_pb2.DeltaArchiveManifest)
return payload
@@ -173,19 +207,20 @@
subreport = 'fake subreport'
# Create a mock message.
- msg = self.mox.CreateMock(update_metadata_pb2._message.Message)
- msg.HasField(name).AndReturn(is_present)
+ msg = mock.create_autospec(update_metadata_pb2._message.Message)
+ self.addPostCheckForMockFunction(msg.HasField, name)
+ msg.HasField.return_value = is_present
setattr(msg, name, val)
-
# Create a mock report.
- report = self.mox.CreateMock(checker._PayloadReport)
+ report = mock.create_autospec(checker._PayloadReport)
if is_present:
if is_submsg:
- report.AddSubReport(name).AndReturn(subreport)
+ self.addPostCheckForMockFunction(report.AddSubReport, name)
+ report.AddSubReport.return_value = subreport
else:
- report.AddField(name, convert(val), linebreak=linebreak, indent=indent)
+ self.addPostCheckForMockFunction(report.AddField, name, convert(val),
+ linebreak=linebreak, indent=indent)
- self.mox.ReplayAll()
return (msg, report, subreport, name, val)
def DoAddElemTest(self, is_present, is_mandatory, is_submsg, convert,
@@ -211,9 +246,9 @@
else:
ret_val, ret_subreport = checker.PayloadChecker._CheckElem(*args,
**kwargs)
- self.assertEquals(val if is_present else None, ret_val)
- self.assertEquals(subreport if is_present and is_submsg else None,
- ret_subreport)
+ self.assertEqual(val if is_present else None, ret_val)
+ self.assertEqual(subreport if is_present and is_submsg else None,
+ ret_subreport)
def DoAddFieldTest(self, is_mandatory, is_present, convert, linebreak,
indent):
@@ -243,7 +278,7 @@
self.assertRaises(PayloadError, tested_func, *args, **kwargs)
else:
ret_val = tested_func(*args, **kwargs)
- self.assertEquals(val if is_present else None, ret_val)
+ self.assertEqual(val if is_present else None, ret_val)
def DoAddSubMsgTest(self, is_mandatory, is_present):
"""Parametrized testing of _Check{Mandatory,Optional}SubMsg().
@@ -267,8 +302,8 @@
self.assertRaises(PayloadError, tested_func, *args)
else:
ret_val, ret_subreport = tested_func(*args)
- self.assertEquals(val if is_present else None, ret_val)
- self.assertEquals(subreport if is_present else None, ret_subreport)
+ self.assertEqual(val if is_present else None, ret_val)
+ self.assertEqual(subreport if is_present else None, ret_subreport)
def testCheckPresentIff(self):
"""Tests _CheckPresentIff()."""
@@ -294,15 +329,14 @@
returned_signed_hash: The signed hash data retuned by openssl.
expected_signed_hash: The signed hash data to compare against.
"""
- try:
- # Stub out the subprocess invocation.
- self.mox.StubOutWithMock(checker.PayloadChecker, '_Run')
+ # Stub out the subprocess invocation.
+ with mock.patch.object(checker.PayloadChecker, '_Run') \
+ as mock_payload_checker:
if expect_subprocess_call:
- checker.PayloadChecker._Run(
- mox.IsA(list), send_data=sig_data).AndReturn(
- (sig_asn1_header + returned_signed_hash, None))
+ mock_payload_checker([], send_data=sig_data)
+ mock_payload_checker.return_value = (
+ sig_asn1_header + returned_signed_hash, None)
- self.mox.ReplayAll()
if expect_pass:
self.assertIsNone(checker.PayloadChecker._CheckSha256Signature(
sig_data, 'foo', expected_signed_hash, 'bar'))
@@ -310,13 +344,11 @@
self.assertRaises(PayloadError,
checker.PayloadChecker._CheckSha256Signature,
sig_data, 'foo', expected_signed_hash, 'bar')
- finally:
- self.mox.UnsetStubs()
def testCheckSha256Signature_Pass(self):
"""Tests _CheckSha256Signature(); pass case."""
sig_data = 'fake-signature'.ljust(256)
- signed_hash = hashlib.sha256('fake-data').digest()
+ signed_hash = hashlib.sha256(b'fake-data').digest()
self.DoCheckSha256SignatureTest(True, True, sig_data,
common.SIG_ASN1_HEADER, signed_hash,
signed_hash)
@@ -324,7 +356,7 @@
def testCheckSha256Signature_FailBadSignature(self):
"""Tests _CheckSha256Signature(); fails due to malformed signature."""
sig_data = 'fake-signature' # Malformed (not 256 bytes in length).
- signed_hash = hashlib.sha256('fake-data').digest()
+ signed_hash = hashlib.sha256(b'fake-data').digest()
self.DoCheckSha256SignatureTest(False, False, sig_data,
common.SIG_ASN1_HEADER, signed_hash,
signed_hash)
@@ -332,7 +364,7 @@
def testCheckSha256Signature_FailBadOutputLength(self):
"""Tests _CheckSha256Signature(); fails due to unexpected output length."""
sig_data = 'fake-signature'.ljust(256)
- signed_hash = 'fake-hash' # Malformed (not 32 bytes in length).
+ signed_hash = b'fake-hash' # Malformed (not 32 bytes in length).
self.DoCheckSha256SignatureTest(False, True, sig_data,
common.SIG_ASN1_HEADER, signed_hash,
signed_hash)
@@ -340,16 +372,16 @@
def testCheckSha256Signature_FailBadAsnHeader(self):
"""Tests _CheckSha256Signature(); fails due to bad ASN1 header."""
sig_data = 'fake-signature'.ljust(256)
- signed_hash = hashlib.sha256('fake-data').digest()
- bad_asn1_header = 'bad-asn-header'.ljust(len(common.SIG_ASN1_HEADER))
+ signed_hash = hashlib.sha256(b'fake-data').digest()
+ bad_asn1_header = b'bad-asn-header'.ljust(len(common.SIG_ASN1_HEADER))
self.DoCheckSha256SignatureTest(False, True, sig_data, bad_asn1_header,
signed_hash, signed_hash)
def testCheckSha256Signature_FailBadHash(self):
"""Tests _CheckSha256Signature(); fails due to bad hash returned."""
sig_data = 'fake-signature'.ljust(256)
- expected_signed_hash = hashlib.sha256('fake-data').digest()
- returned_signed_hash = hashlib.sha256('bad-fake-data').digest()
+ expected_signed_hash = hashlib.sha256(b'fake-data').digest()
+ returned_signed_hash = hashlib.sha256(b'bad-fake-data').digest()
self.DoCheckSha256SignatureTest(False, True, sig_data,
common.SIG_ASN1_HEADER,
expected_signed_hash, returned_signed_hash)
@@ -455,23 +487,23 @@
# Add old kernel/rootfs partition info, as required.
if fail_mismatched_oki_ori or fail_old_kernel_fs_size or fail_bad_oki:
oki_hash = (None if fail_bad_oki
- else hashlib.sha256('fake-oki-content').digest())
+ else hashlib.sha256(b'fake-oki-content').digest())
payload_gen.SetPartInfo(common.KERNEL, False, old_kernel_fs_size,
oki_hash)
if not fail_mismatched_oki_ori and (fail_old_rootfs_fs_size or
fail_bad_ori):
ori_hash = (None if fail_bad_ori
- else hashlib.sha256('fake-ori-content').digest())
+ else hashlib.sha256(b'fake-ori-content').digest())
payload_gen.SetPartInfo(common.ROOTFS, False, old_rootfs_fs_size,
ori_hash)
# Add new kernel/rootfs partition info.
payload_gen.SetPartInfo(
common.KERNEL, True, new_kernel_fs_size,
- None if fail_bad_nki else hashlib.sha256('fake-nki-content').digest())
+ None if fail_bad_nki else hashlib.sha256(b'fake-nki-content').digest())
payload_gen.SetPartInfo(
common.ROOTFS, True, new_rootfs_fs_size,
- None if fail_bad_nri else hashlib.sha256('fake-nri-content').digest())
+ None if fail_bad_nri else hashlib.sha256(b'fake-nri-content').digest())
# Set the minor version.
payload_gen.SetMinorVersion(0)
@@ -518,7 +550,7 @@
# Passes w/ all real extents.
extents = self.NewExtentList((0, 4), (8, 3), (1024, 16))
- self.assertEquals(
+ self.assertEqual(
23,
payload_checker._CheckExtents(extents, (1024 + 16) * block_size,
collections.defaultdict(int), 'foo'))
@@ -553,34 +585,34 @@
block_size = payload_checker.block_size
data_length = 10000
- op = self.mox.CreateMock(
- update_metadata_pb2.InstallOperation)
+ op = mock.create_autospec(update_metadata_pb2.InstallOperation)
op.type = common.OpType.REPLACE
# Pass.
op.src_extents = []
self.assertIsNone(
payload_checker._CheckReplaceOperation(
- op, data_length, (data_length + block_size - 1) / block_size,
+ op, data_length, (data_length + block_size - 1) // block_size,
'foo'))
# Fail, src extents founds.
op.src_extents = ['bar']
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, data_length, (data_length + block_size - 1) / block_size, 'foo')
+ op, data_length, (data_length + block_size - 1) // block_size, 'foo')
# Fail, missing data.
op.src_extents = []
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, None, (data_length + block_size - 1) / block_size, 'foo')
+ op, None, (data_length + block_size - 1) // block_size, 'foo')
# Fail, length / block number mismatch.
op.src_extents = ['bar']
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, data_length, (data_length + block_size - 1) / block_size + 1, 'foo')
+ op, data_length, (data_length + block_size - 1) // block_size + 1,
+ 'foo')
def testCheckReplaceBzOperation(self):
"""Tests _CheckReplaceOperation() where op.type == REPLACE_BZ."""
@@ -588,7 +620,7 @@
block_size = payload_checker.block_size
data_length = block_size * 3
- op = self.mox.CreateMock(
+ op = mock.create_autospec(
update_metadata_pb2.InstallOperation)
op.type = common.OpType.REPLACE_BZ
@@ -596,25 +628,32 @@
op.src_extents = []
self.assertIsNone(
payload_checker._CheckReplaceOperation(
- op, data_length, (data_length + block_size - 1) / block_size + 5,
+ op, data_length, (data_length + block_size - 1) // block_size + 5,
'foo'))
# Fail, src extents founds.
op.src_extents = ['bar']
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, data_length, (data_length + block_size - 1) / block_size + 5, 'foo')
+ op, data_length, (data_length + block_size - 1) // block_size + 5,
+ 'foo')
# Fail, missing data.
op.src_extents = []
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, None, (data_length + block_size - 1) / block_size, 'foo')
+ op, None, (data_length + block_size - 1) // block_size, 'foo')
# Fail, too few blocks to justify BZ.
op.src_extents = []
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
+ op, data_length, (data_length + block_size - 1) // block_size, 'foo')
+
+ # Fail, total_dst_blocks is a floating point value.
+ op.src_extents = []
+ self.assertRaises(
+ PayloadError, payload_checker._CheckReplaceOperation,
op, data_length, (data_length + block_size - 1) / block_size, 'foo')
def testCheckReplaceXzOperation(self):
@@ -623,7 +662,7 @@
block_size = payload_checker.block_size
data_length = block_size * 3
- op = self.mox.CreateMock(
+ op = mock.create_autospec(
update_metadata_pb2.InstallOperation)
op.type = common.OpType.REPLACE_XZ
@@ -631,25 +670,32 @@
op.src_extents = []
self.assertIsNone(
payload_checker._CheckReplaceOperation(
- op, data_length, (data_length + block_size - 1) / block_size + 5,
+ op, data_length, (data_length + block_size - 1) // block_size + 5,
'foo'))
# Fail, src extents founds.
op.src_extents = ['bar']
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, data_length, (data_length + block_size - 1) / block_size + 5, 'foo')
+ op, data_length, (data_length + block_size - 1) // block_size + 5,
+ 'foo')
# Fail, missing data.
op.src_extents = []
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
- op, None, (data_length + block_size - 1) / block_size, 'foo')
+ op, None, (data_length + block_size - 1) // block_size, 'foo')
# Fail, too few blocks to justify XZ.
op.src_extents = []
self.assertRaises(
PayloadError, payload_checker._CheckReplaceOperation,
+ op, data_length, (data_length + block_size - 1) // block_size, 'foo')
+
+ # Fail, total_dst_blocks is a floating point value.
+ op.src_extents = []
+ self.assertRaises(
+ PayloadError, payload_checker._CheckReplaceOperation,
op, data_length, (data_length + block_size - 1) / block_size, 'foo')
def testCheckAnyDiff(self):
@@ -724,9 +770,9 @@
old_part_size = test_utils.MiB(4)
new_part_size = test_utils.MiB(8)
old_block_counters = array.array(
- 'B', [0] * ((old_part_size + block_size - 1) / block_size))
+ 'B', [0] * ((old_part_size + block_size - 1) // block_size))
new_block_counters = array.array(
- 'B', [0] * ((new_part_size + block_size - 1) / block_size))
+ 'B', [0] * ((new_part_size + block_size - 1) // block_size))
prev_data_offset = 1876
blob_hash_counts = collections.defaultdict(int)
@@ -769,16 +815,14 @@
fake_data = 'fake-data'.ljust(op.data_length)
if not allow_unhashed and not fail_data_hash:
# Create a valid data blob hash.
- op.data_sha256_hash = hashlib.sha256(fake_data).digest()
- payload.ReadDataBlob(op.data_offset, op.data_length).AndReturn(
- fake_data)
+ op.data_sha256_hash = hashlib.sha256(fake_data.encode('utf-8')).digest()
+ payload.ReadDataBlob.return_value = fake_data.encode('utf-8')
elif fail_data_hash:
# Create an invalid data blob hash.
op.data_sha256_hash = hashlib.sha256(
- fake_data.replace(' ', '-')).digest()
- payload.ReadDataBlob(op.data_offset, op.data_length).AndReturn(
- fake_data)
+ fake_data.replace(' ', '-').encode('utf-8')).digest()
+ payload.ReadDataBlob.return_value = fake_data.encode('utf-8')
total_dst_blocks = 0
if not fail_missing_dst_extents:
@@ -807,7 +851,6 @@
payload_checker.minor_version <= 3):
op.dst_length = total_dst_blocks * block_size
- self.mox.ReplayAll()
should_fail = (fail_src_extents or fail_dst_extents or
fail_mismatched_data_offset_length or
fail_missing_dst_extents or fail_src_length or
@@ -857,7 +900,8 @@
rootfs_data_length -= block_size
payload_gen.AddOperation(common.ROOTFS, rootfs_op_type,
- dst_extents=[(0, rootfs_data_length / block_size)],
+ dst_extents=
+ [(0, rootfs_data_length // block_size)],
data_offset=0,
data_length=rootfs_data_length)
@@ -889,13 +933,13 @@
rootfs_part_size = test_utils.MiB(2)
kernel_part_size = test_utils.KiB(16)
payload_gen.SetPartInfo(common.ROOTFS, True, rootfs_part_size,
- hashlib.sha256('fake-new-rootfs-content').digest())
+ hashlib.sha256(b'fake-new-rootfs-content').digest())
payload_gen.SetPartInfo(common.KERNEL, True, kernel_part_size,
- hashlib.sha256('fake-new-kernel-content').digest())
+ hashlib.sha256(b'fake-new-kernel-content').digest())
payload_gen.SetMinorVersion(0)
payload_gen.AddOperationWithData(
common.ROOTFS, common.OpType.REPLACE,
- dst_extents=[(0, rootfs_part_size / block_size)],
+ dst_extents=[(0, rootfs_part_size // block_size)],
data_blob=os.urandom(rootfs_part_size))
do_forge_sigs_data = (fail_empty_sigs_blob or fail_sig_missing_fields or
@@ -908,7 +952,7 @@
if fail_sig_missing_fields:
sig_data = None
else:
- sig_data = test_utils.SignSha256('fake-payload-content',
+ sig_data = test_utils.SignSha256(b'fake-payload-content',
test_utils._PRIVKEY_FILE_NAME)
sigs_gen.AddSig(5 if fail_unknown_sig_version else 1, sig_data)
@@ -984,9 +1028,9 @@
kernel_filesystem_size = test_utils.KiB(16)
rootfs_filesystem_size = test_utils.MiB(2)
payload_gen.SetPartInfo(common.ROOTFS, True, rootfs_filesystem_size,
- hashlib.sha256('fake-new-rootfs-content').digest())
+ hashlib.sha256(b'fake-new-rootfs-content').digest())
payload_gen.SetPartInfo(common.KERNEL, True, kernel_filesystem_size,
- hashlib.sha256('fake-new-kernel-content').digest())
+ hashlib.sha256(b'fake-new-kernel-content').digest())
payload_gen.SetMinorVersion(0)
rootfs_part_size = 0
@@ -997,7 +1041,7 @@
rootfs_op_size += block_size
payload_gen.AddOperationWithData(
common.ROOTFS, common.OpType.REPLACE,
- dst_extents=[(0, rootfs_op_size / block_size)],
+ dst_extents=[(0, rootfs_op_size // block_size)],
data_blob=os.urandom(rootfs_op_size))
kernel_part_size = 0
@@ -1008,7 +1052,7 @@
kernel_op_size += block_size
payload_gen.AddOperationWithData(
common.KERNEL, common.OpType.REPLACE,
- dst_extents=[(0, kernel_op_size / block_size)],
+ dst_extents=[(0, kernel_op_size // block_size)],
data_blob=os.urandom(kernel_op_size))
# Generate payload (complete w/ signature) and create the test object.
@@ -1054,6 +1098,7 @@
else:
self.assertIsNone(payload_checker.Run(**kwargs2))
+
# This implements a generic API, hence the occasional unused args.
# pylint: disable=W0613
def ValidateCheckOperationTest(op_type_name, allow_unhashed,
@@ -1104,13 +1149,13 @@
(values) associated with them.
validate_func: A function used for validating test argument combinations.
"""
- for value_tuple in itertools.product(*arg_space.itervalues()):
- run_dargs = dict(zip(arg_space.iterkeys(), value_tuple))
+ for value_tuple in itertools.product(*iter(arg_space.values())):
+ run_dargs = dict(zip(iter(arg_space.keys()), value_tuple))
if validate_func and not validate_func(**run_dargs):
continue
run_method_name = 'Do%sTest' % tested_method_name
test_method_name = 'test%s' % tested_method_name
- for arg_key, arg_val in run_dargs.iteritems():
+ for arg_key, arg_val in run_dargs.items():
if arg_val or isinstance(arg_val, int):
test_method_name += '__%s=%s' % (arg_key, arg_val)
setattr(PayloadCheckerTest, test_method_name,
diff --git a/scripts/update_payload/common.py b/scripts/update_payload/common.py
index dfb8181..b934cf8 100644
--- a/scripts/update_payload/common.py
+++ b/scripts/update_payload/common.py
@@ -16,8 +16,11 @@
"""Utilities for update payload processing."""
+from __future__ import absolute_import
from __future__ import print_function
+import base64
+
from update_payload import update_metadata_pb2
from update_payload.error import PayloadError
@@ -26,9 +29,9 @@
# Constants.
#
SIG_ASN1_HEADER = (
- '\x30\x31\x30\x0d\x06\x09\x60\x86'
- '\x48\x01\x65\x03\x04\x02\x01\x05'
- '\x00\x04\x20'
+ b'\x30\x31\x30\x0d\x06\x09\x60\x86'
+ b'\x48\x01\x65\x03\x04\x02\x01\x05'
+ b'\x00\x04\x20'
)
BRILLO_MAJOR_PAYLOAD_VERSION = 2
@@ -43,6 +46,7 @@
# Tuple of (name in system, name in protobuf).
CROS_PARTITIONS = ((KERNEL, KERNEL), (ROOTFS, 'rootfs'))
+
#
# Payload operation types.
#
@@ -138,7 +142,7 @@
try:
data = file_obj.read(length)
- except IOError, e:
+ except IOError as e:
raise PayloadError('error reading from file (%s): %s' % (file_obj.name, e))
if len(data) != length:
@@ -164,7 +168,7 @@
def FormatSha256(digest):
"""Returns a canonical string representation of a SHA256 digest."""
- return digest.encode('base64').strip()
+ return base64.b64encode(digest).decode('utf-8')
#
diff --git a/scripts/update_payload/format_utils.py b/scripts/update_payload/format_utils.py
index 6248ba9..e73badf 100644
--- a/scripts/update_payload/format_utils.py
+++ b/scripts/update_payload/format_utils.py
@@ -16,6 +16,8 @@
"""Various formatting functions."""
+from __future__ import division
+
def NumToPercent(num, total, min_precision=1, max_precision=5):
"""Returns the percentage (string) of |num| out of |total|.
@@ -50,7 +52,7 @@
precision = min(min_precision, max_precision)
factor = 10 ** precision
while precision <= max_precision:
- percent = num * 100 * factor / total
+ percent = num * 100 * factor // total
if percent:
break
factor *= 10
@@ -102,8 +104,8 @@
magnitude = next_magnitude
if exp != 0:
- whole = size / magnitude
- frac = (size % magnitude) * (10 ** precision) / magnitude
+ whole = size // magnitude
+ frac = (size % magnitude) * (10 ** precision) // magnitude
while frac and not frac % 10:
frac /= 10
return '%d%s %s' % (whole, '.%d' % frac if frac else '', suffixes[exp - 1])
diff --git a/scripts/update_payload/format_utils_unittest.py b/scripts/update_payload/format_utils_unittest.py
index 42ea621..4dcd652 100755
--- a/scripts/update_payload/format_utils_unittest.py
+++ b/scripts/update_payload/format_utils_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2013 The Android Open Source Project
#
@@ -17,6 +17,11 @@
"""Unit tests for format_utils.py."""
+# Disable check for function names to avoid errors based on old code
+# pylint: disable-msg=invalid-name
+
+from __future__ import absolute_import
+
import unittest
from update_payload import format_utils
diff --git a/scripts/update_payload/histogram.py b/scripts/update_payload/histogram.py
index 1ac2ab5..bad2dc3 100644
--- a/scripts/update_payload/histogram.py
+++ b/scripts/update_payload/histogram.py
@@ -16,6 +16,9 @@
"""Histogram generation tools."""
+from __future__ import absolute_import
+from __future__ import division
+
from collections import defaultdict
from update_payload import format_utils
@@ -110,7 +113,7 @@
hist_bar = '|'
for key, count in self.data:
if self.total:
- bar_len = count * self.scale / self.total
+ bar_len = count * self.scale // self.total
hist_bar = '|%s|' % ('#' * bar_len).ljust(self.scale)
line = '%s %s %s' % (
diff --git a/scripts/update_payload/histogram_unittest.py b/scripts/update_payload/histogram_unittest.py
index e757dd0..ccde2bb 100755
--- a/scripts/update_payload/histogram_unittest.py
+++ b/scripts/update_payload/histogram_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
#
# Copyright (C) 2013 The Android Open Source Project
#
@@ -17,6 +17,11 @@
"""Unit tests for histogram.py."""
+# Disable check for function names to avoid errors based on old code
+# pylint: disable-msg=invalid-name
+
+from __future__ import absolute_import
+
import unittest
from update_payload import format_utils
diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py
index 1ed5f99..ea5ed30 100644
--- a/scripts/update_payload/payload.py
+++ b/scripts/update_payload/payload.py
@@ -16,6 +16,7 @@
"""Tools for reading, verifying and applying Chrome OS update payloads."""
+from __future__ import absolute_import
from __future__ import print_function
import hashlib
@@ -64,7 +65,7 @@
"""Update payload header struct."""
# Header constants; sizes are in bytes.
- _MAGIC = 'CrAU'
+ _MAGIC = b'CrAU'
_VERSION_SIZE = 8
_MANIFEST_LEN_SIZE = 8
_METADATA_SIGNATURE_LEN_SIZE = 4
@@ -111,7 +112,6 @@
payload_file, self._METADATA_SIGNATURE_LEN_SIZE, True,
hasher=hasher)
-
def __init__(self, payload_file, payload_file_offset=0):
"""Initialize the payload object.
diff --git a/scripts/update_payload/test_utils.py b/scripts/update_payload/test_utils.py
index 4f5fed0..e153669 100644
--- a/scripts/update_payload/test_utils.py
+++ b/scripts/update_payload/test_utils.py
@@ -16,9 +16,10 @@
"""Utilities for unit testing."""
+from __future__ import absolute_import
from __future__ import print_function
-import cStringIO
+import io
import hashlib
import os
import struct
@@ -70,7 +71,7 @@
"""
try:
file_obj.write(struct.pack(common.IntPackingFmtStr(size, is_unsigned), val))
- except IOError, e:
+ except IOError as e:
raise payload.PayloadError('error writing to file (%s): %s' %
(file_obj.name, e))
@@ -335,7 +336,7 @@
if do_generate_sigs_data:
# First, sign some arbitrary data to obtain the size of a signature blob.
- fake_sig = SignSha256('fake-payload-data', privkey_file_name)
+ fake_sig = SignSha256(b'fake-payload-data', privkey_file_name)
fake_sigs_gen = SignaturesGenerator()
fake_sigs_gen.AddSig(1, fake_sig)
sigs_len = len(fake_sigs_gen.ToBinary())
@@ -345,7 +346,7 @@
if do_generate_sigs_data:
# Once all payload fields are updated, dump and sign it.
- temp_payload_file = cStringIO.StringIO()
+ temp_payload_file = io.BytesIO()
self.WriteToFile(temp_payload_file, data_blobs=self.data_blobs)
sig = SignSha256(temp_payload_file.getvalue(), privkey_file_name)
sigs_gen = SignaturesGenerator()
