Add --compressor_types option
E2E test time of OTA is approaching ~40 minutes. Add an option to
disable brotli compression to speed up unit tests. From our past
experience, compressino algorithms are hardly sources of bugs. So no
need to test compression algorihtms in presubmit test.
Test: th
Change-Id: I7660efd6eb3d5c305c74d1a9fef63f8121653ea0
diff --git a/payload_generator/delta_diff_utils.cc b/payload_generator/delta_diff_utils.cc
index 781b33b..c32f2b0 100644
--- a/payload_generator/delta_diff_utils.cc
+++ b/payload_generator/delta_diff_utils.cc
@@ -45,6 +45,7 @@
#include <base/threading/simple_thread.h>
#include <brillo/data_encoding.h>
#include <bsdiff/bsdiff.h>
+#include <bsdiff/constants.h>
#include <bsdiff/control_entry.h>
#include <bsdiff/patch_reader.h>
#include <bsdiff/patch_writer_factory.h>
@@ -201,6 +202,11 @@
return GenerateBestDiffOperation(diff_candidates, aop, data_blob);
}
+std::vector<bsdiff::CompressorType>
+BestDiffGenerator::GetUsableCompressorTypes() const {
+ return config_.compressors;
+}
+
bool BestDiffGenerator::GenerateBestDiffOperation(
const std::vector<std::pair<InstallOperation_Type, size_t>>&
diff_candidates,
@@ -260,10 +266,8 @@
std::unique_ptr<bsdiff::PatchWriterInterface> bsdiff_patch_writer;
if (operation_type == InstallOperation::BROTLI_BSDIFF) {
- bsdiff_patch_writer =
- bsdiff::CreateBSDF2PatchWriter(patch.value(),
- bsdiff::CompressorType::kBrotli,
- kBrotliCompressionQuality);
+ bsdiff_patch_writer = bsdiff::CreateBSDF2PatchWriter(
+ patch.value(), GetUsableCompressorTypes(), kBrotliCompressionQuality);
} else {
bsdiff_patch_writer = bsdiff::CreateBsdiffPatchWriter(patch.value());
}
@@ -328,6 +332,7 @@
new_data_,
src_deflates,
dst_deflates,
+ GetUsableCompressorTypes(),
temp_file.path(),
&puffdiff_delta));
TEST_AND_RETURN_FALSE(!puffdiff_delta.empty());
@@ -1142,7 +1147,7 @@
const brillo::Blob& hash = hasher.raw_hash();
info->set_hash(hash.data(), hash.size());
LOG(INFO) << part.path << ": size=" << part.size
- << " hash=" << brillo::data_encoding::Base64Encode(hash);
+ << " hash=" << HexEncode(hash);
return true;
}
diff --git a/payload_generator/delta_diff_utils.h b/payload_generator/delta_diff_utils.h
index b335e2a..8d1dca6 100644
--- a/payload_generator/delta_diff_utils.h
+++ b/payload_generator/delta_diff_utils.h
@@ -192,6 +192,7 @@
brillo::Blob* data_blob);
private:
+ std::vector<bsdiff::CompressorType> GetUsableCompressorTypes() const;
bool TryBsdiffAndUpdateOperation(InstallOperation_Type operation_type,
AnnotatedOperation* aop,
brillo::Blob* data_blob);
diff --git a/payload_generator/generate_delta_main.cc b/payload_generator/generate_delta_main.cc
index 1919a7e..4ede12b 100644
--- a/payload_generator/generate_delta_main.cc
+++ b/payload_generator/generate_delta_main.cc
@@ -430,6 +430,10 @@
"Whether to use Virtual AB Compression XOR feature");
DEFINE_string(
apex_info_file, "", "Path to META/apex_info.pb found in target build");
+ DEFINE_string(compressor_types,
+ "bz2:brotli",
+ "Colon ':' separated list of compressors. Allowed valures are "
+ "bz2 and brotli.");
brillo::FlagHelper::Init(
argc,
@@ -547,6 +551,7 @@
}
payload_config.enable_vabc_xor = FLAGS_enable_vabc_xor;
+ payload_config.ParseCompressorTypes(FLAGS_compressor_types);
if (!FLAGS_new_partitions.empty()) {
LOG_IF(FATAL, !FLAGS_new_image.empty() || !FLAGS_new_kernel.empty())
diff --git a/payload_generator/payload_generation_config.cc b/payload_generator/payload_generation_config.cc
index 7484755..178261e 100644
--- a/payload_generator/payload_generation_config.cc
+++ b/payload_generator/payload_generation_config.cc
@@ -25,6 +25,7 @@
#include <brillo/strings/string_utils.h>
#include <libsnapshot/cow_format.h>
+#include "bsdiff/constants.h"
#include "update_engine/common/utils.h"
#include "update_engine/payload_consumer/delta_performer.h"
#include "update_engine/payload_generator/boot_img_filesystem.h"
@@ -326,4 +327,23 @@
return true;
}
+void PayloadGenerationConfig::ParseCompressorTypes(
+ const std::string& compressor_types) {
+ auto types = brillo::string_utils::Split(compressor_types, ":");
+ CHECK_LE(types.size(), 2UL)
+ << "Only two compressor types are allowed: bz2 and brotli";
+ CHECK_GT(types.size(), 0UL) << "Please pass in at least 1 valid compressor. "
+ "Allowed values are bz2 and brotli.";
+ compressors.clear();
+ for (const auto& type : types) {
+ if (type == "bz2") {
+ compressors.emplace_back(bsdiff::CompressorType::kBZ2);
+ } else if (type == "brotli") {
+ compressors.emplace_back(bsdiff::CompressorType::kBrotli);
+ } else {
+ LOG(FATAL) << "Unknown compressor type: " << type;
+ }
+ }
+}
+
} // namespace chromeos_update_engine
diff --git a/payload_generator/payload_generation_config.h b/payload_generator/payload_generation_config.h
index c15ac78..36ec676 100644
--- a/payload_generator/payload_generation_config.h
+++ b/payload_generator/payload_generation_config.h
@@ -26,6 +26,7 @@
#include <brillo/key_value_store.h>
#include <brillo/secure_blob.h>
+#include "bsdiff/constants.h"
#include "update_engine/payload_consumer/payload_constants.h"
#include "update_engine/payload_generator/filesystem_interface.h"
#include "update_engine/update_metadata.pb.h"
@@ -184,6 +185,8 @@
// Returns whether the PayloadGenerationConfig is valid.
bool Validate() const;
+ void ParseCompressorTypes(const std::string& compressor_types);
+
// Image information about the new image that's the target of this payload.
ImageConfig target;
@@ -236,6 +239,9 @@
// Whether to enable VABC xor op
bool enable_vabc_xor = false;
+
+ std::vector<bsdiff::CompressorType> compressors{
+ bsdiff::CompressorType::kBZ2, bsdiff::CompressorType::kBrotli};
};
} // namespace chromeos_update_engine
diff --git a/scripts/brillo_update_payload b/scripts/brillo_update_payload
index 0b1b463..2f5ef22 100755
--- a/scripts/brillo_update_payload
+++ b/scripts/brillo_update_payload
@@ -210,6 +210,8 @@
"Optional: Enable the use of Virtual AB Compression XOR feature"
DEFINE_string force_minor_version "" \
"Optional: Override the minor version for the delta generation."
+ DEFINE_string compressor_types "" \
+ "Optional: allowed compressor types. Colon separated, allowe values are bz2 and brotli"
fi
if [[ "${COMMAND}" == "hash" || "${COMMAND}" == "sign" ]]; then
DEFINE_string unsigned_payload "" "Path to the input unsigned payload."
@@ -717,6 +719,10 @@
GENERATOR_ARGS+=(
--disable_verity_computation="${FLAGS_disable_verity_computation}" )
fi
+ if [[ -n "${FLAGS_compressor_types}" ]]; then
+ GENERATOR_ARGS+=(
+ --compressor_types="${FLAGS_compressor_types}" )
+ fi
fi
if [[ -n "${FLAGS_enable_vabc_xor}" ]]; then
