update_engine: Use the FilesystemInterface to generate deltas.
This patch removes the FilesystemIterator used to iterate the file data
blocks, the metadata.{cc,h} files used to iterate the metadata blocks and
the ReadUnwrittenBlocks() method used to process the rest of the blocks.
Instead, these three cases are handled by the same DeltaReadFilesystem()
method using the FilesystemInterface abstraction for ext2.
The sub-file block handling was changed from (filename, offset, size) to
just the list of blocks (or Extents) and the parsing of the filesystem
was removed from the methods that produce operations. The filename is
kept just as a label for logging purposes.
This patch implies that the filesystem doesn't need to be mounted in
order to generate the payload, as all the data is accessed as blocks
in a partition. The mountpoint was removed from all the operations
generators, but it is still required to mount the old filesystem to
detect the minor version supported by the old update_engine. Also,
since the list of blocks is never accessed using the FIGETBSZ ioctl,
no payload_generator/ test require root privileges, nor the
delta_generator binary.
BUG=chromium:305832,chromium:331965
TEST=Updated unittest to use extents.
Change-Id: Ia2ea9433190258f70e3bce09f896b18326a7abf9
Reviewed-on: https://chromium-review.googlesource.com/275804
Reviewed-by: Alex Deymo <[email protected]>
Commit-Queue: Alex Deymo <[email protected]>
Tested-by: Alex Deymo <[email protected]>
diff --git a/payload_generator/annotated_operation.cc b/payload_generator/annotated_operation.cc
index 7f8a36c..2aa79fc 100644
--- a/payload_generator/annotated_operation.cc
+++ b/payload_generator/annotated_operation.cc
@@ -25,19 +25,6 @@
}
} // namespace
-void AnnotatedOperation::SetNameFromFileAndChunk(
- const string& filename, off_t chunk_offset, off_t chunk_size) {
- name = filename;
- if (chunk_offset != 0 || chunk_size != -1) {
- if (chunk_size != -1) {
- base::StringAppendF(&name, " [%" PRId64 ", %" PRId64 ")",
- chunk_offset, chunk_offset + chunk_size);
- } else {
- base::StringAppendF(&name, " [%" PRId64 ", end)", chunk_offset);
- }
- }
-}
-
bool AnnotatedOperation::SetOperationBlob(chromeos::Blob* blob, int data_fd,
off_t* data_file_size) {
TEST_AND_RETURN_FALSE(utils::PWriteAll(data_fd,
diff --git a/payload_generator/annotated_operation.h b/payload_generator/annotated_operation.h
index d07b6ae..bacc0ab 100644
--- a/payload_generator/annotated_operation.h
+++ b/payload_generator/annotated_operation.h
@@ -21,10 +21,6 @@
// The InstallOperation, as defined by the protobuf.
DeltaArchiveManifest_InstallOperation op;
- // Sets |name| to a human readable representation of a chunk in a file.
- void SetNameFromFileAndChunk(const std::string& filename,
- off_t chunk_offset, off_t chunk_size);
-
// Writes |blob| to the end of |data_fd|, and updates |data_file_size| to
// match the new size of |data_fd|. It sets the data_offset and data_length
// in AnnotatedOperation to match the offset and size of |blob| in |data_fd|.
diff --git a/payload_generator/delta_diff_generator.cc b/payload_generator/delta_diff_generator.cc
index c9259b5..9ff78a9 100644
--- a/payload_generator/delta_diff_generator.cc
+++ b/payload_generator/delta_diff_generator.cc
@@ -27,25 +27,21 @@
#include "update_engine/bzip.h"
#include "update_engine/delta_performer.h"
+#include "update_engine/extent_ranges.h"
#include "update_engine/file_writer.h"
#include "update_engine/omaha_hash_calculator.h"
#include "update_engine/payload_constants.h"
-#include "update_engine/payload_generator/extent_mapper.h"
-#include "update_engine/payload_generator/filesystem_iterator.h"
+#include "update_engine/payload_generator/ext2_filesystem.h"
#include "update_engine/payload_generator/full_update_generator.h"
-#include "update_engine/payload_generator/graph_types.h"
-#include "update_engine/payload_generator/graph_utils.h"
#include "update_engine/payload_generator/inplace_generator.h"
-#include "update_engine/payload_generator/metadata.h"
#include "update_engine/payload_generator/payload_signer.h"
+#include "update_engine/payload_generator/raw_filesystem.h"
#include "update_engine/payload_verifier.h"
#include "update_engine/subprocess.h"
#include "update_engine/update_metadata.pb.h"
#include "update_engine/utils.h"
using std::map;
-using std::max;
-using std::min;
using std::set;
using std::sort;
using std::string;
@@ -61,7 +57,7 @@
// application requires a significant amount of RAM. We put a hard-limit of
// 200 MiB that should not affect any released board, but will limit the
// Chrome binary in ASan builders.
-const off_t kMaxBsdiffDestinationSize = 200 * 1024 * 1024; // bytes
+const uint64_t kMaxBsdiffDestinationSize = 200 * 1024 * 1024; // bytes
static const char* kInstallOperationTypes[] = {
"REPLACE",
@@ -76,7 +72,6 @@
namespace chromeos_update_engine {
-typedef DeltaDiffGenerator::Block Block;
typedef map<const DeltaArchiveManifest_InstallOperation*,
string> OperationNameMap;
@@ -86,30 +81,8 @@
const char* const kEmptyPath = "";
const char* const kBsdiffPath = "bsdiff";
-// Needed for testing purposes, in case we can't use actual filesystem objects.
-// TODO(garnold) (chromium:331965) Replace this hack with a properly injected
-// parameter in form of a mockable abstract class.
-bool (*get_extents_with_chunk_func)(const string&, off_t, off_t,
- vector<Extent>*) =
- extent_mapper::ExtentsForFileChunkFibmap;
-
namespace {
-bool IsSparseHole(const Extent &extent) {
- return (extent.start_block() == kSparseHole);
-}
-
-// Stores all the extents of |path| into |extents|. Returns true on success.
-bool GatherExtents(const string& path,
- off_t chunk_offset,
- off_t chunk_size,
- vector<Extent>* extents) {
- extents->clear();
- TEST_AND_RETURN_FALSE(
- get_extents_with_chunk_func(path, chunk_offset, chunk_size, extents));
- return true;
-}
-
// Writes the uint64_t passed in in host-endian to the file as big-endian.
// Returns true on success.
bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) {
@@ -277,8 +250,8 @@
uint64_t src_num_blocks = (*src_extents)[src_idx].num_blocks();
uint64_t dst_num_blocks = (*dst_extents)[dst_idx].num_blocks();
- uint64_t min_num_blocks = min(src_num_blocks - src_offset,
- dst_num_blocks - dst_offset);
+ uint64_t min_num_blocks = std::min(src_num_blocks - src_offset,
+ dst_num_blocks - dst_offset);
uint64_t prev_src_offset = src_offset;
uint64_t prev_dst_offset = dst_offset;
src_offset += min_num_blocks;
@@ -315,34 +288,51 @@
} // namespace
-bool DeltaDiffGenerator::DeltaReadFiles(Graph* graph,
- vector<Block>* blocks,
- const string& old_part,
- const string& new_part,
- const string& old_root,
- const string& new_root,
- off_t chunk_size,
- int data_fd,
- off_t* data_file_size,
- bool src_ops_allowed) {
- set<ino_t> visited_inodes;
- set<ino_t> visited_src_inodes;
- for (FilesystemIterator fs_iter(new_root,
- set<string>{"/lost+found"});
- !fs_iter.IsEnd(); fs_iter.Increment()) {
- // We never diff symlinks (here, we check that dst file is not a symlink).
- if (!S_ISREG(fs_iter.GetStat().st_mode))
+bool DeltaDiffGenerator::DeltaReadFilesystem(
+ vector<AnnotatedOperation>* aops,
+ const string& old_part,
+ const string& new_part,
+ FilesystemInterface* old_fs,
+ FilesystemInterface* new_fs,
+ off_t chunk_blocks,
+ int data_fd,
+ off_t* data_file_size,
+ bool src_ops_allowed) {
+ ExtentRanges old_visited_blocks;
+ ExtentRanges new_visited_blocks;
+
+ map<string, vector<Extent>> old_files_map;
+ if (old_fs) {
+ vector<FilesystemInterface::File> old_files;
+ old_fs->GetFiles(&old_files);
+ for (const FilesystemInterface::File& file : old_files)
+ old_files_map[file.name] = file.extents;
+ }
+
+ vector<FilesystemInterface::File> new_files;
+ new_fs->GetFiles(&new_files);
+
+ // The processing is very straightforward here, we generate operations for
+ // every file (and pseudo-file such as the metadata) in the new filesystem
+ // based on the file with the same name in the old filesystem, if any.
+ // Files with overlapping data blocks (like hardlinks or filesystems with tail
+ // packing or compression where the blocks store more than one file) are only
+ // generated once in the new image, but are also used only once from the old
+ // image due to some simplifications (see below).
+ for (const FilesystemInterface::File& new_file : new_files) {
+ // Ignore the files in the new filesystem without blocks. Symlinks with
+ // data blocks (for example, symlinks bigger than 60 bytes in ext2) are
+ // handled as normal files. We also ignore blocks that were already
+ // processed by a previous file.
+ vector<Extent> new_file_extents = FilterExtentRanges(
+ new_file.extents, new_visited_blocks);
+ new_visited_blocks.AddExtents(new_file_extents);
+
+ if (new_file_extents.empty())
continue;
- // Make sure we visit each inode only once.
- if (utils::SetContainsKey(visited_inodes, fs_iter.GetStat().st_ino))
- continue;
- visited_inodes.insert(fs_iter.GetStat().st_ino);
- off_t dst_size = fs_iter.GetFileSize();
- if (dst_size == 0)
- continue;
-
- LOG(INFO) << "Encoding file " << fs_iter.GetPartialPath();
+ LOG(INFO) << "Encoding file " << new_file.name << " ("
+ << BlocksInExtents(new_file_extents) << " blocks)";
// We can't visit each dst image inode more than once, as that would
// duplicate work. Here, we avoid visiting each source image inode
@@ -352,235 +342,191 @@
// from using a graph/cycle detection/etc to generate diffs, and at that
// time, it will be easy (non-complex) to have many operations read
// from the same source blocks. At that time, this code can die. -adlr
- bool should_diff_from_source = false;
- string src_path = old_root + fs_iter.GetPartialPath();
- struct stat src_stbuf;
- // We never diff symlinks (here, we check that src file is not a symlink).
- if (0 == lstat(src_path.c_str(), &src_stbuf) &&
- S_ISREG(src_stbuf.st_mode)) {
- should_diff_from_source = !utils::SetContainsKey(visited_src_inodes,
- src_stbuf.st_ino);
- visited_src_inodes.insert(src_stbuf.st_ino);
- }
+ vector<Extent> old_file_extents = FilterExtentRanges(
+ old_files_map[new_file.name], old_visited_blocks);
+ old_visited_blocks.AddExtents(old_file_extents);
- off_t size = chunk_size == -1 ? dst_size : chunk_size;
- off_t step = size;
- for (off_t offset = 0; offset < dst_size; offset += step) {
- if (offset + size >= dst_size) {
- size = -1; // Read through the end of the file.
- }
- TEST_AND_RETURN_FALSE(DeltaDiffGenerator::DeltaReadFile(
- graph,
- Vertex::kInvalidIndex,
- blocks,
- old_part,
- new_part,
- (should_diff_from_source ? old_root : kEmptyPath),
- new_root,
- fs_iter.GetPartialPath(),
- offset,
- size,
- data_fd,
- data_file_size,
- src_ops_allowed));
- }
+ TEST_AND_RETURN_FALSE(DeltaReadFile(
+ aops,
+ old_part,
+ new_part,
+ old_file_extents,
+ new_file_extents,
+ new_file.name, // operation name
+ chunk_blocks,
+ data_fd,
+ data_file_size,
+ src_ops_allowed));
}
+ // Process all the blocks not included in any file. We provided all the unused
+ // blocks in the old partition as available data.
+ vector<Extent> new_unvisited = { ExtentForRange(0, new_fs->GetBlockCount()) };
+ new_unvisited = FilterExtentRanges(new_unvisited, new_visited_blocks);
+ if (new_unvisited.empty())
+ return true;
+
+ vector<Extent> old_unvisited;
+ if (old_fs) {
+ old_unvisited.push_back(ExtentForRange(0, old_fs->GetBlockCount()));
+ old_unvisited = FilterExtentRanges(old_unvisited, old_visited_blocks);
+ }
+
+ LOG(INFO) << "Scanning " << BlocksInExtents(new_unvisited)
+ << " unwritten blocks";
+ TEST_AND_RETURN_FALSE(DeltaReadFile(
+ aops,
+ old_part,
+ new_part,
+ old_unvisited,
+ new_unvisited,
+ "<non-file-data>", // operation name
+ chunk_blocks,
+ data_fd,
+ data_file_size,
+ src_ops_allowed));
+
return true;
}
-bool DeltaDiffGenerator::DeltaReadFile(Graph* graph,
- Vertex::Index existing_vertex,
- vector<Block>* blocks,
- const string& old_part,
- const string& new_part,
- const string& old_root,
- const string& new_root,
- const string& path, // within new_root
- off_t chunk_offset,
- off_t chunk_size,
- int data_fd,
- off_t* data_file_size,
- bool src_ops_allowed) {
+bool DeltaDiffGenerator::DeltaReadFile(
+ vector<AnnotatedOperation>* aops,
+ const string& old_part,
+ const string& new_part,
+ const vector<Extent>& old_extents,
+ const vector<Extent>& new_extents,
+ const string& name,
+ off_t chunk_blocks,
+ int data_fd,
+ off_t* data_file_size,
+ bool src_ops_allowed) {
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation operation;
+ uint64_t total_blocks = BlocksInExtents(new_extents);
+ if (chunk_blocks == -1)
+ chunk_blocks = total_blocks;
+
// If bsdiff breaks again, blacklist the problem file by using:
- // bsdiff_allowed = (path != "/foo/bar")
+ // bsdiff_allowed = (name != "/foo/bar")
//
// TODO(dgarrett): chromium-os:15274 connect this test to the command line.
bool bsdiff_allowed = true;
-
- if (utils::FileSize(new_root + path) > kMaxBsdiffDestinationSize)
+ if (static_cast<uint64_t>(chunk_blocks) * kBlockSize >
+ kMaxBsdiffDestinationSize) {
bsdiff_allowed = false;
+ }
- if (!bsdiff_allowed)
- LOG(INFO) << "bsdiff blacklisting: " << path;
+ if (!bsdiff_allowed) {
+ LOG(INFO) << "bsdiff blacklisting: " << name;
+ }
- string old_filename = (old_root == kEmptyPath) ? kEmptyPath : old_root + path;
+ for (uint64_t block_offset = 0; block_offset < total_blocks;
+ block_offset += chunk_blocks) {
+ // Split the old/new file in the same chunks. Note that this could drop
+ // some information from the old file used for the new chunk. If the old
+ // file is smaller (or even empty when there's no old file) the chunk will
+ // also be empty.
+ vector<Extent> old_extents_chunk = ExtentsSublist(
+ old_extents, block_offset, chunk_blocks);
+ vector<Extent> new_extents_chunk = ExtentsSublist(
+ new_extents, block_offset, chunk_blocks);
+ NormalizeExtents(&old_extents_chunk);
+ NormalizeExtents(&new_extents_chunk);
- TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_part,
- new_part,
- chunk_offset,
- chunk_size,
- bsdiff_allowed,
- &data,
- &operation,
- true,
- src_ops_allowed,
- old_filename,
- new_root + path));
+ TEST_AND_RETURN_FALSE(ReadExtentsToDiff(old_part,
+ new_part,
+ old_extents_chunk,
+ new_extents_chunk,
+ bsdiff_allowed,
+ &data,
+ &operation,
+ src_ops_allowed));
- // Check if the operation writes nothing.
- if (operation.dst_extents_size() == 0) {
- if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) {
- LOG(INFO) << "Empty MOVE operation ("
- << new_root + path << "), skipping";
- return true;
- } else {
- LOG(ERROR) << "Empty non-MOVE operation";
- return false;
+ // Check if the operation writes nothing.
+ if (operation.dst_extents_size() == 0) {
+ if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) {
+ LOG(INFO) << "Empty MOVE operation ("
+ << name << "), skipping";
+ continue;
+ } else {
+ LOG(ERROR) << "Empty non-MOVE operation";
+ return false;
+ }
}
+
+ // Write the data
+ if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE &&
+ operation.type() !=
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) {
+ operation.set_data_offset(*data_file_size);
+ operation.set_data_length(data.size());
+ }
+
+ TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, data.data(), data.size()));
+ *data_file_size += data.size();
+
+ // Now, insert into the list of operations.
+ AnnotatedOperation aop;
+ aop.name = name;
+ if (static_cast<uint64_t>(chunk_blocks) < total_blocks) {
+ aop.name = base::StringPrintf("%s:%" PRIu64,
+ name.c_str(), block_offset / chunk_blocks);
+ }
+ aop.op = operation;
+ aops->emplace_back(aop);
}
-
- // Write the data
- if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE &&
- operation.type() !=
- DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) {
- operation.set_data_offset(*data_file_size);
- operation.set_data_length(data.size());
- }
-
- TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, data.data(), data.size()));
- *data_file_size += data.size();
-
- // Now, insert into graph and blocks vector
- Vertex::Index vertex = existing_vertex;
- if (vertex == Vertex::kInvalidIndex) {
- graph->emplace_back();
- vertex = graph->size() - 1;
- }
- (*graph)[vertex].op = operation;
- CHECK((*graph)[vertex].op.has_type());
- (*graph)[vertex].file_name = path;
- (*graph)[vertex].chunk_offset = chunk_offset;
- (*graph)[vertex].chunk_size = chunk_size;
-
- if (blocks)
- TEST_AND_RETURN_FALSE(InplaceGenerator::AddInstallOpToBlocksVector(
- (*graph)[vertex].op,
- *graph,
- vertex,
- blocks));
return true;
}
-bool DeltaDiffGenerator::ReadFileToDiff(
+bool DeltaDiffGenerator::ReadExtentsToDiff(
const string& old_part,
const string& new_part,
- off_t chunk_offset,
- off_t chunk_size,
+ const vector<Extent>& old_extents,
+ const vector<Extent>& new_extents,
bool bsdiff_allowed,
chromeos::Blob* out_data,
DeltaArchiveManifest_InstallOperation* out_op,
- bool gather_extents,
- bool src_ops_allowed,
- const string& old_filename,
- const string& new_filename) {
-
- // Do we have an original file to consider?
- off_t old_size = 0;
- bool original = !old_filename.empty();
- if (original && (old_size = utils::FileSize(old_filename)) < 0) {
- // If stat-ing the old file fails, it should be because it doesn't exist.
- TEST_AND_RETURN_FALSE(!utils::FileExists(old_filename.c_str()));
- original = false;
- }
-
+ bool src_ops_allowed) {
DeltaArchiveManifest_InstallOperation operation;
- vector<Extent> src_extents, dst_extents;
- // Gather source extents if we have an original file.
- if (original) {
- if (gather_extents) {
- TEST_AND_RETURN_FALSE(
- GatherExtents(old_filename, chunk_offset, chunk_size, &src_extents));
- ClearSparseHoles(&src_extents);
- if (src_extents.size() == 0) {
- // Reading from sparse hole, do nothing.
- operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
- *out_op = operation;
- return true;
- }
- } else {
- // We have a kernel, so make one extent to cover it all.
- Extent* src_extent = operation.add_src_extents();
- src_extent->set_start_block(0);
- src_extent->set_num_blocks(
- (utils::FileSize(old_filename) + (kBlockSize - 1)) / kBlockSize);
- src_extents.push_back(*src_extent);
- }
- }
+ // Data blob that will be written to delta file.
+ const chromeos::Blob* data_blob = nullptr;
- // Gather destination extents.
- if (gather_extents) {
- TEST_AND_RETURN_FALSE(
- GatherExtents(new_filename, chunk_offset, chunk_size, &dst_extents));
- ClearSparseHoles(&dst_extents);
- if (dst_extents.size() == 0) {
- // Make an empty move operation.
- operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
- *out_op = operation;
- return true;
- }
- } else {
- Extent* dst_extent = operation.add_dst_extents();
- dst_extent->set_start_block(0);
- dst_extent->set_num_blocks(
- (utils::FileSize(new_filename) + (kBlockSize - 1)) / kBlockSize);
- dst_extents.push_back(*dst_extent);
- }
+ // We read blocks from old_extents and write blocks to new_extents.
+ uint64_t blocks_to_read = BlocksInExtents(old_extents);
+ uint64_t blocks_to_write = BlocksInExtents(new_extents);
- NormalizeExtents(&src_extents);
- NormalizeExtents(&dst_extents);
-
- // Figure out how many blocks we need to write to dst_extents.
- uint64_t blocks_to_write = 0;
- for (uint32_t i = 0; i < dst_extents.size(); i++)
- blocks_to_write += dst_extents[i].num_blocks();
-
- // Figure out how many blocks we need to read to src_extents.
- uint64_t blocks_to_read = 0;
- for (uint32_t i = 0; i < src_extents.size(); i++)
- blocks_to_read += src_extents[i].num_blocks();
+ // Make copies of the extents so we can modify them.
+ vector<Extent> src_extents = old_extents;
+ vector<Extent> dst_extents = new_extents;
// Read in bytes from new data.
chromeos::Blob new_data;
TEST_AND_RETURN_FALSE(utils::ReadExtents(new_part,
- dst_extents,
+ new_extents,
&new_data,
kBlockSize * blocks_to_write,
kBlockSize));
-
TEST_AND_RETURN_FALSE(!new_data.empty());
- TEST_AND_RETURN_FALSE(chunk_size == -1 ||
- static_cast<off_t>(new_data.size()) <= chunk_size);
+
+ // Using a REPLACE is always an option.
+ operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE);
+ data_blob = &new_data;
+
+ // Try compressing it with bzip2.
chromeos::Blob new_data_bz;
TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz));
CHECK(!new_data_bz.empty());
- chromeos::Blob data; // Data blob that will be written to delta file.
-
- size_t current_best_size = 0;
- if (new_data.size() <= new_data_bz.size()) {
- operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE);
- current_best_size = new_data.size();
- data = new_data;
- } else {
+ if (new_data_bz.size() < data_blob->size()) {
+ // A REPLACE_BZ is better.
operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
- current_best_size = new_data_bz.size();
- data = new_data_bz;
+ data_blob = &new_data_bz;
}
+
chromeos::Blob old_data;
- if (original) {
+ chromeos::Blob empty_blob;
+ chromeos::Blob bsdiff_delta;
+ if (blocks_to_read > 0) {
// Read old data.
TEST_AND_RETURN_FALSE(
utils::ReadExtents(old_part, src_extents, &old_data,
@@ -593,9 +539,8 @@
} else {
operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
}
- current_best_size = 0;
- data.clear();
- } else if (!old_data.empty() && bsdiff_allowed) {
+ data_blob = &empty_blob;
+ } else if (bsdiff_allowed) {
// If the source file is considered bsdiff safe (no bsdiff bugs
// triggered), see if BSDIFF encoding is smaller.
base::FilePath old_chunk;
@@ -611,46 +556,34 @@
utils::WriteFile(new_chunk.value().c_str(),
new_data.data(), new_data.size()));
- chromeos::Blob bsdiff_delta;
TEST_AND_RETURN_FALSE(
BsdiffFiles(old_chunk.value(), new_chunk.value(), &bsdiff_delta));
CHECK_GT(bsdiff_delta.size(), static_cast<chromeos::Blob::size_type>(0));
- if (bsdiff_delta.size() < current_best_size) {
+ if (bsdiff_delta.size() < data_blob->size()) {
if (src_ops_allowed) {
operation.set_type(
DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF);
} else {
operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF);
}
- current_best_size = bsdiff_delta.size();
- data = bsdiff_delta;
+ data_blob = &bsdiff_delta;
}
}
}
- operation.set_src_length(old_data.size());
- operation.set_dst_length(new_data.size());
-
- // Set parameters of the operations
- CHECK_EQ(data.size(), current_best_size);
-
- if (gather_extents) {
- // Remove identical src/dst block ranges in MOVE operations.
- if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) {
- size_t removed_bytes = RemoveIdenticalBlockRanges(
- &src_extents, &dst_extents, new_data.size());
-
- // Adjust the file length field accordingly.
- if (removed_bytes) {
- operation.set_src_length(old_data.size() - removed_bytes);
- operation.set_dst_length(new_data.size() - removed_bytes);
- }
- }
-
- // Embed extents in the operation.
- StoreExtents(src_extents, operation.mutable_src_extents());
- StoreExtents(dst_extents, operation.mutable_dst_extents());
+ size_t removed_bytes = 0;
+ // Remove identical src/dst block ranges in MOVE operations.
+ if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) {
+ removed_bytes = RemoveIdenticalBlockRanges(
+ &src_extents, &dst_extents, new_data.size());
}
+ // Set legacy src_length and dst_length fields.
+ operation.set_src_length(old_data.size() - removed_bytes);
+ operation.set_dst_length(new_data.size() - removed_bytes);
+
+ // Embed extents in the operation.
+ StoreExtents(src_extents, operation.mutable_src_extents());
+ StoreExtents(dst_extents, operation.mutable_dst_extents());
// Replace operations should not have source extents.
if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_REPLACE ||
@@ -660,7 +593,7 @@
operation.clear_src_length();
}
- out_data->swap(data);
+ *out_data = std::move(*data_blob);
*out_op = operation;
return true;
@@ -669,209 +602,37 @@
bool DeltaDiffGenerator::DeltaCompressKernelPartition(
const string& old_kernel_part,
const string& new_kernel_part,
- vector<AnnotatedOperation>* kernel_ops,
+ uint64_t old_kernel_size,
+ uint64_t new_kernel_size,
+ uint64_t block_size,
+ vector<AnnotatedOperation>* aops,
int blobs_fd,
off_t* blobs_length,
bool src_ops_allowed) {
LOG(INFO) << "Delta compressing kernel partition...";
LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update...";
- DeltaArchiveManifest_InstallOperation op;
- chromeos::Blob data;
- TEST_AND_RETURN_FALSE(
- ReadFileToDiff(old_kernel_part,
- new_kernel_part,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- false, // gather_extents
- src_ops_allowed,
- old_kernel_part, // Doesn't matter, kernel has no files.
- new_kernel_part));
+ unique_ptr<RawFilesystem> old_kernel_fs;
+ if (!old_kernel_part.empty())
+ old_kernel_fs = RawFilesystem::Create("<kernel-delta-operation>",
+ block_size,
+ old_kernel_size / block_size);
+ unique_ptr<RawFilesystem> new_kernel_fs = RawFilesystem::Create(
+ "<kernel-delta-operation>",
+ block_size,
+ new_kernel_size / block_size);
- // Check if the operation writes nothing.
- if (op.dst_extents_size() == 0) {
- if (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) {
- LOG(INFO) << "Empty MOVE operation, nothing to do.";
- return true;
- } else {
- LOG(ERROR) << "Empty non-MOVE operation";
- return false;
- }
- }
+ DeltaReadFilesystem(aops,
+ old_kernel_part,
+ new_kernel_part,
+ old_kernel_fs.get(),
+ new_kernel_fs.get(),
+ -1, // chunk_blocks
+ blobs_fd,
+ blobs_length,
+ src_ops_allowed);
- // Write the data.
- if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE &&
- op.type() != DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) {
- op.set_data_offset(*blobs_length);
- op.set_data_length(data.size());
- }
-
- // Add the new install operation.
- kernel_ops->clear();
- kernel_ops->emplace_back();
- kernel_ops->back().op = op;
- kernel_ops->back().name = "<kernel-delta-operation>";
-
- TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, data.data(), data.size()));
- *blobs_length += data.size();
-
- LOG(INFO) << "Done delta compressing kernel partition: "
- << kInstallOperationTypes[op.type()];
- return true;
-}
-
-// TODO(deymo): Replace Vertex with AnnotatedOperation. This requires to move
-// out the code that adds the reader dependencies on the new vertex.
-bool DeltaDiffGenerator::ReadUnwrittenBlocks(
- const vector<Block>& blocks,
- int blobs_fd,
- off_t* blobs_length,
- const string& old_image_path,
- const uint64_t old_image_size,
- const string& new_image_path,
- Vertex* vertex,
- uint32_t minor_version) {
- vertex->file_name = "<rootfs-non-file-data>";
-
- DeltaArchiveManifest_InstallOperation* out_op = &vertex->op;
- int new_image_fd = open(new_image_path.c_str(), O_RDONLY, 000);
- TEST_AND_RETURN_FALSE_ERRNO(new_image_fd >= 0);
- ScopedFdCloser new_image_fd_closer(&new_image_fd);
- int old_image_fd = open(old_image_path.c_str(), O_RDONLY, 000);
- TEST_AND_RETURN_FALSE_ERRNO(old_image_fd >= 0);
- ScopedFdCloser old_image_fd_closer(&old_image_fd);
-
- string temp_file_path;
- TEST_AND_RETURN_FALSE(utils::MakeTempFile("CrAU_temp_data.XXXXXX",
- &temp_file_path,
- nullptr));
-
- FILE* file = fopen(temp_file_path.c_str(), "w");
- TEST_AND_RETURN_FALSE(file);
- int err = BZ_OK;
-
- BZFILE* bz_file = BZ2_bzWriteOpen(&err,
- file,
- 9, // max compression
- 0, // verbosity
- 0); // default work factor
- TEST_AND_RETURN_FALSE(err == BZ_OK);
-
- vector<Extent> extents;
- vector<Block>::size_type block_count = 0;
-
- LOG(INFO) << "Appending unwritten blocks to extents";
- for (vector<Block>::size_type i = 0; i < blocks.size(); i++) {
- if (blocks[i].writer != Vertex::kInvalidIndex)
- continue;
- AppendBlockToExtents(&extents, i);
- block_count++;
- }
-
- // Code will handle buffers of any size that's a multiple of kBlockSize,
- // so we arbitrarily set it to 1024 * kBlockSize.
- chromeos::Blob new_buf(1024 * kBlockSize);
- chromeos::Blob old_buf(1024 * kBlockSize);
-
- LOG(INFO) << "Scanning " << block_count << " unwritten blocks";
- vector<Extent> changed_extents;
- vector<Block>::size_type changed_block_count = 0;
- vector<Block>::size_type blocks_copied_count = 0;
-
- // For each extent in extents, write the unchanged blocks into BZ2_bzWrite,
- // which sends it to an output file. We use the temporary buffers to hold the
- // old and new data, which may be smaller than the extent, so in that case we
- // have to loop to get the extent's data (that's the inner while loop).
- for (const Extent& extent : extents) {
- vector<Block>::size_type blocks_read = 0;
- float printed_progress = -1;
- while (blocks_read < extent.num_blocks()) {
- const uint64_t copy_first_block = extent.start_block() + blocks_read;
- const int copy_block_cnt =
- min(new_buf.size() / kBlockSize,
- static_cast<chromeos::Blob::size_type>(
- extent.num_blocks() - blocks_read));
- const size_t count = copy_block_cnt * kBlockSize;
- const off_t offset = copy_first_block * kBlockSize;
- ssize_t rc = pread(new_image_fd, new_buf.data(), count, offset);
- TEST_AND_RETURN_FALSE_ERRNO(rc >= 0);
- TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count);
-
- const off_t old_offset = min(offset, static_cast<off_t>(old_image_size));
- const size_t old_count =
- min(static_cast<size_t>(offset + count),
- static_cast<size_t>(old_image_size)) - old_offset;
- rc = pread(old_image_fd, old_buf.data(), old_count, old_offset);
- TEST_AND_RETURN_FALSE_ERRNO(rc >= 0);
- TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == old_count);
-
- // Compare each block in the buffer to its counterpart in the old image
- // and only compress it if its content has changed.
- int buf_offset = 0;
- for (int i = 0; i < copy_block_cnt; ++i) {
- int buf_end_offset = buf_offset + kBlockSize;
- if (minor_version == kSourceMinorPayloadVersion ||
- static_cast<size_t>(buf_end_offset) > old_count ||
- !std::equal(new_buf.begin() + buf_offset,
- new_buf.begin() + buf_end_offset,
- old_buf.begin() + buf_offset)) {
- BZ2_bzWrite(&err, bz_file, &new_buf[buf_offset], kBlockSize);
- TEST_AND_RETURN_FALSE(err == BZ_OK);
- const uint64_t block_idx = copy_first_block + i;
- if (blocks[block_idx].reader != Vertex::kInvalidIndex) {
- graph_utils::AddReadBeforeDep(vertex, blocks[block_idx].reader,
- block_idx);
- }
- AppendBlockToExtents(&changed_extents, block_idx);
- changed_block_count++;
- }
- buf_offset = buf_end_offset;
- }
-
- blocks_read += copy_block_cnt;
- blocks_copied_count += copy_block_cnt;
- float current_progress =
- static_cast<float>(blocks_copied_count) / block_count;
- if (printed_progress + 0.1 < current_progress ||
- blocks_copied_count == block_count) {
- LOG(INFO) << "progress: " << current_progress;
- printed_progress = current_progress;
- }
- }
- }
- BZ2_bzWriteClose(&err, bz_file, 0, nullptr, nullptr);
- TEST_AND_RETURN_FALSE(err == BZ_OK);
- bz_file = nullptr;
- TEST_AND_RETURN_FALSE_ERRNO(0 == fclose(file));
- file = nullptr;
-
- LOG(INFO) << "Compressed " << changed_block_count << " blocks ("
- << block_count - changed_block_count << " blocks unchanged)";
- chromeos::Blob compressed_data;
- if (changed_block_count > 0) {
- LOG(INFO) << "Reading compressed data off disk";
- TEST_AND_RETURN_FALSE(utils::ReadFile(temp_file_path, &compressed_data));
- }
- TEST_AND_RETURN_FALSE(unlink(temp_file_path.c_str()) == 0);
-
- // Add node to graph to write these blocks
- out_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
- out_op->set_data_offset(*blobs_length);
- out_op->set_data_length(compressed_data.size());
- LOG(INFO) << "Rootfs non-data blocks compressed take up "
- << compressed_data.size();
- *blobs_length += compressed_data.size();
- out_op->set_dst_length(kBlockSize * changed_block_count);
- DeltaDiffGenerator::StoreExtents(changed_extents,
- out_op->mutable_dst_extents());
-
- TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd,
- compressed_data.data(),
- compressed_data.size()));
- LOG(INFO) << "Done processing unwritten blocks";
+ LOG(INFO) << "Done delta compressing kernel partition.";
return true;
}
@@ -1019,62 +780,39 @@
off_t* data_file_size,
vector<AnnotatedOperation>* rootfs_ops,
vector<AnnotatedOperation>* kernel_ops) {
- // List of blocks in the target partition, with the operation that needs to
- // write it and the operation that needs to read it. This is used here to
- // keep track of the blocks that no operation is writing it.
- vector<Block> blocks(config.target.rootfs_size / config.block_size);
+ unique_ptr<Ext2Filesystem> old_fs = Ext2Filesystem::CreateFromFile(
+ config.source.rootfs_part);
+ unique_ptr<Ext2Filesystem> new_fs = Ext2Filesystem::CreateFromFile(
+ config.target.rootfs_part);
- // TODO(deymo): DeltaReadFiles() should not use a graph to generate the
- // operations, either in the in-place or source uprate. Split out the
- // graph dependency generation.
- Graph graph;
- TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
- &blocks,
- config.source.rootfs_part,
- config.target.rootfs_part,
- config.source.rootfs_mountpt,
- config.target.rootfs_mountpt,
- config.chunk_size,
- data_file_fd,
- data_file_size,
- true)); // src_ops_allowed
+ off_t chunk_blocks = config.chunk_size == -1 ? -1 : (
+ config.chunk_size / config.block_size);
+
rootfs_ops->clear();
- for (const Vertex& v : graph) {
- rootfs_ops->emplace_back();
- AnnotatedOperation& aop = rootfs_ops->back();
- aop.op = v.op;
- aop.SetNameFromFileAndChunk(v.file_name, v.chunk_offset, v.chunk_size);
- }
-
+ TEST_AND_RETURN_FALSE(DeltaReadFilesystem(rootfs_ops,
+ config.source.rootfs_part,
+ config.target.rootfs_part,
+ old_fs.get(),
+ new_fs.get(),
+ chunk_blocks,
+ data_file_fd,
+ data_file_size,
+ true)); // src_ops_allowed
LOG(INFO) << "done reading normal files";
// Read kernel partition
TEST_AND_RETURN_FALSE(
DeltaCompressKernelPartition(config.source.kernel_part,
config.target.kernel_part,
+ config.source.kernel_size,
+ config.target.kernel_size,
+ config.block_size,
kernel_ops,
data_file_fd,
data_file_size,
true)); // src_ops_allowed
LOG(INFO) << "done reading kernel";
- Vertex unwritten_vertex;
- TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
- data_file_fd,
- data_file_size,
- config.source.rootfs_part,
- config.source.rootfs_size,
- config.target.rootfs_part,
- &unwritten_vertex,
- config.minor_version));
- if (unwritten_vertex.op.data_length() == 0) {
- LOG(INFO) << "No unwritten blocks to write, omitting operation";
- } else {
- rootfs_ops->emplace_back();
- rootfs_ops->back().op = unwritten_vertex.op;
- rootfs_ops->back().name = unwritten_vertex.file_name;
- }
-
TEST_AND_RETURN_FALSE(FragmentOperations(rootfs_ops,
config.target.rootfs_part,
data_file_fd,
@@ -1118,7 +856,6 @@
LOG(INFO) << "Rootfs partition size: " << config.rootfs_partition_size;
LOG(INFO) << "Actual filesystem size: " << config.target.rootfs_size;
- LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex;
LOG(INFO) << "Block count: "
<< config.target.rootfs_size / config.block_size;
@@ -1359,11 +1096,6 @@
kBlockSize);
}
-void DeltaDiffGenerator::ClearSparseHoles(vector<Extent>* extents) {
- extents->erase(std::remove_if(extents->begin(), extents->end(), IsSparseHole),
- extents->end());
-}
-
bool DeltaDiffGenerator::FragmentOperations(
vector<AnnotatedOperation>* aops,
const string& target_part_path,
diff --git a/payload_generator/delta_diff_generator.h b/payload_generator/delta_diff_generator.h
index 06f84a3..1d37d5b 100644
--- a/payload_generator/delta_diff_generator.h
+++ b/payload_generator/delta_diff_generator.h
@@ -14,7 +14,7 @@
#include "update_engine/payload_constants.h"
#include "update_engine/payload_generator/extent_utils.h"
-#include "update_engine/payload_generator/graph_types.h"
+#include "update_engine/payload_generator/filesystem_interface.h"
#include "update_engine/payload_generator/operations_generator.h"
#include "update_engine/payload_generator/payload_generation_config.h"
#include "update_engine/update_metadata.pb.h"
@@ -36,21 +36,6 @@
public:
DeltaDiffGenerator() = default;
- // Represents a disk block on the install partition.
- struct Block {
- // During install, each block on the install partition will be written
- // and some may be read (in all likelihood, many will be read).
- // The reading and writing will be performed by InstallOperations,
- // each of which has a corresponding vertex in a graph.
- // A Block object tells which vertex will read or write this block
- // at install time.
- // Generally, there will be a vector of Block objects whose length
- // is the number of blocks on the install partition.
- Block() : reader(Vertex::kInvalidIndex), writer(Vertex::kInvalidIndex) {}
- Vertex::Index reader;
- Vertex::Index writer;
- };
-
// These functions are public so that the unit tests can access them:
// Generate the update payload operations for the kernel and rootfs using
@@ -70,93 +55,78 @@
std::vector<AnnotatedOperation>* rootfs_ops,
std::vector<AnnotatedOperation>* kernel_ops) override;
- // For each regular file within new_root, creates a node in the graph,
- // determines the best way to compress it (REPLACE, REPLACE_BZ, COPY, BSDIFF),
- // and writes any necessary data to the end of data_fd.
- static bool DeltaReadFiles(Graph* graph,
- std::vector<Block>* blocks,
- const std::string& old_part,
- const std::string& new_part,
- const std::string& old_root,
- const std::string& new_root,
- off_t chunk_size,
- int data_fd,
- off_t* data_file_size,
- bool src_ops_allowed);
+ // Create operations in |aops| to produce all the files reported by |new_fs|,
+ // including all the blocks not reported by any file.
+ // It uses the files reported by |old_fs| and the data in |old_part| to
+ // determine the best way to compress the new files (REPLACE, REPLACE_BZ,
+ // COPY, BSDIFF) and writes any necessary data to the end of data_fd updating
+ // data_file_size accordingly.
+ static bool DeltaReadFilesystem(std::vector<AnnotatedOperation>* aops,
+ const std::string& old_part,
+ const std::string& new_part,
+ FilesystemInterface* old_fs,
+ FilesystemInterface* new_fs,
+ off_t chunk_blocks,
+ int data_fd,
+ off_t* data_file_size,
+ bool src_ops_allowed);
- // For a given regular file which must exist at new_root + path, and
- // may exist at old_root + path, creates a new InstallOperation and
- // adds it to the graph. Also, populates the |blocks| array as
- // necessary, if |blocks| is non-null. Also, writes the data
- // necessary to send the file down to the client into data_fd, which
- // has length *data_file_size. *data_file_size is updated
- // appropriately. If |existing_vertex| is no kInvalidIndex, use that
- // rather than allocating a new vertex. Returns true on success.
- static bool DeltaReadFile(Graph* graph,
- Vertex::Index existing_vertex,
- std::vector<Block>* blocks,
+ // For a given file |name| append operations to |aops| to produce it in the
+ // |new_part|. The file will be split in chunks of |chunk_blocks| blocks each
+ // or treated as a single chunk if |chunk_blocks| is -1. The file data is
+ // stored in |new_part| in the blocks described by |new_extents| and, if it
+ // exists, the old version exists in |old_part| in the blocks described by
+ // |old_extents|. The operations added to |aops| reference the data blob
+ // in the file |data_fd|, which has length *data_file_size. *data_file_size is
+ // updated appropriately. Returns true on success.
+ static bool DeltaReadFile(std::vector<AnnotatedOperation>* aops,
const std::string& old_part,
const std::string& new_part,
- const std::string& old_root,
- const std::string& new_root,
- const std::string& path,
- off_t chunk_offset,
- off_t chunk_size,
+ const std::vector<Extent>& old_extents,
+ const std::vector<Extent>& new_extents,
+ const std::string& name,
+ off_t chunk_blocks,
int data_fd,
off_t* data_file_size,
bool src_ops_allowed);
- // Reads old_filename (if it exists) and a new_filename and determines
- // the smallest way to encode this file for the diff. It reads extents from
- // |old_part| and |new_part|. It stores necessary data in out_data and fills
- // in out_op. If there's no change in old and new files, it creates a MOVE
- // operation. If there is a change, or the old file doesn't exist,
- // the smallest of REPLACE, REPLACE_BZ, or BSDIFF wins.
- // new_filename must contain at least one byte.
- // |new_filename| is read starting at |chunk_offset|.
- // If |chunk_size| is not -1, only up to |chunk_size| bytes are diffed.
+ // Reads the blocks |old_extents| from |old_part| (if it exists) and the
+ // |new_extents| from |new_part| and determines the smallest way to encode
+ // this |new_extents| for the diff. It stores necessary data in |out_data| and
+ // fills in |out_op|. If there's no change in old and new files, it creates a
+ // MOVE operation. If there is a change, the smallest of REPLACE, REPLACE_BZ,
+ // or BSDIFF wins. |new_extents| must not be empty.
// If |src_ops_allowed| is true, it will emit SOURCE_COPY and SOURCE_BSDIFF
// operations instead of MOVE and BSDIFF, respectively.
// Returns true on success.
- static bool ReadFileToDiff(const std::string& old_part,
- const std::string& new_part,
- off_t chunk_offset,
- off_t chunk_size,
- bool bsdiff_allowed,
- chromeos::Blob* out_data,
- DeltaArchiveManifest_InstallOperation* out_op,
- bool gather_extents,
- bool src_ops_allowed,
- const std::string& old_filename,
- const std::string& new_filename);
+ static bool ReadExtentsToDiff(const std::string& old_part,
+ const std::string& new_part,
+ const std::vector<Extent>& old_extents,
+ const std::vector<Extent>& new_extents,
+ bool bsdiff_allowed,
+ chromeos::Blob* out_data,
+ DeltaArchiveManifest_InstallOperation* out_op,
+ bool src_ops_allowed);
// Delta compresses a kernel partition |new_kernel_part| with knowledge of the
// old kernel partition |old_kernel_part|. If |old_kernel_part| is an empty
- // string, generates a full update of the partition.
+ // string, generates a full update of the partition. The size of the old and
+ // new kernel is passed in |old_kernel_size| and |new_kernel_size|. The
+ // operations used to generate the new kernel are stored in the |aops|
+ // vector, and the blob associated to those operations is written at the end
+ // of the |blobs_fd| file, adding to the value pointed by |blobs_length| the
+ // bytes written to |blobs_fd|.
static bool DeltaCompressKernelPartition(
const std::string& old_kernel_part,
const std::string& new_kernel_part,
- std::vector<AnnotatedOperation>* ops,
+ uint64_t old_kernel_size,
+ uint64_t new_kernel_size,
+ uint64_t block_size,
+ std::vector<AnnotatedOperation>* aops,
int blobs_fd,
off_t* blobs_length,
bool src_ops_allowed);
- // Reads blocks from image_path that are not yet marked as being written in
- // the blocks array. These blocks that remain are either unchanged files or
- // non-file-data blocks. We compare each of them to the old image, and
- // compress the ones that changed into a single REPLACE_BZ operation. This
- // updates a newly created node in the graph to write these blocks and writes
- // the appropriate blob to blobs_fd. Reads and updates blobs_length.
- static bool ReadUnwrittenBlocks(
- const std::vector<Block>& blocks,
- int blobs_fd,
- off_t* blobs_length,
- const std::string& old_image_path,
- const uint64_t old_image_size,
- const std::string& new_image_path,
- Vertex* vertex,
- uint32_t minor_version);
-
// Stores all Extents in 'extents' into 'out'.
static void StoreExtents(const std::vector<Extent>& extents,
google::protobuf::RepeatedPtrField<Extent>* out);
@@ -230,9 +200,6 @@
return ret;
}
- // Takes a vector of extents and removes extents that begin in a sparse hole.
- static void ClearSparseHoles(std::vector<Extent>* extents);
-
// Takes a vector of AnnotatedOperations |aops| and fragments those operations
// such that there is only one dst extent per operation. Sets |aops| to a
// vector of the new fragmented operations.
diff --git a/payload_generator/delta_diff_generator_unittest.cc b/payload_generator/delta_diff_generator_unittest.cc
index da82d14..c198916 100644
--- a/payload_generator/delta_diff_generator_unittest.cc
+++ b/payload_generator/delta_diff_generator_unittest.cc
@@ -10,6 +10,7 @@
#include <unistd.h>
#include <algorithm>
+#include <cstdio>
#include <map>
#include <set>
#include <sstream>
@@ -17,6 +18,7 @@
#include <utility>
#include <vector>
+#include <base/files/scoped_file.h>
#include <base/logging.h>
#include <base/strings/string_util.h>
#include <bzlib.h>
@@ -27,8 +29,8 @@
#include "update_engine/delta_performer.h"
#include "update_engine/extent_ranges.h"
#include "update_engine/payload_constants.h"
-#include "update_engine/payload_generator/extent_mapper.h"
#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/fake_filesystem.h"
#include "update_engine/payload_generator/graph_types.h"
#include "update_engine/subprocess.h"
#include "update_engine/test_utils.h"
@@ -37,72 +39,38 @@
using chromeos_update_engine::test_utils::kRandomString;
using std::set;
using std::string;
+using std::unique_ptr;
using std::vector;
namespace chromeos_update_engine {
-typedef DeltaDiffGenerator::Block Block;
-
-typedef bool (*GetExtentsWithChunk)(const string&, off_t, off_t,
- vector<Extent>*);
-extern GetExtentsWithChunk get_extents_with_chunk_func;
-
namespace {
-uint64_t AddExtent(uint64_t start_block, uint64_t num_blocks,
- vector<Extent>* extents) {
- extents->push_back(ExtentForRange(start_block, num_blocks));
- return num_blocks;
-}
+// Writes the |data| in the blocks specified by |extents| on the partition
+// |part_path|. The |data| size could be smaller than the size of the blocks
+// passed.
+bool WriteExtents(const string& part_path,
+ const vector<Extent>& extents,
+ off_t block_size,
+ const chromeos::Blob& data) {
+ uint64_t offset = 0;
+ base::ScopedFILE fp(fopen(part_path.c_str(), "r+"));
+ TEST_AND_RETURN_FALSE(fp.get());
-// Used to provide a fake list of extents for a given path.
-std::map<string, vector<Extent>> fake_file_extents;
-
-// Fake function for GatherExtents in delta_diff_generator. This function
-// returns whatever is specified in the global fake_file_extents variable.
-bool FakeGetExtents(const string& path, off_t chunk_offset, off_t chunk_size,
- vector<Extent>* out) {
- if (fake_file_extents.find(path) != fake_file_extents.end()) {
- *out = fake_file_extents[path];
- return true;
- } else {
- return false;
+ for (const Extent& extent : extents) {
+ if (offset >= data.size())
+ break;
+ TEST_AND_RETURN_FALSE(
+ fseek(fp.get(), extent.start_block() * block_size, SEEK_SET) == 0);
+ uint64_t to_write = std::min(extent.num_blocks() * block_size,
+ data.size() - offset);
+ TEST_AND_RETURN_FALSE(
+ fwrite(data.data() + offset, 1, to_write, fp.get()) == to_write);
+ offset += extent.num_blocks() * block_size;
}
-}
-
-// Inserts |data| at block |offset| in |result|. Fills in the remaining blocks
-// so that there are |num_blocks| blocks in |result|.
-bool MakePartition(uint64_t num_blocks, const string& data, off_t offset,
- chromeos::Blob* result) {
- TEST_AND_RETURN_FALSE(
- static_cast<uint64_t>((kBlockSize * offset) + data.size()) <=
- kBlockSize * num_blocks);
- chromeos::Blob out(kBlockSize * num_blocks, 0);
- chromeos::Blob::iterator start = out.begin() + (kBlockSize * offset);
- copy(data.begin(), data.end(), start);
- *result = out;
return true;
}
-// Copies from |target| to |result|. Gets the substring of |target| starting
-// at block |start_block| of length |num_blocks| blocks and inserts it at block
-// |block_offset| in |result|.
-void UpdatePartition(const string& target, uint64_t start_block,
- uint64_t num_blocks, off_t block_offset,
- chromeos::Blob* result) {
- uint64_t num_insert = num_blocks * kBlockSize;
- off_t target_offset = block_offset * kBlockSize;
-
- const string target_substr = target.substr(kBlockSize * start_block,
- num_insert);
- ASSERT_EQ(target_substr.size(), num_insert);
-
- for (uint64_t i = 0; i < num_insert; i++) {
- result->at(target_offset + i) =
- static_cast<unsigned char>(target_substr[i]);
- }
-}
-
bool ExtentEquals(Extent ext, uint64_t start_block, uint64_t num_blocks) {
return ext.start_block() == start_block && ext.num_blocks() == num_blocks;
}
@@ -391,116 +359,63 @@
class DeltaDiffGeneratorTest : public ::testing::Test {
protected:
- void SetUp() override {
- ASSERT_TRUE(utils::MakeTempFile("DeltaDiffGeneratorTest-old_path-XXXXXX",
- &old_path_, nullptr));
- ASSERT_TRUE(utils::MakeTempFile("DeltaDiffGeneratorTest-new_path-XXXXXX",
- &new_path_, nullptr));
- ASSERT_TRUE(utils::MakeTempFile(
- "DeltaDiffGeneratorTest-old_part_path-XXXXXX",
- &old_part_path_, nullptr));
- ASSERT_TRUE(utils::MakeTempFile(
- "DeltaDiffGeneratorTest-new_part_path-XXXXXX",
- &new_part_path_, nullptr));
+ const uint64_t kFilesystemSize = kBlockSize * 1024;
- // Mock out the extent gathering function.
- orig_get_extents_with_chunk_func_ = get_extents_with_chunk_func;
- get_extents_with_chunk_func = FakeGetExtents;
- // Ensure we start with a clean list of fake extents.
- fake_file_extents.clear();
+ void SetUp() override {
+ old_part_path_ = "DeltaDiffGeneratorTest-old_part_path-XXXXXX";
+ CreateFilesystem(&old_fs_, &old_part_path_, kFilesystemSize);
+
+ new_part_path_ = "DeltaDiffGeneratorTest-new_part_path-XXXXXX";
+ CreateFilesystem(&old_fs_, &new_part_path_, kFilesystemSize);
}
void TearDown() override {
- unlink(old_path_.c_str());
- unlink(new_path_.c_str());
unlink(old_part_path_.c_str());
unlink(new_part_path_.c_str());
- get_extents_with_chunk_func = orig_get_extents_with_chunk_func_;
}
- // Updates the FakeGetExtents for old_path_ and new_path_ to match the current
- // file size.
- void UpdateFakeExtents() {
- UpdateFakeFileExtents(old_path_, kBlockSize);
- UpdateFakeFileExtents(new_path_, kBlockSize);
+ // Create a fake filesystem of the given size and initialize the partition
+ // holding it.
+ void CreateFilesystem(unique_ptr<FakeFilesystem>* fs, string* filename,
+ uint64_t size) {
+ string pattern = *filename;
+ ASSERT_TRUE(utils::MakeTempFile(pattern.c_str(), filename, nullptr));
+ ASSERT_EQ(0, truncate(filename->c_str(), size));
+ fs->reset(new FakeFilesystem(kBlockSize, size / kBlockSize));
}
- // Update the fake_file_extents for |path| using the file size of that file,
- // assuming a block size of |block_size| and an optional |start_block|.
- // The faked extent list will have only one extent with the list of all the
- // contiguous blocks.
- void UpdateFakeFileExtents(const string& path, size_t block_size,
- size_t start_block) {
- size_t num_blocks = (utils::FileSize(path) + block_size - 1) / block_size;
- if (num_blocks == 0) {
- fake_file_extents[path] = vector<Extent>{};
- } else {
- fake_file_extents[path] =
- vector<Extent>{1, ExtentForRange(start_block, num_blocks)};
- }
- }
-
- void UpdateFakeFileExtents(const string& path, size_t block_size) {
- UpdateFakeFileExtents(path, block_size, 0);
- }
-
- // Paths to old and new temporary files used in all the tests.
- string old_path_;
- string new_path_;
-
// Paths to old and new temporary filesystems used in the tests.
string old_part_path_;
string new_part_path_;
- GetExtentsWithChunk orig_get_extents_with_chunk_func_;
+ // FilesystemInterface fake implementations used to mock out the file/block
+ // distribution.
+ unique_ptr<FakeFilesystem> old_fs_;
+ unique_ptr<FakeFilesystem> new_fs_;
};
-TEST_F(DeltaDiffGeneratorTest, BlockDefaultValues) {
- // Tests that a Block is initialized with the default values as a
- // Vertex::kInvalidIndex. This is required by the delta generators.
- DeltaDiffGenerator::Block block;
- EXPECT_EQ(Vertex::kInvalidIndex, block.reader);
- EXPECT_EQ(Vertex::kInvalidIndex, block.writer);
-}
-
TEST_F(DeltaDiffGeneratorTest, MoveSmallTest) {
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
+ chromeos::Blob data_blob(kBlockSize);
+ test_utils::FillWithData(&data_blob);
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(11, random_string, 10, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(11, 1) };
+ vector<Extent> new_extents = { ExtentForRange(1, 1) };
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
-
- // Force the old file to be on a different block.
- UpdateFakeFileExtents(old_path_, kBlockSize, 10);
- UpdateFakeFileExtents(new_path_, kBlockSize);
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, data_blob));
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, data_blob));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
-
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ false)); // src_ops_allowed
EXPECT_TRUE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -527,84 +442,40 @@
// Old: [ 20 21 22 23 24 25 ] [ 28 29 ]
// New: [ 18 ] [ 21 22 ] [ 20 ] [ 24 25 26 ] [ 29 ]
// Same: ^^ ^^ ^^ ^^ ^^
- vector<Extent> old_extents;
- uint64_t num_extents = 0;
- num_extents += AddExtent(20, 6, &old_extents);
- num_extents += AddExtent(28, 2, &old_extents);
- fake_file_extents[old_path_] = old_extents;
- vector<Extent> new_extents;
- AddExtent(18, 1, &new_extents);
- AddExtent(21, 2, &new_extents);
- AddExtent(20, 1, &new_extents);
- AddExtent(24, 3, &new_extents);
- AddExtent(29, 1, &new_extents);
- fake_file_extents[new_path_] = new_extents;
+ vector<Extent> old_extents = {
+ ExtentForRange(20, 6),
+ ExtentForRange(28, 2) };
+ vector<Extent> new_extents = {
+ ExtentForRange(18, 1),
+ ExtentForRange(21, 2),
+ ExtentForRange(20, 1),
+ ExtentForRange(24, 3),
+ ExtentForRange(29, 1) };
- // The size of the data should match the total number of blocks; the last
- // block is only partly filled.
- size_t file_len = 7 * 4096 + 3333;
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- string random_data;
- while (random_data.size() < file_len)
- random_data += random_string;
- if (random_data.size() > file_len) {
- random_data.erase(file_len);
- random_data.insert(random_data.end(), (4096 - 3333), 0);
+ uint64_t num_blocks = BlocksInExtents(old_extents);
+ EXPECT_EQ(num_blocks, BlocksInExtents(new_extents));
+
+ // The size of the data should match the total number of blocks. Each block
+ // has a different content.
+ chromeos::Blob file_data;
+ for (uint64_t i = 0; i < num_blocks; ++i) {
+ file_data.resize(file_data.size() + kBlockSize, 'a' + i);
}
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_data.c_str(), file_len));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_data.c_str(), file_len));
-
- // Make partitions that match the extents and random_data.
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(30, "", 0, &old_part));
- EXPECT_TRUE(MakePartition(30, "", 0, &new_part));
-
- UpdatePartition(random_data, 0, 6, 20, &old_part);
- UpdatePartition(random_data, 6, 2, 28, &old_part);
-
- UpdatePartition(random_data, 0, 1, 18, &new_part);
- UpdatePartition(random_data, 1, 2, 21, &new_part);
- UpdatePartition(random_data, 3, 1, 20, &new_part);
- UpdatePartition(random_data, 4, 3, 24, &new_part);
- UpdatePartition(random_data, 7, 1, 29, &new_part);
-
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, file_data));
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, file_data));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
-
- // Adjust the old/new extents to remove duplicates.
- old_extents[0].set_num_blocks(1);
- Extent e;
- e.set_start_block(23);
- e.set_num_blocks(1);
- old_extents.insert(old_extents.begin() + 1, e);
- old_extents[2].set_num_blocks(1);
- new_extents.erase(new_extents.begin() + 1);
- new_extents[2].set_start_block(26);
- new_extents[2].set_num_blocks(1);
- new_extents.erase(new_extents.begin() + 3);
- num_extents -= 5;
- file_len -= 4 * 4096 + 3333;
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ false)); // src_ops_allowed
EXPECT_TRUE(data.empty());
@@ -613,8 +484,20 @@
EXPECT_FALSE(op.has_data_offset());
EXPECT_FALSE(op.has_data_length());
- EXPECT_EQ(file_len, op.src_length());
- EXPECT_EQ(num_extents, BlocksInExtents(op.src_extents()));
+ // The expected old and new extents that actually moved. See comment above.
+ old_extents = {
+ ExtentForRange(20, 1),
+ ExtentForRange(23, 1),
+ ExtentForRange(28, 1) };
+ new_extents = {
+ ExtentForRange(18, 1),
+ ExtentForRange(20, 1),
+ ExtentForRange(26, 1) };
+ num_blocks = BlocksInExtents(old_extents);
+
+ EXPECT_EQ(num_blocks * kBlockSize, op.src_length());
+ EXPECT_EQ(num_blocks * kBlockSize, op.dst_length());
+
EXPECT_EQ(old_extents.size(), op.src_extents_size());
for (int i = 0; i < op.src_extents_size(); i++) {
EXPECT_EQ(old_extents[i].start_block(), op.src_extents(i).start_block())
@@ -623,8 +506,6 @@
<< "i == " << i;
}
- EXPECT_EQ(file_len, op.dst_length());
- EXPECT_EQ(num_extents, BlocksInExtents(op.dst_extents()));
EXPECT_EQ(new_extents.size(), op.dst_extents_size());
for (int i = 0; i < op.dst_extents_size(); i++) {
EXPECT_EQ(new_extents[i].start_block(), op.dst_extents(i).start_block())
@@ -635,40 +516,30 @@
}
TEST_F(DeltaDiffGeneratorTest, BsdiffSmallTest) {
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.c_str(),
- random_string.size() - 1));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- UpdateFakeExtents();
+ // Test a BSDIFF operation from block 1 to block 2.
+ chromeos::Blob data_blob(kBlockSize);
+ test_utils::FillWithData(&data_blob);
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(
- 1, random_string.substr(0, random_string.size() - 1), 0, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(1, 1) };
+ vector<Extent> new_extents = { ExtentForRange(2, 1) };
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, data_blob));
+ // Modify one byte in the new file.
+ data_blob[0]++;
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, data_blob));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
@@ -686,41 +557,31 @@
}
TEST_F(DeltaDiffGeneratorTest, BsdiffNotAllowedTest) {
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.c_str(),
- random_string.size() - 1));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- UpdateFakeExtents();
+ // Same setup as the previous test, but this time BSDIFF operations are not
+ // allowed.
+ chromeos::Blob data_blob(kBlockSize);
+ test_utils::FillWithData(&data_blob);
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(
- 1, random_string.substr(0, random_string.size() - 1), 0, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(1, 1) };
+ vector<Extent> new_extents = { ExtentForRange(2, 1) };
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, data_blob));
+ // Modify one byte in the new file.
+ data_blob[0]++;
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, data_blob));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
-
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- false, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ false, // bsdiff_allowed
+ &data,
+ &op,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
@@ -731,40 +592,27 @@
}
TEST_F(DeltaDiffGeneratorTest, BsdiffNotAllowedMoveTest) {
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- UpdateFakeExtents();
+ chromeos::Blob data_blob(kBlockSize);
+ test_utils::FillWithData(&data_blob);
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(1, random_string, 0, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(1, 1) };
+ vector<Extent> new_extents = { ExtentForRange(2, 1) };
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, data_blob));
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, data_blob));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
-
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- false, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ false, // bsdiff_allowed
+ &data,
+ &op,
+ false)); // src_ops_allowed
EXPECT_TRUE(data.empty());
@@ -775,54 +623,38 @@
}
TEST_F(DeltaDiffGeneratorTest, ReplaceSmallTest) {
- chromeos::Blob new_part;
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(1, 1) };
+ vector<Extent> new_extents = { ExtentForRange(2, 1) };
- chromeos::Blob old_part(kBlockSize, 0);
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
+ // Make a blob that's just 1's that will compress well.
+ chromeos::Blob ones(kBlockSize, 1);
- // Fill old_path with zeroes.
- chromeos::Blob old_data(kBlockSize, 0);
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- old_data.data(),
- old_data.size()));
-
- chromeos::Blob random_data;
// Make a blob with random data that won't compress well.
+ chromeos::Blob random_data;
std::mt19937 gen(12345);
std::uniform_int_distribution<uint8_t> dis(0, 255);
for (uint32_t i = 0; i < kBlockSize; i++) {
random_data.push_back(dis(gen));
}
- // Make a blob that's just 1's that will compress well.
- chromeos::Blob ones(kBlockSize, 1);
-
for (int i = 0; i < 2; i++) {
chromeos::Blob data_to_test = i == 0 ? random_data : ones;
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- data_to_test.data(),
- data_to_test.size()));
- UpdateFakeExtents();
-
- string data_str(data_to_test.begin(), data_to_test.end());
- EXPECT_TRUE(MakePartition(1, data_str, 0, &new_part));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
+ // The old_extents will be initialized with 0.
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize,
+ data_to_test));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -840,94 +672,31 @@
}
}
-TEST_F(DeltaDiffGeneratorTest, BsdiffNoGatherExtentsSmallTest) {
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.data(),
- random_string.size() - 1));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
-
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(
- 1, random_string.substr(0, random_string.size() - 1), 0, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
-
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
- chromeos::Blob data;
- DeltaArchiveManifest_InstallOperation op;
-
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- false, // gather_extents
- false, // src_ops_allowed
- old_path_,
- new_path_));
- EXPECT_FALSE(data.empty());
-
- EXPECT_TRUE(op.has_type());
- EXPECT_EQ(DeltaArchiveManifest_InstallOperation_Type_BSDIFF, op.type());
- EXPECT_FALSE(op.has_data_offset());
- EXPECT_FALSE(op.has_data_length());
- EXPECT_EQ(1, op.src_extents_size());
- EXPECT_EQ(0, op.src_extents().Get(0).start_block());
- EXPECT_EQ(1, op.src_extents().Get(0).num_blocks());
- EXPECT_EQ(kBlockSize, op.src_length());
- EXPECT_EQ(1, op.dst_extents_size());
- EXPECT_EQ(0, op.dst_extents().Get(0).start_block());
- EXPECT_EQ(1, op.dst_extents().Get(0).num_blocks());
- EXPECT_EQ(kBlockSize, op.dst_length());
-}
-
TEST_F(DeltaDiffGeneratorTest, SourceCopyTest) {
// Makes sure SOURCE_COPY operations are emitted whenever src_ops_allowed
// is true. It is the same setup as MoveSmallTest, which checks that
// the operation is well-formed.
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.data(),
- random_string.size()));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- UpdateFakeExtents();
+ chromeos::Blob data_blob(kBlockSize);
+ test_utils::FillWithData(&data_blob);
+
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(11, 1) };
+ vector<Extent> new_extents = { ExtentForRange(1, 1) };
+
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, data_blob));
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, data_blob));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
-
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(1, random_string, 0, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
-
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
-
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- true, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ true)); // src_ops_allowed
EXPECT_TRUE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -938,41 +707,29 @@
// Makes sure SOURCE_BSDIFF operations are emitted whenever src_ops_allowed
// is true. It is the same setup as BsdiffSmallTest, which checks
// that the operation is well-formed.
- const string random_string(reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString));
- EXPECT_TRUE(utils::WriteFile(old_path_.c_str(),
- random_string.data(),
- random_string.size() - 1));
- EXPECT_TRUE(utils::WriteFile(new_path_.c_str(),
- random_string.c_str(),
- random_string.size()));
- UpdateFakeExtents();
+ chromeos::Blob data_blob(kBlockSize);
+ test_utils::FillWithData(&data_blob);
- chromeos::Blob old_part;
- chromeos::Blob new_part;
- EXPECT_TRUE(MakePartition(
- 1, random_string.substr(0, random_string.size() - 1), 0, &old_part));
- EXPECT_TRUE(MakePartition(1, random_string, 0, &new_part));
+ // The old file is on a different block than the new one.
+ vector<Extent> old_extents = { ExtentForRange(1, 1) };
+ vector<Extent> new_extents = { ExtentForRange(2, 1) };
- EXPECT_TRUE(utils::WriteFile(old_part_path_.c_str(),
- old_part.data(), old_part.size()));
- EXPECT_TRUE(utils::WriteFile(new_part_path_.c_str(),
- new_part.data(), new_part.size()));
+ EXPECT_TRUE(WriteExtents(old_part_path_, old_extents, kBlockSize, data_blob));
+ // Modify one byte in the new file.
+ data_blob[0]++;
+ EXPECT_TRUE(WriteExtents(new_part_path_, new_extents, kBlockSize, data_blob));
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation op;
-
- EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_part_path_,
- new_part_path_,
- 0, // chunk_offset
- -1, // chunk_size
- true, // bsdiff_allowed
- &data,
- &op,
- true, // gather_extents
- true, // src_ops_allowed
- old_path_,
- new_path_));
+ EXPECT_TRUE(DeltaDiffGenerator::ReadExtentsToDiff(
+ old_part_path_,
+ new_part_path_,
+ old_extents,
+ new_extents,
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ true)); // src_ops_allowed
EXPECT_FALSE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -1062,20 +819,6 @@
EXPECT_EQ("aop2", ops[1].name);
}
-TEST_F(DeltaDiffGeneratorTest, SparseHolesFilteredTest) {
- // Test to see that extents starting with a sparse hole are filtered out by
- // ClearSparseHoles.
- vector<Extent> extents;
- AddExtent(kSparseHole, 1, &extents);
- AddExtent(21, 2, &extents);
- AddExtent(kSparseHole, 3, &extents);
- AddExtent(29, 1, &extents);
- DeltaDiffGenerator::ClearSparseHoles(&extents);
- EXPECT_EQ(extents.size(), 2);
- EXPECT_EQ(extents[0], ExtentForRange(21, 2));
- EXPECT_EQ(extents[1], ExtentForRange(29, 1));
-}
-
TEST_F(DeltaDiffGeneratorTest, SplitSourceCopyTest) {
DeltaArchiveManifest_InstallOperation op;
op.set_type(DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY);
diff --git a/payload_generator/extent_mapper.cc b/payload_generator/extent_mapper.cc
deleted file mode 100644
index c936450..0000000
--- a/payload_generator/extent_mapper.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "update_engine/payload_generator/extent_mapper.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <linux/fs.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include <algorithm>
-
-#include "update_engine/payload_constants.h"
-#include "update_engine/payload_generator/extent_utils.h"
-#include "update_engine/utils.h"
-
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-namespace extent_mapper {
-
-namespace {
-const int kBlockSize = 4096;
-}
-
-bool ExtentsForFileChunkFibmap(const string& path,
- off_t chunk_offset,
- off_t chunk_size,
- vector<Extent>* out) {
- CHECK(out);
- CHECK_EQ(0, chunk_offset % kBlockSize);
- CHECK(chunk_size == -1 || chunk_size >= 0);
- struct stat stbuf;
- int rc = stat(path.c_str(), &stbuf);
- TEST_AND_RETURN_FALSE_ERRNO(rc == 0);
- TEST_AND_RETURN_FALSE(S_ISREG(stbuf.st_mode));
-
- int fd = open(path.c_str(), O_RDONLY, 0);
- TEST_AND_RETURN_FALSE_ERRNO(fd >= 0);
- ScopedFdCloser fd_closer(&fd);
-
- // Get file size in blocks
- off_t file_size = utils::FileSize(fd);
- if (file_size < 0) {
- return false;
- }
- CHECK_LE(chunk_offset, file_size);
- off_t size = file_size - chunk_offset;
- if (chunk_size != -1) {
- size = std::min(size, chunk_size);
- }
- const int block_count = (size + kBlockSize - 1) / kBlockSize;
- const int start_block = chunk_offset / kBlockSize;
- Extent current;
- current.set_start_block(0);
- current.set_num_blocks(0);
-
- for (int i = start_block; i < start_block + block_count; i++) {
- unsigned int block32 = i;
- rc = ioctl(fd, FIBMAP, &block32);
- TEST_AND_RETURN_FALSE_ERRNO(rc == 0);
-
- const uint64_t block = (block32 == 0 ? kSparseHole : block32);
-
- AppendBlockToExtents(out, block);
- }
- return true;
-}
-
-bool ExtentsForFileFibmap(const string& path, vector<Extent>* out) {
- return ExtentsForFileChunkFibmap(path, 0, -1, out);
-}
-
-bool GetFilesystemBlockSize(const string& path, uint32_t* out_blocksize) {
- int fd = open(path.c_str(), O_RDONLY, 0);
- TEST_AND_RETURN_FALSE_ERRNO(fd >= 0);
- ScopedFdCloser fd_closer(&fd);
- int rc = ioctl(fd, FIGETBSZ, out_blocksize);
- TEST_AND_RETURN_FALSE_ERRNO(rc != -1);
- return true;
-}
-
-} // namespace extent_mapper
-
-} // namespace chromeos_update_engine
diff --git a/payload_generator/extent_mapper.h b/payload_generator/extent_mapper.h
deleted file mode 100644
index 98a5452..0000000
--- a/payload_generator/extent_mapper.h
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_EXTENT_MAPPER_H_
-#define UPDATE_ENGINE_PAYLOAD_GENERATOR_EXTENT_MAPPER_H_
-
-#include <string>
-#include <vector>
-
-#include "update_engine/update_metadata.pb.h"
-
-namespace chromeos_update_engine {
-
-namespace extent_mapper {
-
-// Uses the FIBMAP ioctl to get all blocks used by a file and return them
-// as extents. Blocks are relative to the start of the filesystem. If
-// there is a sparse "hole" in the file, the blocks for that will be
-// represented by an extent whose start block is kSpareseHole.
-// The resulting extents are stored in 'out'. Keep in mind that while
-// the blocksize of a filesystem is often 4096 bytes, that is not always
-// the case, so one should consult GetFilesystemBlockSize(), too.
-// Returns true on success.
-//
-// ExtentsForFileChunkFibmap gets the blocks starting from
-// |chunk_offset|. |chunk_offset| must be a multiple of the block size. If
-// |chunk_size| is not -1, only blocks covering up to |chunk_size| bytes are
-// returned.
-bool ExtentsForFileFibmap(const std::string& path, std::vector<Extent>* out);
-bool ExtentsForFileChunkFibmap(const std::string& path,
- off_t chunk_offset,
- off_t chunk_size,
- std::vector<Extent>* out);
-
-// Puts the blocksize of the filesystem, as used by the FIBMAP ioctl, into
-// out_blocksize by using the FIGETBSZ ioctl. Returns true on success.
-bool GetFilesystemBlockSize(const std::string& path, uint32_t* out_blocksize);
-
-} // namespace extent_mapper
-
-} // namespace chromeos_update_engine
-
-#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_EXTENT_MAPPER_H_
diff --git a/payload_generator/extent_mapper_unittest.cc b/payload_generator/extent_mapper_unittest.cc
deleted file mode 100644
index 3e87cdc..0000000
--- a/payload_generator/extent_mapper_unittest.cc
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "update_engine/payload_generator/extent_mapper.h"
-
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include <set>
-#include <string>
-#include <vector>
-
-#include <base/macros.h>
-#include <gtest/gtest.h>
-
-#include "update_engine/payload_constants.h"
-#include "update_engine/utils.h"
-
-using std::set;
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-class ExtentMapperTest : public ::testing::Test {};
-
-TEST(ExtentMapperTest, RunAsRootSimpleTest) {
- // It's hard to have a concrete test for extent mapping without including
- // a specific filesystem image.
- // In lieu of this, we do a weak test: make sure the extents of the unittest
- // executable are consistent and they match with the size of the file.
- const string kFilename = "/proc/self/exe";
-
- uint32_t block_size = 0;
- EXPECT_TRUE(extent_mapper::GetFilesystemBlockSize(kFilename, &block_size));
- EXPECT_GT(block_size, 0);
-
- vector<Extent> extents;
-
- ASSERT_TRUE(extent_mapper::ExtentsForFileFibmap(kFilename, &extents));
-
- EXPECT_FALSE(extents.empty());
- set<uint64_t> blocks;
-
- for (vector<Extent>::const_iterator it = extents.begin();
- it != extents.end(); ++it) {
- for (uint64_t block = it->start_block();
- block < it->start_block() + it->num_blocks();
- block++) {
- EXPECT_FALSE(utils::SetContainsKey(blocks, block));
- blocks.insert(block);
- }
- }
-
- off_t file_size = utils::FileSize(kFilename);
- EXPECT_GT(file_size, 0);
- EXPECT_EQ(blocks.size(), (file_size + block_size - 1)/block_size);
-
- // Map a 2-block chunk at offset |block_size|.
- vector<Extent> chunk_extents;
- ASSERT_TRUE(
- extent_mapper::ExtentsForFileChunkFibmap(kFilename,
- block_size,
- block_size + 1,
- &chunk_extents));
- EXPECT_FALSE(chunk_extents.empty());
- int chunk_blocks = 0;
- for (vector<Extent>::const_iterator it = chunk_extents.begin();
- it != chunk_extents.end(); ++it) {
- chunk_blocks += it->num_blocks();
- }
- EXPECT_EQ(2, chunk_blocks);
-}
-
-TEST(ExtentMapperTest, RunAsRootSparseFileTest) {
- // Create sparse file with one real block, then two sparse ones, then a real
- // block at the end.
- const char tmp_name_template[] =
- "/tmp/ExtentMapperTest.RunAsRootSparseFileTest.XXXXXX";
- char buf[sizeof(tmp_name_template)];
- strncpy(buf, tmp_name_template, sizeof(buf));
- COMPILE_ASSERT(sizeof(buf) > 8, buf_size_incorrect);
- ASSERT_EQ('\0', buf[sizeof(buf) - 1]);
-
- int fd = mkstemp(buf);
- ASSERT_GE(fd, 0);
-
- uint32_t block_size = 0;
- EXPECT_TRUE(extent_mapper::GetFilesystemBlockSize(buf, &block_size));
- EXPECT_GT(block_size, 0);
-
- EXPECT_EQ(1, pwrite(fd, "x", 1, 0));
- EXPECT_EQ(1, pwrite(fd, "x", 1, 3 * block_size));
- close(fd);
-
- vector<Extent> extents;
- EXPECT_TRUE(extent_mapper::ExtentsForFileFibmap(buf, &extents));
- unlink(buf);
- EXPECT_EQ(3, extents.size());
- EXPECT_EQ(1, extents[0].num_blocks());
- EXPECT_EQ(2, extents[1].num_blocks());
- EXPECT_EQ(1, extents[2].num_blocks());
- EXPECT_NE(kSparseHole, extents[0].start_block());
- EXPECT_EQ(kSparseHole, extents[1].start_block());
- EXPECT_NE(kSparseHole, extents[2].start_block());
- EXPECT_NE(extents[2].start_block(), extents[0].start_block());
-}
-
-} // namespace chromeos_update_engine
diff --git a/payload_generator/filesystem_iterator.cc b/payload_generator/filesystem_iterator.cc
deleted file mode 100644
index 073b7a3..0000000
--- a/payload_generator/filesystem_iterator.cc
+++ /dev/null
@@ -1,157 +0,0 @@
-// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "update_engine/payload_generator/filesystem_iterator.h"
-
-#include <dirent.h>
-#include <errno.h>
-#include <sys/types.h>
-
-#include <set>
-#include <string>
-#include <vector>
-
-#include <base/logging.h>
-#include <base/strings/string_util.h>
-
-#include "update_engine/utils.h"
-
-using std::set;
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-// We use a macro here for two reasons:
-// 1. We want to be able to return from the caller's function.
-// 2. We can use the #macro_arg ism to get a string of the calling statement,
-// which we can log.
-
-#define RETURN_ERROR_IF_FALSE(_statement) \
- do { \
- bool _result = (_statement); \
- if (!_result) { \
- string _message = utils::ErrnoNumberAsString(errno); \
- LOG(INFO) << #_statement << " failed: " << _message << ". Aborting"; \
- is_end_ = true; \
- is_err_ = true; \
- return; \
- } \
- } while (0)
-
-FilesystemIterator::FilesystemIterator(
- const string& path,
- const set<string>& excl_prefixes)
- : excl_prefixes_(excl_prefixes),
- is_end_(false),
- is_err_(false) {
- root_path_ = utils::NormalizePath(path, true);
- RETURN_ERROR_IF_FALSE(lstat(root_path_.c_str(), &stbuf_) == 0);
- root_dev_ = stbuf_.st_dev;
-}
-
-FilesystemIterator::~FilesystemIterator() {
- for (vector<DIR*>::iterator it = dirs_.begin(); it != dirs_.end(); ++it) {
- LOG_IF(ERROR, closedir(*it) != 0) << "closedir failed";
- }
-}
-
-// Returns the size of the current file
-off_t FilesystemIterator::GetFileSize() const {
- return utils::FileSize(GetFullPath());
-}
-
-// Returns full path for current file
-string FilesystemIterator::GetFullPath() const {
- return root_path_ + GetPartialPath();
-}
-
-string FilesystemIterator::GetPartialPath() const {
- string ret;
- for (vector<string>::const_iterator it = names_.begin();
- it != names_.end(); ++it) {
- ret += "/";
- ret += *it;
- }
- return ret;
-}
-
-// Increments to the next file
-void FilesystemIterator::Increment() {
- // If we're currently on a dir, descend into children, but only if
- // we're on the same device as the root device
-
- bool entering_dir = false; // true if we're entering into a new dir
- if (S_ISDIR(stbuf_.st_mode) && (stbuf_.st_dev == root_dev_)) {
- DIR* dir = opendir(GetFullPath().c_str());
- if ((!dir) && ((errno == ENOTDIR) || (errno == ENOENT))) {
- // opendir failed b/c either it's not a dir or it doesn't exist.
- // that's fine. let's just skip over this.
- LOG(ERROR) << "Can't descend into " << GetFullPath();
- } else {
- RETURN_ERROR_IF_FALSE(dir);
- entering_dir = true;
- dirs_.push_back(dir);
- }
- }
-
- if (!entering_dir && names_.empty()) {
- // root disappeared while we tried to descend into it
- is_end_ = true;
- return;
- }
-
- if (!entering_dir)
- names_.pop_back();
-
- IncrementInternal();
- for (set<string>::const_iterator it = excl_prefixes_.begin();
- it != excl_prefixes_.end(); ++it) {
- if (StartsWithASCII(GetPartialPath(), *it, true)) {
- Increment();
- break;
- }
- }
- return;
-}
-
-// Assumes that we need to find the next child of dirs_.back(), or if
-// there are none more, go up the chain
-void FilesystemIterator::IncrementInternal() {
- CHECK_EQ(dirs_.size(), names_.size() + 1);
- for (;;) {
- struct dirent dir_entry;
- struct dirent* dir_entry_pointer;
- int r;
- RETURN_ERROR_IF_FALSE(
- (r = readdir_r(dirs_.back(), &dir_entry, &dir_entry_pointer)) == 0);
- if (dir_entry_pointer) {
- // Found an entry
- names_.push_back(dir_entry_pointer->d_name);
- // Validate
- RETURN_ERROR_IF_FALSE(lstat(GetFullPath().c_str(), &stbuf_) == 0);
- if (strcmp(dir_entry_pointer->d_name, ".") &&
- strcmp(dir_entry_pointer->d_name, "..")) {
- // Done
- return;
- }
- // Child didn't work out. Try again
- names_.pop_back();
- } else {
- // No more children in this dir. Pop it and try again
- RETURN_ERROR_IF_FALSE(closedir(dirs_.back()) == 0);
- dirs_.pop_back();
- if (dirs_.empty()) {
- CHECK(names_.empty());
- // Done with the entire iteration
- is_end_ = true;
- return;
- }
- CHECK(!names_.empty());
- names_.pop_back();
- }
- }
-}
-
-} // namespace chromeos_update_engine
diff --git a/payload_generator/filesystem_iterator.h b/payload_generator/filesystem_iterator.h
deleted file mode 100644
index 8248479..0000000
--- a/payload_generator/filesystem_iterator.h
+++ /dev/null
@@ -1,131 +0,0 @@
-// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_FILESYSTEM_ITERATOR_H_
-#define UPDATE_ENGINE_PAYLOAD_GENERATOR_FILESYSTEM_ITERATOR_H_
-
-// This class is used to walk a filesystem. It will iterate over every file
-// on the same device as the file passed in the ctor. Directories will be
-// visited before their children. Children will be visited in no particular
-// order.
-
-// The iterator is a forward iterator. It's not random access nor can it be
-// decremented.
-
-// Note: If the iterator comes across a mount point where another filesystem
-// is mounted, that mount point will be present, but none of its children
-// will be. Technically the mount point is on the other filesystem (and
-// the Stat() call will verify that), but we return it anyway since:
-// 1. Such a folder must exist in the first filesystem if it got used
-// as a mount point.
-// 2. You probably want to copy if it you're using the iterator to do a
-// filesystem copy
-// 3. If you don't want that, you can just check Stat().st_dev and skip
-// foreign filesystems manually.
-
-#include <dirent.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include <set>
-#include <string>
-#include <vector>
-
-namespace chromeos_update_engine {
-
-class FilesystemIterator {
- public:
- FilesystemIterator(const std::string& path,
- const std::set<std::string>& excl_prefixes);
-
- ~FilesystemIterator();
-
- // Returns stat struct for the current file.
- struct stat GetStat() const {
- return stbuf_;
- }
-
- // Returns the size of the current file.
- off_t GetFileSize() const;
-
- // Returns full path for current file.
- std::string GetFullPath() const;
-
- // Returns the path that's part of the iterator. For example, if
- // the object were constructed by passing in "/foo/bar" and Path()
- // returns "/foo/bar/baz/bat.txt", IterPath would return
- // "/baz/bat.txt". When this object is on root (ie, the very first
- // path), IterPath will return "", otherwise the first character of
- // IterPath will be "/".
- std::string GetPartialPath() const;
-
- // Returns name for current file.
- std::string GetBasename() const {
- return names_.back();
- }
-
- // Increments to the next file.
- void Increment();
-
- // If we're at the end. If at the end, do not call Stat(), Path(), etc.,
- // since this iterator currently isn't pointing to any file at all.
- bool IsEnd() const {
- return is_end_;
- }
-
- // Returns true if the iterator is in an error state.
- bool IsErr() const {
- return is_err_;
- }
-
- private:
- // Helper for Increment.
- void IncrementInternal();
-
- // Returns true if path exists and it's a directory.
- bool DirectoryExists(const std::string& path);
-
- // In general (i.e., not midway through a call to Increment()), there is a
- // relationship between dirs_ and names_: dirs[i] == names_[i - 1].
- // For example, say we are asked to iterate "/usr/local" and we're currently
- // at /usr/local/share/dict/words. dirs_ contains DIR* variables for the
- // dirs at: {"/usr/local", ".../share", ".../dict"} and names_ contains:
- // {"share", "dict", "words"}. root_path_ contains "/usr/local".
- // root_dev_ would be the dev for root_path_
- // (and /usr/local/share/dict/words). stbuf_ would be the stbuf for
- // /usr/local/share/dict/words.
-
- // All opened directories. If this is empty, we're currently on the root,
- // but not descended into the root.
- // This will always contain the current directory and all it's ancestors
- // in root-to-leaf order. For more details, see comment above.
- std::vector<DIR*> dirs_;
-
- // The list of all filenames for the current path that we've descended into.
- std::vector<std::string> names_;
-
- // The device of the root path we've been asked to iterate.
- dev_t root_dev_;
-
- // The root path we've been asked to iterate.
- std::string root_path_;
-
- // Exclude items w/ this prefix.
- std::set<std::string> excl_prefixes_;
-
- // The struct stat of the current file we're at.
- struct stat stbuf_;
-
- // Generally false; set to true when we reach the end of files to iterate
- // or error occurs.
- bool is_end_;
-
- // Generally false; set to true if an error occurs.
- bool is_err_;
-};
-
-} // namespace chromeos_update_engine
-
-#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_FILESYSTEM_ITERATOR_H_
diff --git a/payload_generator/filesystem_iterator_unittest.cc b/payload_generator/filesystem_iterator_unittest.cc
deleted file mode 100644
index 3e72069..0000000
--- a/payload_generator/filesystem_iterator_unittest.cc
+++ /dev/null
@@ -1,147 +0,0 @@
-// Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "update_engine/payload_generator/filesystem_iterator.h"
-
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include <set>
-#include <string>
-#include <vector>
-
-#include <base/strings/string_util.h>
-#include <base/strings/stringprintf.h>
-#include <gtest/gtest.h>
-
-#include "update_engine/test_utils.h"
-#include "update_engine/utils.h"
-
-using chromeos_update_engine::test_utils::System;
-using std::set;
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-class FilesystemIteratorTest : public ::testing::Test {
- protected:
- void SetUp() override {
- ASSERT_TRUE(utils::MakeTempDirectory("FilesystemIteratorTest-XXXXXX",
- &test_dir_));
- LOG(INFO) << "SetUp() mkdir " << test_dir_;
- }
-
- void TearDown() override {
- LOG(INFO) << "TearDown() rmdir " << test_dir_;
- EXPECT_EQ(0, System(base::StringPrintf("rm -rf %s", TestDir())));
- }
-
- const char* TestDir() {
- return test_dir_.c_str();
- }
-
- private:
- string test_dir_;
-};
-
-TEST_F(FilesystemIteratorTest, RunAsRootSuccessTest) {
- ASSERT_EQ(0, getuid());
-
- // Create uniquely named main/sub images.
- string main_image;
- ASSERT_TRUE(utils::MakeTempFile("FilesystemIteratorTest.image1-XXXXXX",
- &main_image, nullptr));
- ScopedPathUnlinker main_image_unlinker(main_image);
-
- string sub_image;
- ASSERT_TRUE(utils::MakeTempFile("FilesystemIteratorTest.image2-XXXXXX",
- &sub_image, nullptr));
- ScopedPathUnlinker sub_image_unlinker(sub_image);
-
- // Create uniquely named main/sub mount points.
- string main_image_mount_point;
- ASSERT_TRUE(utils::MakeTempDirectory(
- "FilesystemIteratorTest.mount-XXXXXX",
- &main_image_mount_point));
- ScopedPathUnlinker main_image_mount_point_unlinker(main_image_mount_point);
- const string sub_image_mount_point = main_image_mount_point + "/some_dir/mnt";
-
- vector<string> expected_paths_vector;
- test_utils::CreateExtImageAtPath(main_image, &expected_paths_vector);
- test_utils::CreateExtImageAtPath(sub_image, nullptr);
- ASSERT_EQ(0, System(string("mount -o loop ") + main_image + " " +
- main_image_mount_point));
- ASSERT_EQ(0, System(string("mount -o loop ") + sub_image + " " +
- sub_image_mount_point));
- for (vector<string>::iterator it = expected_paths_vector.begin();
- it != expected_paths_vector.end(); ++it)
- *it = main_image_mount_point + *it;
- set<string> expected_paths(expected_paths_vector.begin(),
- expected_paths_vector.end());
- test_utils::VerifyAllPaths(main_image_mount_point, expected_paths);
-
- EXPECT_TRUE(utils::UnmountFilesystem(sub_image_mount_point));
- EXPECT_TRUE(utils::UnmountFilesystem(main_image_mount_point));
-}
-
-TEST_F(FilesystemIteratorTest, NegativeTest) {
- {
- FilesystemIterator iter("/non/existent/path", set<string>());
- EXPECT_TRUE(iter.IsEnd());
- EXPECT_TRUE(iter.IsErr());
- }
-
- {
- FilesystemIterator iter(TestDir(), set<string>());
- EXPECT_FALSE(iter.IsEnd());
- EXPECT_FALSE(iter.IsErr());
- // Here I'm deleting the exact directory that iterator is point at,
- // then incrementing (which normally would descend into that directory).
- EXPECT_EQ(0, rmdir(TestDir()));
- iter.Increment();
- EXPECT_TRUE(iter.IsEnd());
- EXPECT_FALSE(iter.IsErr());
- }
-}
-
-TEST_F(FilesystemIteratorTest, DeleteWhileTraverseTest) {
- const string dir_name = TestDir();
- ASSERT_EQ(0, chmod(dir_name.c_str(), 0755));
- const string sub_dir_name(dir_name + "/a");
- ASSERT_EQ(0, mkdir(sub_dir_name.c_str(), 0755));
- const string sub_sub_dir_name(sub_dir_name + "/b");
- ASSERT_EQ(0, mkdir(sub_sub_dir_name.c_str(), 0755));
- ASSERT_EQ(0, mkdir((dir_name + "/b").c_str(), 0755));
- ASSERT_EQ(0, mkdir((dir_name + "/c").c_str(), 0755));
-
- string expected_paths_arr[] = {
- "",
- "/a",
- "/b",
- "/c"
- };
- set<string> expected_paths(expected_paths_arr,
- expected_paths_arr +
- arraysize(expected_paths_arr));
-
- FilesystemIterator iter(dir_name, set<string>());
- while (!iter.IsEnd()) {
- string path = iter.GetPartialPath();
- EXPECT_TRUE(expected_paths.find(path) != expected_paths.end());
- if (expected_paths.find(path) != expected_paths.end()) {
- expected_paths.erase(path);
- }
- if (path == "/a") {
- EXPECT_EQ(0, rmdir(sub_sub_dir_name.c_str()));
- EXPECT_EQ(0, rmdir(sub_dir_name.c_str()));
- }
- iter.Increment();
- }
- EXPECT_FALSE(iter.IsErr());
- EXPECT_TRUE(expected_paths.empty());
-}
-
-} // namespace chromeos_update_engine
diff --git a/payload_generator/graph_types.h b/payload_generator/graph_types.h
index 9d3a6fc..7b20bb5 100644
--- a/payload_generator/graph_types.h
+++ b/payload_generator/graph_types.h
@@ -13,6 +13,7 @@
#include <base/macros.h>
+#include "update_engine/payload_generator/extent_utils.h"
#include "update_engine/update_metadata.pb.h"
// A few classes that help in generating delta images use these types
@@ -20,8 +21,6 @@
namespace chromeos_update_engine {
-bool operator==(const Extent& a, const Extent& b);
-
struct EdgeProperties {
// Read-before extents. I.e., blocks in |extents| must be read by the
// node pointed to before the pointing node runs (presumably b/c it
@@ -42,9 +41,7 @@
Vertex() :
valid(true),
index(-1),
- lowlink(-1),
- chunk_offset(0),
- chunk_size(-1) {}
+ lowlink(-1) {}
bool valid;
typedef std::map<std::vector<Vertex>::size_type, EdgeProperties> EdgeMap;
@@ -64,8 +61,6 @@
// Other Vertex properties:
DeltaArchiveManifest_InstallOperation op;
std::string file_name;
- off_t chunk_offset;
- off_t chunk_size;
typedef std::vector<Vertex>::size_type Index;
static const Vertex::Index kInvalidIndex;
diff --git a/payload_generator/graph_utils.cc b/payload_generator/graph_utils.cc
index 7b03236..9166560 100644
--- a/payload_generator/graph_utils.cc
+++ b/payload_generator/graph_utils.cc
@@ -115,7 +115,6 @@
LOG(INFO) << i
<< (graph[i].valid ? "" : "-INV")
<< ": " << graph[i].file_name
- << " " << graph[i].chunk_size << "@" << graph[i].chunk_offset
<< ": " << InstallOperationTypeName(graph[i].op.type());
LOG(INFO) << " src_extents:";
DumpExtents(graph[i].op.src_extents(), 4);
diff --git a/payload_generator/inplace_generator.cc b/payload_generator/inplace_generator.cc
index 29d53e3..827f63b 100644
--- a/payload_generator/inplace_generator.cc
+++ b/payload_generator/inplace_generator.cc
@@ -15,9 +15,9 @@
#include "update_engine/payload_constants.h"
#include "update_engine/payload_generator/cycle_breaker.h"
#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/ext2_filesystem.h"
#include "update_engine/payload_generator/graph_types.h"
#include "update_engine/payload_generator/graph_utils.h"
-#include "update_engine/payload_generator/metadata.h"
#include "update_engine/payload_generator/topological_sort.h"
#include "update_engine/update_metadata.pb.h"
#include "update_engine/utils.h"
@@ -27,11 +27,12 @@
using std::pair;
using std::set;
using std::string;
+using std::unique_ptr;
using std::vector;
namespace chromeos_update_engine {
-using Block = DeltaDiffGenerator::Block;
+using Block = InplaceGenerator::Block;
// This class allocates non-existent temp blocks, starting from
// kTempBlockStart. Other code is responsible for converting these
@@ -174,8 +175,8 @@
// must complete before A executes.
void InplaceGenerator::CreateEdges(
Graph* graph,
- const vector<DeltaDiffGenerator::Block>& blocks) {
- for (vector<DeltaDiffGenerator::Block>::size_type i = 0;
+ const vector<Block>& blocks) {
+ for (vector<Block>::size_type i = 0;
i < blocks.size(); i++) {
// Blocks with both a reader and writer get an edge
if (blocks[i].reader == Vertex::kInvalidIndex ||
@@ -288,7 +289,6 @@
// all temp nodes invalid.
bool ConvertCutsToFull(
Graph* graph,
- const string& new_root,
const string& new_part,
int data_fd,
off_t* data_file_size,
@@ -302,7 +302,6 @@
graph,
cut,
new_part,
- new_root,
data_fd,
data_file_size));
deleted_nodes.insert(cut.new_vertex);
@@ -330,7 +329,6 @@
// on exceptional error cases.
bool AssignBlockForAdjoiningCuts(
Graph* graph,
- const string& new_root,
const string& new_part,
int data_fd,
off_t* data_file_size,
@@ -395,7 +393,6 @@
if (scratch_ranges.blocks() < blocks_needed) {
LOG(INFO) << "Unable to find sufficient scratch";
TEST_AND_RETURN_FALSE(ConvertCutsToFull(graph,
- new_root,
new_part,
data_fd,
data_file_size,
@@ -442,7 +439,6 @@
bool InplaceGenerator::AssignTempBlocks(
Graph* graph,
- const string& new_root,
const string& new_part,
int data_fd,
off_t* data_file_size,
@@ -466,7 +462,6 @@
} else {
CHECK(!cuts_group.empty());
TEST_AND_RETURN_FALSE(AssignBlockForAdjoiningCuts(graph,
- new_root,
new_part,
data_fd,
data_file_size,
@@ -484,7 +479,6 @@
}
CHECK(!cuts_group.empty());
TEST_AND_RETURN_FALSE(AssignBlockForAdjoiningCuts(graph,
- new_root,
new_part,
data_fd,
data_file_size,
@@ -524,7 +518,6 @@
bool InplaceGenerator::ConvertCutToFullOp(Graph* graph,
const CutEdgeVertexes& cut,
const string& new_part,
- const string& new_root,
int data_fd,
off_t* data_file_size) {
// Drop all incoming edges, keep all outgoing edges
@@ -537,20 +530,26 @@
Vertex::EdgeMap out_edges = (*graph)[cut.old_dst].out_edges;
graph_utils::DropWriteBeforeDeps(&out_edges);
+ // Replace the operation with a REPLACE or REPLACE_BZ to generate the same
+ // |new_extents| list of blocks and update the graph.
+ vector<AnnotatedOperation> new_aop;
+ vector<Extent> new_extents;
+ DeltaDiffGenerator::ExtentsToVector((*graph)[cut.old_dst].op.dst_extents(),
+ &new_extents);
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::DeltaReadFile(
- graph,
- cut.old_dst,
- nullptr,
+ &new_aop,
kEmptyPath, // old_part
new_part,
- kEmptyPath,
- new_root,
+ vector<Extent>(), // old_extents
+ new_extents,
(*graph)[cut.old_dst].file_name,
- (*graph)[cut.old_dst].chunk_offset,
- (*graph)[cut.old_dst].chunk_size,
+ -1, // chunk_blocks, forces to have a single operation.
data_fd,
data_file_size,
false)); // src_ops_allowed
+ TEST_AND_RETURN_FALSE(new_aop.size() == 1);
+ TEST_AND_RETURN_FALSE(AddInstallOpToGraph(
+ graph, cut.old_dst, nullptr, new_aop.front().op, new_aop.front().name));
(*graph)[cut.old_dst].out_edges = out_edges;
@@ -569,12 +568,11 @@
}
bool InplaceGenerator::ConvertGraphToDag(Graph* graph,
- const string& new_part,
- const string& new_root,
- int fd,
- off_t* data_file_size,
- vector<Vertex::Index>* final_order,
- Vertex::Index scratch_vertex) {
+ const string& new_part,
+ int fd,
+ off_t* data_file_size,
+ vector<Vertex::Index>* final_order,
+ Vertex::Index scratch_vertex) {
CycleBreaker cycle_breaker;
LOG(INFO) << "Finding cycles...";
set<Edge> cut_edges;
@@ -608,7 +606,6 @@
if (!cuts.empty())
TEST_AND_RETURN_FALSE(AssignTempBlocks(graph,
new_part,
- new_root,
fd,
data_file_size,
final_order,
@@ -646,7 +643,7 @@
const DeltaArchiveManifest_InstallOperation& operation,
const Graph& graph,
Vertex::Index vertex,
- vector<DeltaDiffGenerator::Block>* blocks) {
+ vector<Block>* blocks) {
// See if this is already present.
TEST_AND_RETURN_FALSE(operation.dst_extents_size() > 0);
@@ -658,9 +655,8 @@
const char* past_participle = (field == READER) ? "read" : "written";
const google::protobuf::RepeatedPtrField<Extent>& extents =
(field == READER) ? operation.src_extents() : operation.dst_extents();
- Vertex::Index DeltaDiffGenerator::Block::*access_type =
- (field == READER) ? &DeltaDiffGenerator::Block::reader
- : &DeltaDiffGenerator::Block::writer;
+ Vertex::Index Block::*access_type = (field == READER) ?
+ &Block::reader : &Block::writer;
for (int i = 0; i < extents_size; i++) {
const Extent& extent = extents.Get(i);
@@ -681,56 +677,67 @@
return true;
}
+bool InplaceGenerator::AddInstallOpToGraph(
+ Graph* graph,
+ Vertex::Index existing_vertex,
+ vector<Block>* blocks,
+ const DeltaArchiveManifest_InstallOperation operation,
+ const string& op_name) {
+ Vertex::Index vertex = existing_vertex;
+ if (vertex == Vertex::kInvalidIndex) {
+ graph->emplace_back();
+ vertex = graph->size() - 1;
+ }
+ (*graph)[vertex].op = operation;
+ CHECK((*graph)[vertex].op.has_type());
+ (*graph)[vertex].file_name = op_name;
+
+ if (blocks)
+ TEST_AND_RETURN_FALSE(InplaceGenerator::AddInstallOpToBlocksVector(
+ (*graph)[vertex].op,
+ *graph,
+ vertex,
+ blocks));
+ return true;
+}
+
bool InplaceGenerator::GenerateOperations(
const PayloadGenerationConfig& config,
int data_file_fd,
off_t* data_file_size,
vector<AnnotatedOperation>* rootfs_ops,
vector<AnnotatedOperation>* kernel_ops) {
+ unique_ptr<Ext2Filesystem> old_fs = Ext2Filesystem::CreateFromFile(
+ config.source.rootfs_part);
+ unique_ptr<Ext2Filesystem> new_fs = Ext2Filesystem::CreateFromFile(
+ config.target.rootfs_part);
+
+ off_t chunk_blocks = (config.chunk_size == -1 ? -1 :
+ config.chunk_size / config.block_size);
+
+ // Temporary list of operations used to construct the dependency graph.
+ vector<AnnotatedOperation> aops;
+ TEST_AND_RETURN_FALSE(
+ DeltaDiffGenerator::DeltaReadFilesystem(&aops,
+ config.source.rootfs_part,
+ config.target.rootfs_part,
+ old_fs.get(),
+ new_fs.get(),
+ chunk_blocks,
+ data_file_fd,
+ data_file_size,
+ false)); // src_ops_allowed
+ // Convert the rootfs operations to the graph.
Graph graph;
CheckGraph(graph);
vector<Block> blocks(config.target.rootfs_size / config.block_size);
-
- TEST_AND_RETURN_FALSE(
- DeltaDiffGenerator::DeltaReadFiles(&graph,
- &blocks,
- config.source.rootfs_part,
- config.target.rootfs_part,
- config.source.rootfs_mountpt,
- config.target.rootfs_mountpt,
- config.chunk_size,
- data_file_fd,
- data_file_size,
- false)); // src_ops_allowed
+ for (const auto& aop : aops) {
+ AddInstallOpToGraph(
+ &graph, Vertex::kInvalidIndex, &blocks, aop.op, aop.name);
+ }
LOG(INFO) << "done reading normal files";
CheckGraph(graph);
- LOG(INFO) << "Starting metadata processing";
- TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(
- &graph,
- &blocks,
- config.source.rootfs_part,
- config.target.rootfs_part,
- data_file_fd,
- data_file_size));
- LOG(INFO) << "Done metadata processing";
- CheckGraph(graph);
-
- graph.emplace_back();
- TEST_AND_RETURN_FALSE(
- DeltaDiffGenerator::ReadUnwrittenBlocks(blocks,
- data_file_fd,
- data_file_size,
- config.source.rootfs_part,
- config.source.rootfs_size,
- config.target.rootfs_part,
- &graph.back(),
- config.minor_version));
- if (graph.back().op.data_length() == 0) {
- LOG(INFO) << "No unwritten blocks to write, omitting operation";
- graph.pop_back();
- }
-
// Final scratch block (if there's space)
Vertex::Index scratch_vertex = Vertex::kInvalidIndex;
if (blocks.size() < (config.rootfs_partition_size / kBlockSize)) {
@@ -747,11 +754,13 @@
DeltaDiffGenerator::DeltaCompressKernelPartition(
config.source.kernel_part,
config.target.kernel_part,
+ config.source.kernel_size,
+ config.target.kernel_size,
+ config.block_size,
kernel_ops,
data_file_fd,
data_file_size,
false)); // src_ops_allowed
-
LOG(INFO) << "done reading kernel";
CheckGraph(graph);
@@ -764,7 +773,6 @@
TEST_AND_RETURN_FALSE(ConvertGraphToDag(
&graph,
config.target.rootfs_part,
- config.target.rootfs_mountpt,
data_file_fd,
data_file_size,
&final_order,
@@ -777,8 +785,7 @@
const Vertex& vertex = graph[vertex_index];
rootfs_ops->emplace_back();
rootfs_ops->back().op = vertex.op;
- rootfs_ops->back().SetNameFromFileAndChunk(
- vertex.file_name, vertex.chunk_offset, vertex.chunk_size);
+ rootfs_ops->back().name = vertex.file_name;
}
return true;
diff --git a/payload_generator/inplace_generator.h b/payload_generator/inplace_generator.h
index 9d10548..90c1fd4 100644
--- a/payload_generator/inplace_generator.h
+++ b/payload_generator/inplace_generator.h
@@ -37,6 +37,21 @@
class InplaceGenerator : public OperationsGenerator {
public:
+ // Represents a disk block on the install partition.
+ struct Block {
+ // During install, each block on the install partition will be written
+ // and some may be read (in all likelihood, many will be read).
+ // The reading and writing will be performed by InstallOperations,
+ // each of which has a corresponding vertex in a graph.
+ // A Block object tells which vertex will read or write this block
+ // at install time.
+ // Generally, there will be a vector of Block objects whose length
+ // is the number of blocks on the install partition.
+ Block() : reader(Vertex::kInvalidIndex), writer(Vertex::kInvalidIndex) {}
+ Vertex::Index reader;
+ Vertex::Index writer;
+ };
+
InplaceGenerator() = default;
// Checks all the operations in the graph have a type assigned.
@@ -69,7 +84,7 @@
// readers of the same block. This is because for an edge A->B, B
// must complete before A executes.
static void CreateEdges(Graph* graph,
- const std::vector<DeltaDiffGenerator::Block>& blocks);
+ const std::vector<Block>& blocks);
// Takes |op_indexes|, which is effectively a mapping from order in
// which the op is performed -> graph vertex index, and produces the
@@ -104,7 +119,6 @@
static bool AssignTempBlocks(
Graph* graph,
const std::string& new_part,
- const std::string& new_root,
int data_fd,
off_t* data_file_size,
std::vector<Vertex::Index>* op_indexes,
@@ -121,7 +135,6 @@
static bool ConvertCutToFullOp(Graph* graph,
const CutEdgeVertexes& cut,
const std::string& new_part,
- const std::string& new_root,
int data_fd,
off_t* data_file_size);
@@ -136,7 +149,6 @@
// Returns true on success.
static bool ConvertGraphToDag(Graph* graph,
const std::string& new_part,
- const std::string& new_root,
int fd,
off_t* data_file_size,
std::vector<Vertex::Index>* final_order,
@@ -161,7 +173,19 @@
const DeltaArchiveManifest_InstallOperation& operation,
const Graph& graph,
Vertex::Index vertex,
- std::vector<DeltaDiffGenerator::Block>* blocks);
+ std::vector<Block>* blocks);
+
+ // Add a vertex (if |existing_vertex| is kInvalidVertex) or update an
+ // |existing_vertex| with the passed |operation|.
+ // This method will also register the vertex as the reader or writer of the
+ // blocks involved in the operation updating the |blocks| vector. The
+ // |op_name| associated with the Vertex is used for logging purposes.
+ static bool AddInstallOpToGraph(
+ Graph* graph,
+ Vertex::Index existing_vertex,
+ std::vector<Block>* blocks,
+ const DeltaArchiveManifest_InstallOperation operation,
+ const std::string& op_name);
// Generate the update payload operations for the kernel and rootfs using
// only operations that read from the target and/or write to the target,
diff --git a/payload_generator/inplace_generator_unittest.cc b/payload_generator/inplace_generator_unittest.cc
index 6c4f36b..5c563c5 100644
--- a/payload_generator/inplace_generator_unittest.cc
+++ b/payload_generator/inplace_generator_unittest.cc
@@ -29,7 +29,7 @@
namespace chromeos_update_engine {
-typedef DeltaDiffGenerator::Block Block;
+using Block = InplaceGenerator::Block;
namespace {
@@ -94,6 +94,14 @@
class InplaceGeneratorTest : public ::testing::Test {
};
+TEST_F(InplaceGeneratorTest, BlockDefaultValues) {
+ // Tests that a Block is initialized with the default values as a
+ // Vertex::kInvalidIndex. This is required by the delta generators.
+ Block block;
+ EXPECT_EQ(Vertex::kInvalidIndex, block.reader);
+ EXPECT_EQ(Vertex::kInvalidIndex, block.writer);
+}
+
TEST_F(InplaceGeneratorTest, SubstituteBlocksTest) {
vector<Extent> remove_blocks;
AppendExtent(&remove_blocks, 3, 3);
@@ -247,14 +255,7 @@
EXPECT_TRUE(graph[1].out_edges.end() != graph[1].out_edges.find(2));
}
-TEST_F(InplaceGeneratorTest, RunAsRootAssignTempBlocksReuseTest) {
- // AssignTempBlocks(Graph* graph,
- // const string& new_root,
- // int data_fd,
- // off_t* data_file_size,
- // vector<Vertex::Index>* op_indexes,
- // vector<vector<Vertex::Index>::size_type>* reverse_op_indexes,
- // const vector<CutEdgeVertexes>& cuts
+TEST_F(InplaceGeneratorTest, AssignTempBlocksReuseTest) {
Graph graph(9);
const vector<Extent> empt;
@@ -321,17 +322,6 @@
InplaceGenerator::GenerateReverseTopoOrderMap(op_indexes,
&reverse_op_indexes);
- // Prepare the filesystem with the minimum required for this to work
- string temp_dir;
- EXPECT_TRUE(utils::MakeTempDirectory("AssignTempBlocksReuseTest.XXXXXX",
- &temp_dir));
- ScopedDirRemover temp_dir_remover(temp_dir);
-
- chromeos::Blob temp_data(kBlockSize * 3);
- test_utils::FillWithData(&temp_data);
- EXPECT_TRUE(test_utils::WriteFileVector(temp_dir + kFilename, temp_data));
- ScopedPathUnlinker filename_unlinker(temp_dir + kFilename);
-
int fd;
EXPECT_TRUE(utils::MakeTempFile("AssignTempBlocksReuseTest.XXXXXX",
nullptr,
@@ -340,7 +330,6 @@
off_t data_file_size = 0;
EXPECT_TRUE(InplaceGenerator::AssignTempBlocks(&graph,
- temp_dir,
"/dev/zero",
fd,
&data_file_size,
@@ -379,7 +368,7 @@
EXPECT_EQ(graph[vect[3]].file_name, "C");
}
-TEST_F(InplaceGeneratorTest, RunAsRootAssignTempBlocksTest) {
+TEST_F(InplaceGeneratorTest, AssignTempBlocksTest) {
Graph graph(9);
const vector<Extent> empt; // empty
const string kFilename = "/foo";
@@ -419,18 +408,6 @@
vector<Vertex::Index> final_order;
-
- // Prepare the filesystem with the minimum required for this to work
- string temp_dir;
- EXPECT_TRUE(utils::MakeTempDirectory("AssignTempBlocksTest.XXXXXX",
- &temp_dir));
- ScopedDirRemover temp_dir_remover(temp_dir);
-
- chromeos::Blob temp_data(kBlockSize * 50);
- test_utils::FillWithData(&temp_data);
- EXPECT_TRUE(test_utils::WriteFileVector(temp_dir + kFilename, temp_data));
- ScopedPathUnlinker filename_unlinker(temp_dir + kFilename);
-
int fd;
EXPECT_TRUE(utils::MakeTempFile("AssignTempBlocksTestData.XXXXXX",
nullptr,
@@ -439,14 +416,12 @@
off_t data_file_size = 0;
EXPECT_TRUE(InplaceGenerator::ConvertGraphToDag(&graph,
- temp_dir,
"/dev/zero",
fd,
&data_file_size,
&final_order,
Vertex::kInvalidIndex));
-
Graph expected_graph(12);
GenVertex(&expected_graph[0], empt, VectOfExt(200, 1), "", OP_REPLACE);
GenVertex(&expected_graph[1], empt, VectOfExt(210, 10), "", OP_REPLACE);
diff --git a/payload_generator/metadata.cc b/payload_generator/metadata.cc
deleted file mode 100644
index f185465..0000000
--- a/payload_generator/metadata.cc
+++ /dev/null
@@ -1,498 +0,0 @@
-// Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "update_engine/payload_generator/metadata.h"
-
-#include <algorithm>
-#include <string>
-#include <vector>
-
-#include <base/strings/string_util.h>
-#include <base/strings/stringprintf.h>
-#include <et/com_err.h>
-#include <ext2fs/ext2_io.h>
-#include <ext2fs/ext2fs.h>
-
-#include "update_engine/bzip.h"
-#include "update_engine/extent_ranges.h"
-#include "update_engine/payload_generator/delta_diff_generator.h"
-#include "update_engine/payload_generator/ext2_utils.h"
-#include "update_engine/payload_generator/extent_utils.h"
-#include "update_engine/payload_generator/inplace_generator.h"
-#include "update_engine/utils.h"
-
-using base::StringPrintf;
-using std::min;
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-namespace {
-const size_t kBlockSize = 4096;
-
-typedef DeltaDiffGenerator::Block Block;
-
-// Read data from the specified extents.
-bool ReadExtentsData(const ext2_filsys fs,
- const vector<Extent>& extents,
- chromeos::Blob* data) {
- // Resize the data buffer to hold all data in the extents
- size_t num_data_blocks = 0;
- for (const Extent& extent : extents) {
- num_data_blocks += extent.num_blocks();
- }
-
- data->resize(num_data_blocks * kBlockSize);
-
- // Read in the data blocks
- const size_t kMaxReadBlocks = 256;
- vector<Block>::size_type blocks_copied_count = 0;
- for (const Extent& extent : extents) {
- vector<Block>::size_type blocks_read = 0;
- while (blocks_read < extent.num_blocks()) {
- const int copy_block_cnt =
- min(kMaxReadBlocks,
- static_cast<size_t>(
- extent.num_blocks() - blocks_read));
- TEST_AND_RETURN_FALSE_ERRCODE(
- io_channel_read_blk(fs->io,
- extent.start_block() + blocks_read,
- copy_block_cnt,
- &(*data)[blocks_copied_count * kBlockSize]));
- blocks_read += copy_block_cnt;
- blocks_copied_count += copy_block_cnt;
- }
- }
-
- return true;
-}
-
-// Compute the bsdiff between two metadata blobs.
-bool ComputeMetadataBsdiff(const chromeos::Blob& old_metadata,
- const chromeos::Blob& new_metadata,
- chromeos::Blob* bsdiff_delta) {
- const string kTempFileTemplate("CrAU_temp_data.XXXXXX");
-
- // Write the metadata buffers to temporary files
- int old_fd;
- string temp_old_file_path;
- TEST_AND_RETURN_FALSE(
- utils::MakeTempFile(kTempFileTemplate, &temp_old_file_path, &old_fd));
- TEST_AND_RETURN_FALSE(old_fd >= 0);
- ScopedPathUnlinker temp_old_file_path_unlinker(temp_old_file_path);
- ScopedFdCloser old_fd_closer(&old_fd);
- TEST_AND_RETURN_FALSE(utils::WriteAll(old_fd,
- old_metadata.data(),
- old_metadata.size()));
-
- int new_fd;
- string temp_new_file_path;
- TEST_AND_RETURN_FALSE(
- utils::MakeTempFile(kTempFileTemplate, &temp_new_file_path, &new_fd));
- TEST_AND_RETURN_FALSE(new_fd >= 0);
- ScopedPathUnlinker temp_new_file_path_unlinker(temp_new_file_path);
- ScopedFdCloser new_fd_closer(&new_fd);
- TEST_AND_RETURN_FALSE(utils::WriteAll(new_fd,
- new_metadata.data(),
- new_metadata.size()));
-
- // Perform bsdiff on these files
- TEST_AND_RETURN_FALSE(
- DeltaDiffGenerator::BsdiffFiles(temp_old_file_path,
- temp_new_file_path,
- bsdiff_delta));
-
- return true;
-}
-
-// Add the specified metadata extents to the graph and blocks vector.
-bool AddMetadataExtents(Graph* graph,
- vector<Block>* blocks,
- const ext2_filsys fs_old,
- const ext2_filsys fs_new,
- const string& metadata_name,
- const vector<Extent>& extents,
- int data_fd,
- off_t* data_file_size) {
- chromeos::Blob data; // Data blob that will be written to delta file.
- DeltaArchiveManifest_InstallOperation op;
-
- {
- // Read in the metadata blocks from the old and new image.
- chromeos::Blob old_data;
- TEST_AND_RETURN_FALSE(ReadExtentsData(fs_old, extents, &old_data));
-
- chromeos::Blob new_data;
- TEST_AND_RETURN_FALSE(ReadExtentsData(fs_new, extents, &new_data));
-
- // Determine the best way to compress this.
- chromeos::Blob new_data_bz;
- TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz));
- CHECK(!new_data_bz.empty());
-
- size_t current_best_size = 0;
- if (new_data.size() <= new_data_bz.size()) {
- op.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE);
- current_best_size = new_data.size();
- data = new_data;
- } else {
- op.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
- current_best_size = new_data_bz.size();
- data = new_data_bz;
- }
-
- if (old_data == new_data) {
- // No change in data.
- op.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
- current_best_size = 0;
- data.clear();
- } else {
- // Try bsdiff of old to new data
- chromeos::Blob bsdiff_delta;
- TEST_AND_RETURN_FALSE(ComputeMetadataBsdiff(old_data,
- new_data,
- &bsdiff_delta));
- CHECK_GT(bsdiff_delta.size(), 0u);
-
- if (bsdiff_delta.size() < current_best_size) {
- op.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF);
- current_best_size = bsdiff_delta.size();
- data = bsdiff_delta;
- }
- }
-
- CHECK_EQ(data.size(), current_best_size);
-
- // Set the source and dest extents to be the same since the filesystem
- // structures are identical
- if (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE ||
- op.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) {
- DeltaDiffGenerator::StoreExtents(extents, op.mutable_src_extents());
- op.set_src_length(old_data.size());
- }
-
- DeltaDiffGenerator::StoreExtents(extents, op.mutable_dst_extents());
- op.set_dst_length(new_data.size());
- }
-
- // Write data to output file
- if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) {
- op.set_data_offset(*data_file_size);
- op.set_data_length(data.size());
- }
-
- TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, data.data(), data.size()));
- *data_file_size += data.size();
-
- // Now, insert into graph and blocks vector
- graph->emplace_back();
- Vertex::Index vertex = graph->size() - 1;
- (*graph)[vertex].op = op;
- CHECK((*graph)[vertex].op.has_type());
- (*graph)[vertex].file_name = metadata_name;
-
- TEST_AND_RETURN_FALSE(InplaceGenerator::AddInstallOpToBlocksVector(
- (*graph)[vertex].op,
- *graph,
- vertex,
- blocks));
-
- return true;
-}
-
-// Reads the file system metadata extents.
-bool ReadFilesystemMetadata(Graph* graph,
- vector<Block>* blocks,
- const ext2_filsys fs_old,
- const ext2_filsys fs_new,
- int data_fd,
- off_t* data_file_size) {
- LOG(INFO) << "Processing <rootfs-metadata>";
-
- // Read all the extents that belong to the main file system metadata.
- // The metadata blocks are at the start of each block group and goes
- // until the end of the inode table.
- for (dgrp_t bg = 0; bg < fs_old->group_desc_count; bg++) {
- struct ext2_group_desc* group_desc = ext2fs_group_desc(fs_old,
- fs_old->group_desc,
- bg);
- __u32 bg_start_block = bg * fs_old->super->s_blocks_per_group;
-
- if (group_desc->bg_inode_table < bg_start_block ||
- (group_desc->bg_inode_table + fs_old->inode_blocks_per_group >
- bg_start_block + fs_old->super->s_blocks_per_group)) {
- LOG(WARNING) << "The inode table for this block group is outside the "
- << "block group. Skipping.";
- continue;
- }
- // We consider metadata all the blocks from the beginning of the block group
- // up to and including the inode table. This includes also the inode and
- // block bitmaps stored before it.
- __u32 num_metadata_blocks = (group_desc->bg_inode_table +
- fs_old->inode_blocks_per_group) -
- bg_start_block;
-
- // Due to bsdiff slowness, we're going to break each block group down
- // into metadata chunks and feed them to bsdiff.
- __u32 num_chunks = 10;
- __u32 blocks_per_chunk = num_metadata_blocks / num_chunks;
- __u32 curr_block = bg_start_block;
- for (__u32 chunk = 0; chunk < num_chunks; chunk++) {
- Extent extent;
- if (chunk < num_chunks - 1) {
- extent = ExtentForRange(curr_block, blocks_per_chunk);
- } else {
- extent = ExtentForRange(curr_block,
- bg_start_block + num_metadata_blocks -
- curr_block);
- }
-
- vector<Extent> extents;
- extents.push_back(extent);
-
- string metadata_name = StringPrintf("<rootfs-bg-%d-%d-metadata>",
- bg, chunk);
-
- LOG(INFO) << "Processing " << metadata_name;
-
- TEST_AND_RETURN_FALSE(AddMetadataExtents(graph,
- blocks,
- fs_old,
- fs_new,
- metadata_name,
- extents,
- data_fd,
- data_file_size));
-
- curr_block += blocks_per_chunk;
- }
- }
-
- return true;
-}
-
-// Processes all blocks belonging to an inode
-int ProcessInodeAllBlocks(ext2_filsys fs,
- blk_t* blocknr,
- e2_blkcnt_t blockcnt,
- blk_t ref_blk,
- int ref_offset,
- void* priv) {
- vector<Extent>* extents = static_cast<vector<Extent>*>(priv);
- AppendBlockToExtents(extents, *blocknr);
- return 0;
-}
-
-// Processes only indirect, double indirect or triple indirect metadata
-// blocks belonging to an inode
-int ProcessInodeMetadataBlocks(ext2_filsys fs,
- blk_t* blocknr,
- e2_blkcnt_t blockcnt,
- blk_t ref_blk,
- int ref_offset,
- void* priv) {
- vector<Extent>* extents = static_cast<vector<Extent>*>(priv);
- if (blockcnt < 0) {
- AppendBlockToExtents(extents, *blocknr);
- }
- return 0;
-}
-
-// Read inode metadata blocks.
-bool ReadInodeMetadata(Graph* graph,
- vector<Block>* blocks,
- const ext2_filsys fs_old,
- const ext2_filsys fs_new,
- int data_fd,
- off_t* data_file_size) {
- TEST_AND_RETURN_FALSE_ERRCODE(ext2fs_read_inode_bitmap(fs_old));
- TEST_AND_RETURN_FALSE_ERRCODE(ext2fs_read_inode_bitmap(fs_new));
-
- ext2_inode_scan iscan;
- TEST_AND_RETURN_FALSE_ERRCODE(ext2fs_open_inode_scan(fs_old, 0, &iscan));
-
- ext2_ino_t ino;
- ext2_inode old_inode;
- while (true) {
- // Get the next inode on both file systems
- errcode_t error = ext2fs_get_next_inode(iscan, &ino, &old_inode);
-
- // If we get an error enumerating the inodes, we'll just log the error
- // and exit from our loop which will eventually return a success code
- // back to the caller. The inode blocks that we cannot account for will
- // be handled by DeltaDiffGenerator::ReadUnwrittenBlocks().
- if (error) {
- LOG(ERROR) << "Failed to retrieve next inode (" << error << ")";
- break;
- }
-
- if (ino == 0) {
- break;
- }
-
- if (ino == EXT2_RESIZE_INO) {
- continue;
- }
-
- ext2_inode new_inode;
- error = ext2fs_read_inode(fs_new, ino, &new_inode);
- if (error) {
- LOG(ERROR) << "Failed to retrieve new inode (" << error << ")";
- continue;
- }
-
- // Skip inodes that are not in use
- if (!ext2fs_test_inode_bitmap(fs_old->inode_map, ino) ||
- !ext2fs_test_inode_bitmap(fs_new->inode_map, ino)) {
- continue;
- }
-
- // Skip inodes that have no data blocks
- if (old_inode.i_blocks == 0 || new_inode.i_blocks == 0) {
- continue;
- }
-
- // Skip inodes that are not the same type
- bool is_old_dir = (ext2fs_check_directory(fs_old, ino) == 0);
- bool is_new_dir = (ext2fs_check_directory(fs_new, ino) == 0);
- if (is_old_dir != is_new_dir) {
- continue;
- }
-
- // Process the inodes metadata blocks
- // For normal files, metadata blocks are indirect, double indirect
- // and triple indirect blocks (no data blocks). For directories and
- // the journal, all blocks are considered metadata blocks.
- LOG(INFO) << "Processing inode " << ino << " metadata";
-
- bool all_blocks = ((ino == EXT2_JOURNAL_INO) || is_old_dir || is_new_dir);
-
- vector<Extent> old_extents;
- error = ext2fs_block_iterate2(fs_old, ino, 0, nullptr,
- all_blocks ? ProcessInodeAllBlocks :
- ProcessInodeMetadataBlocks,
- &old_extents);
- if (error) {
- LOG(ERROR) << "Failed to enumerate old inode " << ino
- << " blocks (" << error << ")";
- continue;
- }
-
- vector<Extent> new_extents;
- error = ext2fs_block_iterate2(fs_new, ino, 0, nullptr,
- all_blocks ? ProcessInodeAllBlocks :
- ProcessInodeMetadataBlocks,
- &new_extents);
- if (error) {
- LOG(ERROR) << "Failed to enumerate new inode " << ino
- << " blocks (" << error << ")";
- continue;
- }
-
- // Skip inode if there are no metadata blocks
- if (old_extents.size() == 0 || new_extents.size() == 0) {
- continue;
- }
-
- // Make sure the two inodes have the same metadata blocks
- if (old_extents.size() != new_extents.size()) {
- continue;
- }
-
- bool same_metadata_extents = true;
- vector<Extent>::iterator it_old;
- vector<Extent>::iterator it_new;
- for (it_old = old_extents.begin(),
- it_new = new_extents.begin();
- it_old != old_extents.end() && it_new != new_extents.end();
- it_old++, it_new++) {
- if (it_old->start_block() != it_new->start_block() ||
- it_old->num_blocks() != it_new->num_blocks()) {
- same_metadata_extents = false;
- break;
- }
- }
-
- if (!same_metadata_extents) {
- continue;
- }
-
- // We have identical inode metadata blocks, we can now add them to
- // our graph and blocks vector
- string metadata_name = StringPrintf("<rootfs-inode-%d-metadata>", ino);
- TEST_AND_RETURN_FALSE(AddMetadataExtents(graph,
- blocks,
- fs_old,
- fs_new,
- metadata_name,
- old_extents,
- data_fd,
- data_file_size));
- }
-
- ext2fs_close_inode_scan(iscan);
-
- return true;
-}
-
-} // namespace
-
-// Reads metadata from old image and new image and determines
-// the smallest way to encode the metadata for the diff.
-// If there's no change in the metadata, it creates a MOVE
-// operation. If there is a change, the smallest of REPLACE, REPLACE_BZ,
-// or BSDIFF wins. It writes the diff to data_fd and updates data_file_size
-// accordingly. It also adds the required operation to the graph and adds the
-// metadata extents to blocks.
-// Returns true on success.
-bool Metadata::DeltaReadMetadata(Graph* graph,
- vector<Block>* blocks,
- const string& old_image,
- const string& new_image,
- int data_fd,
- off_t* data_file_size) {
- // Open the two file systems.
- ext2_filsys fs_old;
- TEST_AND_RETURN_FALSE_ERRCODE(ext2fs_open(old_image.c_str(), 0, 0, 0,
- unix_io_manager, &fs_old));
- ScopedExt2fsCloser fs_old_closer(fs_old);
-
- ext2_filsys fs_new;
- TEST_AND_RETURN_FALSE_ERRCODE(ext2fs_open(new_image.c_str(), 0, 0, 0,
- unix_io_manager, &fs_new));
- ScopedExt2fsCloser fs_new_closer(fs_new);
-
- // Make sure these two file systems are the same.
- // If they are not the same, the metadata blocks will be packaged up in its
- // entirety by ReadUnwrittenBlocks().
- if (fs_old->blocksize != fs_new->blocksize ||
- fs_old->fragsize != fs_new->fragsize ||
- fs_old->group_desc_count != fs_new->group_desc_count ||
- fs_old->inode_blocks_per_group != fs_new->inode_blocks_per_group ||
- fs_old->super->s_inodes_count != fs_new->super->s_inodes_count ||
- fs_old->super->s_blocks_count != fs_new->super->s_blocks_count) {
- return true;
- }
-
- // Process the main file system metadata (superblock, inode tables, etc)
- TEST_AND_RETURN_FALSE(ReadFilesystemMetadata(graph,
- blocks,
- fs_old,
- fs_new,
- data_fd,
- data_file_size));
-
- // Process each inode metadata blocks.
- TEST_AND_RETURN_FALSE(ReadInodeMetadata(graph,
- blocks,
- fs_old,
- fs_new,
- data_fd,
- data_file_size));
-
- return true;
-}
-
-}; // namespace chromeos_update_engine
diff --git a/payload_generator/metadata.h b/payload_generator/metadata.h
deleted file mode 100644
index cf3d043..0000000
--- a/payload_generator/metadata.h
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_METADATA_H_
-#define UPDATE_ENGINE_PAYLOAD_GENERATOR_METADATA_H_
-
-#include <string>
-#include <vector>
-
-#include "update_engine/payload_generator/delta_diff_generator.h"
-#include "update_engine/payload_generator/graph_types.h"
-
-namespace chromeos_update_engine {
-
-class Metadata {
- public:
- // Reads metadata from old image and new image and determines
- // the smallest way to encode the metadata for the diff.
- // If there's no change in the metadata, it creates a MOVE
- // operation. If there is a change, the smallest of REPLACE, REPLACE_BZ,
- // or BSDIFF wins. It writes the diff to data_fd and updates data_file_size
- // accordingly. It also adds the required operation to the graph and adds the
- // metadata extents to blocks.
- // Returns true on success.
- static bool DeltaReadMetadata(Graph* graph,
- std::vector<DeltaDiffGenerator::Block>* blocks,
- const std::string& old_image,
- const std::string& new_image,
- int data_fd,
- off_t* data_file_size);
-
- private:
- // This should never be constructed.
- DISALLOW_IMPLICIT_CONSTRUCTORS(Metadata);
-};
-
-}; // namespace chromeos_update_engine
-
-#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_METADATA_H_
diff --git a/payload_generator/metadata_unittest.cc b/payload_generator/metadata_unittest.cc
deleted file mode 100644
index 492fd69..0000000
--- a/payload_generator/metadata_unittest.cc
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "update_engine/payload_generator/metadata.h"
-
-#include <ext2fs/ext2fs.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include <string>
-#include <vector>
-
-#include <base/strings/string_util.h>
-#include <base/strings/stringprintf.h>
-#include <gtest/gtest.h>
-
-#include "update_engine/payload_generator/delta_diff_generator.h"
-#include "update_engine/payload_generator/graph_types.h"
-#include "update_engine/test_utils.h"
-#include "update_engine/utils.h"
-
-using chromeos_update_engine::test_utils::CreateEmptyExtImageAtPath;
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-typedef DeltaDiffGenerator::Block Block;
-
-class MetadataTest : public ::testing::Test {
-};
-
-TEST_F(MetadataTest, RunAsRootReadMetadataDissimilarFileSystems) {
- string a_img, b_img;
- EXPECT_TRUE(utils::MakeTempFile("a_img.XXXXXX", &a_img, nullptr));
- ScopedPathUnlinker a_img_unlinker(a_img);
- EXPECT_TRUE(utils::MakeTempFile("b_img.XXXXXX", &b_img, nullptr));
- ScopedPathUnlinker b_img_unlinker(b_img);
-
- CreateEmptyExtImageAtPath(a_img, 10485759, 4096);
- CreateEmptyExtImageAtPath(b_img, 11534336, 4096);
-
- Graph graph;
- vector<Block> blocks;
- EXPECT_TRUE(Metadata::DeltaReadMetadata(&graph,
- &blocks,
- a_img,
- b_img,
- 0,
- nullptr));
- EXPECT_EQ(graph.size(), 0);
-
- CreateEmptyExtImageAtPath(a_img, 10485759, 4096);
- CreateEmptyExtImageAtPath(b_img, 10485759, 8192);
-
- graph.clear();
- blocks.clear();
- EXPECT_TRUE(Metadata::DeltaReadMetadata(&graph,
- &blocks,
- a_img,
- b_img,
- 0,
- nullptr));
- EXPECT_EQ(graph.size(), 0);
-}
-
-TEST_F(MetadataTest, RunAsRootReadMetadata) {
- string a_img, b_img, data_file;
- EXPECT_TRUE(utils::MakeTempFile("a_img.XXXXXX", &a_img, nullptr));
- ScopedPathUnlinker a_img_unlinker(a_img);
- EXPECT_TRUE(utils::MakeTempFile("b_img.XXXXXX", &b_img, nullptr));
- ScopedPathUnlinker b_img_unlinker(b_img);
- EXPECT_TRUE(utils::MakeTempFile("data_file.XXXXXX", &data_file, nullptr));
- ScopedPathUnlinker data_file_unlinker(data_file);
-
- const size_t image_size = (256 * 1024 * 1024); // Enough for 2 block groups
- const int block_size = 4096;
- CreateEmptyExtImageAtPath(a_img, image_size, block_size);
-
- // Create a file large enough to create an indirect block
- {
- string a_img_mnt;
- test_utils::ScopedLoopMounter a_img_mount(a_img, &a_img_mnt, 0);
- test_utils::System(
- base::StringPrintf(
- "dd if=/dev/zero of=%s/test_file bs=%d count=%d status=none",
- a_img_mnt.c_str(), block_size, EXT2_NDIR_BLOCKS + 1));
- }
-
- test_utils::System(
- base::StringPrintf("cp %s %s", a_img.c_str(), b_img.c_str()));
-
- int fd = open(data_file.c_str(), O_RDWR | O_CREAT, S_IRWXU);
- EXPECT_NE(fd, -1);
- ScopedFdCloser fd_closer(&fd);
-
- Graph graph;
- vector<Block> blocks(image_size / block_size);
- off_t data_file_size;
- EXPECT_TRUE(Metadata::DeltaReadMetadata(&graph,
- &blocks,
- a_img,
- b_img,
- fd,
- &data_file_size));
-
- // There are 12 metadata that we look for:
- // - Block group 0 metadata (superblock, group descriptor, bitmaps, etc)
- // - Chunk 0, 1, 2, 3
- // - Block group 1 metadata
- // - Chunk 0, 1, 2, 3
- // - Root directory (inode 2)
- // - Journal (inode 8)
- // - lost+found directory (inode 11)
- // - test_file indirect block (inode 12)
- struct {
- string metadata_name;
- off_t start_block; // Set to -1 to skip start block verification
- off_t num_blocks; // Set to -1 to skip num blocks verification
- } exp_results[] =
- {{"<rootfs-bg-0-0-metadata>", 0, 104},
- {"<rootfs-bg-0-1-metadata>", 104, 104},
- {"<rootfs-bg-0-2-metadata>", 208, 104},
- {"<rootfs-bg-0-3-metadata>", 312, 104},
- {"<rootfs-bg-0-4-metadata>", 416, 104},
- {"<rootfs-bg-0-5-metadata>", 520, 104},
- {"<rootfs-bg-0-6-metadata>", 624, 104},
- {"<rootfs-bg-0-7-metadata>", 728, 104},
- {"<rootfs-bg-0-8-metadata>", 832, 104},
- {"<rootfs-bg-0-9-metadata>", 936, 107},
- {"<rootfs-bg-1-0-metadata>", 32768, 104},
- {"<rootfs-bg-1-1-metadata>", 32872, 104},
- {"<rootfs-bg-1-2-metadata>", 32976, 104},
- {"<rootfs-bg-1-3-metadata>", 33080, 104},
- {"<rootfs-bg-1-4-metadata>", 33184, 104},
- {"<rootfs-bg-1-5-metadata>", 33288, 104},
- {"<rootfs-bg-1-6-metadata>", 33392, 104},
- {"<rootfs-bg-1-7-metadata>", 33496, 104},
- {"<rootfs-bg-1-8-metadata>", 33600, 104},
- {"<rootfs-bg-1-9-metadata>", 33704, 107},
- {"<rootfs-inode-2-metadata>", -1, 1},
- {"<rootfs-inode-8-metadata>", -1, 4101},
- {"<rootfs-inode-11-metadata>", -1, 4},
- {"<rootfs-inode-12-metadata>", -1, 1}};
-
- int num_exp_results = sizeof(exp_results) / sizeof(exp_results[0]);
- EXPECT_EQ(graph.size(), num_exp_results);
-
- for (int i = 0; i < num_exp_results; i++) {
- Vertex& vertex = graph[i];
- DeltaArchiveManifest_InstallOperation& op = vertex.op;
-
- EXPECT_STRCASEEQ(vertex.file_name.c_str(),
- exp_results[i].metadata_name.c_str());
-
- EXPECT_EQ(op.src_extents().size(), op.dst_extents().size());
- for (int e = 0; e < op.src_extents().size(); e++) {
- EXPECT_EQ(op.src_extents(e).start_block(),
- op.dst_extents(e).start_block());
- EXPECT_EQ(op.src_extents(e).num_blocks(),
- op.dst_extents(e).num_blocks());
- }
-
- if (exp_results[i].start_block != -1) {
- EXPECT_EQ(op.src_extents(0).start_block(), exp_results[i].start_block);
- }
-
- if (exp_results[i].num_blocks != -1) {
- EXPECT_EQ(op.src_extents(0).num_blocks(), exp_results[i].num_blocks);
- }
- }
-}
-
-} // namespace chromeos_update_engine
diff --git a/test_utils.cc b/test_utils.cc
index 66db978..afc073c 100644
--- a/test_utils.cc
+++ b/test_utils.cc
@@ -5,6 +5,7 @@
#include "update_engine/test_utils.h"
#include <attr/xattr.h>
+#include <dirent.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
@@ -22,7 +23,6 @@
#include <base/strings/string_util.h>
#include "update_engine/file_writer.h"
-#include "update_engine/payload_generator/filesystem_iterator.h"
#include "update_engine/utils.h"
using base::StringPrintf;
@@ -232,51 +232,6 @@
}
}
-void VerifyAllPaths(const string& parent, set<string> expected_paths) {
- FilesystemIterator iter(parent, set<string>());
- ino_t test_ino = 0;
- ino_t testlink_ino = 0;
- while (!iter.IsEnd()) {
- string path = iter.GetFullPath();
- EXPECT_TRUE(expected_paths.find(path) != expected_paths.end()) << path;
- EXPECT_EQ(1, expected_paths.erase(path));
- if (EndsWith(path, "/hi", true) ||
- EndsWith(path, "/hello", true) ||
- EndsWith(path, "/test", true) ||
- EndsWith(path, "/testlink", true)) {
- EXPECT_TRUE(S_ISREG(iter.GetStat().st_mode));
- if (EndsWith(path, "/test", true))
- test_ino = iter.GetStat().st_ino;
- else if (EndsWith(path, "/testlink", true))
- testlink_ino = iter.GetStat().st_ino;
- } else if (EndsWith(path, "/some_dir", true) ||
- EndsWith(path, "/empty_dir", true) ||
- EndsWith(path, "/mnt", true) ||
- EndsWith(path, "/lost+found", true) ||
- parent == path) {
- EXPECT_TRUE(S_ISDIR(iter.GetStat().st_mode));
- } else if (EndsWith(path, "/fifo", true)) {
- EXPECT_TRUE(S_ISFIFO(iter.GetStat().st_mode));
- } else if (EndsWith(path, "/cdev", true)) {
- EXPECT_TRUE(S_ISCHR(iter.GetStat().st_mode));
- } else if (EndsWith(path, "/sym", true)) {
- EXPECT_TRUE(S_ISLNK(iter.GetStat().st_mode));
- } else {
- LOG(INFO) << "got non hardcoded path: " << path;
- }
- iter.Increment();
- }
- EXPECT_EQ(testlink_ino, test_ino);
- EXPECT_NE(0, test_ino);
- EXPECT_FALSE(iter.IsErr());
- EXPECT_TRUE(expected_paths.empty());
- if (!expected_paths.empty()) {
- for (const string& path : expected_paths) {
- LOG(INFO) << "extra path: " << path;
- }
- }
-}
-
ScopedLoopMounter::ScopedLoopMounter(const string& file_path,
string* mnt_path,
unsigned long flags) { // NOLINT - long
diff --git a/test_utils.h b/test_utils.h
index 5453d27..726deb4 100644
--- a/test_utils.h
+++ b/test_utils.h
@@ -88,13 +88,6 @@
void CreateExtImageAtPath(const std::string& path,
std::vector<std::string>* out_paths);
-// Verifies that for each path in paths, it exists in the filesystem under
-// parent. Also, verifies that no additional paths are present under parent.
-// Also tests properties of various files created by CreateExtImageAtPath().
-// Intentionally copies expected_paths.
-void VerifyAllPaths(const std::string& parent,
- std::set<std::string> expected_paths);
-
// Class to unmount FS when object goes out of scope
class ScopedFilesystemUnmounter {
public:
diff --git a/update_engine.gyp b/update_engine.gyp
index 1731757..3ab459c 100644
--- a/update_engine.gyp
+++ b/update_engine.gyp
@@ -280,14 +280,11 @@
'payload_generator/cycle_breaker.cc',
'payload_generator/delta_diff_generator.cc',
'payload_generator/ext2_filesystem.cc',
- 'payload_generator/extent_mapper.cc',
'payload_generator/extent_utils.cc',
- 'payload_generator/filesystem_iterator.cc',
'payload_generator/full_update_generator.cc',
'payload_generator/graph_types.cc',
'payload_generator/graph_utils.cc',
'payload_generator/inplace_generator.cc',
- 'payload_generator/metadata.cc',
'payload_generator/payload_generation_config.cc',
'payload_generator/payload_signer.cc',
'payload_generator/raw_filesystem.cc',
@@ -389,14 +386,11 @@
'payload_generator/cycle_breaker_unittest.cc',
'payload_generator/delta_diff_generator_unittest.cc',
'payload_generator/ext2_filesystem_unittest.cc',
- 'payload_generator/extent_mapper_unittest.cc',
'payload_generator/extent_utils_unittest.cc',
'payload_generator/fake_filesystem.cc',
- 'payload_generator/filesystem_iterator_unittest.cc',
'payload_generator/full_update_generator_unittest.cc',
'payload_generator/graph_utils_unittest.cc',
'payload_generator/inplace_generator_unittest.cc',
- 'payload_generator/metadata_unittest.cc',
'payload_generator/payload_signer_unittest.cc',
'payload_generator/tarjan_unittest.cc',
'payload_generator/topological_sort_unittest.cc',
