Skip to content

Generate block based OTA for product partitions #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: c10
Choose a base branch
from
Open

Generate block based OTA for product partitions #3

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
99b753f
Generate block based OTA for product partitions
PCMasterRace007 Jun 13, 2020
0a3c570
Merge branch 'c10' into ImpulsiveCoder007-dynamicpatch-1-1
PCMasterRace007 Jun 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
259 changes: 131 additions & 128 deletions tools/releasetools/ota_from_target_files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@

from __future__ import print_function

import collections
import logging
import multiprocessing
import os.path
Expand Down Expand Up @@ -825,6 +826,92 @@ def CopyInstallTools(output_zip):
install_target = os.path.join("install", os.path.relpath(root, install_path), f)
output_zip.write(install_source, install_target)

def GetBlockDifferences(target_zip, source_zip, target_info, source_info,
device_specific):
"""Returns a ordered dict of block differences with partition name as key."""

def GetIncrementalBlockDifferenceForPartition(name):
if not HasPartition(source_zip, name):
raise RuntimeError("can't generate incremental that adds {}".format(name))

partition_src = common.GetUserImage(name, OPTIONS.source_tmp, source_zip,
info_dict=source_info,
allow_shared_blocks=allow_shared_blocks)

hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator(
name, 4096, target_info)
partition_tgt = common.GetUserImage(name, OPTIONS.target_tmp, target_zip,
info_dict=target_info,
allow_shared_blocks=allow_shared_blocks,
hashtree_info_generator=
hashtree_info_generator)

# Check the first block of the source system partition for remount R/W only
# if the filesystem is ext4.
partition_source_info = source_info["fstab"]["/" + name]
check_first_block = partition_source_info.fs_type == "ext4"
# Disable using imgdiff for squashfs. 'imgdiff -z' expects input files to be
# in zip formats. However with squashfs, a) all files are compressed in LZ4;
# b) the blocks listed in block map may not contain all the bytes for a
# given file (because they're rounded to be 4K-aligned).
partition_target_info = target_info["fstab"]["/" + name]
disable_imgdiff = (partition_source_info.fs_type == "squashfs" or
partition_target_info.fs_type == "squashfs")
return common.BlockDifference(name, partition_src, partition_tgt,
check_first_block,
version=blockimgdiff_version,
disable_imgdiff=disable_imgdiff)

if source_zip:
# See notes in common.GetUserImage()
allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or
target_info.get('ext4_share_dup_blocks') == "true")
blockimgdiff_version = max(
int(i) for i in target_info.get(
"blockimgdiff_versions", "1").split(","))
assert blockimgdiff_version >= 3

block_diff_dict = collections.OrderedDict()
partition_names = ["system", "vendor", "product", "odm", "system_ext"]
for partition in partition_names:
if not HasPartition(target_zip, partition):
continue
# Full OTA update.
if not source_zip:
tgt = common.GetUserImage(partition, OPTIONS.input_tmp, target_zip,
info_dict=target_info,
reset_file_map=True)
block_diff_dict[partition] = common.BlockDifference(partition, tgt,
src=None)
# Incremental OTA update.
else:
block_diff_dict[partition] = GetIncrementalBlockDifferenceForPartition(
partition)
assert "system" in block_diff_dict

# Get the block diffs from the device specific script. If there is a
# duplicate block diff for a partition, ignore the diff in the generic script
# and use the one in the device specific script instead.
if source_zip:
device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences()
function_name = "IncrementalOTA_GetBlockDifferences"
else:
device_specific_diffs = device_specific.FullOTA_GetBlockDifferences()
function_name = "FullOTA_GetBlockDifferences"

if device_specific_diffs:
assert all(isinstance(diff, common.BlockDifference)
for diff in device_specific_diffs), \
"{} is not returning a list of BlockDifference objects".format(
function_name)
for diff in device_specific_diffs:
if diff.partition in block_diff_dict:
logger.warning("Duplicate block difference found. Device specific block"
" diff for partition '%s' overrides the one in generic"
" script.", diff.partition)
block_diff_dict[diff.partition] = diff

return block_diff_dict

def WriteFullOTAPackage(input_zip, output_file):
target_info = BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts)
Expand Down Expand Up @@ -867,6 +954,13 @@ def WriteFullOTAPackage(input_zip, output_file):
target_info.WriteDeviceAssertions(script, OPTIONS.oem_no_mount)
device_specific.FullOTA_Assertions()

block_diff_dict = GetBlockDifferences(target_zip=input_zip, source_zip=None,
target_info=target_info,
source_info=None,
device_specific=device_specific)



# Two-step package strategy (in chronological order, which is *not*
# the order in which the generated script has things):
#
Expand Down Expand Up @@ -936,8 +1030,8 @@ def WriteFullOTAPackage(input_zip, output_file):

script.Print("BackupTool Work Done");

system_progress = 0.75

#system_progress = 0.75
script.Print("**************************************");
script.Print("** ___ _ _ ___ __ **");
script.Print("** / __\___ | | |_ /___\/ _\ **");
Expand Down Expand Up @@ -975,51 +1069,34 @@ def WriteFullOTAPackage(input_zip, output_file):
script.Print(" Manufacturer : %s"%(manufacturer));
script.Print("***********************************************");

# All other partitions as well as the data wipe use 10% of the progress, and
# the update of the system partition takes the remaining progress.
system_progress = 0.9 - (len(block_diff_dict) - 1) * 0.1

if OPTIONS.wipe_user_data:
system_progress -= 0.1
if HasVendorPartition(input_zip):
system_progress -= 0.1

script.ShowProgress(system_progress, 0)

def GetBlockDifference(partition):
# Full OTA is done as an "incremental" against an empty source image. This
# has the effect of writing new data from the package to the entire
# partition, but lets us reuse the updater code that writes incrementals to
# do it.
tgt = common.GetUserImage(partition, OPTIONS.input_tmp, input_zip,
info_dict=target_info,
reset_file_map=True)
diff = common.BlockDifference(partition, tgt, src=None)
return diff

device_specific_diffs = device_specific.FullOTA_GetBlockDifferences()
if device_specific_diffs:
assert all(isinstance(diff, common.BlockDifference)
for diff in device_specific_diffs), \
"FullOTA_GetBlockDifferences is not returning a list of " \
"BlockDifference objects"

progress_dict = dict()
block_diffs = [GetBlockDifference("system")]
if HasVendorPartition(input_zip):
block_diffs.append(GetBlockDifference("vendor"))
progress_dict["vendor"] = 0.1
if device_specific_diffs:
block_diffs += device_specific_diffs

progress_dict = {partition: 0.1 for partition in block_diff_dict}
progress_dict["system"] = system_progress

if target_info.get('use_dynamic_partitions') == "true":
# Use empty source_info_dict to indicate that all partitions / groups must
# be re-added.
dynamic_partitions_diff = common.DynamicPartitionsDifference(
info_dict=OPTIONS.info_dict,
block_diffs=block_diffs,
block_diffs=block_diff_dict.values(),
progress_dict=progress_dict,
build_without_vendor=(not HasVendorPartition(input_zip)))
# In DynamicPartitionsDifference, we have no direct information
# whether the package is FullOTA or not, so we should pass the
# build_without_vendor parameter here instead of detecting it
# automatically in DynamicPartitionsDifference.
# (A non-FullOTA build may also contain no vendor image if there
# is no change)
build_without_vendor=('vendor' not in block_diff_dict.keys()))
dynamic_partitions_diff.WriteScript(script, output_zip,
write_verify_script=OPTIONS.verify)
else:
for block_diff in block_diffs:
for block_diff in block_diff_dict.values():
block_diff.WriteScript(script, output_zip,
progress=progress_dict.get(block_diff.partition),
write_verify_script=OPTIONS.verify)
Expand All @@ -1035,10 +1112,9 @@ def GetBlockDifference(partition):
script.ShowProgress(0.02, 10)
script.RunBackup("restore", sysmount)

script.ShowProgress(0.05, 5)
script.WriteRawImage("/boot", "boot.img")

script.ShowProgress(0.2, 10)
script.ShowProgress(0.1, 10)
device_specific.FullOTA_InstallEnd()

script.Print("Welcome to ColtOS : A Custom ROM by TeamColt");
Expand Down Expand Up @@ -1600,66 +1676,11 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
target_recovery = common.GetBootableImage(
"/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")

# See notes in common.GetUserImage()
allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or
target_info.get('ext4_share_dup_blocks') == "true")
system_src = common.GetUserImage("system", OPTIONS.source_tmp, source_zip,
info_dict=source_info,
allow_shared_blocks=allow_shared_blocks)

hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator(
"system", 4096, target_info)
system_tgt = common.GetUserImage("system", OPTIONS.target_tmp, target_zip,
info_dict=target_info,
allow_shared_blocks=allow_shared_blocks,
hashtree_info_generator=
hashtree_info_generator)

blockimgdiff_version = max(
int(i) for i in target_info.get("blockimgdiff_versions", "1").split(","))
assert blockimgdiff_version >= 3

# Check the first block of the source system partition for remount R/W only
# if the filesystem is ext4.
system_src_partition = source_info["fstab"]["/system"]
check_first_block = system_src_partition.fs_type == "ext4"
# Disable using imgdiff for squashfs. 'imgdiff -z' expects input files to be
# in zip formats. However with squashfs, a) all files are compressed in LZ4;
# b) the blocks listed in block map may not contain all the bytes for a given
# file (because they're rounded to be 4K-aligned).
system_tgt_partition = target_info["fstab"]["/system"]
disable_imgdiff = (system_src_partition.fs_type == "squashfs" or
system_tgt_partition.fs_type == "squashfs")
system_diff = common.BlockDifference("system", system_tgt, system_src,
check_first_block,
version=blockimgdiff_version,
disable_imgdiff=disable_imgdiff)

if HasVendorPartition(target_zip):
if not HasVendorPartition(source_zip):
raise RuntimeError("can't generate incremental that adds /vendor")
vendor_src = common.GetUserImage("vendor", OPTIONS.source_tmp, source_zip,
info_dict=source_info,
allow_shared_blocks=allow_shared_blocks)
hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator(
"vendor", 4096, target_info)
vendor_tgt = common.GetUserImage(
"vendor", OPTIONS.target_tmp, target_zip,
info_dict=target_info,
allow_shared_blocks=allow_shared_blocks,
hashtree_info_generator=hashtree_info_generator)

# Check first block of vendor partition for remount R/W only if
# disk type is ext4
vendor_partition = source_info["fstab"]["/vendor"]
check_first_block = vendor_partition.fs_type == "ext4"
disable_imgdiff = vendor_partition.fs_type == "squashfs"
vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src,
check_first_block,
version=blockimgdiff_version,
disable_imgdiff=disable_imgdiff)
else:
vendor_diff = None
block_diff_dict = GetBlockDifferences(target_zip=target_zip,
source_zip=source_zip,
target_info=target_info,
source_info=source_info,
device_specific=device_specific)

# Assertions (e.g. device properties check).
target_info.WriteDeviceAssertions(script, OPTIONS.oem_no_mount)
Expand Down Expand Up @@ -1723,12 +1744,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
WriteFingerprintAssertion(script, target_info, source_info)

# Check the required cache size (i.e. stashed blocks).
size = []
if system_diff:
size.append(system_diff.required_cache)
if vendor_diff:
size.append(vendor_diff.required_cache)

required_cache_sizes = [diff.required_cache for diff in
block_diff_dict.values()]
if updating_boot:
boot_type, boot_device = common.GetTypeAndDevice("/boot", source_info)
d = common.Difference(target_boot, source_boot)
Expand All @@ -1751,10 +1768,14 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
"{}:{}:{}:{}".format(
boot_type, boot_device, source_boot.size, source_boot.sha1))

size.append(target_boot.size)
required_cache_sizes.append(target_boot.size)

if size:
script.CacheFreeSpaceCheck(max(size))
if required_cache_sizes:
script.CacheFreeSpaceCheck(max(required_cache_sizes))

# Verify the existing partitions.
for diff in block_diff_dict.values():
diff.WriteVerifyScript(script, touched_blocks_only=True)

device_specific.IncrementalOTA_VerifyEnd()

Expand All @@ -1771,30 +1792,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
# Stage 3/3: Make changes.
script.Comment("Stage 3/3")

# Verify the existing partitions.
system_diff.WriteVerifyScript(script, touched_blocks_only=True)
if vendor_diff:
vendor_diff.WriteVerifyScript(script, touched_blocks_only=True)
device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences()
if device_specific_diffs:
assert all(isinstance(diff, common.BlockDifference)
for diff in device_specific_diffs), \
"IncrementalOTA_GetBlockDifferences is not returning a list of " \
"BlockDifference objects"
for diff in device_specific_diffs:
diff.WriteVerifyScript(script, touched_blocks_only=True)

script.Comment("---- start making changes here ----")

device_specific.IncrementalOTA_InstallBegin()

block_diffs = [system_diff]
progress_dict = {"system": 0.8 if vendor_diff else 0.9}
if vendor_diff:
block_diffs.append(vendor_diff)
progress_dict["vendor"] = 0.1
if device_specific_diffs:
block_diffs += device_specific_diffs
progress_dict = {partition: 0.1 for partition in block_diff_dict}
progress_dict["system"] = 1 - len(block_diff_dict) * 0.1

if OPTIONS.source_info_dict.get("use_dynamic_partitions") == "true":
if OPTIONS.target_info_dict.get("use_dynamic_partitions") != "true":
Expand All @@ -1803,12 +1806,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
dynamic_partitions_diff = common.DynamicPartitionsDifference(
info_dict=OPTIONS.target_info_dict,
source_info_dict=OPTIONS.source_info_dict,
block_diffs=block_diffs,
block_diffs=block_diff_dict.values(),
progress_dict=progress_dict)
dynamic_partitions_diff.WriteScript(
script, output_zip, write_verify_script=OPTIONS.verify)
else:
for block_diff in block_diffs:
for block_diff in block_diff_dict.values():
block_diff.WriteScript(script, output_zip,
progress=progress_dict.get(block_diff.partition),
write_verify_script=OPTIONS.verify)
Expand Down