diff options
| -rw-r--r-- | CleanSpec.mk | 3 | ||||
| -rw-r--r-- | core/android_soong_config_vars.mk | 17 | ||||
| -rw-r--r-- | core/config.mk | 5 | ||||
| -rw-r--r-- | core/definitions.mk | 13 | ||||
| -rw-r--r-- | core/main.mk | 5 | ||||
| -rw-r--r-- | core/product_config.rbc | 66 | ||||
| -rw-r--r-- | core/version_defaults.mk | 2 | ||||
| -rw-r--r-- | envsetup.sh | 10 | ||||
| -rw-r--r-- | rbesetup.sh | 7 | ||||
| -rw-r--r-- | target/board/generic_arm64/BoardConfig.mk | 2 | ||||
| -rw-r--r-- | target/board/generic_arm64/device.mk | 13 | ||||
| -rwxr-xr-x | target/board/generic_x86_64/device.mk | 4 | ||||
| -rw-r--r-- | target/product/generic_ramdisk.mk | 1 | ||||
| -rw-r--r-- | tests/run.rbc | 17 | ||||
| -rw-r--r-- | tools/fs_config/Android.mk | 65 | ||||
| -rwxr-xr-x | tools/releasetools/merge_target_files.py | 132 | ||||
| -rwxr-xr-x | tools/releasetools/ota_from_target_files.py | 2 | ||||
| -rw-r--r-- | tools/releasetools/test_merge_target_files.py | 4 | ||||
| -rw-r--r-- | tools/releasetools/test_ota_from_target_files.py | 2 |
19 files changed, 306 insertions, 64 deletions
diff --git a/CleanSpec.mk b/CleanSpec.mk index 3beadff9b0..e96735b051 100644 --- a/CleanSpec.mk +++ b/CleanSpec.mk @@ -757,6 +757,9 @@ $(call add-clean-step, rm -rf $(OUT_DIR)/soong/host/*/lib*/libconscrypt_openjdk_ # vendor-ramdisk renamed to vendor_ramdisk $(call add-clean-step, rm -rf $(PRODUCT_OUT)/vendor-ramdisk) +# Common R directory has been removed. +$(call add-clean-step, rm -rf $(OUT_DIR)/target/common/R) + # ************************************************ # NEWER CLEAN STEPS MUST BE AT THE END OF THE LIST # ************************************************ diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk index 6c39b2b499..64a6449e04 100644 --- a/core/android_soong_config_vars.mk +++ b/core/android_soong_config_vars.mk @@ -39,6 +39,10 @@ endif ifneq (,$(findstring .android.art,$(TARGET_BUILD_APPS))) # Build ART modules from source if they are listed in TARGET_BUILD_APPS. SOONG_CONFIG_art_module_source_build := true +else ifeq (,$(filter-out modules_% mainline_modules_%,$(TARGET_PRODUCT))) + # Always build from source for the module targets. This ought to be covered by + # the TARGET_BUILD_APPS check above, but there are test builds that don't set it. + SOONG_CONFIG_art_module_source_build := true else ifneq (,$(filter true,$(NATIVE_COVERAGE) $(CLANG_COVERAGE))) # Always build ART APEXes from source in coverage builds since the prebuilts # aren't built with instrumentation. @@ -51,6 +55,19 @@ else ifneq (,$(PRODUCT_FUCHSIA)) # Fuchsia picks out ART internal packages that aren't available in the # prebuilt. SOONG_CONFIG_art_module_source_build := true +else ifeq (,$(filter x86 x86_64,$(HOST_CROSS_ARCH))) + # We currently only provide prebuilts for x86 on host. This skips prebuilts in + # cuttlefish builds for ARM servers. + SOONG_CONFIG_art_module_source_build := true +else ifneq (,$(filter dex2oatds dex2oats,$(PRODUCT_HOST_PACKAGES))) + # Some products depend on host tools that aren't available as prebuilts. + SOONG_CONFIG_art_module_source_build := true +else ifeq (,$(filter com.google.android.art,$(PRODUCT_PACKAGES))) + # TODO(b/192006406): There is currently no good way to control which prebuilt + # APEX (com.google.android.art or com.android.art) gets picked for deapexing + # to provide dex jars for hiddenapi and dexpreopting. Instead the AOSP APEX is + # completely disabled, and we build from source for AOSP products. + SOONG_CONFIG_art_module_source_build := true else # This sets the default for building ART APEXes from source rather than # prebuilts (in packages/modules/ArtPrebuilt and prebuilt/module_sdk/art) in diff --git a/core/config.mk b/core/config.mk index e30ff3ff34..acdf15e3b1 100644 --- a/core/config.mk +++ b/core/config.mk @@ -445,6 +445,11 @@ endif ifneq ($(filter true,$(SOONG_ALLOW_MISSING_DEPENDENCIES)),) ALLOW_MISSING_DEPENDENCIES := true endif +# Mac builds default to ALLOW_MISSING_DEPENDENCIES, at least until the host +# tools aren't enabled by default for Mac. +ifeq ($(HOST_OS),darwin) + ALLOW_MISSING_DEPENDENCIES := true +endif .KATI_READONLY := ALLOW_MISSING_DEPENDENCIES TARGET_BUILD_USE_PREBUILT_SDKS := diff --git a/core/definitions.mk b/core/definitions.mk index c44e7180a6..0fd023abf5 100644 --- a/core/definitions.mk +++ b/core/definitions.mk @@ -1928,21 +1928,10 @@ endef # b/37750224 AAPT_ASAN_OPTIONS := ASAN_OPTIONS=detect_leaks=0 -# Search for generated R.java/Manifest.java in $1, copy the found R.java as $2. -# Also copy them to a central 'R' directory to make it easier to add the files to an IDE. +# Search for generated R.java in $1, copy the found R.java as $2. define find-generated-R.java -$(hide) for GENERATED_MANIFEST_FILE in `find $(1) \ - -name Manifest.java 2> /dev/null`; do \ - dir=`awk '/package/{gsub(/\./,"/",$$2);gsub(/;/,"",$$2);print $$2;exit}' $$GENERATED_MANIFEST_FILE`; \ - mkdir -p $(TARGET_COMMON_OUT_ROOT)/R/$$dir; \ - cp $$GENERATED_MANIFEST_FILE $(TARGET_COMMON_OUT_ROOT)/R/$$dir; \ - done; $(hide) for GENERATED_R_FILE in `find $(1) \ -name R.java 2> /dev/null`; do \ - dir=`awk '/package/{gsub(/\./,"/",$$2);gsub(/;/,"",$$2);print $$2;exit}' $$GENERATED_R_FILE`; \ - mkdir -p $(TARGET_COMMON_OUT_ROOT)/R/$$dir; \ - cp $$GENERATED_R_FILE $(TARGET_COMMON_OUT_ROOT)/R/$$dir \ - || exit 31; \ cp $$GENERATED_R_FILE $(2) || exit 32; \ done; @# Ensure that the target file is always created, i.e. also in case we did not diff --git a/core/main.mk b/core/main.mk index 0393beecdb..70c7dd2bd8 100644 --- a/core/main.mk +++ b/core/main.mk @@ -547,7 +547,12 @@ subdir_makefiles_total := $(words int $(subdir_makefiles) post finish) $(foreach mk,$(subdir_makefiles),$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] including $(mk) ...)$(eval include $(mk))) +# For an unbundled image, we can skip blueprint_tools because unbundled image +# aims to remove a large number framework projects from the manifest, the +# sources or dependencies for these tools may be missing from the tree. +ifeq (,$(TARGET_BUILD_UNBUNDLED_IMAGE)) droid_targets : blueprint_tools +endif endif # dont_bother diff --git a/core/product_config.rbc b/core/product_config.rbc index 111e759ae6..8e85c4b406 100644 --- a/core/product_config.rbc +++ b/core/product_config.rbc @@ -62,7 +62,8 @@ def __print_attr(attr, value): elif _options.format == "pretty": print(attr, "=", repr(value)) elif _options.format == "make": - print(attr, ":=", value) + # Trim all spacing to a single space + print(attr, ":=", _mkstrip(value)) else: fail("bad output format", _options.format) @@ -432,6 +433,66 @@ def _mkinfo(file, message = ""): """Prints info.""" print(message) + +def __mkpatsubst_word(parsed_pattern,parsed_subst, word): + (before, after) = parsed_pattern + if not word.startswith(before): + return word + if not word.endswith(after): + return word + if len(parsed_subst) < 2: + return parsed_subst[0] + return parsed_subst[0] + word[len(before):len(word) - len(after)] + parsed_subst[1] + + +def _mkpatsubst(pattern, replacement, s): + """Emulates Make's patsubst. + + Tokenizes `s` (unless it is already a list), and then performs a simple + wildcard substitution (in other words, `foo%bar` pattern is equivalent to + the regular expression `^foo(.*)bar$, and the first `%` in replacement is + $1 in regex terms). Escaping % is not supported + """ + if pattern.find("\\") >= 0: + fail("'\\' in pattern is not allowed") + parsed_pattern = pattern.split("%", 1) + words = s if type(s) == "list" else _mkstrip(s).split(" ") + if len(parsed_pattern) == 1: + out_words = [ replacement if x == pattern else x for x in words] + else: + parsed_replacement = replacement.split("%", 1) + out_words = [__mkpatsubst_word(parsed_pattern, parsed_replacement, x) for x in words] + return out_words if type(s) == "list" else " ".join(out_words) + + +def _mkstrip(s): + """Emulates Make's strip. + + That is, removes string's leading and trailing whitespace characters and + replaces any sequence of whitespace characters with with a single space. + """ + result = "" + was_space = False + for ch in s.strip().elems(): + is_space = ch.isspace() + if not is_space: + if was_space: + result += " " + result += ch + was_space = is_space + return result + +def _mksubst(old, new, s): + """Emulates Make's subst. + + Replaces each occurence of 'old' with 'new'. + If 's' is a list, applies substitution to each item. + """ + if type(s) == "list": + return [e.replace(old, new) for e in s] + return s.replace(old, new) + + def __get_options(): """Returns struct containing runtime global settings.""" settings = dict( @@ -478,7 +539,10 @@ rblf = struct( indirect = _indirect, mkinfo = _mkinfo, mkerror = _mkerror, + mkpatsubst = _mkpatsubst, mkwarning = _mkwarning, + mkstrip = _mkstrip, + mksubst = _mksubst, printvars = _printvars, product_configuration = _product_configuration, require_artifacts_in_path = _require_artifacts_in_path, diff --git a/core/version_defaults.mk b/core/version_defaults.mk index 6ba86dee15..8e6c306ea3 100644 --- a/core/version_defaults.mk +++ b/core/version_defaults.mk @@ -241,7 +241,7 @@ ifndef PLATFORM_SECURITY_PATCH # It must be of the form "YYYY-MM-DD" on production devices. # It must match one of the Android Security Patch Level strings of the Public Security Bulletins. # If there is no $PLATFORM_SECURITY_PATCH set, keep it empty. - PLATFORM_SECURITY_PATCH := 2021-06-05 + PLATFORM_SECURITY_PATCH := 2021-07-05 endif .KATI_READONLY := PLATFORM_SECURITY_PATCH diff --git a/envsetup.sh b/envsetup.sh index 4f9440e9fb..77b22477a6 100644 --- a/envsetup.sh +++ b/envsetup.sh @@ -1871,6 +1871,16 @@ function showcommands() { fi } +# Source necessary setup scripts needed to run the build with Remote Execution. +function source_rbe() { + local T=$(gettop) + + if [[ "x$USE_RBE" != "x" && "$USE_RBE" != "false" ]]; then + . $T/build/make/rbesetup.sh --skip-envsetup + fi +} + validate_current_shell source_vendorsetup +source_rbe addcompletions diff --git a/rbesetup.sh b/rbesetup.sh index ec39e6ef17..3b0e7cf7dc 100644 --- a/rbesetup.sh +++ b/rbesetup.sh @@ -24,8 +24,11 @@ function _source_env_setup_script() { } # This function needs to run first as the remaining defining functions may be -# using the envsetup.sh defined functions. -_source_env_setup_script || return +# using the envsetup.sh defined functions. Skip this part if this script is already +# being invoked from envsetup.sh. +if [[ "$1" != "--skip-envsetup" ]]; then + _source_env_setup_script || return +fi # This function prefixes the given command with appropriate variables needed # for the build to be executed with RBE. diff --git a/target/board/generic_arm64/BoardConfig.mk b/target/board/generic_arm64/BoardConfig.mk index 21b4065487..1133564619 100644 --- a/target/board/generic_arm64/BoardConfig.mk +++ b/target/board/generic_arm64/BoardConfig.mk @@ -56,6 +56,7 @@ include build/make/target/board/BoardConfigGsiCommon.mk include build/make/target/board/BoardConfigGkiCommon.mk BOARD_KERNEL-4.19-GZ_BOOTIMAGE_PARTITION_SIZE := 47185920 +BOARD_KERNEL-4.19-GZ-ALLSYMS_BOOTIMAGE_PARTITION_SIZE := 47185920 BOARD_KERNEL-5.4_BOOTIMAGE_PARTITION_SIZE := 67108864 BOARD_KERNEL-5.4-ALLSYMS_BOOTIMAGE_PARTITION_SIZE := 67108864 BOARD_KERNEL-5.4-GZ_BOOTIMAGE_PARTITION_SIZE := 47185920 @@ -82,6 +83,7 @@ BOARD_KERNEL_BINARIES := \ ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT))) BOARD_KERNEL_BINARIES += \ + kernel-4.19-gz-allsyms \ kernel-5.4-allsyms kernel-5.4-gz-allsyms kernel-5.4-lz4-allsyms \ kernel-5.10-allsyms kernel-5.10-gz-allsyms kernel-5.10-lz4-allsyms \ diff --git a/target/board/generic_arm64/device.mk b/target/board/generic_arm64/device.mk index 27dc158fbd..0064aecadc 100644 --- a/target/board/generic_arm64/device.mk +++ b/target/board/generic_arm64/device.mk @@ -33,12 +33,13 @@ $(call dist-for-goals, dist_files, kernel/prebuilts/mainline/arm64/prebuilt-info ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT))) PRODUCT_COPY_FILES += \ - kernel/prebuilts/5.4/arm64/kernel-5.4:kernel-5.4-allsyms \ - kernel/prebuilts/5.4/arm64/kernel-5.4-gz:kernel-5.4-gz-allsyms \ - kernel/prebuilts/5.4/arm64/kernel-5.4-lz4:kernel-5.4-lz4-allsyms \ - kernel/prebuilts/5.10/arm64/kernel-5.10:kernel-5.10-allsyms \ - kernel/prebuilts/5.10/arm64/kernel-5.10-gz:kernel-5.10-gz-allsyms \ - kernel/prebuilts/5.10/arm64/kernel-5.10-lz4:kernel-5.10-lz4-allsyms \ + kernel/prebuilts/4.19/arm64/kernel-4.19-gz-allsyms:kernel-4.19-gz-allsyms \ + kernel/prebuilts/5.4/arm64/kernel-5.4-allsyms:kernel-5.4-allsyms \ + kernel/prebuilts/5.4/arm64/kernel-5.4-gz-allsyms:kernel-5.4-gz-allsyms \ + kernel/prebuilts/5.4/arm64/kernel-5.4-lz4-allsyms:kernel-5.4-lz4-allsyms \ + kernel/prebuilts/5.10/arm64/kernel-5.10-allsyms:kernel-5.10-allsyms \ + kernel/prebuilts/5.10/arm64/kernel-5.10-gz-allsyms:kernel-5.10-gz-allsyms \ + kernel/prebuilts/5.10/arm64/kernel-5.10-lz4-allsyms:kernel-5.10-lz4-allsyms \ endif diff --git a/target/board/generic_x86_64/device.mk b/target/board/generic_x86_64/device.mk index e195bd3fad..f31a49155b 100755 --- a/target/board/generic_x86_64/device.mk +++ b/target/board/generic_x86_64/device.mk @@ -23,8 +23,8 @@ $(call dist-for-goals, dist_files, kernel/prebuilts/5.10/x86_64/prebuilt-info.tx ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT))) PRODUCT_COPY_FILES += \ - kernel/prebuilts/5.4/x86_64/kernel-5.4:kernel-5.4-allsyms \ - kernel/prebuilts/5.10/x86_64/kernel-5.10:kernel-5.10-allsyms \ + kernel/prebuilts/5.4/x86_64/kernel-5.4-allsyms:kernel-5.4-allsyms \ + kernel/prebuilts/5.10/x86_64/kernel-5.10-allsyms:kernel-5.10-allsyms \ endif diff --git a/target/product/generic_ramdisk.mk b/target/product/generic_ramdisk.mk index ae81329c68..80d34bea30 100644 --- a/target/product/generic_ramdisk.mk +++ b/target/product/generic_ramdisk.mk @@ -25,6 +25,7 @@ PRODUCT_PACKAGES += \ # Debug ramdisk PRODUCT_PACKAGES += \ + adb_debug.prop \ userdebug_plat_sepolicy.cil \ _my_paths := \ diff --git a/tests/run.rbc b/tests/run.rbc index b13f835bc1..4cda1807bb 100644 --- a/tests/run.rbc +++ b/tests/run.rbc @@ -1,4 +1,3 @@ - # Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +27,22 @@ def assert_eq(expected, actual): if expected != actual: fail("Expected %s, got %s" % (expected, actual)) +# Unit tests for non-trivial runtime functions +assert_eq("", rblf.mkstrip(" \n \t ")) +assert_eq("a b c", rblf.mkstrip(" a b \n c \t")) + +assert_eq("b1 b2", rblf.mksubst("a", "b", "a1 a2")) +assert_eq(["b1", "x2"], rblf.mksubst("a", "b", ["a1", "x2"])) + +assert_eq("ABcdYZ", rblf.mkpatsubst("ab%yz", "AB%YZ", "abcdyz")) +assert_eq("bcz", rblf.mkpatsubst("a%z", "A%Z", "bcz")) +assert_eq(["Ay", "Az"], rblf.mkpatsubst("a%", "A%", ["ay", "az"])) +assert_eq("AcZ bcz", rblf.mkpatsubst("a%z", "A%Z", "acz bcz")) +assert_eq("Abcd", rblf.mkpatsubst("a%", "A%", "abcd")) +assert_eq("abcZ", rblf.mkpatsubst("%z", "%Z", "abcz")) +assert_eq("azx b", rblf.mkpatsubst("az", "AZ", "azx b")) +assert_eq(["azx", "b"], rblf.mkpatsubst("az", "AZ", ["azx", "b"])) +assert_eq("ABC", rblf.mkpatsubst("abc", "ABC", "abc")) globals, config = rblf.product_configuration("test/device", init) assert_eq( diff --git a/tools/fs_config/Android.mk b/tools/fs_config/Android.mk index 10d25e04cc..63cb4eb8b1 100644 --- a/tools/fs_config/Android.mk +++ b/tools/fs_config/Android.mk @@ -27,7 +27,22 @@ endif system_android_filesystem_config := system/core/libcutils/include/private/android_filesystem_config.h system_capability_header := bionic/libc/kernel/uapi/linux/capability.h -# List of supported vendor, oem, odm, vendor_dlkm, odm_dlkm, product and system_ext Partitions +# Use snapshots if exist +vendor_android_filesystem_config := $(strip \ + $(if $(filter-out current,$(BOARD_VNDK_VERSION)), \ + $(SOONG_VENDOR_$(BOARD_VNDK_VERSION)_SNAPSHOT_DIR)/include/$(system_android_filesystem_config))) +ifeq (,$(wildcard $(vendor_android_filesystem_config))) +vendor_android_filesystem_config := $(system_android_filesystem_config) +endif + +vendor_capability_header := $(strip \ + $(if $(filter-out current,$(BOARD_VNDK_VERSION)), \ + $(SOONG_VENDOR_$(BOARD_VNDK_VERSION)_SNAPSHOT_DIR)/include/$(system_capability_header))) +ifeq (,$(wildcard $(vendor_capability_header))) +vendor_capability_header := $(system_capability_header) +endif + +# List of supported vendor, oem, odm, vendor_dlkm and odm_dlkm Partitions fs_config_generate_extra_partition_list := $(strip \ $(if $(BOARD_USES_VENDORIMAGE)$(BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE),vendor) \ $(if $(BOARD_USES_OEMIMAGE)$(BOARD_OEMIMAGE_FILE_SYSTEM_TYPE),oem) \ @@ -206,10 +221,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -232,10 +247,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -316,10 +331,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_ODM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -342,10 +357,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_ODM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -371,10 +386,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -397,10 +412,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -426,10 +441,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_ODM_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -452,10 +467,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_ODM_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ diff --git a/tools/releasetools/merge_target_files.py b/tools/releasetools/merge_target_files.py index 5e6c42d84e..c1fa9e7d32 100755 --- a/tools/releasetools/merge_target_files.py +++ b/tools/releasetools/merge_target_files.py @@ -78,6 +78,14 @@ Usage: merge_target_files [args] If provided, duplicate APK/APEX keys are ignored and the value from the framework is used. + --rebuild-sepolicy + If provided, rebuilds odm.img or vendor.img to include merged sepolicy + files. If odm is present then odm is preferred. + + --vendor-otatools otatools.zip + If provided, use this otatools.zip when recompiling the odm or vendor + image to include sepolicy. + --keep-tmp Keep tempoary files for debugging purposes. """ @@ -129,6 +137,8 @@ OPTIONS.output_super_empty = None OPTIONS.rebuild_recovery = False # TODO(b/150582573): Remove this option. OPTIONS.allow_duplicate_apkapex_keys = False +OPTIONS.vendor_otatools = None +OPTIONS.rebuild_sepolicy = False OPTIONS.keep_tmp = False # In an item list (framework or vendor), we may see entries that select whole @@ -666,7 +676,7 @@ def copy_file_contexts(framework_target_files_dir, vendor_target_files_dir, os.path.join(output_target_files_dir, 'META', 'vendor_file_contexts.bin')) -def compile_split_sepolicy(product_out, partition_map, output_policy): +def compile_split_sepolicy(product_out, partition_map): """Uses secilc to compile a split sepolicy file. Depends on various */etc/selinux/* and */etc/vintf/* files within partitions. @@ -674,7 +684,6 @@ def compile_split_sepolicy(product_out, partition_map, output_policy): Args: product_out: PRODUCT_OUT directory, containing partition directories. partition_map: A map of partition name -> relative path within product_out. - output_policy: The name of the output policy created by secilc. Returns: A command list that can be executed to create the compiled sepolicy. @@ -709,7 +718,7 @@ def compile_split_sepolicy(product_out, partition_map, output_policy): # Use the same flags and arguments as selinux.cpp OpenSplitPolicy(). cmd = ['secilc', '-m', '-M', 'true', '-G', '-N'] cmd.extend(['-c', kernel_sepolicy_version]) - cmd.extend(['-o', output_policy]) + cmd.extend(['-o', os.path.join(product_out, 'META/combined_sepolicy')]) cmd.extend(['-f', '/dev/null']) required_policy_files = ( @@ -747,7 +756,8 @@ def validate_merged_apex_info(output_target_files_dir, partitions): Depends on the <partition>/apex/* APEX files within partitions. Args: - output_target_files_dir: Output directory containing merged partition directories. + output_target_files_dir: Output directory containing merged partition + directories. partitions: A list of all the partitions in the output directory. Raises: @@ -965,6 +975,92 @@ def generate_images(target_files_dir, rebuild_recovery): add_img_to_target_files.main(add_img_args) +def rebuild_image_with_sepolicy(target_files_dir, + vendor_otatools=None, + vendor_target_files=None): + """Rebuilds odm.img or vendor.img to include merged sepolicy files. + + If odm is present then odm is preferred -- otherwise vendor is used. + + Args: + target_files_dir: Path to the extracted merged target-files package. + vendor_otatools: If not None, path to an otatools.zip from the vendor build + that is used when recompiling the image. + vendor_target_files: Expected if vendor_otatools is not None. Path to the + vendor target-files zip. + """ + partition = 'vendor' + if os.path.exists(os.path.join(target_files_dir, 'ODM')) or os.path.exists( + os.path.join(target_files_dir, 'IMAGES/odm.img')): + partition = 'odm' + partition_img = '{}.img'.format(partition) + + logger.info('Recompiling %s using the merged sepolicy files.', partition_img) + + # Copy the combined SEPolicy file and framework hashes to the image that is + # being rebuilt. + def copy_selinux_file(input_path, output_filename): + shutil.copy( + os.path.join(target_files_dir, input_path), + os.path.join(target_files_dir, partition.upper(), 'etc/selinux', + output_filename)) + + copy_selinux_file('META/combined_sepolicy', 'precompiled_sepolicy') + copy_selinux_file('SYSTEM/etc/selinux/plat_sepolicy_and_mapping.sha256', + 'precompiled_sepolicy.plat_sepolicy_and_mapping.sha256') + copy_selinux_file( + 'SYSTEM_EXT/etc/selinux/system_ext_sepolicy_and_mapping.sha256', + 'precompiled_sepolicy.system_ext_sepolicy_and_mapping.sha256') + copy_selinux_file('PRODUCT/etc/selinux/product_sepolicy_and_mapping.sha256', + 'precompiled_sepolicy.product_sepolicy_and_mapping.sha256') + + if not vendor_otatools: + # Remove the partition from the merged target-files archive. It will be + # rebuilt later automatically by generate_images(). + os.remove(os.path.join(target_files_dir, 'IMAGES', partition_img)) + else: + # TODO(b/192253131): Remove the need for vendor_otatools by fixing + # backwards-compatibility issues when compiling images on R from S+. + if not vendor_target_files: + raise ValueError( + 'Expected vendor_target_files if vendor_otatools is not None.') + logger.info( + '%s recompilation will be performed using the vendor otatools.zip', + partition_img) + + # Unzip the vendor build's otatools.zip and target-files archive. + vendor_otatools_dir = common.MakeTempDir( + prefix='merge_target_files_vendor_otatools_') + vendor_target_files_dir = common.MakeTempDir( + prefix='merge_target_files_vendor_target_files_') + common.UnzipToDir(vendor_otatools, vendor_otatools_dir) + common.UnzipToDir(vendor_target_files, vendor_target_files_dir) + + # Copy the partition contents from the merged target-files archive to the + # vendor target-files archive. + shutil.rmtree(os.path.join(vendor_target_files_dir, partition.upper())) + shutil.copytree( + os.path.join(target_files_dir, partition.upper()), + os.path.join(vendor_target_files_dir, partition.upper())) + + # Delete then rebuild the partition. + os.remove(os.path.join(vendor_target_files_dir, 'IMAGES', partition_img)) + rebuild_partition_command = [ + os.path.join(vendor_otatools_dir, 'bin', 'add_img_to_target_files'), + '--verbose', + '--add_missing', + vendor_target_files_dir, + ] + logger.info('Recompiling %s: %s', partition_img, + ' '.join(rebuild_partition_command)) + common.RunAndCheckOutput(rebuild_partition_command, verbose=True) + + # Move the newly-created image to the merged target files dir. + shutil.move( + os.path.join(vendor_target_files_dir, 'IMAGES', partition_img), + os.path.join(target_files_dir, 'IMAGES', partition_img)) + + def generate_super_empty_image(target_dir, output_super_empty): """Generates super_empty image from target package. @@ -1049,7 +1145,8 @@ def merge_target_files(temp_dir, framework_target_files, framework_item_list, framework_misc_info_keys, vendor_target_files, vendor_item_list, output_target_files, output_dir, output_item_list, output_ota, output_img, - output_super_empty, rebuild_recovery): + output_super_empty, rebuild_recovery, vendor_otatools, + rebuild_sepolicy): """Merges two target files packages together. This function takes framework and vendor target files packages as input, @@ -1085,6 +1182,9 @@ def merge_target_files(temp_dir, framework_target_files, framework_item_list, merged target files package and saves it at this path. rebuild_recovery: If true, rebuild the recovery patch used by non-A/B devices and write it to the system image. + vendor_otatools: Path to an otatools zip used for recompiling vendor images. + rebuild_sepolicy: If true, rebuild odm.img (if target uses ODM) or + vendor.img using a merged precompiled_sepolicy file. """ logger.info('starting: merge framework %s and vendor %s into output %s', @@ -1137,14 +1237,14 @@ def merge_target_files(temp_dir, framework_target_files, framework_item_list, partition_map=filtered_partitions) # Check that the split sepolicy from the multiple builds can compile. - split_sepolicy_cmd = compile_split_sepolicy( - product_out=output_target_files_temp_dir, - partition_map=filtered_partitions, - output_policy=os.path.join(output_target_files_temp_dir, - 'META/combined.policy')) + split_sepolicy_cmd = compile_split_sepolicy(output_target_files_temp_dir, + filtered_partitions) logger.info('Compiling split sepolicy: %s', ' '.join(split_sepolicy_cmd)) common.RunAndCheckOutput(split_sepolicy_cmd) - # TODO(b/178864050): Run tests on the combined.policy file. + # Include the compiled policy in an image if requested. + if rebuild_sepolicy: + rebuild_image_with_sepolicy(output_target_files_temp_dir, vendor_otatools, + vendor_target_files) # Run validation checks on the pre-installed APEX files. validate_merged_apex_info(output_target_files_temp_dir, partition_map.keys()) @@ -1261,6 +1361,10 @@ def main(): OPTIONS.rebuild_recovery = True elif o == '--allow-duplicate-apkapex-keys': OPTIONS.allow_duplicate_apkapex_keys = True + elif o == '--vendor-otatools': + OPTIONS.vendor_otatools = a + elif o == '--rebuild-sepolicy': + OPTIONS.rebuild_sepolicy = True elif o == '--keep-tmp': OPTIONS.keep_tmp = True else: @@ -1289,6 +1393,8 @@ def main(): 'output-super-empty=', 'rebuild_recovery', 'allow-duplicate-apkapex-keys', + 'vendor-otatools=', + 'rebuild-sepolicy', 'keep-tmp', ], extra_option_handler=option_handler) @@ -1342,7 +1448,9 @@ def main(): output_ota=OPTIONS.output_ota, output_img=OPTIONS.output_img, output_super_empty=OPTIONS.output_super_empty, - rebuild_recovery=OPTIONS.rebuild_recovery), OPTIONS.keep_tmp) + rebuild_recovery=OPTIONS.rebuild_recovery, + vendor_otatools=OPTIONS.vendor_otatools, + rebuild_sepolicy=OPTIONS.rebuild_sepolicy), OPTIONS.keep_tmp) if __name__ == '__main__': diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index bf0b8f123e..8face66a2e 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -530,6 +530,8 @@ class StreamingPropertyFiles(PropertyFiles): 'payload_properties.txt', ) self.optional = ( + # apex_info.pb isn't directly used in the update flow + 'apex_info.pb', # care_map is available only if dm-verity is enabled. 'care_map.pb', 'care_map.txt', diff --git a/tools/releasetools/test_merge_target_files.py b/tools/releasetools/test_merge_target_files.py index 4f61472571..835edab713 100644 --- a/tools/releasetools/test_merge_target_files.py +++ b/tools/releasetools/test_merge_target_files.py @@ -265,10 +265,10 @@ class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase): 'system': 'system', 'product': 'product', 'vendor': 'vendor', - }, os.path.join(product_out_dir, 'policy')) + }) self.assertEqual(' '.join(cmd), ('secilc -m -M true -G -N -c 30 ' - '-o {OTP}/policy -f /dev/null ' + '-o {OTP}/META/combined_sepolicy -f /dev/null ' '{OTP}/system/etc/selinux/plat_sepolicy.cil ' '{OTP}/system/etc/selinux/mapping/30.0.cil ' '{OTP}/vendor/etc/selinux/vendor_sepolicy.cil ' diff --git a/tools/releasetools/test_ota_from_target_files.py b/tools/releasetools/test_ota_from_target_files.py index 51def30629..11cfee185a 100644 --- a/tools/releasetools/test_ota_from_target_files.py +++ b/tools/releasetools/test_ota_from_target_files.py @@ -863,6 +863,7 @@ class StreamingPropertyFilesTest(PropertyFilesTestCase): property_files.required) self.assertEqual( ( + 'apex_info.pb', 'care_map.pb', 'care_map.txt', 'compatibility.zip', @@ -962,6 +963,7 @@ class AbOtaPropertyFilesTest(PropertyFilesTestCase): property_files.required) self.assertEqual( ( + 'apex_info.pb', 'care_map.pb', 'care_map.txt', 'compatibility.zip', |