aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndroid Build Coastguard Worker <android-build-coastguard-worker@google.com>2024-01-08 17:35:42 +0000
committerAndroid Build Coastguard Worker <android-build-coastguard-worker@google.com>2024-01-08 18:15:43 +0000
commit840db711e6a6dcffe4517a6575b30cf2e78b500d (patch)
tree043eaf3be3acd0351114767464967919dd13983c
parent2d945e9269c867c45d3b019f37f6caa10319322f (diff)
parente14f4862bb439a73214bef4d909e9bdf35de8e40 (diff)
downloadbuild-aml_cfg_341510000.tar.gz
Make change and version bump to aml_cfg_341510000 for mainline module file: Noneaml_cfg_341510000
Snap for 11285496 from e14f4862bb439a73214bef4d909e9bdf35de8e40 to mainline-configinfrastructure-release Change-Id: Ibfaff76c11c9c074cccfa876912c91e5a1ab4d02
-rw-r--r--core/Makefile12
-rw-r--r--core/OWNERS2
-rw-r--r--core/android_soong_config_vars.mk17
-rw-r--r--core/build_id.mk2
-rw-r--r--core/main.mk76
-rw-r--r--core/sbom.mk11
-rw-r--r--core/version_defaults.mk4
-rw-r--r--tools/metadata/Android.bp14
-rw-r--r--tools/metadata/OWNERS4
-rw-r--r--tools/metadata/generator.go195
-rw-r--r--tools/metadata/go.mod7
-rw-r--r--tools/metadata/go.work10
-rw-r--r--tools/metadata/testdata/emptyInputFile.txt1
-rw-r--r--tools/metadata/testdata/expectedOutputFile.txt22
-rw-r--r--tools/metadata/testdata/file1.txt13
-rw-r--r--tools/metadata/testdata/file2.txt25
-rw-r--r--tools/metadata/testdata/file3.txt13
-rw-r--r--tools/metadata/testdata/file4.txt25
-rw-r--r--tools/metadata/testdata/generatedEmptyOutputFile.txt1
-rw-r--r--tools/metadata/testdata/generatedOutputFile.txt22
-rw-r--r--tools/metadata/testdata/inputFiles.txt1
-rw-r--r--tools/metadata/testdata/inputFilesNegativeCase.txt1
-rw-r--r--tools/metadata/testdata/metadata_test.go89
-rw-r--r--tools/metadata/testdata/outputFile.txt22
-rw-r--r--tools/releasetools/Android.bp3
-rwxr-xr-xtools/releasetools/check_target_files_signatures.py5
-rw-r--r--tools/releasetools/common.py71
-rwxr-xr-xtools/sbom/generate-sbom.py46
-rw-r--r--tools/sbom/sbom_data.py16
-rw-r--r--tools/sbom/sbom_writers.py71
-rw-r--r--tools/sbom/sbom_writers_test.py15
-rw-r--r--tools/sbom/testdata/expected_json_sbom.spdx.json15
-rw-r--r--tools/sbom/testdata/expected_tagvalue_sbom.spdx5
-rw-r--r--tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx70
34 files changed, 815 insertions, 91 deletions
diff --git a/core/Makefile b/core/Makefile
index 4208672300..38dc37b260 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -4883,7 +4883,9 @@ my_board_extracted_kernel := true
endif # INSTALLED_BOOTIMAGE_TARGET
endif # my_board_extracted_kernel
-ifneq ($(my_board_extracted_kernel),true)
+ifeq ($(my_board_extracted_kernel),true)
+$(call dist-for-goals, droid_targets, $(BUILT_KERNEL_VERSION_FILE))
+else
$(warning Neither INSTALLED_KERNEL_TARGET nor INSTALLED_BOOTIMAGE_TARGET is defined when \
PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS is true. Information about the updated kernel \
cannot be built into OTA update package. You can fix this by: \
@@ -6586,6 +6588,14 @@ $(JACOCO_REPORT_CLASSES_ALL) :
ifeq (,$(TARGET_BUILD_UNBUNDLED))
$(JACOCO_REPORT_CLASSES_ALL): $(INTERNAL_ALLIMAGES_FILES)
endif
+
+# This is not ideal, but it is difficult to correctly figure out the actual jacoco report
+# jars we need to add here as dependencies, so we add the device-tests as a dependency when
+# the env variable is set and this should guarantee thaat all the jacoco report jars are ready
+# when we package the final report jar here.
+ifeq ($(JACOCO_PACKAGING_INCLUDE_DEVICE_TESTS),true)
+ $(JACOCO_REPORT_CLASSES_ALL): $(COMPATIBILITY.device-tests.FILES)
+endif
endif # EMMA_INSTRUMENT=true
diff --git a/core/OWNERS b/core/OWNERS
index eb1d5c3c46..88f6d068bb 100644
--- a/core/OWNERS
+++ b/core/OWNERS
@@ -5,3 +5,5 @@ per-file proguard*.flags = jdduke@google.com
# For version updates
per-file version_defaults.mk = aseaton@google.com,lubomir@google.com,pscovanner@google.com,bkhalife@google.com,jainne@google.com
+# For sdk extensions version updates
+per-file version_defaults.mk = amhk@google.com,gurpreetgs@google.com,mkhokhlova@google.com,robertogil@google.com
diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk
index 140acf0f16..304a734b5a 100644
--- a/core/android_soong_config_vars.mk
+++ b/core/android_soong_config_vars.mk
@@ -87,9 +87,9 @@ endif
ifneq (,$(MODULE_BUILD_FROM_SOURCE))
# Keep an explicit setting.
-else ifeq (,$(filter docs sdk win_sdk sdk_addon,$(MAKECMDGOALS))$(findstring com.google.android.conscrypt,$(PRODUCT_PACKAGES)))
+else ifeq (,$(filter docs sdk win_sdk sdk_addon,$(MAKECMDGOALS))$(findstring com.google.android.conscrypt,$(PRODUCT_PACKAGES))$(findstring com.google.android.go.conscrypt,$(PRODUCT_PACKAGES)))
# Prebuilt module SDKs require prebuilt modules to work, and currently
- # prebuilt modules are only provided for com.google.android.xxx. If we can't
+ # prebuilt modules are only provided for com.google.android(.go)?.xxx. If we can't
# find one of them in PRODUCT_PACKAGES then assume com.android.xxx are in use,
# and disable prebuilt SDKs. In particular this applies to AOSP builds.
#
@@ -123,9 +123,22 @@ endif
# are controlled by the MODULE_BUILD_FROM_SOURCE environment variable by
# default.
INDIVIDUALLY_TOGGLEABLE_PREBUILT_MODULES := \
+ adservices \
+ appsearch \
btservices \
+ configinfrastructure \
+ conscrypt \
+ healthfitness \
+ ipsec \
+ media \
+ mediaprovider \
+ ondevicepersonalization \
permission \
rkpd \
+ scheduling \
+ sdkext \
+ statsd \
+ tethering \
uwb \
wifi \
diff --git a/core/build_id.mk b/core/build_id.mk
index 99991f05a8..8e362e4409 100644
--- a/core/build_id.mk
+++ b/core/build_id.mk
@@ -18,4 +18,4 @@
# (like "CRB01"). It must be a single word, and is
# capitalized by convention.
-BUILD_ID=340911000
+BUILD_ID=341510000
diff --git a/core/main.mk b/core/main.mk
index cb4dca6b41..04b6b33228 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -2157,13 +2157,21 @@ endif # TARGET_BUILD_APPS
# is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile.
# is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile.
# is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile.
-
+# build_output_path: the path of the built file, used to calculate checksum
+# static_libraries/whole_static_libraries: list of module name of the static libraries the file links against, e.g. libclang_rt.builtins or libclang_rt.builtins_32
+# Info of all static libraries of all installed files are collected in variable _all_static_libs that is used to list all the static library files in sbom-metadata.csv.
+# See the second foreach loop in the rule of sbom-metadata.csv for the detailed info of static libraries collected in _all_static_libs.
+# is_static_lib: whether the file is a static library
+
+metadata_list := $(OUT_DIR)/.module_paths/METADATA.list
+metadata_files := $(subst $(newline),$(space),$(file <$(metadata_list)))
# (TODO: b/272358583 find another way of always rebuilding this target)
# Remove the sbom-metadata.csv whenever makefile is evaluated
$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1)
-$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)
+$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) $(metadata_list) $(metadata_files)
rm -f $@
- @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@
+ echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $@
+ $(eval _all_static_libs :=)
$(foreach f,$(installed_files),\
$(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \
$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \
@@ -2185,11 +2193,25 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)
$(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \
$(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \
$(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \
- @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \
+ $(eval _static_libs := $(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES)) \
+ $(eval _whole_static_libs := $(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES)) \
+ $(foreach l,$(_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \
+ $(foreach l,$(_whole_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \
+ echo /$(_path_on_device),$(_module_path),$(_soong_module_type),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_build_output_path),$(_static_libs),$(_whole_static_libs), >> $@; \
$(if $(_post_installed_dexpreopt_zip), \
- for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \
+ for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i$(comma)$(_static_libs)$(comma)$(_whole_static_libs)$(comma) >> $@ ; done ; \
) \
)
+ $(foreach l,$(sort $(_all_static_libs)), \
+ $(eval _lib_stem := $(call word-colon,1,$l)) \
+ $(eval _module_path := $(call word-colon,2,$l)) \
+ $(eval _soong_module_type := $(call word-colon,3,$l)) \
+ $(eval _built_file := $(call word-colon,4,$l)) \
+ $(eval _static_libs := $(ALL_STATIC_LIBRARIES.$l.STATIC_LIBRARIES)) \
+ $(eval _whole_static_libs := $(ALL_STATIC_LIBRARIES.$l.WHOLE_STATIC_LIBRARIES)) \
+ $(eval _is_static_lib := Y) \
+ echo $(_lib_stem).a,$(_module_path),$(_soong_module_type),,,,,$(_built_file),$(_static_libs),$(_whole_static_libs),$(_is_static_lib) >> $@; \
+ )
.PHONY: sbom
ifeq ($(TARGET_BUILD_APPS),)
@@ -2201,17 +2223,47 @@ $(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)
$(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json)
else
-apps_only_sbom_files := $(sort $(patsubst %,%.spdx.json,$(filter %.apk,$(apps_only_installed_files))))
-$(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)
- rm -rf $@
- $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --unbundled_apk
+# Create build rules for generating SBOMs of unbundled APKs and APEXs
+# $1: sbom file
+# $2: sbom fragment file
+# $3: installed file
+# $4: sbom-metadata.csv file
+define generate-app-sbom
+$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$(3)))
+$(eval _module_name := $(ALL_INSTALLED_FILES.$(3)))
+$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH))))
+$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE))))
+$(eval _dep_modules := $(filter %.$(_module_name),$(ALL_MODULES)) $(filter %.$(_module_name)$(TARGET_2ND_ARCH_MODULE_SUFFIX),$(ALL_MODULES)))
+$(eval _is_apex := $(filter %.apex,$(3)))
+
+$(4): $(3) $(metadata_list) $(metadata_files)
+ rm -rf $$@
+ echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $$@
+ echo /$(_path_on_device),$(_module_path),$(_soong_module_type),,,,,$(3),,, >> $$@
+ $(if $(filter %.apex,$(3)),\
+ $(foreach m,$(_dep_modules),\
+ echo $(patsubst $(PRODUCT_OUT)/apex/$(_module_name)/%,%,$(ALL_MODULES.$m.INSTALLED)),$(sort $(ALL_MODULES.$m.PATH)),$(sort $(ALL_MODULES.$m.SOONG_MODULE_TYPE)),,,,,$(strip $(ALL_MODULES.$m.BUILT)),,, >> $$@;))
+
+$(2): $(1)
+$(1): $(4) $(GEN_SBOM)
+ rm -rf $$@
+ $(GEN_SBOM) --output_file $$@ --metadata $(4) --build_version $$(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(if $(filter %.apk,$(3)),--unbundled_apk,--unbundled_apex)
+endef
+
+apps_only_sbom_files :=
+apps_only_fragment_files :=
+$(foreach f,$(filter %.apk %.apex,$(installed_files)), \
+ $(eval _metadata_csv_file := $(patsubst %,%-sbom-metadata.csv,$f)) \
+ $(eval _sbom_file := $(patsubst %,%.spdx.json,$f)) \
+ $(eval _fragment_file := $(patsubst %,%-fragment.spdx,$f)) \
+ $(eval apps_only_sbom_files += $(_sbom_file)) \
+ $(eval apps_only_fragment_files += $(_fragment_file)) \
+ $(eval $(call generate-app-sbom,$(_sbom_file),$(_fragment_file),$f,$(_metadata_csv_file))) \
+)
sbom: $(apps_only_sbom_files)
-$(foreach f,$(apps_only_sbom_files),$(eval $(patsubst %.spdx.json,%-fragment.spdx,$f): $f))
-apps_only_fragment_files := $(patsubst %.spdx.json,%-fragment.spdx,$(apps_only_sbom_files))
$(foreach f,$(apps_only_fragment_files),$(eval apps_only_fragment_dist_files += :sbom/$(notdir $f)))
-
$(foreach f,$(apps_only_sbom_files),$(eval apps_only_sbom_dist_files += :sbom/$(notdir $f)))
$(call dist-for-goals,apps_only,$(join $(apps_only_sbom_files),$(apps_only_sbom_dist_files)) $(join $(apps_only_fragment_files),$(apps_only_fragment_dist_files)))
endif
diff --git a/core/sbom.mk b/core/sbom.mk
index e23bbc132f..39c251ae0c 100644
--- a/core/sbom.mk
+++ b/core/sbom.mk
@@ -3,9 +3,20 @@
# unless a .mk file changes its installed file after including base_rules.mk.
ifdef my_register_name
+ # ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle libclang_rt.builtins, for primary arch
+ # ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle_32 libclang_rt.builtins_32, for 2nd arch.
ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED)))
$(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\
$(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\
+ $(eval ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\
+ $(eval ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\
)
endif
+ ifeq (STATIC_LIBRARIES,$(LOCAL_MODULE_CLASS))
+ ALL_STATIC_LIBRARIES.$(my_register_name).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))
+ ALL_STATIC_LIBRARIES.$(my_register_name).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))
+ ifdef LOCAL_SOONG_MODULE_TYPE
+ ALL_STATIC_LIBRARIES.$(my_register_name).BUILT_FILE := $(LOCAL_PREBUILT_MODULE_FILE)
+ endif
+ endif
endif \ No newline at end of file
diff --git a/core/version_defaults.mk b/core/version_defaults.mk
index 8dc425760f..160c6455ca 100644
--- a/core/version_defaults.mk
+++ b/core/version_defaults.mk
@@ -79,7 +79,7 @@ endif
.KATI_READONLY := PLATFORM_SDK_VERSION
# This is the sdk extension version of this tree.
-PLATFORM_SDK_EXTENSION_VERSION := 7
+PLATFORM_SDK_EXTENSION_VERSION := 11
.KATI_READONLY := PLATFORM_SDK_EXTENSION_VERSION
# This is the sdk extension version that PLATFORM_SDK_VERSION ships with.
@@ -104,7 +104,7 @@ ifndef PLATFORM_SECURITY_PATCH
# It must be of the form "YYYY-MM-DD" on production devices.
# It must match one of the Android Security Patch Level strings of the Public Security Bulletins.
# If there is no $PLATFORM_SECURITY_PATCH set, keep it empty.
- PLATFORM_SECURITY_PATCH := 2023-08-05
+ PLATFORM_SECURITY_PATCH := 2024-01-01
endif
include $(BUILD_SYSTEM)/version_util.mk
diff --git a/tools/metadata/Android.bp b/tools/metadata/Android.bp
new file mode 100644
index 0000000000..b2fabecb96
--- /dev/null
+++ b/tools/metadata/Android.bp
@@ -0,0 +1,14 @@
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+blueprint_go_binary {
+ name: "metadata",
+ deps: [
+ "soong-testing-test_spec_proto",
+ "golang-protobuf-proto",
+ ],
+ srcs: [
+ "generator.go",
+ ]
+} \ No newline at end of file
diff --git a/tools/metadata/OWNERS b/tools/metadata/OWNERS
new file mode 100644
index 0000000000..03bcdf1c40
--- /dev/null
+++ b/tools/metadata/OWNERS
@@ -0,0 +1,4 @@
+dariofreni@google.com
+joeo@google.com
+ronish@google.com
+caditya@google.com
diff --git a/tools/metadata/generator.go b/tools/metadata/generator.go
new file mode 100644
index 0000000000..e970e1708f
--- /dev/null
+++ b/tools/metadata/generator.go
@@ -0,0 +1,195 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "sort"
+ "strings"
+ "sync"
+
+ "android/soong/testing/test_spec_proto"
+ "google.golang.org/protobuf/proto"
+)
+
+type keyToLocksMap struct {
+ locks sync.Map
+}
+
+func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex {
+ mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{})
+ return mutex.(*sync.Mutex)
+}
+
+func getSortedKeys(syncMap *sync.Map) []string {
+ var allKeys []string
+ syncMap.Range(
+ func(key, _ interface{}) bool {
+ allKeys = append(allKeys, key.(string))
+ return true
+ },
+ )
+
+ sort.Strings(allKeys)
+ return allKeys
+}
+
+func writeOutput(
+ outputFile string,
+ allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata,
+) {
+ testSpec := &test_spec_proto.TestSpec{
+ OwnershipMetadataList: allMetadata,
+ }
+ data, err := proto.Marshal(testSpec)
+ if err != nil {
+ log.Fatal(err)
+ }
+ file, err := os.Create(outputFile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer file.Close()
+
+ _, err = file.Write(data)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func readFileToString(filePath string) string {
+ file, err := os.Open(filePath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer file.Close()
+
+ data, err := io.ReadAll(file)
+ if err != nil {
+ log.Fatal(err)
+ }
+ return string(data)
+}
+
+func writeNewlineToOutputFile(outputFile string) {
+ file, err := os.Create(outputFile)
+ data := "\n"
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer file.Close()
+
+ _, err = file.Write([]byte(data))
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func processTestSpecProtobuf(
+ filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap,
+ errCh chan error, wg *sync.WaitGroup,
+) {
+ defer wg.Done()
+
+ fileContent := strings.TrimRight(readFileToString(filePath), "\n")
+ testData := test_spec_proto.TestSpec{}
+ err := proto.Unmarshal([]byte(fileContent), &testData)
+ if err != nil {
+ errCh <- err
+ return
+ }
+
+ ownershipMetadata := testData.GetOwnershipMetadataList()
+ for _, metadata := range ownershipMetadata {
+ key := metadata.GetTargetName()
+ lock := keyLocks.GetLockForKey(key)
+ lock.Lock()
+
+ value, loaded := ownershipMetadataMap.LoadOrStore(
+ key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata},
+ )
+ if loaded {
+ existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
+ isDuplicate := false
+ for _, existing := range existingMetadata {
+ if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() {
+ errCh <- fmt.Errorf(
+ "Conflicting trendy team IDs found for %s at:\n%s with teamId"+
+ ": %s,\n%s with teamId: %s",
+ key,
+ metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(),
+ existing.GetTrendyTeamId(),
+ )
+
+ lock.Unlock()
+ return
+ }
+ if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() {
+ isDuplicate = true
+ break
+ }
+ }
+ if !isDuplicate {
+ existingMetadata = append(existingMetadata, metadata)
+ ownershipMetadataMap.Store(key, existingMetadata)
+ }
+ }
+
+ lock.Unlock()
+ }
+}
+
+func main() {
+ inputFile := flag.String("inputFile", "", "Input file path")
+ outputFile := flag.String("outputFile", "", "Output file path")
+ rule := flag.String("rule", "", "Metadata rule (Hint: test_spec or code_metadata)")
+ flag.Parse()
+
+ if *inputFile == "" || *outputFile == "" || *rule == "" {
+ fmt.Println("Usage: metadata -rule <rule> -inputFile <input file path> -outputFile <output file path>")
+ os.Exit(1)
+ }
+
+ inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n")
+ filePaths := strings.Split(inputFileData, " ")
+ if len(filePaths) == 1 && filePaths[0] == "" {
+ writeNewlineToOutputFile(*outputFile)
+ return
+ }
+ ownershipMetadataMap := &sync.Map{}
+ keyLocks := &keyToLocksMap{}
+ errCh := make(chan error, len(filePaths))
+ var wg sync.WaitGroup
+
+ switch *rule {
+ case "test_spec":
+ for _, filePath := range filePaths {
+ wg.Add(1)
+ go processTestSpecProtobuf(filePath, ownershipMetadataMap, keyLocks, errCh, &wg)
+ }
+
+ wg.Wait()
+ close(errCh)
+
+ for err := range errCh {
+ log.Fatal(err)
+ }
+
+ allKeys := getSortedKeys(ownershipMetadataMap)
+ var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata
+
+ for _, key := range allKeys {
+ value, _ := ownershipMetadataMap.Load(key)
+ metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
+ allMetadata = append(allMetadata, metadataList...)
+ }
+
+ writeOutput(*outputFile, allMetadata)
+ break
+ case "code_metadata":
+ default:
+ log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule)
+ }
+}
diff --git a/tools/metadata/go.mod b/tools/metadata/go.mod
new file mode 100644
index 0000000000..e9d04b16f6
--- /dev/null
+++ b/tools/metadata/go.mod
@@ -0,0 +1,7 @@
+module android/soong/tools/metadata
+
+require google.golang.org/protobuf v0.0.0
+
+replace google.golang.org/protobuf v0.0.0 => ../../../external/golang-protobuf
+
+go 1.18 \ No newline at end of file
diff --git a/tools/metadata/go.work b/tools/metadata/go.work
new file mode 100644
index 0000000000..23875daf3d
--- /dev/null
+++ b/tools/metadata/go.work
@@ -0,0 +1,10 @@
+go 1.18
+
+use (
+ .
+ ../../../../external/golang-protobuf
+ ../../../soong/testing/test_spec_proto
+
+)
+
+replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf
diff --git a/tools/metadata/testdata/emptyInputFile.txt b/tools/metadata/testdata/emptyInputFile.txt
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++ b/tools/metadata/testdata/emptyInputFile.txt
@@ -0,0 +1 @@
+
diff --git a/tools/metadata/testdata/expectedOutputFile.txt b/tools/metadata/testdata/expectedOutputFile.txt
new file mode 100644
index 0000000000..b0d382f279
--- /dev/null
+++ b/tools/metadata/testdata/expectedOutputFile.txt
@@ -0,0 +1,22 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345 \ No newline at end of file
diff --git a/tools/metadata/testdata/file1.txt b/tools/metadata/testdata/file1.txt
new file mode 100644
index 0000000000..81beed00ab
--- /dev/null
+++ b/tools/metadata/testdata/file1.txt
@@ -0,0 +1,13 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345
diff --git a/tools/metadata/testdata/file2.txt b/tools/metadata/testdata/file2.txt
new file mode 100644
index 0000000000..32a753fef5
--- /dev/null
+++ b/tools/metadata/testdata/file2.txt
@@ -0,0 +1,25 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
diff --git a/tools/metadata/testdata/file3.txt b/tools/metadata/testdata/file3.txt
new file mode 100644
index 0000000000..81beed00ab
--- /dev/null
+++ b/tools/metadata/testdata/file3.txt
@@ -0,0 +1,13 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345
diff --git a/tools/metadata/testdata/file4.txt b/tools/metadata/testdata/file4.txt
new file mode 100644
index 0000000000..6a7590021d
--- /dev/null
+++ b/tools/metadata/testdata/file4.txt
@@ -0,0 +1,25 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-one
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
diff --git a/tools/metadata/testdata/generatedEmptyOutputFile.txt b/tools/metadata/testdata/generatedEmptyOutputFile.txt
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++ b/tools/metadata/testdata/generatedEmptyOutputFile.txt
@@ -0,0 +1 @@
+
diff --git a/tools/metadata/testdata/generatedOutputFile.txt b/tools/metadata/testdata/generatedOutputFile.txt
new file mode 100644
index 0000000000..b0d382f279
--- /dev/null
+++ b/tools/metadata/testdata/generatedOutputFile.txt
@@ -0,0 +1,22 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345 \ No newline at end of file
diff --git a/tools/metadata/testdata/inputFiles.txt b/tools/metadata/testdata/inputFiles.txt
new file mode 100644
index 0000000000..e44bc94d32
--- /dev/null
+++ b/tools/metadata/testdata/inputFiles.txt
@@ -0,0 +1 @@
+file1.txt file2.txt \ No newline at end of file
diff --git a/tools/metadata/testdata/inputFilesNegativeCase.txt b/tools/metadata/testdata/inputFilesNegativeCase.txt
new file mode 100644
index 0000000000..a37aa3fd5d
--- /dev/null
+++ b/tools/metadata/testdata/inputFilesNegativeCase.txt
@@ -0,0 +1 @@
+file3.txt file4.txt \ No newline at end of file
diff --git a/tools/metadata/testdata/metadata_test.go b/tools/metadata/testdata/metadata_test.go
new file mode 100644
index 0000000000..71856fe606
--- /dev/null
+++ b/tools/metadata/testdata/metadata_test.go
@@ -0,0 +1,89 @@
+package main
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os/exec"
+ "strings"
+ "testing"
+)
+
+func TestMetadata(t *testing.T) {
+ cmd := exec.Command(
+ "metadata", "-rule", "test_spec", "-inputFile", "./inputFiles.txt", "-outputFile",
+ "./generatedOutputFile.txt",
+ )
+ stderr, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err)
+ }
+
+ // Read the contents of the expected output file
+ expectedOutput, err := ioutil.ReadFile("./expectedOutputFile.txt")
+ if err != nil {
+ t.Fatalf("Error reading expected output file: %s", err)
+ }
+
+ // Read the contents of the generated output file
+ generatedOutput, err := ioutil.ReadFile("./generatedOutputFile.txt")
+ if err != nil {
+ t.Fatalf("Error reading generated output file: %s", err)
+ }
+
+ fmt.Println()
+
+ // Compare the contents
+ if string(expectedOutput) != string(generatedOutput) {
+ t.Errorf("Generated file contents do not match the expected output")
+ }
+}
+
+func TestMetadataNegativeCase(t *testing.T) {
+ cmd := exec.Command(
+ "metadata", "-rule", "test_spec", "-inputFile", "./inputFilesNegativeCase.txt", "-outputFile",
+ "./generatedOutputFileNegativeCase.txt",
+ )
+ stderr, err := cmd.CombinedOutput()
+ if err == nil {
+ t.Fatalf(
+ "Expected an error, but the metadata command executed successfully. Output: %s",
+ stderr,
+ )
+ }
+
+ expectedError := "Conflicting trendy team IDs found for java-test-module" +
+ "-name-one at:\nAndroid.bp with teamId: 12346," +
+ "\nAndroid.bp with teamId: 12345"
+ if !strings.Contains(
+ strings.TrimSpace(string(stderr)), strings.TrimSpace(expectedError),
+ ) {
+ t.Errorf(
+ "Unexpected error message. Expected to contain: %s, Got: %s",
+ expectedError, stderr,
+ )
+ }
+}
+
+func TestEmptyInputFile(t *testing.T) {
+ cmd := exec.Command(
+ "metadata", "-rule", "test_spec", "-inputFile", "./emptyInputFile.txt", "-outputFile",
+ "./generatedEmptyOutputFile.txt",
+ )
+ stderr, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err)
+ }
+
+ // Read the contents of the generated output file
+ generatedOutput, err := ioutil.ReadFile("./generatedEmptyOutputFile.txt")
+ if err != nil {
+ t.Fatalf("Error reading generated output file: %s", err)
+ }
+
+ fmt.Println()
+
+ // Compare the contents
+ if string(generatedOutput) != "\n" {
+ t.Errorf("Generated file contents do not match the expected output")
+ }
+}
diff --git a/tools/metadata/testdata/outputFile.txt b/tools/metadata/testdata/outputFile.txt
new file mode 100644
index 0000000000..b0d382f279
--- /dev/null
+++ b/tools/metadata/testdata/outputFile.txt
@@ -0,0 +1,22 @@
+
+.
+java-test-module-name-one
+Android.bp12345
+.
+java-test-module-name-six
+Android.bp12346
+.
+java-test-module-name-six
+Aqwerty.bp12346
+.
+java-test-module-name-six
+Apoiuyt.bp12346
+.
+java-test-module-name-two
+Android.bp12345
+.
+java-test-module-name-two
+Asdfghj.bp12345
+.
+java-test-module-name-two
+Azxcvbn.bp12345 \ No newline at end of file
diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp
index a76dc8a8bc..d07292a082 100644
--- a/tools/releasetools/Android.bp
+++ b/tools/releasetools/Android.bp
@@ -236,6 +236,9 @@ python_library_host {
"rangelib.py",
"sparse_img.py",
],
+ data: [
+ ":zip2zip",
+ ],
// Only the tools that are referenced directly are listed as required modules. For example,
// `avbtool` is not here, as the script always uses the one from info_dict['avb_avbtool'].
required: [
diff --git a/tools/releasetools/check_target_files_signatures.py b/tools/releasetools/check_target_files_signatures.py
index d935607e43..a7b35230ea 100755
--- a/tools/releasetools/check_target_files_signatures.py
+++ b/tools/releasetools/check_target_files_signatures.py
@@ -241,7 +241,8 @@ class APK(object):
# Signer (minSdkVersion=24, maxSdkVersion=32) certificate SHA-1 digest: 19da94896ce4078c38ca695701f1dec741ec6d67
# ...
certs_info = {}
- certificate_regex = re.compile(r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)")
+ certificate_regex = re.compile(
+ r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)")
for line in output.splitlines():
m = certificate_regex.match(line)
if not m:
@@ -312,7 +313,7 @@ class TargetFiles(object):
# This is the list of wildcards of files we extract from |filename|.
apk_extensions = ['*.apk', '*.apex']
- with zipfile.ZipFile(filename) as input_zip:
+ with zipfile.ZipFile(filename, "r") as input_zip:
self.certmap, compressed_extension = common.ReadApkCerts(input_zip)
if compressed_extension:
apk_extensions.append('*.apk' + compressed_extension)
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index b50caaa95e..abedecfa76 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -35,6 +35,7 @@ import shlex
import shutil
import subprocess
import sys
+import stat
import tempfile
import threading
import time
@@ -2102,6 +2103,26 @@ def Gunzip(in_filename, out_filename):
shutil.copyfileobj(in_file, out_file)
+def UnzipSingleFile(input_zip: zipfile.ZipFile, info: zipfile.ZipInfo, dirname: str):
+ # According to https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip/6297838#6297838
+ # higher bits of |external_attr| are unix file permission and types
+ unix_filetype = info.external_attr >> 16
+
+ def CheckMask(a, mask):
+ return (a & mask) == mask
+
+ def IsSymlink(a):
+ return CheckMask(a, stat.S_IFLNK)
+ # python3.11 zipfile implementation doesn't handle symlink correctly
+ if not IsSymlink(unix_filetype):
+ return input_zip.extract(info, dirname)
+ if dirname is None:
+ dirname = os.getcwd()
+ target = os.path.join(dirname, info.filename)
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ os.symlink(input_zip.read(info).decode(), target)
+
+
def UnzipToDir(filename, dirname, patterns=None):
"""Unzips the archive to the given directory.
@@ -2112,20 +2133,46 @@ def UnzipToDir(filename, dirname, patterns=None):
archvie. Non-matching patterns will be filtered out. If there's no match
after the filtering, no file will be unzipped.
"""
- cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
- if patterns is not None:
+ with zipfile.ZipFile(filename, allowZip64=True, mode="r") as input_zip:
# Filter out non-matching patterns. unzip will complain otherwise.
- with zipfile.ZipFile(filename, allowZip64=True) as input_zip:
- names = input_zip.namelist()
- filtered = [
- pattern for pattern in patterns if fnmatch.filter(names, pattern)]
-
- # There isn't any matching files. Don't unzip anything.
- if not filtered:
- return
- cmd.extend(filtered)
+ entries = input_zip.infolist()
+ # b/283033491
+ # Per https://en.wikipedia.org/wiki/ZIP_(file_format)#Central_directory_file_header
+ # In zip64 mode, central directory record's header_offset field might be
+ # set to 0xFFFFFFFF if header offset is > 2^32. In this case, the extra
+ # fields will contain an 8 byte little endian integer at offset 20
+ # to indicate the actual local header offset.
+ # As of python3.11, python does not handle zip64 central directories
+ # correctly, so we will manually do the parsing here.
+
+ # ZIP64 central directory extra field has two required fields:
+ # 2 bytes header ID and 2 bytes size field. Thes two require fields have
+ # a total size of 4 bytes. Then it has three other 8 bytes field, followed
+ # by a 4 byte disk number field. The last disk number field is not required
+ # to be present, but if it is present, the total size of extra field will be
+ # divisible by 8(because 2+2+4+8*n is always going to be multiple of 8)
+ # Most extra fields are optional, but when they appear, their must appear
+ # in the order defined by zip64 spec. Since file header offset is the 2nd
+ # to last field in zip64 spec, it will only be at last 8 bytes or last 12-4
+ # bytes, depending on whether disk number is present.
+ for entry in entries:
+ if entry.header_offset == 0xFFFFFFFF:
+ if len(entry.extra) % 8 == 0:
+ entry.header_offset = int.from_bytes(entry.extra[-12:-4], "little")
+ else:
+ entry.header_offset = int.from_bytes(entry.extra[-8:], "little")
+ if patterns is not None:
+ filtered = [info for info in entries if any(
+ [fnmatch.fnmatch(info.filename, p) for p in patterns])]
- RunAndCheckOutput(cmd)
+ # There isn't any matching files. Don't unzip anything.
+ if not filtered:
+ return
+ for info in filtered:
+ UnzipSingleFile(input_zip, info, dirname)
+ else:
+ for info in entries:
+ UnzipSingleFile(input_zip, info, dirname)
def UnzipTemp(filename, patterns=None):
diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py
index 2415f7e270..b19be87666 100755
--- a/tools/sbom/generate-sbom.py
+++ b/tools/sbom/generate-sbom.py
@@ -332,14 +332,6 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path):
return external_doc_ref, packages, relationships
-def generate_package_verification_code(files):
- checksums = [file.checksum for file in files]
- checksums.sort()
- h = hashlib.sha1()
- h.update(''.join(checksums).encode(encoding='utf-8'))
- return h.hexdigest()
-
-
def save_report(report_file_path, report):
with open(report_file_path, 'w', encoding='utf-8') as report_file:
for type, issues in report.items():
@@ -487,20 +479,32 @@ def main():
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
build_output_path = installed_file_metadata['build_output_path']
+ is_static_lib = installed_file_metadata['is_static_lib']
if not installed_file_has_metadata(installed_file_metadata, report):
continue
- if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
+ if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
+ # Ignore non-existing static library files for now since they are not shipped on devices.
report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
continue
file_id = new_file_id(installed_file)
- doc.files.append(
- sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))
- if not args.unbundled_apex:
- product_package.file_ids.append(file_id)
- elif len(doc.files) > 1:
- doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
+ # TODO(b/285453664): Soong should report the information of statically linked libraries to Make.
+ # This happens when a different sanitized version of static libraries is used in linking.
+ # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be
+ # located correctly because Soong doesn't report the information to Make.
+ sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string
+ if os.path.islink(build_output_path) or os.path.isfile(build_output_path):
+ sha1 = checksum(build_output_path)
+ doc.files.append(sbom_data.File(id=file_id,
+ name=installed_file,
+ checksum=sha1))
+
+ if not is_static_lib:
+ if not args.unbundled_apex:
+ product_package.file_ids.append(file_id)
+ elif len(doc.files) > 1:
+ doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
metadata_file_path = get_metadata_file_path(installed_file_metadata)
@@ -544,13 +548,21 @@ def main():
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
- if not args.unbundled_apex:
- product_package.verification_code = generate_package_verification_code(doc.files)
+ # Process static libraries and whole static libraries the installed file links to
+ static_libs = installed_file_metadata['static_libraries']
+ whole_static_libs = installed_file_metadata['whole_static_libraries']
+ all_static_libs = (static_libs + ' ' + whole_static_libs).strip()
+ if all_static_libs:
+ for lib in all_static_libs.split(' '):
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.STATIC_LINK,
+ id2=new_file_id(lib + '.a')))
if args.unbundled_apex:
doc.describes = doc.files[0].id
# Save SBOM records to output file
+ doc.generate_packages_verification_code()
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
prefix = args.output_file
if prefix.endswith('.spdx'):
diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py
index 14c4eb231d..ea38e364dc 100644
--- a/tools/sbom/sbom_data.py
+++ b/tools/sbom/sbom_data.py
@@ -25,6 +25,7 @@ fields in each data class.
from dataclasses import dataclass, field
from typing import List
+import hashlib
SPDXID_DOC = 'SPDXRef-DOCUMENT'
SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
@@ -81,6 +82,7 @@ class RelationshipType:
VARIANT_OF = 'VARIANT_OF'
GENERATED_FROM = 'GENERATED_FROM'
CONTAINS = 'CONTAINS'
+ STATIC_LINK = 'STATIC_LINK'
@dataclass
@@ -122,3 +124,17 @@ class Document:
if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
for r in self.relationships):
self.relationships.append(rel)
+
+ def generate_packages_verification_code(self):
+ for package in self.packages:
+ if not package.file_ids:
+ continue
+
+ checksums = []
+ for file in self.files:
+ if file.id in package.file_ids:
+ checksums.append(file.checksum)
+ checksums.sort()
+ h = hashlib.sha1()
+ h.update(''.join(checksums).encode(encoding='utf-8'))
+ package.verification_code = h.hexdigest()
diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py
index 85dee9d1a5..1cb864db75 100644
--- a/tools/sbom/sbom_writers.py
+++ b/tools/sbom/sbom_writers.py
@@ -85,7 +85,7 @@ class TagValueWriter:
return headers
@staticmethod
- def marshal_package(package):
+ def marshal_package(sbom_doc, package, fragment):
download_location = sbom_data.VALUE_NOASSERTION
if package.download_location:
download_location = package.download_location
@@ -107,50 +107,32 @@ class TagValueWriter:
f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')
tagvalues.append('')
- return tagvalues
-
- @staticmethod
- def marshal_described_element(sbom_doc, fragment):
- if not sbom_doc.describes:
- return None
-
- product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
- if product_package:
- tagvalues = TagValueWriter.marshal_package(product_package[0])
- if not fragment:
- tagvalues.append(
- f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
+ if package.id == sbom_doc.describes and not fragment:
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
- return tagvalues
- file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
- if file:
- tagvalues = TagValueWriter.marshal_file(file[0])
- if not fragment:
- tagvalues.append(
- f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
-
- return tagvalues
+ for file in sbom_doc.files:
+ if file.id in package.file_ids:
+ tagvalues += TagValueWriter.marshal_file(file)
- return None
+ return tagvalues
@staticmethod
- def marshal_packages(sbom_doc):
+ def marshal_packages(sbom_doc, fragment):
tagvalues = []
marshaled_relationships = []
i = 0
packages = sbom_doc.packages
while i < len(packages):
- if packages[i].id == sbom_doc.describes:
- i += 1
- continue
-
- if i + 1 < len(packages) \
- and packages[i].id.startswith('SPDXRef-SOURCE-') \
- and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
- tagvalues += TagValueWriter.marshal_package(packages[i])
- tagvalues += TagValueWriter.marshal_package(packages[i + 1])
+ if (i + 1 < len(packages)
+ and packages[i].id.startswith('SPDXRef-SOURCE-')
+ and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')):
+ # Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other
+ # in SBOMs in tagvalue format.
+ tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
+ tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment)
rel = next((r for r in sbom_doc.relationships if
r.id1 == packages[i].id and
r.id2 == packages[i + 1].id and
@@ -162,7 +144,7 @@ class TagValueWriter:
i += 2
else:
- tagvalues += TagValueWriter.marshal_package(packages[i])
+ tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)
i += 1
return tagvalues, marshaled_relationships
@@ -179,12 +161,20 @@ class TagValueWriter:
return tagvalues
@staticmethod
- def marshal_files(sbom_doc):
+ def marshal_files(sbom_doc, fragment):
tagvalues = []
+ files_in_packages = []
+ for package in sbom_doc.packages:
+ files_in_packages += package.file_ids
for file in sbom_doc.files:
- if file.id == sbom_doc.describes:
+ if file.id in files_in_packages:
continue
tagvalues += TagValueWriter.marshal_file(file)
+ if file.id == sbom_doc.describes and not fragment:
+ # Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable.
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
+ tagvalues.append('')
return tagvalues
@staticmethod
@@ -208,11 +198,8 @@ class TagValueWriter:
content = []
if not fragment:
content += TagValueWriter.marshal_doc_headers(sbom_doc)
- described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)
- if described_element:
- content += described_element
- content += TagValueWriter.marshal_files(sbom_doc)
- tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
+ content += TagValueWriter.marshal_files(sbom_doc, fragment)
+ tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)
content += tagvalues
content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
file.write('\n'.join(content))
diff --git a/tools/sbom/sbom_writers_test.py b/tools/sbom/sbom_writers_test.py
index 361dae6f5d..cf85e014b7 100644
--- a/tools/sbom/sbom_writers_test.py
+++ b/tools/sbom/sbom_writers_test.py
@@ -31,6 +31,7 @@ SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1'
SPDXID_FILE1 = 'SPDXRef-file1'
SPDXID_FILE2 = 'SPDXRef-file2'
SPDXID_FILE3 = 'SPDXRef-file3'
+SPDXID_FILE4 = 'SPDXRef-file4'
class SBOMWritersTest(unittest.TestCase):
@@ -101,6 +102,8 @@ class SBOMWritersTest(unittest.TestCase):
sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
self.sbom_doc.files.append(
sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
+ self.sbom_doc.files.append(
+ sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444'))
self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
@@ -112,6 +115,10 @@ class SBOMWritersTest(unittest.TestCase):
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=SPDXID_SOURCE_PACKAGE1
))
+ self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1,
+ relationship=sbom_data.RelationshipType.STATIC_LINK,
+ id2=SPDXID_FILE4
+ ))
# SBOM fragment of a APK
self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
@@ -139,6 +146,14 @@ class SBOMWritersTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(expected_output, output.getvalue())
+ def test_tagvalue_writer_doc_describes_file(self):
+ with io.StringIO() as output:
+ self.sbom_doc.describes = SPDXID_FILE4
+ sbom_writers.TagValueWriter.write(self.sbom_doc, output)
+ expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_doc_describes_file.spdx').read_text()
+ self.maxDiff = None
+ self.assertEqual(expected_output, output.getvalue())
+
def test_tagvalue_writer_unbundled(self):
with io.StringIO() as output:
sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True)
diff --git a/tools/sbom/testdata/expected_json_sbom.spdx.json b/tools/sbom/testdata/expected_json_sbom.spdx.json
index 32715a5392..53936c5c9d 100644
--- a/tools/sbom/testdata/expected_json_sbom.spdx.json
+++ b/tools/sbom/testdata/expected_json_sbom.spdx.json
@@ -110,6 +110,16 @@
"checksumValue": "33333"
}
]
+ },
+ {
+ "fileName": "file4.a",
+ "SPDXID": "SPDXRef-file4",
+ "checksums": [
+ {
+ "algorithm": "SHA1",
+ "checksumValue": "44444"
+ }
+ ]
}
],
"relationships": [
@@ -129,6 +139,11 @@
"relationshipType": "GENERATED_FROM"
},
{
+ "spdxElementId": "SPDXRef-file1",
+ "relatedSpdxElement": "SPDXRef-file4",
+ "relationshipType": "STATIC_LINK"
+ },
+ {
"spdxElementId": "SPDXRef-SOURCE-package1",
"relatedSpdxElement": "SPDXRef-UPSTREAM-package1",
"relationshipType": "VARIANT_OF"
diff --git a/tools/sbom/testdata/expected_tagvalue_sbom.spdx b/tools/sbom/testdata/expected_tagvalue_sbom.spdx
index ee39e82fcf..e6fd17e9c3 100644
--- a/tools/sbom/testdata/expected_tagvalue_sbom.spdx
+++ b/tools/sbom/testdata/expected_tagvalue_sbom.spdx
@@ -7,6 +7,10 @@ Creator: Organization: Google
Created: 2023-03-31T22:17:58Z
ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
+FileName: file4.a
+SPDXID: SPDXRef-file4
+FileChecksum: SHA1: 44444
+
PackageName: PRODUCT
SPDXID: SPDXRef-PRODUCT
PackageDownloadLocation: NONE
@@ -63,3 +67,4 @@ Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
+Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4
diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx
new file mode 100644
index 0000000000..428d7e3ddb
--- /dev/null
+++ b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx
@@ -0,0 +1,70 @@
+SPDXVersion: SPDX-2.3
+DataLicense: CC0-1.0
+SPDXID: SPDXRef-DOCUMENT
+DocumentName: test doc
+DocumentNamespace: http://www.google.com/sbom/spdx/android
+Creator: Organization: Google
+Created: 2023-03-31T22:17:58Z
+ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890
+
+FileName: file4.a
+SPDXID: SPDXRef-file4
+FileChecksum: SHA1: 44444
+
+Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-file4
+
+PackageName: PRODUCT
+SPDXID: SPDXRef-PRODUCT
+PackageDownloadLocation: NONE
+FilesAnalyzed: true
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+PackageVerificationCode: 123456
+
+FileName: /bin/file1
+SPDXID: SPDXRef-file1
+FileChecksum: SHA1: 11111
+
+FileName: /bin/file2
+SPDXID: SPDXRef-file2
+FileChecksum: SHA1: 22222
+
+FileName: /bin/file3
+SPDXID: SPDXRef-file3
+FileChecksum: SHA1: 33333
+
+PackageName: PLATFORM
+SPDXID: SPDXRef-PLATFORM
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+PackageName: Prebuilt package1
+SPDXID: SPDXRef-PREBUILT-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+
+PackageName: Source package1
+SPDXID: SPDXRef-SOURCE-package1
+PackageDownloadLocation: NONE
+FilesAnalyzed: false
+PackageVersion: build_finger_print
+PackageSupplier: Organization: Google
+ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4
+
+PackageName: Upstream package1
+SPDXID: SPDXRef-UPSTREAM-package1
+PackageDownloadLocation: NOASSERTION
+FilesAnalyzed: false
+PackageVersion: 1.1
+PackageSupplier: Organization: upstream
+
+Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1
+
+Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM
+Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1
+Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1
+Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4