diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-01-04 19:47:10 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-01-04 20:28:06 +0000 |
commit | 547ef5eff4782f963410910dff032e223930f5f0 (patch) | |
tree | 1b31c1b42acacc516df1ebafad70ff4a27569920 | |
parent | 2e45054dcfa82f84adc9e3209d7ebc95cc09894e (diff) | |
parent | 0be2581e80073ff12fc3d1a056488563e6c10516 (diff) | |
download | build-547ef5eff4782f963410910dff032e223930f5f0.tar.gz |
Make change and version bump to aml_ips_341510000 for mainline module file: Noneaml_ips_341510000
Snap for 11273583 from 0be2581e80073ff12fc3d1a056488563e6c10516 to mainline-ipsec-release
Change-Id: I4a3c7239275a6c2baf2c5421df4ad5703e3c6ce6
34 files changed, 803 insertions, 91 deletions
diff --git a/core/Makefile b/core/Makefile index 4208672300..38dc37b260 100644 --- a/core/Makefile +++ b/core/Makefile @@ -4883,7 +4883,9 @@ my_board_extracted_kernel := true endif # INSTALLED_BOOTIMAGE_TARGET endif # my_board_extracted_kernel -ifneq ($(my_board_extracted_kernel),true) +ifeq ($(my_board_extracted_kernel),true) +$(call dist-for-goals, droid_targets, $(BUILT_KERNEL_VERSION_FILE)) +else $(warning Neither INSTALLED_KERNEL_TARGET nor INSTALLED_BOOTIMAGE_TARGET is defined when \ PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS is true. Information about the updated kernel \ cannot be built into OTA update package. You can fix this by: \ @@ -6586,6 +6588,14 @@ $(JACOCO_REPORT_CLASSES_ALL) : ifeq (,$(TARGET_BUILD_UNBUNDLED)) $(JACOCO_REPORT_CLASSES_ALL): $(INTERNAL_ALLIMAGES_FILES) endif + +# This is not ideal, but it is difficult to correctly figure out the actual jacoco report +# jars we need to add here as dependencies, so we add the device-tests as a dependency when +# the env variable is set and this should guarantee thaat all the jacoco report jars are ready +# when we package the final report jar here. +ifeq ($(JACOCO_PACKAGING_INCLUDE_DEVICE_TESTS),true) + $(JACOCO_REPORT_CLASSES_ALL): $(COMPATIBILITY.device-tests.FILES) +endif endif # EMMA_INSTRUMENT=true diff --git a/core/OWNERS b/core/OWNERS index eb1d5c3c46..88f6d068bb 100644 --- a/core/OWNERS +++ b/core/OWNERS @@ -5,3 +5,5 @@ per-file proguard*.flags = jdduke@google.com # For version updates per-file version_defaults.mk = aseaton@google.com,lubomir@google.com,pscovanner@google.com,bkhalife@google.com,jainne@google.com +# For sdk extensions version updates +per-file version_defaults.mk = amhk@google.com,gurpreetgs@google.com,mkhokhlova@google.com,robertogil@google.com diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk index 140acf0f16..29238d569b 100644 --- a/core/android_soong_config_vars.mk +++ b/core/android_soong_config_vars.mk @@ -87,9 +87,9 @@ endif ifneq (,$(MODULE_BUILD_FROM_SOURCE)) # Keep an explicit setting. -else ifeq (,$(filter docs sdk win_sdk sdk_addon,$(MAKECMDGOALS))$(findstring com.google.android.conscrypt,$(PRODUCT_PACKAGES))) +else ifeq (,$(filter docs sdk win_sdk sdk_addon,$(MAKECMDGOALS))$(findstring com.google.android.conscrypt,$(PRODUCT_PACKAGES))$(findstring com.google.android.go.conscrypt,$(PRODUCT_PACKAGES))) # Prebuilt module SDKs require prebuilt modules to work, and currently - # prebuilt modules are only provided for com.google.android.xxx. If we can't + # prebuilt modules are only provided for com.google.android(.go)?.xxx. If we can't # find one of them in PRODUCT_PACKAGES then assume com.android.xxx are in use, # and disable prebuilt SDKs. In particular this applies to AOSP builds. # @@ -128,6 +128,7 @@ INDIVIDUALLY_TOGGLEABLE_PREBUILT_MODULES := \ rkpd \ uwb \ wifi \ + mediaprovider \ $(foreach m, $(INDIVIDUALLY_TOGGLEABLE_PREBUILT_MODULES),\ $(if $(call soong_config_get,$(m)_module,source_build),,\ diff --git a/core/build_id.mk b/core/build_id.mk index a4463a6a8d..8e362e4409 100644 --- a/core/build_id.mk +++ b/core/build_id.mk @@ -18,4 +18,4 @@ # (like "CRB01"). It must be a single word, and is # capitalized by convention. -BUILD_ID=340914000 +BUILD_ID=341510000 diff --git a/core/main.mk b/core/main.mk index cb4dca6b41..04b6b33228 100644 --- a/core/main.mk +++ b/core/main.mk @@ -2157,13 +2157,21 @@ endif # TARGET_BUILD_APPS # is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile. # is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile. # is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile. - +# build_output_path: the path of the built file, used to calculate checksum +# static_libraries/whole_static_libraries: list of module name of the static libraries the file links against, e.g. libclang_rt.builtins or libclang_rt.builtins_32 +# Info of all static libraries of all installed files are collected in variable _all_static_libs that is used to list all the static library files in sbom-metadata.csv. +# See the second foreach loop in the rule of sbom-metadata.csv for the detailed info of static libraries collected in _all_static_libs. +# is_static_lib: whether the file is a static library + +metadata_list := $(OUT_DIR)/.module_paths/METADATA.list +metadata_files := $(subst $(newline),$(space),$(file <$(metadata_list))) # (TODO: b/272358583 find another way of always rebuilding this target) # Remove the sbom-metadata.csv whenever makefile is evaluated $(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1) -$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) +$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) $(metadata_list) $(metadata_files) rm -f $@ - @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@ + echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $@ + $(eval _all_static_libs :=) $(foreach f,$(installed_files),\ $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \ $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \ @@ -2185,11 +2193,25 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \ $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \ $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \ - @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \ + $(eval _static_libs := $(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES)) \ + $(eval _whole_static_libs := $(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES)) \ + $(foreach l,$(_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \ + $(foreach l,$(_whole_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \ + echo /$(_path_on_device),$(_module_path),$(_soong_module_type),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_build_output_path),$(_static_libs),$(_whole_static_libs), >> $@; \ $(if $(_post_installed_dexpreopt_zip), \ - for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \ + for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i$(comma)$(_static_libs)$(comma)$(_whole_static_libs)$(comma) >> $@ ; done ; \ ) \ ) + $(foreach l,$(sort $(_all_static_libs)), \ + $(eval _lib_stem := $(call word-colon,1,$l)) \ + $(eval _module_path := $(call word-colon,2,$l)) \ + $(eval _soong_module_type := $(call word-colon,3,$l)) \ + $(eval _built_file := $(call word-colon,4,$l)) \ + $(eval _static_libs := $(ALL_STATIC_LIBRARIES.$l.STATIC_LIBRARIES)) \ + $(eval _whole_static_libs := $(ALL_STATIC_LIBRARIES.$l.WHOLE_STATIC_LIBRARIES)) \ + $(eval _is_static_lib := Y) \ + echo $(_lib_stem).a,$(_module_path),$(_soong_module_type),,,,,$(_built_file),$(_static_libs),$(_whole_static_libs),$(_is_static_lib) >> $@; \ + ) .PHONY: sbom ifeq ($(TARGET_BUILD_APPS),) @@ -2201,17 +2223,47 @@ $(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) $(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json) else -apps_only_sbom_files := $(sort $(patsubst %,%.spdx.json,$(filter %.apk,$(apps_only_installed_files)))) -$(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) - rm -rf $@ - $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --unbundled_apk +# Create build rules for generating SBOMs of unbundled APKs and APEXs +# $1: sbom file +# $2: sbom fragment file +# $3: installed file +# $4: sbom-metadata.csv file +define generate-app-sbom +$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$(3))) +$(eval _module_name := $(ALL_INSTALLED_FILES.$(3))) +$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) +$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) +$(eval _dep_modules := $(filter %.$(_module_name),$(ALL_MODULES)) $(filter %.$(_module_name)$(TARGET_2ND_ARCH_MODULE_SUFFIX),$(ALL_MODULES))) +$(eval _is_apex := $(filter %.apex,$(3))) + +$(4): $(3) $(metadata_list) $(metadata_files) + rm -rf $$@ + echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $$@ + echo /$(_path_on_device),$(_module_path),$(_soong_module_type),,,,,$(3),,, >> $$@ + $(if $(filter %.apex,$(3)),\ + $(foreach m,$(_dep_modules),\ + echo $(patsubst $(PRODUCT_OUT)/apex/$(_module_name)/%,%,$(ALL_MODULES.$m.INSTALLED)),$(sort $(ALL_MODULES.$m.PATH)),$(sort $(ALL_MODULES.$m.SOONG_MODULE_TYPE)),,,,,$(strip $(ALL_MODULES.$m.BUILT)),,, >> $$@;)) + +$(2): $(1) +$(1): $(4) $(GEN_SBOM) + rm -rf $$@ + $(GEN_SBOM) --output_file $$@ --metadata $(4) --build_version $$(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(if $(filter %.apk,$(3)),--unbundled_apk,--unbundled_apex) +endef + +apps_only_sbom_files := +apps_only_fragment_files := +$(foreach f,$(filter %.apk %.apex,$(installed_files)), \ + $(eval _metadata_csv_file := $(patsubst %,%-sbom-metadata.csv,$f)) \ + $(eval _sbom_file := $(patsubst %,%.spdx.json,$f)) \ + $(eval _fragment_file := $(patsubst %,%-fragment.spdx,$f)) \ + $(eval apps_only_sbom_files += $(_sbom_file)) \ + $(eval apps_only_fragment_files += $(_fragment_file)) \ + $(eval $(call generate-app-sbom,$(_sbom_file),$(_fragment_file),$f,$(_metadata_csv_file))) \ +) sbom: $(apps_only_sbom_files) -$(foreach f,$(apps_only_sbom_files),$(eval $(patsubst %.spdx.json,%-fragment.spdx,$f): $f)) -apps_only_fragment_files := $(patsubst %.spdx.json,%-fragment.spdx,$(apps_only_sbom_files)) $(foreach f,$(apps_only_fragment_files),$(eval apps_only_fragment_dist_files += :sbom/$(notdir $f))) - $(foreach f,$(apps_only_sbom_files),$(eval apps_only_sbom_dist_files += :sbom/$(notdir $f))) $(call dist-for-goals,apps_only,$(join $(apps_only_sbom_files),$(apps_only_sbom_dist_files)) $(join $(apps_only_fragment_files),$(apps_only_fragment_dist_files))) endif diff --git a/core/sbom.mk b/core/sbom.mk index e23bbc132f..39c251ae0c 100644 --- a/core/sbom.mk +++ b/core/sbom.mk @@ -3,9 +3,20 @@ # unless a .mk file changes its installed file after including base_rules.mk. ifdef my_register_name + # ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle libclang_rt.builtins, for primary arch + # ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle_32 libclang_rt.builtins_32, for 2nd arch. ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED))) $(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\ $(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\ + $(eval ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\ + $(eval ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\ ) endif + ifeq (STATIC_LIBRARIES,$(LOCAL_MODULE_CLASS)) + ALL_STATIC_LIBRARIES.$(my_register_name).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX)) + ALL_STATIC_LIBRARIES.$(my_register_name).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX)) + ifdef LOCAL_SOONG_MODULE_TYPE + ALL_STATIC_LIBRARIES.$(my_register_name).BUILT_FILE := $(LOCAL_PREBUILT_MODULE_FILE) + endif + endif endif
\ No newline at end of file diff --git a/core/version_defaults.mk b/core/version_defaults.mk index 4442b10729..160c6455ca 100644 --- a/core/version_defaults.mk +++ b/core/version_defaults.mk @@ -79,7 +79,7 @@ endif .KATI_READONLY := PLATFORM_SDK_VERSION # This is the sdk extension version of this tree. -PLATFORM_SDK_EXTENSION_VERSION := 8 +PLATFORM_SDK_EXTENSION_VERSION := 11 .KATI_READONLY := PLATFORM_SDK_EXTENSION_VERSION # This is the sdk extension version that PLATFORM_SDK_VERSION ships with. @@ -104,7 +104,7 @@ ifndef PLATFORM_SECURITY_PATCH # It must be of the form "YYYY-MM-DD" on production devices. # It must match one of the Android Security Patch Level strings of the Public Security Bulletins. # If there is no $PLATFORM_SECURITY_PATCH set, keep it empty. - PLATFORM_SECURITY_PATCH := 2023-08-05 + PLATFORM_SECURITY_PATCH := 2024-01-01 endif include $(BUILD_SYSTEM)/version_util.mk diff --git a/tools/metadata/Android.bp b/tools/metadata/Android.bp new file mode 100644 index 0000000000..b2fabecb96 --- /dev/null +++ b/tools/metadata/Android.bp @@ -0,0 +1,14 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "metadata", + deps: [ + "soong-testing-test_spec_proto", + "golang-protobuf-proto", + ], + srcs: [ + "generator.go", + ] +}
\ No newline at end of file diff --git a/tools/metadata/OWNERS b/tools/metadata/OWNERS new file mode 100644 index 0000000000..03bcdf1c40 --- /dev/null +++ b/tools/metadata/OWNERS @@ -0,0 +1,4 @@ +dariofreni@google.com +joeo@google.com +ronish@google.com +caditya@google.com diff --git a/tools/metadata/generator.go b/tools/metadata/generator.go new file mode 100644 index 0000000000..e970e1708f --- /dev/null +++ b/tools/metadata/generator.go @@ -0,0 +1,195 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "sort" + "strings" + "sync" + + "android/soong/testing/test_spec_proto" + "google.golang.org/protobuf/proto" +) + +type keyToLocksMap struct { + locks sync.Map +} + +func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex { + mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{}) + return mutex.(*sync.Mutex) +} + +func getSortedKeys(syncMap *sync.Map) []string { + var allKeys []string + syncMap.Range( + func(key, _ interface{}) bool { + allKeys = append(allKeys, key.(string)) + return true + }, + ) + + sort.Strings(allKeys) + return allKeys +} + +func writeOutput( + outputFile string, + allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata, +) { + testSpec := &test_spec_proto.TestSpec{ + OwnershipMetadataList: allMetadata, + } + data, err := proto.Marshal(testSpec) + if err != nil { + log.Fatal(err) + } + file, err := os.Create(outputFile) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + _, err = file.Write(data) + if err != nil { + log.Fatal(err) + } +} + +func readFileToString(filePath string) string { + file, err := os.Open(filePath) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + log.Fatal(err) + } + return string(data) +} + +func writeNewlineToOutputFile(outputFile string) { + file, err := os.Create(outputFile) + data := "\n" + if err != nil { + log.Fatal(err) + } + defer file.Close() + + _, err = file.Write([]byte(data)) + if err != nil { + log.Fatal(err) + } +} + +func processTestSpecProtobuf( + filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap, + errCh chan error, wg *sync.WaitGroup, +) { + defer wg.Done() + + fileContent := strings.TrimRight(readFileToString(filePath), "\n") + testData := test_spec_proto.TestSpec{} + err := proto.Unmarshal([]byte(fileContent), &testData) + if err != nil { + errCh <- err + return + } + + ownershipMetadata := testData.GetOwnershipMetadataList() + for _, metadata := range ownershipMetadata { + key := metadata.GetTargetName() + lock := keyLocks.GetLockForKey(key) + lock.Lock() + + value, loaded := ownershipMetadataMap.LoadOrStore( + key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata}, + ) + if loaded { + existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata) + isDuplicate := false + for _, existing := range existingMetadata { + if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() { + errCh <- fmt.Errorf( + "Conflicting trendy team IDs found for %s at:\n%s with teamId"+ + ": %s,\n%s with teamId: %s", + key, + metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(), + existing.GetTrendyTeamId(), + ) + + lock.Unlock() + return + } + if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() { + isDuplicate = true + break + } + } + if !isDuplicate { + existingMetadata = append(existingMetadata, metadata) + ownershipMetadataMap.Store(key, existingMetadata) + } + } + + lock.Unlock() + } +} + +func main() { + inputFile := flag.String("inputFile", "", "Input file path") + outputFile := flag.String("outputFile", "", "Output file path") + rule := flag.String("rule", "", "Metadata rule (Hint: test_spec or code_metadata)") + flag.Parse() + + if *inputFile == "" || *outputFile == "" || *rule == "" { + fmt.Println("Usage: metadata -rule <rule> -inputFile <input file path> -outputFile <output file path>") + os.Exit(1) + } + + inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n") + filePaths := strings.Split(inputFileData, " ") + if len(filePaths) == 1 && filePaths[0] == "" { + writeNewlineToOutputFile(*outputFile) + return + } + ownershipMetadataMap := &sync.Map{} + keyLocks := &keyToLocksMap{} + errCh := make(chan error, len(filePaths)) + var wg sync.WaitGroup + + switch *rule { + case "test_spec": + for _, filePath := range filePaths { + wg.Add(1) + go processTestSpecProtobuf(filePath, ownershipMetadataMap, keyLocks, errCh, &wg) + } + + wg.Wait() + close(errCh) + + for err := range errCh { + log.Fatal(err) + } + + allKeys := getSortedKeys(ownershipMetadataMap) + var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata + + for _, key := range allKeys { + value, _ := ownershipMetadataMap.Load(key) + metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata) + allMetadata = append(allMetadata, metadataList...) + } + + writeOutput(*outputFile, allMetadata) + break + case "code_metadata": + default: + log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule) + } +} diff --git a/tools/metadata/go.mod b/tools/metadata/go.mod new file mode 100644 index 0000000000..e9d04b16f6 --- /dev/null +++ b/tools/metadata/go.mod @@ -0,0 +1,7 @@ +module android/soong/tools/metadata + +require google.golang.org/protobuf v0.0.0 + +replace google.golang.org/protobuf v0.0.0 => ../../../external/golang-protobuf + +go 1.18
\ No newline at end of file diff --git a/tools/metadata/go.work b/tools/metadata/go.work new file mode 100644 index 0000000000..23875daf3d --- /dev/null +++ b/tools/metadata/go.work @@ -0,0 +1,10 @@ +go 1.18 + +use ( + . + ../../../../external/golang-protobuf + ../../../soong/testing/test_spec_proto + +) + +replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf diff --git a/tools/metadata/testdata/emptyInputFile.txt b/tools/metadata/testdata/emptyInputFile.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/tools/metadata/testdata/emptyInputFile.txt @@ -0,0 +1 @@ + diff --git a/tools/metadata/testdata/expectedOutputFile.txt b/tools/metadata/testdata/expectedOutputFile.txt new file mode 100644 index 0000000000..b0d382f279 --- /dev/null +++ b/tools/metadata/testdata/expectedOutputFile.txt @@ -0,0 +1,22 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/metadata/testdata/file1.txt b/tools/metadata/testdata/file1.txt new file mode 100644 index 0000000000..81beed00ab --- /dev/null +++ b/tools/metadata/testdata/file1.txt @@ -0,0 +1,13 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345 diff --git a/tools/metadata/testdata/file2.txt b/tools/metadata/testdata/file2.txt new file mode 100644 index 0000000000..32a753fef5 --- /dev/null +++ b/tools/metadata/testdata/file2.txt @@ -0,0 +1,25 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 diff --git a/tools/metadata/testdata/file3.txt b/tools/metadata/testdata/file3.txt new file mode 100644 index 0000000000..81beed00ab --- /dev/null +++ b/tools/metadata/testdata/file3.txt @@ -0,0 +1,13 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345 diff --git a/tools/metadata/testdata/file4.txt b/tools/metadata/testdata/file4.txt new file mode 100644 index 0000000000..6a7590021d --- /dev/null +++ b/tools/metadata/testdata/file4.txt @@ -0,0 +1,25 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-one +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 diff --git a/tools/metadata/testdata/generatedEmptyOutputFile.txt b/tools/metadata/testdata/generatedEmptyOutputFile.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/tools/metadata/testdata/generatedEmptyOutputFile.txt @@ -0,0 +1 @@ + diff --git a/tools/metadata/testdata/generatedOutputFile.txt b/tools/metadata/testdata/generatedOutputFile.txt new file mode 100644 index 0000000000..b0d382f279 --- /dev/null +++ b/tools/metadata/testdata/generatedOutputFile.txt @@ -0,0 +1,22 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/metadata/testdata/inputFiles.txt b/tools/metadata/testdata/inputFiles.txt new file mode 100644 index 0000000000..e44bc94d32 --- /dev/null +++ b/tools/metadata/testdata/inputFiles.txt @@ -0,0 +1 @@ +file1.txt file2.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputFilesNegativeCase.txt b/tools/metadata/testdata/inputFilesNegativeCase.txt new file mode 100644 index 0000000000..a37aa3fd5d --- /dev/null +++ b/tools/metadata/testdata/inputFilesNegativeCase.txt @@ -0,0 +1 @@ +file3.txt file4.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/metadata_test.go b/tools/metadata/testdata/metadata_test.go new file mode 100644 index 0000000000..71856fe606 --- /dev/null +++ b/tools/metadata/testdata/metadata_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os/exec" + "strings" + "testing" +) + +func TestMetadata(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "test_spec", "-inputFile", "./inputFiles.txt", "-outputFile", + "./generatedOutputFile.txt", + ) + stderr, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) + } + + // Read the contents of the expected output file + expectedOutput, err := ioutil.ReadFile("./expectedOutputFile.txt") + if err != nil { + t.Fatalf("Error reading expected output file: %s", err) + } + + // Read the contents of the generated output file + generatedOutput, err := ioutil.ReadFile("./generatedOutputFile.txt") + if err != nil { + t.Fatalf("Error reading generated output file: %s", err) + } + + fmt.Println() + + // Compare the contents + if string(expectedOutput) != string(generatedOutput) { + t.Errorf("Generated file contents do not match the expected output") + } +} + +func TestMetadataNegativeCase(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "test_spec", "-inputFile", "./inputFilesNegativeCase.txt", "-outputFile", + "./generatedOutputFileNegativeCase.txt", + ) + stderr, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf( + "Expected an error, but the metadata command executed successfully. Output: %s", + stderr, + ) + } + + expectedError := "Conflicting trendy team IDs found for java-test-module" + + "-name-one at:\nAndroid.bp with teamId: 12346," + + "\nAndroid.bp with teamId: 12345" + if !strings.Contains( + strings.TrimSpace(string(stderr)), strings.TrimSpace(expectedError), + ) { + t.Errorf( + "Unexpected error message. Expected to contain: %s, Got: %s", + expectedError, stderr, + ) + } +} + +func TestEmptyInputFile(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "test_spec", "-inputFile", "./emptyInputFile.txt", "-outputFile", + "./generatedEmptyOutputFile.txt", + ) + stderr, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) + } + + // Read the contents of the generated output file + generatedOutput, err := ioutil.ReadFile("./generatedEmptyOutputFile.txt") + if err != nil { + t.Fatalf("Error reading generated output file: %s", err) + } + + fmt.Println() + + // Compare the contents + if string(generatedOutput) != "\n" { + t.Errorf("Generated file contents do not match the expected output") + } +} diff --git a/tools/metadata/testdata/outputFile.txt b/tools/metadata/testdata/outputFile.txt new file mode 100644 index 0000000000..b0d382f279 --- /dev/null +++ b/tools/metadata/testdata/outputFile.txt @@ -0,0 +1,22 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp index a76dc8a8bc..d07292a082 100644 --- a/tools/releasetools/Android.bp +++ b/tools/releasetools/Android.bp @@ -236,6 +236,9 @@ python_library_host { "rangelib.py", "sparse_img.py", ], + data: [ + ":zip2zip", + ], // Only the tools that are referenced directly are listed as required modules. For example, // `avbtool` is not here, as the script always uses the one from info_dict['avb_avbtool']. required: [ diff --git a/tools/releasetools/check_target_files_signatures.py b/tools/releasetools/check_target_files_signatures.py index d935607e43..a7b35230ea 100755 --- a/tools/releasetools/check_target_files_signatures.py +++ b/tools/releasetools/check_target_files_signatures.py @@ -241,7 +241,8 @@ class APK(object): # Signer (minSdkVersion=24, maxSdkVersion=32) certificate SHA-1 digest: 19da94896ce4078c38ca695701f1dec741ec6d67 # ... certs_info = {} - certificate_regex = re.compile(r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)") + certificate_regex = re.compile( + r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)") for line in output.splitlines(): m = certificate_regex.match(line) if not m: @@ -312,7 +313,7 @@ class TargetFiles(object): # This is the list of wildcards of files we extract from |filename|. apk_extensions = ['*.apk', '*.apex'] - with zipfile.ZipFile(filename) as input_zip: + with zipfile.ZipFile(filename, "r") as input_zip: self.certmap, compressed_extension = common.ReadApkCerts(input_zip) if compressed_extension: apk_extensions.append('*.apk' + compressed_extension) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index b50caaa95e..abedecfa76 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -35,6 +35,7 @@ import shlex import shutil import subprocess import sys +import stat import tempfile import threading import time @@ -2102,6 +2103,26 @@ def Gunzip(in_filename, out_filename): shutil.copyfileobj(in_file, out_file) +def UnzipSingleFile(input_zip: zipfile.ZipFile, info: zipfile.ZipInfo, dirname: str): + # According to https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip/6297838#6297838 + # higher bits of |external_attr| are unix file permission and types + unix_filetype = info.external_attr >> 16 + + def CheckMask(a, mask): + return (a & mask) == mask + + def IsSymlink(a): + return CheckMask(a, stat.S_IFLNK) + # python3.11 zipfile implementation doesn't handle symlink correctly + if not IsSymlink(unix_filetype): + return input_zip.extract(info, dirname) + if dirname is None: + dirname = os.getcwd() + target = os.path.join(dirname, info.filename) + os.makedirs(os.path.dirname(target), exist_ok=True) + os.symlink(input_zip.read(info).decode(), target) + + def UnzipToDir(filename, dirname, patterns=None): """Unzips the archive to the given directory. @@ -2112,20 +2133,46 @@ def UnzipToDir(filename, dirname, patterns=None): archvie. Non-matching patterns will be filtered out. If there's no match after the filtering, no file will be unzipped. """ - cmd = ["unzip", "-o", "-q", filename, "-d", dirname] - if patterns is not None: + with zipfile.ZipFile(filename, allowZip64=True, mode="r") as input_zip: # Filter out non-matching patterns. unzip will complain otherwise. - with zipfile.ZipFile(filename, allowZip64=True) as input_zip: - names = input_zip.namelist() - filtered = [ - pattern for pattern in patterns if fnmatch.filter(names, pattern)] - - # There isn't any matching files. Don't unzip anything. - if not filtered: - return - cmd.extend(filtered) + entries = input_zip.infolist() + # b/283033491 + # Per https://en.wikipedia.org/wiki/ZIP_(file_format)#Central_directory_file_header + # In zip64 mode, central directory record's header_offset field might be + # set to 0xFFFFFFFF if header offset is > 2^32. In this case, the extra + # fields will contain an 8 byte little endian integer at offset 20 + # to indicate the actual local header offset. + # As of python3.11, python does not handle zip64 central directories + # correctly, so we will manually do the parsing here. + + # ZIP64 central directory extra field has two required fields: + # 2 bytes header ID and 2 bytes size field. Thes two require fields have + # a total size of 4 bytes. Then it has three other 8 bytes field, followed + # by a 4 byte disk number field. The last disk number field is not required + # to be present, but if it is present, the total size of extra field will be + # divisible by 8(because 2+2+4+8*n is always going to be multiple of 8) + # Most extra fields are optional, but when they appear, their must appear + # in the order defined by zip64 spec. Since file header offset is the 2nd + # to last field in zip64 spec, it will only be at last 8 bytes or last 12-4 + # bytes, depending on whether disk number is present. + for entry in entries: + if entry.header_offset == 0xFFFFFFFF: + if len(entry.extra) % 8 == 0: + entry.header_offset = int.from_bytes(entry.extra[-12:-4], "little") + else: + entry.header_offset = int.from_bytes(entry.extra[-8:], "little") + if patterns is not None: + filtered = [info for info in entries if any( + [fnmatch.fnmatch(info.filename, p) for p in patterns])] - RunAndCheckOutput(cmd) + # There isn't any matching files. Don't unzip anything. + if not filtered: + return + for info in filtered: + UnzipSingleFile(input_zip, info, dirname) + else: + for info in entries: + UnzipSingleFile(input_zip, info, dirname) def UnzipTemp(filename, patterns=None): diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py index 2415f7e270..b19be87666 100755 --- a/tools/sbom/generate-sbom.py +++ b/tools/sbom/generate-sbom.py @@ -332,14 +332,6 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path): return external_doc_ref, packages, relationships -def generate_package_verification_code(files): - checksums = [file.checksum for file in files] - checksums.sort() - h = hashlib.sha1() - h.update(''.join(checksums).encode(encoding='utf-8')) - return h.hexdigest() - - def save_report(report_file_path, report): with open(report_file_path, 'w', encoding='utf-8') as report_file: for type, issues in report.items(): @@ -487,20 +479,32 @@ def main(): product_copy_files = installed_file_metadata['product_copy_files'] kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] build_output_path = installed_file_metadata['build_output_path'] + is_static_lib = installed_file_metadata['is_static_lib'] if not installed_file_has_metadata(installed_file_metadata, report): continue - if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)): + if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)): + # Ignore non-existing static library files for now since they are not shipped on devices. report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file) continue file_id = new_file_id(installed_file) - doc.files.append( - sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path))) - if not args.unbundled_apex: - product_package.file_ids.append(file_id) - elif len(doc.files) > 1: - doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id)) + # TODO(b/285453664): Soong should report the information of statically linked libraries to Make. + # This happens when a different sanitized version of static libraries is used in linking. + # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be + # located correctly because Soong doesn't report the information to Make. + sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string + if os.path.islink(build_output_path) or os.path.isfile(build_output_path): + sha1 = checksum(build_output_path) + doc.files.append(sbom_data.File(id=file_id, + name=installed_file, + checksum=sha1)) + + if not is_static_lib: + if not args.unbundled_apex: + product_package.file_ids.append(file_id) + elif len(doc.files) > 1: + doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id)) if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): metadata_file_path = get_metadata_file_path(installed_file_metadata) @@ -544,13 +548,21 @@ def main(): relationship=sbom_data.RelationshipType.GENERATED_FROM, id2=sbom_data.SPDXID_PLATFORM)) - if not args.unbundled_apex: - product_package.verification_code = generate_package_verification_code(doc.files) + # Process static libraries and whole static libraries the installed file links to + static_libs = installed_file_metadata['static_libraries'] + whole_static_libs = installed_file_metadata['whole_static_libraries'] + all_static_libs = (static_libs + ' ' + whole_static_libs).strip() + if all_static_libs: + for lib in all_static_libs.split(' '): + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.STATIC_LINK, + id2=new_file_id(lib + '.a'))) if args.unbundled_apex: doc.describes = doc.files[0].id # Save SBOM records to output file + doc.generate_packages_verification_code() doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') prefix = args.output_file if prefix.endswith('.spdx'): diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py index 14c4eb231d..ea38e364dc 100644 --- a/tools/sbom/sbom_data.py +++ b/tools/sbom/sbom_data.py @@ -25,6 +25,7 @@ fields in each data class. from dataclasses import dataclass, field from typing import List +import hashlib SPDXID_DOC = 'SPDXRef-DOCUMENT' SPDXID_PRODUCT = 'SPDXRef-PRODUCT' @@ -81,6 +82,7 @@ class RelationshipType: VARIANT_OF = 'VARIANT_OF' GENERATED_FROM = 'GENERATED_FROM' CONTAINS = 'CONTAINS' + STATIC_LINK = 'STATIC_LINK' @dataclass @@ -122,3 +124,17 @@ class Document: if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship for r in self.relationships): self.relationships.append(rel) + + def generate_packages_verification_code(self): + for package in self.packages: + if not package.file_ids: + continue + + checksums = [] + for file in self.files: + if file.id in package.file_ids: + checksums.append(file.checksum) + checksums.sort() + h = hashlib.sha1() + h.update(''.join(checksums).encode(encoding='utf-8')) + package.verification_code = h.hexdigest() diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py index 85dee9d1a5..1cb864db75 100644 --- a/tools/sbom/sbom_writers.py +++ b/tools/sbom/sbom_writers.py @@ -85,7 +85,7 @@ class TagValueWriter: return headers @staticmethod - def marshal_package(package): + def marshal_package(sbom_doc, package, fragment): download_location = sbom_data.VALUE_NOASSERTION if package.download_location: download_location = package.download_location @@ -107,50 +107,32 @@ class TagValueWriter: f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}') tagvalues.append('') - return tagvalues - - @staticmethod - def marshal_described_element(sbom_doc, fragment): - if not sbom_doc.describes: - return None - - product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes] - if product_package: - tagvalues = TagValueWriter.marshal_package(product_package[0]) - if not fragment: - tagvalues.append( - f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') + if package.id == sbom_doc.describes and not fragment: + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') tagvalues.append('') - return tagvalues - file = [f for f in sbom_doc.files if f.id == sbom_doc.describes] - if file: - tagvalues = TagValueWriter.marshal_file(file[0]) - if not fragment: - tagvalues.append( - f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') - - return tagvalues + for file in sbom_doc.files: + if file.id in package.file_ids: + tagvalues += TagValueWriter.marshal_file(file) - return None + return tagvalues @staticmethod - def marshal_packages(sbom_doc): + def marshal_packages(sbom_doc, fragment): tagvalues = [] marshaled_relationships = [] i = 0 packages = sbom_doc.packages while i < len(packages): - if packages[i].id == sbom_doc.describes: - i += 1 - continue - - if i + 1 < len(packages) \ - and packages[i].id.startswith('SPDXRef-SOURCE-') \ - and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'): - tagvalues += TagValueWriter.marshal_package(packages[i]) - tagvalues += TagValueWriter.marshal_package(packages[i + 1]) + if (i + 1 < len(packages) + and packages[i].id.startswith('SPDXRef-SOURCE-') + and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')): + # Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other + # in SBOMs in tagvalue format. + tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment) + tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment) rel = next((r for r in sbom_doc.relationships if r.id1 == packages[i].id and r.id2 == packages[i + 1].id and @@ -162,7 +144,7 @@ class TagValueWriter: i += 2 else: - tagvalues += TagValueWriter.marshal_package(packages[i]) + tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment) i += 1 return tagvalues, marshaled_relationships @@ -179,12 +161,20 @@ class TagValueWriter: return tagvalues @staticmethod - def marshal_files(sbom_doc): + def marshal_files(sbom_doc, fragment): tagvalues = [] + files_in_packages = [] + for package in sbom_doc.packages: + files_in_packages += package.file_ids for file in sbom_doc.files: - if file.id == sbom_doc.describes: + if file.id in files_in_packages: continue tagvalues += TagValueWriter.marshal_file(file) + if file.id == sbom_doc.describes and not fragment: + # Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable. + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') + tagvalues.append('') return tagvalues @staticmethod @@ -208,11 +198,8 @@ class TagValueWriter: content = [] if not fragment: content += TagValueWriter.marshal_doc_headers(sbom_doc) - described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment) - if described_element: - content += described_element - content += TagValueWriter.marshal_files(sbom_doc) - tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc) + content += TagValueWriter.marshal_files(sbom_doc, fragment) + tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment) content += tagvalues content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships) file.write('\n'.join(content)) diff --git a/tools/sbom/sbom_writers_test.py b/tools/sbom/sbom_writers_test.py index 361dae6f5d..cf85e014b7 100644 --- a/tools/sbom/sbom_writers_test.py +++ b/tools/sbom/sbom_writers_test.py @@ -31,6 +31,7 @@ SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1' SPDXID_FILE1 = 'SPDXRef-file1' SPDXID_FILE2 = 'SPDXRef-file2' SPDXID_FILE3 = 'SPDXRef-file3' +SPDXID_FILE4 = 'SPDXRef-file4' class SBOMWritersTest(unittest.TestCase): @@ -101,6 +102,8 @@ class SBOMWritersTest(unittest.TestCase): sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222')) self.sbom_doc.files.append( sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333')) + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444')) self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, relationship=sbom_data.RelationshipType.GENERATED_FROM, @@ -112,6 +115,10 @@ class SBOMWritersTest(unittest.TestCase): relationship=sbom_data.RelationshipType.GENERATED_FROM, id2=SPDXID_SOURCE_PACKAGE1 )) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.STATIC_LINK, + id2=SPDXID_FILE4 + )) # SBOM fragment of a APK self.unbundled_sbom_doc = sbom_data.Document(name='test doc', @@ -139,6 +146,14 @@ class SBOMWritersTest(unittest.TestCase): self.maxDiff = None self.assertEqual(expected_output, output.getvalue()) + def test_tagvalue_writer_doc_describes_file(self): + with io.StringIO() as output: + self.sbom_doc.describes = SPDXID_FILE4 + sbom_writers.TagValueWriter.write(self.sbom_doc, output) + expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_doc_describes_file.spdx').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + def test_tagvalue_writer_unbundled(self): with io.StringIO() as output: sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True) diff --git a/tools/sbom/testdata/expected_json_sbom.spdx.json b/tools/sbom/testdata/expected_json_sbom.spdx.json index 32715a5392..53936c5c9d 100644 --- a/tools/sbom/testdata/expected_json_sbom.spdx.json +++ b/tools/sbom/testdata/expected_json_sbom.spdx.json @@ -110,6 +110,16 @@ "checksumValue": "33333" } ] + }, + { + "fileName": "file4.a", + "SPDXID": "SPDXRef-file4", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "44444" + } + ] } ], "relationships": [ @@ -129,6 +139,11 @@ "relationshipType": "GENERATED_FROM" }, { + "spdxElementId": "SPDXRef-file1", + "relatedSpdxElement": "SPDXRef-file4", + "relationshipType": "STATIC_LINK" + }, + { "spdxElementId": "SPDXRef-SOURCE-package1", "relatedSpdxElement": "SPDXRef-UPSTREAM-package1", "relationshipType": "VARIANT_OF" diff --git a/tools/sbom/testdata/expected_tagvalue_sbom.spdx b/tools/sbom/testdata/expected_tagvalue_sbom.spdx index ee39e82fcf..e6fd17e9c3 100644 --- a/tools/sbom/testdata/expected_tagvalue_sbom.spdx +++ b/tools/sbom/testdata/expected_tagvalue_sbom.spdx @@ -7,6 +7,10 @@ Creator: Organization: Google Created: 2023-03-31T22:17:58Z ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890 +FileName: file4.a +SPDXID: SPDXRef-file4 +FileChecksum: SHA1: 44444 + PackageName: PRODUCT SPDXID: SPDXRef-PRODUCT PackageDownloadLocation: NONE @@ -63,3 +67,4 @@ Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1 Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1 Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1 +Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4 diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx new file mode 100644 index 0000000000..428d7e3ddb --- /dev/null +++ b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx @@ -0,0 +1,70 @@ +SPDXVersion: SPDX-2.3 +DataLicense: CC0-1.0 +SPDXID: SPDXRef-DOCUMENT +DocumentName: test doc +DocumentNamespace: http://www.google.com/sbom/spdx/android +Creator: Organization: Google +Created: 2023-03-31T22:17:58Z +ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890 + +FileName: file4.a +SPDXID: SPDXRef-file4 +FileChecksum: SHA1: 44444 + +Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-file4 + +PackageName: PRODUCT +SPDXID: SPDXRef-PRODUCT +PackageDownloadLocation: NONE +FilesAnalyzed: true +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +PackageVerificationCode: 123456 + +FileName: /bin/file1 +SPDXID: SPDXRef-file1 +FileChecksum: SHA1: 11111 + +FileName: /bin/file2 +SPDXID: SPDXRef-file2 +FileChecksum: SHA1: 22222 + +FileName: /bin/file3 +SPDXID: SPDXRef-file3 +FileChecksum: SHA1: 33333 + +PackageName: PLATFORM +SPDXID: SPDXRef-PLATFORM +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Prebuilt package1 +SPDXID: SPDXRef-PREBUILT-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Source package1 +SPDXID: SPDXRef-SOURCE-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4 + +PackageName: Upstream package1 +SPDXID: SPDXRef-UPSTREAM-package1 +PackageDownloadLocation: NOASSERTION +FilesAnalyzed: false +PackageVersion: 1.1 +PackageSupplier: Organization: upstream + +Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1 + +Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM +Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1 +Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1 +Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4 |