diff options
author | Kevin Han <kevhan@google.com> | 2023-03-13 21:28:45 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2023-03-13 21:28:45 +0000 |
commit | a27f3cef8b5020b4a4e94e186a59a0a587276f76 (patch) | |
tree | 4458e4e83c12a14bb0c15549fb12d4322871f630 | |
parent | 50aeb8e735aed6f5874b79af828754794328114d (diff) | |
parent | afb2495f6fb1915446865282528f7a850fd86ee9 (diff) | |
download | build-temp_273316506_tm-qpr2.tar.gz |
Merge "Revert "Generate SBOM of the target product in file sbom.spdx in..."" am: afb2495f6ftemp_273316506_tm-qpr2
Original change: https://android-review.googlesource.com/c/platform/build/+/2485539
Change-Id: I6a2405ac28898643a070616d7da3d5f49df9a9b2
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
-rw-r--r-- | core/Makefile | 29 | ||||
-rw-r--r-- | core/app_prebuilt_internal.mk | 4 | ||||
-rw-r--r-- | core/base_rules.mk | 7 | ||||
-rw-r--r-- | core/clear_vars.mk | 1 | ||||
-rw-r--r-- | core/config.mk | 3 | ||||
-rw-r--r-- | core/dex_preopt_odex_install.mk | 1 | ||||
-rw-r--r-- | core/main.mk | 80 | ||||
-rw-r--r-- | core/prebuilt_internal.mk | 3 | ||||
-rw-r--r-- | core/sbom.mk | 11 | ||||
-rw-r--r-- | core/soong_app_prebuilt.mk | 5 | ||||
-rw-r--r-- | core/sysprop.mk | 16 | ||||
-rw-r--r-- | tools/Android.bp | 16 | ||||
-rwxr-xr-x | tools/generate-sbom.py | 684 | ||||
-rw-r--r-- | tools/protos/Android.bp | 32 | ||||
-rw-r--r-- | tools/protos/metadata_file.proto | 281 |
15 files changed, 2 insertions, 1171 deletions
diff --git a/core/Makefile b/core/Makefile index b501c07e9b..e7d0e4c3a5 100644 --- a/core/Makefile +++ b/core/Makefile @@ -474,10 +474,7 @@ $(if $(strip $(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver))$(BOARD_$(1)_KERNEL_MODU $(eval BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver) := $(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)))) \ $(if $(filter false,$(BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver))),\ $(eval BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver) :=),) \ - $(eval _files := $(call build-image-kernel-modules,$(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)),$(2),$(3),$(call intermediates-dir-for,PACKAGING,depmod_$(1)$(_sep)$(_kver)),$(BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver)),$(4),$(BOARD_$(1)_KERNEL_MODULES_ARCHIVE$(_sep)$(_kver)),$(_stripped_staging_dir),$(_kver),$(7),$(8))) \ - $(call copy-many-files,$(_files)) \ - $(eval _modules := $(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)) ANDROID-GEN ANDROID-GEN ANDROID-GEN ANDROID-GEN) \ - $(eval KERNEL_MODULE_COPY_FILES += $(join $(addsuffix :,$(_modules)),$(_files)))) \ + $(call copy-many-files,$(call build-image-kernel-modules,$(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)),$(2),$(3),$(call intermediates-dir-for,PACKAGING,depmod_$(1)$(_sep)$(_kver)),$(BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver)),$(4),$(BOARD_$(1)_KERNEL_MODULES_ARCHIVE$(_sep)$(_kver)),$(_stripped_staging_dir),$(_kver),$(7),$(8)))) \ $(if $(_kver), \ $(eval _dir := $(_kver)/), \ $(eval _dir :=)) \ @@ -490,7 +487,6 @@ $(if $(BOARD_$(1)_KERNEL_MODULES_BLOCKLIST_FILE$(_sep)$(_kver)), \ $(eval $(call build-image-kernel-modules-blocklist-file, \ $(BOARD_$(1)_KERNEL_MODULES_BLOCKLIST_FILE$(_sep)$(_kver)), \ $(2)/lib/modules/$(_dir)modules.blocklist)) \ - $(eval ALL_KERNEL_MODULES_BLOCKLIST += $(2)/lib/modules/$(_dir)modules.blocklist) \ $(2)/lib/modules/$(_dir)modules.blocklist) endef @@ -1639,21 +1635,6 @@ target_system_dlkm_notice_file_txt := $(TARGET_OUT_INTERMEDIATES)/NOTICE_SYSTEM_ target_system_dlkm_notice_file_xml_gz := $(TARGET_OUT_INTERMEDIATES)/NOTICE_SYSTEM_DLKM.xml.gz installed_system_dlkm_notice_xml_gz := $(TARGET_OUT_SYSTEM_DLKM)/etc/NOTICE.xml.gz -ALL_INSTALLED_NOTICE_FILES := \ - $(installed_notice_html_or_xml_gz) \ - $(installed_vendor_notice_xml_gz) \ - $(installed_product_notice_xml_gz) \ - $(installed_system_ext_notice_xml_gz) \ - $(installed_odm_notice_xml_gz) \ - $(installed_vendor_dlkm_notice_xml_gz) \ - $(installed_odm_dlkm_notice_xml_gz) \ - $(installed_system_dlkm_notice_xml_gz) \ - -# $1 installed file path, e.g. out/target/product/vsoc_x86_64/system_ext/etc/NOTICE.xml.gz -define is-notice-file -$(if $(findstring $1,$(ALL_INSTALLED_NOTICE_FILES)),Y) -endef - # Notice files are copied to TARGET_OUT_NOTICE_FILES as a side-effect of their module # being built. A notice xml file must depend on all modules that could potentially # install a license file relevant to it. @@ -3080,19 +3061,15 @@ $(1): $(HOST_OUT_EXECUTABLES)/fsverity_manifest_generator \ --cert $$(PRIVATE_KEY).x509.pem \ --key $$(PRIVATE_KEY).pk8 -$(1).idsig: $(1) - -ALL_DEFAULT_INSTALLED_MODULES += $(1) $(1).idsig +ALL_DEFAULT_INSTALLED_MODULES += $(1) endef # fsverity-generate-and-install-manifest-apk $(eval $(call fsverity-generate-and-install-manifest-apk, \ $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk,system)) -ALL_FSVERITY_BUILD_MANIFEST_APK += $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk.idsig ifdef BUILDING_SYSTEM_EXT_IMAGE $(eval $(call fsverity-generate-and-install-manifest-apk, \ $(TARGET_OUT_SYSTEM_EXT)/etc/security/fsverity/BuildManifestSystemExt.apk,system_ext)) - ALL_FSVERITY_BUILD_MANIFEST_APK += $(TARGET_OUT_SYSTEM_EXT)/etc/security/fsverity/BuildManifestSystemExt.apk $(TARGET_OUT_SYSTEM_EXT)/etc/security/fsverity/BuildManifestSystemExt.apk.idsig endif endif # PRODUCT_FSVERITY_GENERATE_METADATA @@ -3167,7 +3144,6 @@ $(call declare-1p-target,$(SYSTEM_LINKER_CONFIG),) $(call declare-license-deps,$(SYSTEM_LINKER_CONFIG),$(INTERNAL_SYSTEMIMAGE_FILES) $(SYSTEM_LINKER_CONFIG_SOURCE)) FULL_SYSTEMIMAGE_DEPS += $(SYSTEM_LINKER_CONFIG) -ALL_DEFAULT_INSTALLED_MODULES += $(SYSTEM_LINKER_CONFIG) # installed file list # Depending on anything that $(BUILT_SYSTEMIMAGE) depends on. @@ -3554,7 +3530,6 @@ $(vendor_linker_config_file): $(INTERNAL_VENDORIMAGE_FILES) $(PRODUCT_VENDOR_LIN --output $@ --value "$(STUB_LIBRARIES)" --system "$(TARGET_OUT_VENDOR)" $(call define declare-0p-target,$(vendor_linker_config_file),) INTERNAL_VENDORIMAGE_FILES += $(vendor_linker_config_file) -ALL_DEFAULT_INSTALLED_MODULES += $(vendor_linker_config_file) INSTALLED_FILES_FILE_VENDOR := $(PRODUCT_OUT)/installed-files-vendor.txt INSTALLED_FILES_JSON_VENDOR := $(INSTALLED_FILES_FILE_VENDOR:.txt=.json) diff --git a/core/app_prebuilt_internal.mk b/core/app_prebuilt_internal.mk index 9fab44d7c9..eb429cdd5c 100644 --- a/core/app_prebuilt_internal.mk +++ b/core/app_prebuilt_internal.mk @@ -302,7 +302,3 @@ $(my_all_targets): $(installed_apk_splits) endif # LOCAL_PACKAGE_SPLITS -########################################################### -## SBOM generation -########################################################### -include $(BUILD_SBOM_GEN)
\ No newline at end of file diff --git a/core/base_rules.mk b/core/base_rules.mk index c45346989c..ec5a21e917 100644 --- a/core/base_rules.mk +++ b/core/base_rules.mk @@ -949,8 +949,6 @@ ALL_MODULES.$(my_register_name).CHECKED := \ $(ALL_MODULES.$(my_register_name).CHECKED) $(my_checked_module) ALL_MODULES.$(my_register_name).BUILT := \ $(ALL_MODULES.$(my_register_name).BUILT) $(LOCAL_BUILT_MODULE) -ALL_MODULES.$(my_register_name).SOONG_MODULE_TYPE := \ - $(ALL_MODULES.$(my_register_name).SOONG_MODULE_TYPE) $(LOCAL_SOONG_MODULE_TYPE) ifndef LOCAL_IS_HOST_MODULE ALL_MODULES.$(my_register_name).TARGET_BUILT := \ $(ALL_MODULES.$(my_register_name).TARGET_BUILT) $(LOCAL_BUILT_MODULE) @@ -1242,8 +1240,3 @@ endif ########################################################### include $(BUILD_NOTICE_FILE) - -########################################################### -## SBOM generation -########################################################### -include $(BUILD_SBOM_GEN)
\ No newline at end of file diff --git a/core/clear_vars.mk b/core/clear_vars.mk index 8913ad0365..e32576067d 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -293,7 +293,6 @@ LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR := LOCAL_SOONG_LICENSE_METADATA := LOCAL_SOONG_LINK_TYPE := LOCAL_SOONG_LINT_REPORTS := -LOCAL_SOONG_MODULE_TYPE := LOCAL_SOONG_PROGUARD_DICT := LOCAL_SOONG_PROGUARD_USAGE_ZIP := LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE := diff --git a/core/config.mk b/core/config.mk index 1bb9a9698d..025a3a1de5 100644 --- a/core/config.mk +++ b/core/config.mk @@ -232,7 +232,6 @@ BUILD_NATIVE_TEST :=$= $(BUILD_SYSTEM)/native_test.mk BUILD_FUZZ_TEST :=$= $(BUILD_SYSTEM)/fuzz_test.mk BUILD_NOTICE_FILE :=$= $(BUILD_SYSTEM)/notice_files.mk -BUILD_SBOM_GEN :=$= $(BUILD_SYSTEM)/sbom.mk include $(BUILD_SYSTEM)/deprecation.mk @@ -642,8 +641,6 @@ VBOOT_SIGNER := $(HOST_OUT_EXECUTABLES)/vboot_signer DEXDUMP := $(HOST_OUT_EXECUTABLES)/dexdump$(BUILD_EXECUTABLE_SUFFIX) PROFMAN := $(HOST_OUT_EXECUTABLES)/profman -GEN_SBOM := $(HOST_OUT_EXECUTABLES)/generate-sbom - FINDBUGS_DIR := external/owasp/sanitizer/tools/findbugs/bin FINDBUGS := $(FINDBUGS_DIR)/findbugs diff --git a/core/dex_preopt_odex_install.mk b/core/dex_preopt_odex_install.mk index 05bb669b64..b303b52f12 100644 --- a/core/dex_preopt_odex_install.mk +++ b/core/dex_preopt_odex_install.mk @@ -447,7 +447,6 @@ ifdef LOCAL_DEX_PREOPT my_dexpreopt_script := $(intermediates)/dexpreopt.sh my_dexpreopt_zip := $(intermediates)/dexpreopt.zip - DEXPREOPT.$(LOCAL_MODULE).POST_INSTALLED_DEXPREOPT_ZIP := $(my_dexpreopt_zip) .KATI_RESTAT: $(my_dexpreopt_script) $(my_dexpreopt_script): PRIVATE_MODULE := $(LOCAL_MODULE) $(my_dexpreopt_script): PRIVATE_GLOBAL_SOONG_CONFIG := $(DEX_PREOPT_SOONG_CONFIG_FOR_MAKE) diff --git a/core/main.mk b/core/main.mk index 8f36f03ab8..3866037b4f 100644 --- a/core/main.mk +++ b/core/main.mk @@ -2019,86 +2019,6 @@ endif # missing dependency errors. $(call build-license-metadata) -# Generate SBOM in SPDX format -product_copy_files_without_owner := $(foreach pcf,$(PRODUCT_COPY_FILES),$(call word-colon,1,$(pcf)):$(call word-colon,2,$(pcf))) -ifeq ($(TARGET_BUILD_APPS),) -dest_files_without_source := $(sort $(foreach pcf,$(product_copy_files_without_owner),$(if $(wildcard $(call word-colon,1,$(pcf))),,$(call word-colon,2,$(pcf))))) -dest_files_without_source := $(addprefix $(PRODUCT_OUT)/,$(dest_files_without_source)) -installed_files := $(sort $(filter-out $(PRODUCT_OUT)/apex/% $(PRODUCT_OUT)/fake_packages/% $(PRODUCT_OUT)/testcases/% $(dest_files_without_source),$(filter $(PRODUCT_OUT)/%,$(modules_to_install)))) -else -installed_files := $(apps_only_installed_files) -endif - -# sbom-metadata.csv contains all raw data collected in Make for generating SBOM in generate-sbom.py. -# There are multiple columns and each identifies the source of an installed file for a specific case. -# The columns and their uses are described as below: -# installed_file: the file path on device, e.g. /product/app/Browser2/Browser2.apk -# module_path: the path of the module that generates the installed file, e.g. packages/apps/Browser2 -# soong_module_type: Soong module type, e.g. android_app, cc_binary -# is_prebuilt_make_module: Y, if the installed file is from a prebuilt Make module, see prebuilt_internal.mk -# product_copy_files: the installed file is from variable PRODUCT_COPY_FILES, e.g. device/google/cuttlefish/shared/config/init.product.rc:product/etc/init/init.rc -# kernel_module_copy_files: the installed file is from variable KERNEL_MODULE_COPY_FILES, similar to product_copy_files -# is_platform_generated: this is an aggregated value including some small cases instead of adding more columns. It is set to Y if any case is Y -# is_build_prop: build.prop in each partition, see sysprop.mk. -# is_notice_file: NOTICE.xml.gz in each partition, see Makefile. -# is_dexpreopt_image_profile: see the usage of DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED in Soong and Make -# is_product_system_other_avbkey: see INSTALLED_PRODUCT_SYSTEM_OTHER_AVBKEY_TARGET -# is_system_other_odex_marker: see INSTALLED_SYSTEM_OTHER_ODEX_MARKER -# is_event_log_tags_file: see variable event_log_tags_file in Makefile -# is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile. -# is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile. -# is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile. - -# (TODO: b/272358583 find another way of always rebuilding this target) -# Remove the sbom-metadata.csv whenever makefile is evaluated -$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1) -$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) - rm -f $@ - @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated >> $@ - $(foreach f,$(installed_files),\ - $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \ - $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \ - $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \ - $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \ - $(eval _is_prebuilt_make_module := $(ALL_MODULES.$(_module_name).IS_PREBUILT_MAKE_MODULE)) \ - $(eval _post_installed_dexpreopt_zip := $(DEXPREOPT.$(_module_name).POST_INSTALLED_DEXPREOPT_ZIP)) \ - $(eval _product_copy_files := $(sort $(filter %:$(_path_on_device),$(product_copy_files_without_owner)))) \ - $(eval _kernel_module_copy_files := $(sort $(filter %$(_path_on_device),$(KERNEL_MODULE_COPY_FILES)))) \ - $(eval _is_build_prop := $(call is-build-prop,$f)) \ - $(eval _is_notice_file := $(call is-notice-file,$f)) \ - $(eval _is_dexpreopt_image_profile := $(if $(filter %:/$(_path_on_device),$(DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED)),Y)) \ - $(eval _is_product_system_other_avbkey := $(if $(findstring $f,$(INSTALLED_PRODUCT_SYSTEM_OTHER_AVBKEY_TARGET)),Y)) \ - $(eval _is_event_log_tags_file := $(if $(findstring $f,$(event_log_tags_file)),Y)) \ - $(eval _is_system_other_odex_marker := $(if $(findstring $f,$(INSTALLED_SYSTEM_OTHER_ODEX_MARKER)),Y)) \ - $(eval _is_kernel_modules_blocklist := $(if $(findstring $f,$(ALL_KERNEL_MODULES_BLOCKLIST)),Y)) \ - $(eval _is_fsverity_build_manifest_apk := $(if $(findstring $f,$(ALL_FSVERITY_BUILD_MANIFEST_APK)),Y)) \ - $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \ - $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)) \ - @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ $(newline) \ - $(if $(_post_installed_dexpreopt_zip), \ - for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ ; done $(newline) \ - ) \ - ) - -.PHONY: sbom -ifeq ($(TARGET_BUILD_APPS),) -sbom: $(PRODUCT_OUT)/sbom.spdx.json -$(PRODUCT_OUT)/sbom.spdx.json: $(PRODUCT_OUT)/sbom.spdx -$(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) - rm -rf $@ - $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr=$(PRODUCT_MANUFACTURER) --json - -$(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json) -else -apps_only_sbom_files := $(patsubst %,%.spdx,$(apps_only_installed_files)) -$(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) - rm -rf $@ - $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr=$(PRODUCT_MANUFACTURER) --unbundled - -$(call dist-for-goals,apps_only,$(apps_only_sbom_files)) -sbom: $(apps_only_sbom_files) -endif - $(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk) $(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] writing build rules ...) diff --git a/core/prebuilt_internal.mk b/core/prebuilt_internal.mk index 5bea9b620f..ef1471d503 100644 --- a/core/prebuilt_internal.mk +++ b/core/prebuilt_internal.mk @@ -57,9 +57,6 @@ else $(error $(LOCAL_MODULE) : unexpected LOCAL_MODULE_CLASS for prebuilts: $(LOCAL_MODULE_CLASS)) endif -$(if $(filter-out $(SOONG_ANDROID_MK),$(LOCAL_MODULE_MAKEFILE)), \ - $(eval ALL_MODULES.$(my_register_name).IS_PREBUILT_MAKE_MODULE := Y)) - $(built_module) : $(LOCAL_ADDITIONAL_DEPENDENCIES) my_prebuilt_src_file := diff --git a/core/sbom.mk b/core/sbom.mk deleted file mode 100644 index e23bbc132f..0000000000 --- a/core/sbom.mk +++ /dev/null @@ -1,11 +0,0 @@ -# For SBOM generation -# This is included by base_rules.mk and is not necessary to be included in other .mk files -# unless a .mk file changes its installed file after including base_rules.mk. - -ifdef my_register_name - ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED))) - $(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\ - $(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\ - ) - endif -endif
\ No newline at end of file diff --git a/core/soong_app_prebuilt.mk b/core/soong_app_prebuilt.mk index dd550b5808..469a8f1bd7 100644 --- a/core/soong_app_prebuilt.mk +++ b/core/soong_app_prebuilt.mk @@ -269,8 +269,3 @@ ifdef LOCAL_PREBUILT_COVERAGE_ARCHIVE endif SOONG_ALREADY_CONV += $(LOCAL_MODULE) - -########################################################### -## SBOM generation -########################################################### -include $(BUILD_SBOM_GEN)
\ No newline at end of file diff --git a/core/sysprop.mk b/core/sysprop.mk index bd6f3d9fdf..6e2caed318 100644 --- a/core/sysprop.mk +++ b/core/sysprop.mk @@ -543,19 +543,3 @@ $(eval $(call build-properties,\ $(empty))) $(eval $(call declare-1p-target,$(INSTALLED_RAMDISK_BUILD_PROP_TARGET))) - -ALL_INSTALLED_BUILD_PROP_FILES := \ - $(INSTALLED_BUILD_PROP_TARGET) \ - $(INSTALLED_VENDOR_BUILD_PROP_TARGET) \ - $(INSTALLED_PRODUCT_BUILD_PROP_TARGET) \ - $(INSTALLED_ODM_BUILD_PROP_TARGET) \ - $(INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET) \ - $(INSTALLED_ODM_DLKM_BUILD_PROP_TARGET) \ - $(INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET) \ - $(INSTALLED_SYSTEM_EXT_BUILD_PROP_TARGET) \ - $(INSTALLED_RAMDISK_BUILD_PROP_TARGET) - -# $1 installed file path, e.g. out/target/product/vsoc_x86_64/system/build.prop -define is-build-prop -$(if $(findstring $1,$(ALL_INSTALLED_BUILD_PROP_FILES)),Y) -endef
\ No newline at end of file diff --git a/tools/Android.bp b/tools/Android.bp index c5c02c6215..f446973bbc 100644 --- a/tools/Android.bp +++ b/tools/Android.bp @@ -69,19 +69,3 @@ python_binary_host { name: "generate_gts_shared_report", srcs: ["generate_gts_shared_report.py"], } - -python_binary_host { - name: "generate-sbom", - srcs: [ - "generate-sbom.py", - ], - version: { - py3: { - embedded_launcher: true, - }, - }, - libs: [ - "metadata_file_proto_py", - "libprotobuf-python", - ], -} diff --git a/tools/generate-sbom.py b/tools/generate-sbom.py deleted file mode 100755 index 54057c9da6..0000000000 --- a/tools/generate-sbom.py +++ /dev/null @@ -1,684 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2023 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Generate the SBOM of the current target product in SPDX format. -Usage example: - generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \ - --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \ - --product_out_dir=out/target/product/vsoc_x86_64 \ - --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \ - --product_mfr=Google -""" - -import argparse -import csv -import datetime -import google.protobuf.text_format as text_format -import hashlib -import json -import os -import metadata_file_pb2 - -# Common -SPDXID = 'SPDXID' -SPDX_VERSION = 'SPDXVersion' -DATA_LICENSE = 'DataLicense' -DOCUMENT_NAME = 'DocumentName' -DOCUMENT_NAMESPACE = 'DocumentNamespace' -CREATED = 'Created' -CREATOR = 'Creator' -EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef' - -# Package -PACKAGE_NAME = 'PackageName' -PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation' -PACKAGE_VERSION = 'PackageVersion' -PACKAGE_SUPPLIER = 'PackageSupplier' -FILES_ANALYZED = 'FilesAnalyzed' -PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode' -PACKAGE_EXTERNAL_REF = 'ExternalRef' -# Package license -PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded' -PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles' -PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared' -PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments' - -# File -FILE_NAME = 'FileName' -FILE_CHECKSUM = 'FileChecksum' -# File license -FILE_LICENSE_CONCLUDED = 'LicenseConcluded' -FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile' -FILE_LICENSE_COMMENTS = 'LicenseComments' -FILE_COPYRIGHT_TEXT = 'FileCopyrightText' -FILE_NOTICE = 'FileNotice' -FILE_ATTRIBUTION_TEXT = 'FileAttributionText' - -# Relationship -RELATIONSHIP = 'Relationship' -REL_DESCRIBES = 'DESCRIBES' -REL_VARIANT_OF = 'VARIANT_OF' -REL_GENERATED_FROM = 'GENERATED_FROM' - -# Package type -PKG_SOURCE = 'SOURCE' -PKG_UPSTREAM = 'UPSTREAM' -PKG_PREBUILT = 'PREBUILT' - -# Security tag -NVD_CPE23 = 'NVD-CPE2.3:' - -# Report -ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:' -ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:' -ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:' -ISSUE_UNKNOWN_SECURITY_TAG_TYPE = "Unknown security tag type:" -INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:' - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') - parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.') - parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.') - parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.') - parser.add_argument('--build_version', required=True, help='The build version.') - parser.add_argument('--product_mfr', required=True, help='The product manufacturer.') - parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format') - parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module') - - return parser.parse_args() - - -def log(*info): - if args.verbose: - for i in info: - print(i) - - -def new_doc_header(doc_id): - return { - SPDX_VERSION: 'SPDX-2.3', - DATA_LICENSE: 'CC0-1.0', - SPDXID: doc_id, - DOCUMENT_NAME: args.build_version, - DOCUMENT_NAMESPACE: 'https://www.google.com/sbom/spdx/android/' + args.build_version, - CREATOR: 'Organization: Google, LLC', - CREATED: '<timestamp>', - EXTERNAL_DOCUMENT_REF: [], - } - - -def new_package_record(id, name, version, supplier, download_location=None, files_analyzed='false', external_refs=[]): - package = { - PACKAGE_NAME: name, - SPDXID: id, - PACKAGE_DOWNLOAD_LOCATION: download_location if download_location else 'NONE', - FILES_ANALYZED: files_analyzed, - } - if version: - package[PACKAGE_VERSION] = version - if supplier: - package[PACKAGE_SUPPLIER] = 'Organization: ' + supplier - if external_refs: - package[PACKAGE_EXTERNAL_REF] = external_refs - - return package - - -def new_file_record(id, name, checksum): - return { - FILE_NAME: name, - SPDXID: id, - FILE_CHECKSUM: checksum - } - - -def encode_for_spdxid(s): - """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-""" - result = '' - for c in s: - if c.isalnum() or c in '.-': - result += c - elif c in '_@/': - result += '-' - else: - result += '0x' + c.encode('utf-8').hex() - - return result.lstrip('-') - - -def new_package_id(package_name, type): - return 'SPDXRef-{}-{}'.format(type, encode_for_spdxid(package_name)) - - -def new_external_doc_ref(package_name, sbom_url, sbom_checksum): - doc_ref_id = 'DocumentRef-{}-{}'.format(PKG_UPSTREAM, encode_for_spdxid(package_name)) - return '{}: {} {} {}'.format(EXTERNAL_DOCUMENT_REF, doc_ref_id, sbom_url, sbom_checksum), doc_ref_id - - -def new_file_id(file_path): - return 'SPDXRef-' + encode_for_spdxid(file_path) - - -def new_relationship_record(id1, relationship, id2): - return '{}: {} {} {}'.format(RELATIONSHIP, id1, relationship, id2) - - -def checksum(file_path): - file_path = args.product_out_dir + '/' + file_path - h = hashlib.sha1() - if os.path.islink(file_path): - h.update(os.readlink(file_path).encode('utf-8')) - else: - with open(file_path, "rb") as f: - h.update(f.read()) - return "SHA1: " + h.hexdigest() - - -def is_soong_prebuilt_module(file_metadata): - return file_metadata['soong_module_type'] and file_metadata['soong_module_type'] in [ - 'android_app_import', 'android_library_import', 'cc_prebuilt_binary', 'cc_prebuilt_library', - 'cc_prebuilt_library_headers', 'cc_prebuilt_library_shared', 'cc_prebuilt_library_static', 'cc_prebuilt_object', - 'dex_import', 'java_import', 'java_sdk_library_import', 'java_system_modules_import', - 'libclang_rt_prebuilt_library_static', 'libclang_rt_prebuilt_library_shared', 'llvm_prebuilt_library_static', - 'ndk_prebuilt_object', 'ndk_prebuilt_shared_stl', 'nkd_prebuilt_static_stl', 'prebuilt_apex', - 'prebuilt_bootclasspath_fragment', 'prebuilt_dsp', 'prebuilt_firmware', 'prebuilt_kernel_modules', - 'prebuilt_rfsa', 'prebuilt_root', 'rust_prebuilt_dylib', 'rust_prebuilt_library', 'rust_prebuilt_rlib', - 'vndk_prebuilt_shared', - - # 'android_test_import', - # 'cc_prebuilt_test_library_shared', - # 'java_import_host', - # 'java_test_import', - # 'llvm_host_prebuilt_library_shared', - # 'prebuilt_apis', - # 'prebuilt_build_tool', - # 'prebuilt_defaults', - # 'prebuilt_etc', - # 'prebuilt_etc_host', - # 'prebuilt_etc_xml', - # 'prebuilt_font', - # 'prebuilt_hidl_interfaces', - # 'prebuilt_platform_compat_config', - # 'prebuilt_stubs_sources', - # 'prebuilt_usr_share', - # 'prebuilt_usr_share_host', - # 'soong_config_module_type_import', - ] - - -def is_source_package(file_metadata): - module_path = file_metadata['module_path'] - return module_path.startswith('external/') and not is_prebuilt_package(file_metadata) - - -def is_prebuilt_package(file_metadata): - module_path = file_metadata['module_path'] - if module_path: - return (module_path.startswith('prebuilts/') or - is_soong_prebuilt_module(file_metadata) or - file_metadata['is_prebuilt_make_module']) - - kernel_module_copy_files = file_metadata['kernel_module_copy_files'] - if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'): - return True - - return False - - -def get_source_package_info(file_metadata, metadata_file_path): - if not metadata_file_path: - return file_metadata['module_path'], [] - - metadata_proto = metadata_file_protos[metadata_file_path] - external_refs = [] - for tag in metadata_proto.third_party.security.tag: - if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()): - external_refs.append("{}: SECURITY cpe23Type {}".format(PACKAGE_EXTERNAL_REF, tag.removeprefix(NVD_CPE23))) - elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()): - external_refs.append("{}: SECURITY cpe22Type {}".format(PACKAGE_EXTERNAL_REF, tag.removeprefix(NVD_CPE23))) - - if metadata_proto.name: - return metadata_proto.name, external_refs - else: - return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name - - -def get_prebuilt_package_name(file_metadata, metadata_file_path): - name = None - if metadata_file_path: - metadata_proto = metadata_file_protos[metadata_file_path] - if metadata_proto.name: - name = metadata_proto.name - else: - name = metadata_file_path - elif file_metadata['module_path']: - name = file_metadata['module_path'] - elif file_metadata['kernel_module_copy_files']: - src_path = file_metadata['kernel_module_copy_files'].split(':')[0] - name = os.path.dirname(src_path) - - return name.removeprefix('prebuilts/').replace('/', '-') - - -def get_metadata_file_path(file_metadata): - metadata_path = '' - if file_metadata['module_path']: - metadata_path = file_metadata['module_path'] - elif file_metadata['kernel_module_copy_files']: - metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) - - while metadata_path and not os.path.exists(metadata_path + '/METADATA'): - metadata_path = os.path.dirname(metadata_path) - - return metadata_path - - -def get_package_version(metadata_file_path): - if not metadata_file_path: - return None - metadata_proto = metadata_file_protos[metadata_file_path] - return metadata_proto.third_party.version - - -def get_package_homepage(metadata_file_path): - if not metadata_file_path: - return None - metadata_proto = metadata_file_protos[metadata_file_path] - if metadata_proto.third_party.homepage: - return metadata_proto.third_party.homepage - for url in metadata_proto.third_party.url: - if url.type == metadata_file_pb2.URL.Type.HOMEPAGE: - return url.value - - return None - - -def get_package_download_location(metadata_file_path): - if not metadata_file_path: - return None - metadata_proto = metadata_file_protos[metadata_file_path] - if metadata_proto.third_party.url: - urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type) - if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE: - return urls[0].value - elif len(urls) > 1: - return urls[1].value - - return None - - -def get_sbom_fragments(installed_file_metadata, metadata_file_path): - external_doc_ref = None - packages = [] - relationships = [] - - # Info from METADATA file - homepage = get_package_homepage(metadata_file_path) - version = get_package_version(metadata_file_path) - download_location = get_package_download_location(metadata_file_path) - - if is_source_package(installed_file_metadata): - # Source fork packages - name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path) - source_package_id = new_package_id(name, PKG_SOURCE) - source_package = new_package_record(source_package_id, name, args.build_version, args.product_mfr, - external_refs=external_refs) - - upstream_package_id = new_package_id(name, PKG_UPSTREAM) - upstream_package = new_package_record(upstream_package_id, name, version, homepage, download_location) - packages += [source_package, upstream_package] - relationships.append(new_relationship_record(source_package_id, REL_VARIANT_OF, upstream_package_id)) - elif is_prebuilt_package(installed_file_metadata): - # Prebuilt fork packages - name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path) - prebuilt_package_id = new_package_id(name, PKG_PREBUILT) - prebuilt_package = new_package_record(prebuilt_package_id, name, args.build_version, args.product_mfr) - packages.append(prebuilt_package) - - if metadata_file_path: - metadata_proto = metadata_file_protos[metadata_file_path] - if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref': - sbom_url = metadata_proto.third_party.sbom_ref.url - sbom_checksum = metadata_proto.third_party.sbom_ref.checksum - upstream_element_id = metadata_proto.third_party.sbom_ref.element_id - if sbom_url and sbom_checksum and upstream_element_id: - external_doc_ref, doc_ref_id = new_external_doc_ref(name, sbom_url, sbom_checksum) - relationships.append( - new_relationship_record(prebuilt_package_id, REL_VARIANT_OF, doc_ref_id + ':' + upstream_element_id)) - - return external_doc_ref, packages, relationships - - -def generate_package_verification_code(files): - checksums = [file[FILE_CHECKSUM] for file in files] - checksums.sort() - h = hashlib.sha1() - h.update(''.join(checksums).encode(encoding='utf-8')) - return h.hexdigest() - - -def write_record(f, record): - if record.__class__.__name__ == 'dict': - for k, v in record.items(): - if k == EXTERNAL_DOCUMENT_REF or k == PACKAGE_EXTERNAL_REF: - for ref in v: - f.write(ref + '\n') - else: - f.write('{}: {}\n'.format(k, v)) - elif record.__class__.__name__ == 'str': - f.write(record + '\n') - f.write('\n') - - -def write_tagvalue_sbom(all_records): - with open(args.output_file, 'w', encoding="utf-8") as output_file: - for rec in all_records: - write_record(output_file, rec) - - -def write_json_sbom(all_records, product_package_id): - doc = {} - product_package = None - for r in all_records: - if r.__class__.__name__ == 'dict': - if DOCUMENT_NAME in r: # Doc header - doc['spdxVersion'] = r[SPDX_VERSION] - doc['dataLicense'] = r[DATA_LICENSE] - doc[SPDXID] = r[SPDXID] - doc['name'] = r[DOCUMENT_NAME] - doc['documentNamespace'] = r[DOCUMENT_NAMESPACE] - doc['creationInfo'] = { - 'creators': [r[CREATOR]], - 'created': r[CREATED], - } - doc['externalDocumentRefs'] = [] - for ref in r[EXTERNAL_DOCUMENT_REF]: - # ref is 'ExternalDocumentRef: <doc id> <doc url> SHA1: xxxxx' - fields = ref.split(' ') - doc_ref = { - 'externalDocumentId': fields[1], - 'spdxDocument': fields[2], - 'checksum': { - 'algorithm': fields[3][:-1], - 'checksumValue': fields[4] - } - } - doc['externalDocumentRefs'].append(doc_ref) - doc['documentDescribes'] = [] - doc['packages'] = [] - doc['files'] = [] - doc['relationships'] = [] - - elif PACKAGE_NAME in r: # packages - package = { - 'name': r[PACKAGE_NAME], - SPDXID: r[SPDXID], - 'downloadLocation': r[PACKAGE_DOWNLOAD_LOCATION], - 'filesAnalyzed': r[FILES_ANALYZED] == "true" - } - if PACKAGE_VERSION in r: - package['versionInfo'] = r[PACKAGE_VERSION] - if PACKAGE_SUPPLIER in r: - package['supplier'] = r[PACKAGE_SUPPLIER] - if PACKAGE_VERIFICATION_CODE in r: - package['packageVerificationCode'] = { - 'packageVerificationCodeValue': r[PACKAGE_VERIFICATION_CODE] - } - if PACKAGE_EXTERNAL_REF in r: - package['externalRefs'] = [] - for ref in r[PACKAGE_EXTERNAL_REF]: - # ref is 'ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4' - fields = ref.split(' ') - ext_ref = { - 'referenceCategory': fields[1], - 'referenceType': fields[2], - 'referenceLocator': fields[3], - } - package['externalRefs'].append(ext_ref) - - doc['packages'].append(package) - if r[SPDXID] == product_package_id: - product_package = package - product_package['hasFiles'] = [] - - elif FILE_NAME in r: # files - file = { - 'fileName': r[FILE_NAME], - SPDXID: r[SPDXID] - } - checksum = r[FILE_CHECKSUM].split(': ') - file['checksums'] = [{ - 'algorithm': checksum[0], - 'checksumValue': checksum[1], - }] - doc['files'].append(file) - product_package['hasFiles'].append(r[SPDXID]) - - elif r.__class__.__name__ == 'str': - if r.startswith(RELATIONSHIP): - # r is 'Relationship: <spdxid> <relationship> <spdxid>' - fields = r.split(' ') - rel = { - 'spdxElementId': fields[1], - 'relatedSpdxElement': fields[3], - 'relationshipType': fields[2], - } - if fields[2] == REL_DESCRIBES: - doc['documentDescribes'].append(fields[3]) - else: - doc['relationships'].append(rel) - - with open(args.output_file + '.json', 'w', encoding="utf-8") as output_file: - output_file.write(json.dumps(doc, indent=4)) - - -def save_report(report): - prefix, _ = os.path.splitext(args.output_file) - with open(prefix + '-gen-report.txt', 'w', encoding="utf-8") as report_file: - for type, issues in report.items(): - report_file.write(type + '\n') - for issue in issues: - report_file.write('\t' + issue + '\n') - report_file.write('\n') - - -def sort_rels(rel): - # rel = 'Relationship file_id GENERATED_FROM package_id' - fields = rel.split(' ') - return fields[3] + fields[1] - - -# Validate the metadata generated by Make for installed files and report if there is no metadata. -def installed_file_has_metadata(installed_file_metadata, report): - installed_file = installed_file_metadata['installed_file'] - module_path = installed_file_metadata['module_path'] - product_copy_files = installed_file_metadata['product_copy_files'] - kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] - is_platform_generated = installed_file_metadata['is_platform_generated'] - - if (not module_path and - not product_copy_files and - not kernel_module_copy_files and - not is_platform_generated and - not installed_file.endswith('.fsv_meta')): - report[ISSUE_NO_METADATA].append(installed_file) - return False - - return True - - -def report_metadata_file(metadata_file_path, installed_file_metadata, report): - if metadata_file_path: - report[INFO_METADATA_FOUND_FOR_PACKAGE].append( - "installed_file: {}, module_path: {}, METADATA file: {}".format( - installed_file_metadata['installed_file'], - installed_file_metadata['module_path'], - metadata_file_path + '/METADATA')) - - package_metadata = metadata_file_pb2.Metadata() - with open(metadata_file_path + '/METADATA', "rt") as f: - text_format.Parse(f.read(), package_metadata) - - if not metadata_file_path in metadata_file_protos: - metadata_file_protos[metadata_file_path] = package_metadata - if not package_metadata.name: - report[ISSUE_METADATA_FILE_INCOMPLETE].append('{} does not has "name"'.format(metadata_file_path + '/METADATA')) - - if not package_metadata.third_party.version: - report[ISSUE_METADATA_FILE_INCOMPLETE].append( - '{} does not has "third_party.version"'.format(metadata_file_path + '/METADATA')) - - for tag in package_metadata.third_party.security.tag: - if not tag.startswith(NVD_CPE23): - report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append( - "Unknown security tag type: {} in {}".format(tag, metadata_file_path + '/METADATA')) - else: - report[ISSUE_NO_METADATA_FILE].append( - "installed_file: {}, module_path: {}".format( - installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) - - -def generate_fragment(): - with open(args.metadata, newline='') as sbom_metadata_file: - reader = csv.DictReader(sbom_metadata_file) - for installed_file_metadata in reader: - installed_file = installed_file_metadata['installed_file'] - if args.output_file != args.product_out_dir + installed_file + ".spdx": - continue - - module_path = installed_file_metadata['module_path'] - package_id = new_package_id(encode_for_spdxid(module_path), PKG_PREBUILT) - package = new_package_record(package_id, module_path, args.build_version, args.product_mfr) - file_id = new_file_id(installed_file) - file = new_file_record(file_id, installed_file, checksum(installed_file)) - relationship = new_relationship_record(file_id, REL_GENERATED_FROM, package_id) - records = [package, file, relationship] - write_tagvalue_sbom(records) - break - - -def main(): - global args - args = get_args() - log("Args:", vars(args)) - - if args.unbundled: - generate_fragment() - return - - global metadata_file_protos - metadata_file_protos = {} - - doc_id = 'SPDXRef-DOCUMENT' - doc_header = new_doc_header(doc_id) - - product_package_id = 'SPDXRef-PRODUCT' - product_package = new_package_record(product_package_id, 'PRODUCT', args.build_version, args.product_mfr, - files_analyzed='true') - - platform_package_id = 'SPDXRef-PLATFORM' - platform_package = new_package_record(platform_package_id, 'PLATFORM', args.build_version, args.product_mfr) - - # Report on some issues and information - report = { - ISSUE_NO_METADATA: [], - ISSUE_NO_METADATA_FILE: [], - ISSUE_METADATA_FILE_INCOMPLETE: [], - ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], - INFO_METADATA_FOUND_FOR_PACKAGE: [] - } - - # Scan the metadata in CSV file and create the corresponding package and file records in SPDX - product_files = [] - package_ids = [] - package_records = [] - rels_file_gen_from = [] - with open(args.metadata, newline='') as sbom_metadata_file: - reader = csv.DictReader(sbom_metadata_file) - for installed_file_metadata in reader: - installed_file = installed_file_metadata['installed_file'] - module_path = installed_file_metadata['module_path'] - product_copy_files = installed_file_metadata['product_copy_files'] - kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] - - if not installed_file_has_metadata(installed_file_metadata, report): - continue - - file_id = new_file_id(installed_file) - product_files.append(new_file_record(file_id, installed_file, checksum(installed_file))) - - if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): - metadata_file_path = get_metadata_file_path(installed_file_metadata) - report_metadata_file(metadata_file_path, installed_file_metadata, report) - - # File from source fork packages or prebuilt fork packages - external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path) - if len(pkgs) > 0: - if external_doc_ref and external_doc_ref not in doc_header[EXTERNAL_DOCUMENT_REF]: - doc_header[EXTERNAL_DOCUMENT_REF].append(external_doc_ref) - for p in pkgs: - if not p[SPDXID] in package_ids: - package_ids.append(p[SPDXID]) - package_records.append(p) - for rel in rels: - if not rel in package_records: - package_records.append(rel) - fork_package_id = pkgs[0][SPDXID] # The first package should be the source/prebuilt fork package - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, fork_package_id)) - elif module_path or installed_file_metadata['is_platform_generated']: - # File from PLATFORM package - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) - elif product_copy_files: - # Format of product_copy_files: <source path>:<dest path> - src_path = product_copy_files.split(':')[0] - # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device, - # so process them as files from PLATFORM package - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) - elif installed_file.endswith('.fsv_meta'): - # See build/make/core/Makefile:2988 - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) - elif kernel_module_copy_files.startswith('ANDROID-GEN'): - # For the four files generated for _dlkm, _ramdisk partitions - # See build/make/core/Makefile:323 - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) - - product_package[PACKAGE_VERIFICATION_CODE] = generate_package_verification_code(product_files) - - all_records = [ - doc_header, - product_package, - new_relationship_record(doc_id, REL_DESCRIBES, product_package_id), - ] - all_records += product_files - all_records.append(platform_package) - all_records += package_records - rels_file_gen_from.sort(key=sort_rels) - all_records += rels_file_gen_from - - # Save SBOM records to output file - doc_header[CREATED] = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') - write_tagvalue_sbom(all_records) - if args.json: - write_json_sbom(all_records, product_package_id) - - save_report(report) - - -if __name__ == '__main__': - main() diff --git a/tools/protos/Android.bp b/tools/protos/Android.bp deleted file mode 100644 index c6ad19e644..0000000000 --- a/tools/protos/Android.bp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2023 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package { - default_applicable_licenses: ["Android-Apache-2.0"], -} - -python_library_host { - name: "metadata_file_proto_py", - version: { - py3: { - enabled: true, - }, - }, - srcs: [ - "metadata_file.proto", - ], - proto: { - canonical_path_from_root: false, - }, -} diff --git a/tools/protos/metadata_file.proto b/tools/protos/metadata_file.proto deleted file mode 100644 index ac1129abee..0000000000 --- a/tools/protos/metadata_file.proto +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright (C) 2023 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package metadata_file; - -// Proto definition of METADATA files of packages in AOSP codebase. -message Metadata { - // Name of the package. - optional string name = 1; - - // A short description (a few lines) of the package. - // Example: "Handles location lookups, throttling, batching, etc." - optional string description = 2; - - // Specifies additional data about third-party packages. - optional ThirdParty third_party = 3; -} - -message ThirdParty { - // URL(s) associated with the package. - // - // At a minimum, all packages must specify a URL which identifies where it - // came from, containing a type of: ARCHIVE, GIT or OTHER. Typically, - // a package should contain only a single URL from these types. Occasionally, - // a package may be broken across multiple archive files for whatever reason, - // in which case having multiple ARCHIVE URLs is okay. However, this should - // not be used to combine different logical packages that are versioned and - // possibly licensed differently. - repeated URL url = 1; - - // The package version. In order of preference, this should contain: - // - If the package comes from Git or another source control system, - // a specific tag or revision in source control, such as "r123" or - // "58e27d2". This MUST NOT be a mutable ref such as a branch name. - // - a released package version such as "1.0", "2.3-beta", etc. - // - the date the package was retrieved, formatted as "As of YYYY-MM-DD". - optional string version = 2; - - // The date of the change in which the package was last upgraded from - // upstream. - // This should only identify package upgrades from upstream, not local - // modifications. This may identify the date of either the original or - // merged change. - // - // Note: this is NOT the date that this version of the package was released - // externally. - optional Date last_upgrade_date = 3; - - // License type that identifies how the package may be used. - optional LicenseType license_type = 4; - - // An additional note explaining the licensing of this package. This is most - // commonly used with commercial license. - optional string license_note = 5; - - // Description of local changes that have been made to the package. This does - // not need to (and in most cases should not) attempt to include an exhaustive - // list of all changes, but may instead direct readers to review the local - // commit history, a collection of patch files, a separate README.md (or - // similar) document, etc. - // Note: Use of this field to store IDs of advisories fixed with a backported - // patch is deprecated, use "security.mitigated_security_patch" instead. - optional string local_modifications = 6; - - // Security related metadata including risk category and any special - // instructions for using the package, as determined by an ISE-TPS review. - optional Security security = 7; - - // The type of directory this metadata represents. - optional DirectoryType type = 8 [default = PACKAGE]; - - // The homepage for the package. This will eventually replace - // `url { type: HOMEPAGE }` - optional string homepage = 9; - - // SBOM information of the package. It is mandatory for prebuilt packages. - oneof sbom { - // Reference to external SBOM document provided as URL. - SBOMRef sbom_ref = 10; - } - -} - -// URL associated with a third-party package. -message URL { - enum Type { - // The homepage for the package. For example, "https://bazel.io/". This URL - // is optional, but encouraged to help disambiguate similarly named packages - // or to get more information about the package. This is especially helpful - // when no other URLs provide human readable resources (such as git:// or - // sso:// URLs). - HOMEPAGE = 1; - - // The URL of the archive containing the source code for the package, for - // example a zip or tgz file. - ARCHIVE = 2; - - // The URL of the upstream git repository this package is retrieved from. - // For example: - // - https://github.com/git/git.git - // - git://git.kernel.org/pub/scm/git/git.git - // - // Use of a git URL requires that the package "version" value must specify a - // specific git tag or revision. - GIT = 3; - - // The URL of the upstream SVN repository this package is retrieved from. - // For example: - // - http://llvm.org/svn/llvm-project/llvm/ - // - // Use of an SVN URL requires that the package "version" value must specify - // a specific SVN tag or revision. - SVN = 4; - - // The URL of the upstream mercurial repository this package is retrieved - // from. For example: - // - https://mercurial-scm.org/repo/evolve - // - // Use of a mercurial URL requires that the package "version" value must - // specify a specific tag or revision. - HG = 5; - - // The URL of the upstream darcs repository this package is retrieved - // from. For example: - // - https://hub.darcs.net/hu.dwim/hu.dwim.util - // - // Use of a DARCS URL requires that the package "version" value must - // specify a specific tag or revision. - DARCS = 6; - - PIPER = 7; - - // A URL that does not fit any other type. This may also indicate that the - // source code was received via email or some other out-of-band way. This is - // most commonly used with commercial software received directly from the - // vendor. In the case of email, the URL value can be used to provide - // additional information about how it was received. - OTHER = 8; - - // The URL identifying where the local copy of the package source code can - // be found. - // - // Typically, the metadata files describing a package reside in the same - // directory as the source code for the package. In a few rare cases where - // they are separate, the LOCAL_SOURCE URL identifies where to find the - // source code. This only describes where to find the local copy of the - // source; there should always be an additional URL describing where the - // package was retrieved from. - // - // Examples: - // - https://android.googlesource.com/platform/external/apache-http/ - LOCAL_SOURCE = 9; - } - - // The type of resource this URL identifies. - optional Type type = 1; - - // The actual URL value. URLs should be absolute and start with 'http://' or - // 'https://' (or occasionally 'git://' or 'ftp://' where appropriate). - optional string value = 2; -} - -// License type that identifies how the packages may be used. -enum LicenseType { - BY_EXCEPTION_ONLY = 1; - NOTICE = 2; - PERMISSIVE = 3; - RECIPROCAL = 4; - RESTRICTED_IF_STATICALLY_LINKED = 5; - RESTRICTED = 6; - UNENCUMBERED = 7; -} - -// Identifies security related metadata including risk category and any special -// instructions for using the package. -message Security { - // Security risk category for a package, as determined by an ISE-TPS review. - enum Category { - CATEGORY_UNSPECIFIED = 0; - - // Package should only be used in a sandboxed environment. - // Package should have restricted visibility. - SANDBOXED_ONLY = 1; - - // Package should not be used to process user content. It is considered - // safe to use to process trusted data only. Package should have restricted - // visibility. - TRUSTED_DATA_ONLY = 2; - - // Package is considered safe to use. - REVIEWED_AND_SECURE = 3; - } - - // Identifies the security risk category for the package. This will be - // provided by the ISE-TPS team as the result of a security review of the - // package. - optional Category category = 1; - - // An additional security note for the package. - optional string note = 2; - - // Text tag to categorize the package. It's currently used by security to: - // - to disable OSV (https://osv.dev) - // support via the `OSV:disable` tag - // - to attach CPE to their corresponding packages, for vulnerability - // monitoring: - // - // Please do document your usecase here should you want to add one. - repeated string tag = 3; - - // ID of advisories fixed with a mitigated patch, for example CVE-2018-1111. - repeated string mitigated_security_patch = 4; -} - -enum DirectoryType { - UNDEFINED = 0; - - // This directory represents a package. - PACKAGE = 1; - - // This directory is designed to organize multiple third-party PACKAGE - // directories. - GROUP = 2; - - // This directory contains several PACKAGE directories representing - // different versions of the same third-party project. - VERSIONS = 3; -} - -// Represents a whole or partial calendar date, such as a birthday. The time of -// day and time zone are either specified elsewhere or are insignificant. The -// date is relative to the Gregorian Calendar. This can represent one of the -// following: -// -// * A full date, with non-zero year, month, and day values. -// * A month and day, with a zero year (for example, an anniversary). -// * A year on its own, with a zero month and a zero day. -// * A year and month, with a zero day (for example, a credit card expiration -// date). -message Date { - // Year of the date. Must be from 1 to 9999, or 0 to specify a date without - // a year. - optional int32 year = 1; - // Month of a year. Must be from 1 to 12, or 0 to specify a year without a - // month and day. - optional int32 month = 2; - // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 - // to specify a year by itself or a year and month where the day isn't - // significant. - optional int32 day = 3; -} - -// Reference to external SBOM document and element corresponding to the package. -// See https://spdx.github.io/spdx-spec/v2.3/document-creation-information/#66-external-document-references-field -message SBOMRef { - // The URL that points to the SBOM document of the upstream package of this - // third_party package. - optional string url = 1; - // Checksum of the SBOM document the url field points to. - // Format: e.g. SHA1:<checksum>, or any algorithm defined in - // https://spdx.github.io/spdx-spec/v2.3/file-information/#8.4 - optional string checksum = 2; - // SPDXID of the upstream package/file defined in the SBOM document the url field points to. - // Format: SPDXRef-[a-zA-Z0-9.-]+, see - // https://spdx.github.io/spdx-spec/v2.3/package-information/#72-package-spdx-identifier-field or - // https://spdx.github.io/spdx-spec/v2.3/file-information/#82-file-spdx-identifier-field - optional string element_id = 3; -}
\ No newline at end of file |