diff options
author | Wei Li <weiwli@google.com> | 2023-05-05 10:49:28 -0700 |
---|---|---|
committer | Cherrypicker Worker <android-build-cherrypicker-worker@google.com> | 2023-05-19 00:35:20 +0000 |
commit | 233d5b97f84e3639d3c8c78bd93e2e5c2cbbd50c (patch) | |
tree | 3d7ddb7d8dc3d4e6ff4ee21ae2b716215d89582a | |
parent | d637b06d8f445f5aa433df413c5bfcb3943a0c6e (diff) | |
download | build-233d5b97f84e3639d3c8c78bd93e2e5c2cbbd50c.tar.gz |
Some changes to support SBOM generation for b build unbundled APEXs.
1) Use output file path of installed files in build system since there is no PRODUCT_OUT in Bazel
2) Use CONTAINS to describe the relationship between a APEX and files it contains
3) Generate SBOM of APEXs, which is similar to SBOM of products
Bug: 275472038
Test: CIs
(cherry picked from https://android-review.googlesource.com/q/commit:fd7e6517d345d3b8d4af12dae345434d968b83b9)
Merged-In: I41622366e5e6ed9dc78cca7bc7bb69a1f8f9bd9f
Change-Id: I41622366e5e6ed9dc78cca7bc7bb69a1f8f9bd9f
-rw-r--r-- | core/main.mk | 11 | ||||
-rwxr-xr-x | tools/sbom/generate-sbom.py | 68 | ||||
-rw-r--r-- | tools/sbom/sbom_data.py | 1 | ||||
-rw-r--r-- | tools/sbom/sbom_writers.py | 22 |
4 files changed, 63 insertions, 39 deletions
diff --git a/core/main.mk b/core/main.mk index 6a24bd3050..cb4dca6b41 100644 --- a/core/main.mk +++ b/core/main.mk @@ -2163,10 +2163,11 @@ endif # TARGET_BUILD_APPS $(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1) $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) rm -f $@ - @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated >> $@ + @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@ $(foreach f,$(installed_files),\ $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \ $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \ + $(eval _build_output_path := $(PRODUCT_OUT)/$(_path_on_device)) \ $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \ $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \ $(eval _is_prebuilt_make_module := $(ALL_MODULES.$(_module_name).IS_PREBUILT_MAKE_MODULE)) \ @@ -2184,9 +2185,9 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \ $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \ $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \ - @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ $(newline) \ + @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \ $(if $(_post_installed_dexpreopt_zip), \ - for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ ; done $(newline) \ + for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \ ) \ ) @@ -2196,14 +2197,14 @@ sbom: $(PRODUCT_OUT)/sbom.spdx.json $(PRODUCT_OUT)/sbom.spdx.json: $(PRODUCT_OUT)/sbom.spdx $(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) rm -rf $@ - $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr="$(PRODUCT_MANUFACTURER)" --json + $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json) else apps_only_sbom_files := $(sort $(patsubst %,%.spdx.json,$(filter %.apk,$(apps_only_installed_files)))) $(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) rm -rf $@ - $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr="$(PRODUCT_MANUFACTURER)" --unbundled + $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --unbundled_apk sbom: $(apps_only_sbom_files) diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py index 56509c9a11..1e98699a47 100755 --- a/tools/sbom/generate-sbom.py +++ b/tools/sbom/generate-sbom.py @@ -19,7 +19,6 @@ Generate the SBOM of the current target product in SPDX format. Usage example: generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \ --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \ - --product_out_dir=out/target/product/vsoc_x86_64 \ --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \ --product_mfr=Google """ @@ -89,11 +88,11 @@ def get_args(): parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.') parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.') - parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.') parser.add_argument('--build_version', required=True, help='The build version.') parser.add_argument('--product_mfr', required=True, help='The product manufacturer.') parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format') - parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module') + parser.add_argument('--unbundled_apk', action='store_true', default=False, help='Generate SBOM for unbundled APKs') + parser.add_argument('--unbundled_apex', action='store_true', default=False, help='Generate SBOM for unbundled APEXs') return parser.parse_args() @@ -127,7 +126,6 @@ def new_file_id(file_path): def checksum(file_path): - file_path = args.product_out_dir + '/' + file_path h = hashlib.sha1() if os.path.islink(file_path): h.update(os.readlink(file_path).encode('utf-8')) @@ -334,9 +332,8 @@ def generate_package_verification_code(files): return h.hexdigest() -def save_report(report): - prefix, _ = os.path.splitext(args.output_file) - with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file: +def save_report(report_file_path, report): + with open(report_file_path, 'w', encoding='utf-8') as report_file: for type, issues in report.items(): report_file.write(type + '\n') for issue in issues: @@ -394,7 +391,7 @@ def report_metadata_file(metadata_file_path, installed_file_metadata, report): installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) -def generate_sbom_for_unbundled(): +def generate_sbom_for_unbundled_apk(): with open(args.metadata, newline='') as sbom_metadata_file: reader = csv.DictReader(sbom_metadata_file) doc = sbom_data.Document(name=args.build_version, @@ -402,7 +399,7 @@ def generate_sbom_for_unbundled(): creators=['Organization: ' + args.product_mfr]) for installed_file_metadata in reader: installed_file = installed_file_metadata['installed_file'] - if args.output_file != args.product_out_dir + installed_file + '.spdx.json': + if args.output_file != installed_file_metadata['build_output_path'] + '.spdx.json': continue module_path = installed_file_metadata['module_path'] @@ -412,7 +409,9 @@ def generate_sbom_for_unbundled(): version=args.build_version, supplier='Organization: ' + args.product_mfr) file_id = new_file_id(installed_file) - file = sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file)) + file = sbom_data.File(id=file_id, + name=installed_file, + checksum=checksum(installed_file_metadata['build_output_path'])) relationship = sbom_data.Relationship(id1=file_id, relationship=sbom_data.RelationshipType.GENERATED_FROM, id2=package_id) @@ -435,24 +434,25 @@ def main(): args = get_args() log('Args:', vars(args)) - if args.unbundled: - generate_sbom_for_unbundled() + if args.unbundled_apk: + generate_sbom_for_unbundled_apk() return global metadata_file_protos metadata_file_protos = {} - doc = sbom_data.Document(name=args.build_version, - namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', - creators=['Organization: ' + args.product_mfr]) - product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT, name=sbom_data.PACKAGE_NAME_PRODUCT, download_location=sbom_data.VALUE_NONE, version=args.build_version, supplier='Organization: ' + args.product_mfr, files_analyzed=True) - doc.packages.append(product_package) + + doc = sbom_data.Document(name=args.build_version, + namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', + creators=['Organization: ' + args.product_mfr]) + if not args.unbundled_apex: + doc.packages.append(product_package) doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, name=sbom_data.PACKAGE_NAME_PLATFORM, @@ -478,18 +478,21 @@ def main(): module_path = installed_file_metadata['module_path'] product_copy_files = installed_file_metadata['product_copy_files'] kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] + build_output_path = installed_file_metadata['build_output_path'] if not installed_file_has_metadata(installed_file_metadata, report): continue - file_path = args.product_out_dir + '/' + installed_file - if not (os.path.islink(file_path) or os.path.isfile(file_path)): + if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)): report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file) continue file_id = new_file_id(installed_file) doc.files.append( - sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file))) - product_package.file_ids.append(file_id) + sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path))) + if not args.unbundled_apex: + product_package.file_ids.append(file_id) + elif len(doc.files) > 1: + doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id)) if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): metadata_file_path = get_metadata_file_path(installed_file_metadata) @@ -533,16 +536,31 @@ def main(): relationship=sbom_data.RelationshipType.GENERATED_FROM, id2=sbom_data.SPDXID_PLATFORM)) - product_package.verification_code = generate_package_verification_code(doc.files) + if not args.unbundled_apex: + product_package.verification_code = generate_package_verification_code(doc.files) + + if args.unbundled_apex: + doc.describes = doc.files[0].id # Save SBOM records to output file doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') - with open(args.output_file, 'w', encoding="utf-8") as file: - sbom_writers.TagValueWriter.write(doc, file) + prefix = args.output_file + if prefix.endswith('.spdx'): + prefix = prefix.removesuffix('.spdx') + elif prefix.endswith('.spdx.json'): + prefix = prefix.removesuffix('.spdx.json') + + output_file = prefix + '.spdx' + if args.unbundled_apex: + output_file = prefix + '-fragment.spdx' + with open(output_file, 'w', encoding="utf-8") as file: + sbom_writers.TagValueWriter.write(doc, file, fragment=args.unbundled_apex) if args.json: - with open(args.output_file+'.json', 'w', encoding="utf-8") as file: + with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file: sbom_writers.JSONWriter.write(doc, file) + save_report(prefix + '-gen-report.txt', report) + if __name__ == '__main__': main() diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py index d2ef48d52c..14c4eb231d 100644 --- a/tools/sbom/sbom_data.py +++ b/tools/sbom/sbom_data.py @@ -80,6 +80,7 @@ class RelationshipType: DESCRIBES = 'DESCRIBES' VARIANT_OF = 'VARIANT_OF' GENERATED_FROM = 'GENERATED_FROM' + CONTAINS = 'CONTAINS' @dataclass diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py index b1c66c5f80..85dee9d1a5 100644 --- a/tools/sbom/sbom_writers.py +++ b/tools/sbom/sbom_writers.py @@ -110,24 +110,26 @@ class TagValueWriter: return tagvalues @staticmethod - def marshal_described_element(sbom_doc): + def marshal_described_element(sbom_doc, fragment): if not sbom_doc.describes: return None product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes] if product_package: tagvalues = TagValueWriter.marshal_package(product_package[0]) - tagvalues.append( - f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') + if not fragment: + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') tagvalues.append('') return tagvalues file = [f for f in sbom_doc.files if f.id == sbom_doc.describes] if file: - tagvalues = [ - f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}' - ] + tagvalues = TagValueWriter.marshal_file(file[0]) + if not fragment: + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') return tagvalues @@ -180,6 +182,8 @@ class TagValueWriter: def marshal_files(sbom_doc): tagvalues = [] for file in sbom_doc.files: + if file.id == sbom_doc.describes: + continue tagvalues += TagValueWriter.marshal_file(file) return tagvalues @@ -204,9 +208,9 @@ class TagValueWriter: content = [] if not fragment: content += TagValueWriter.marshal_doc_headers(sbom_doc) - described_element = TagValueWriter.marshal_described_element(sbom_doc) - if described_element: - content += described_element + described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment) + if described_element: + content += described_element content += TagValueWriter.marshal_files(sbom_doc) tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc) content += tagvalues |