diff options
| author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-05-10 16:02:02 +0000 |
|---|---|---|
| committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2024-05-10 16:02:02 +0000 |
| commit | c74f4e7bea3cd914626b3a39f5fe7170cfe6604e (patch) | |
| tree | 5f77910436b3a54a2c589fcd943a4d657864f8e0 | |
| parent | 37c6fb5659dc2a588344a84d230cc5321f334223 (diff) | |
| parent | e07a784cf34257f4fe88420b8f3f31875b4e2fb4 (diff) | |
| download | build-busytown-mac-infra-release.tar.gz | |
Merge "Snap for 11819167 from 326c0d00cf54e5772e548d8617ba19bc384a677b to busytown-mac-infra-release" into busytown-mac-infra-releasebusytown-mac-infra-release
558 files changed, 43448 insertions, 8097 deletions
diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000000..cd5c426a04 --- /dev/null +++ b/Android.bp @@ -0,0 +1,36 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +// Package the minimal files required to run envsetup.sh in the test +// environment. +genrule { + name: "envsetup_minimum.zip", + visibility: [ + "//build/make/tests:__subpackages__", + ], + tools: [ + "soong_zip", + ], + srcs: [ + "envsetup.sh", + "shell_utils.sh", + "core/envsetup.mk", + ], + out: ["envsetup.zip"], + cmd: "$(location soong_zip) -o $(out) -D build/make", +} diff --git a/Changes.md b/Changes.md index 8979e3091c..fc15e601b6 100644 --- a/Changes.md +++ b/Changes.md @@ -1,4 +1,58 @@ -# Build System Changes for Android.mk Writers +# Build System Changes for Android.mk/Android.bp Writers + +## Soong genrules are now sandboxed + +Previously, soong genrules could access any files in the source tree, without specifying them as +inputs. This makes them incorrect in incremental builds, and incompatible with RBE and Bazel. + +Now, genrules are sandboxed so they can only access their listed srcs. Modules denylisted in +genrule/allowlists.go are exempt from this. You can also set `BUILD_BROKEN_GENRULE_SANDBOXING` +in board config to disable this behavior. + +## Partitions are no longer affected by previous builds + +Partition builds used to include everything in their staging directories, and building an +individual module will install it to the staging directory. Thus, previously, `m mymodule` followed +by `m` would cause `mymodule` to be presinstalled on the device, even if it wasn't listed in +`PRODUCT_PACKAGES`. + +This behavior has been changed, and now the partition images only include what they'd have if you +did a clean build. This behavior can be disabled by setting the +`BUILD_BROKEN_INCORRECT_PARTITION_IMAGES` environment variable or board config variable. + +Manually adding make rules that build to the staging directories without going through the make +module system will not be compatible with this change. This includes many usages of +`LOCAL_POST_INSTALL_CMD`. + +## Perform validation of Soong plugins + +Each Soong plugin will require manual work to migrate to Bazel. In order to +minimize the manual work outside of build/soong, we are restricting plugins to +those that exist today and those in vendor or hardware directories. + +If you need to extend the build system via a plugin, please reach out to the +build team via email android-building@googlegroups.com (external) for any +questions, or see [go/soong](http://go/soong) (internal). + +To omit the validation, `BUILD_BROKEN_PLUGIN_VALIDATION` expects a +space-separated list of plugins to omit from the validation. This must be set +within a product configuration .mk file, board config .mk file, or buildspec.mk. + +## Python 2 to 3 migration + +The path set when running builds now makes the `python` executable point to python 3, +whereas on previous versions it pointed to python 2. If you still have python 2 scripts, +you can change the shebang line to use `python2` explicitly. This only applies for +scripts run directly from makefiles, or from soong genrules. This behavior can be +temporarily overridden by setting the `BUILD_BROKEN_PYTHON_IS_PYTHON2` environment +variable to `true`. It's only an environment variable and not a product config variable +because product config sometimes calls python code. + +In addition, `python_*` soong modules no longer allow python 2. This can be temporarily +overridden by setting the `BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES` product configuration +variable to `true`. + +Python 2 is slated for complete removal in V. ## Stop referencing sysprop_library directly from cc modules @@ -477,6 +531,24 @@ $(call dist-for-goals,foo,bar/baz) will copy `bar/baz` into `$DIST_DIR/baz` when `m foo dist` is run. +#### FILE_NAME_TAG {#FILE_NAME_TAG} + +To embed the `BUILD_NUMBER` (or for local builds, `eng.${USER}`), include +`FILE_NAME_TAG_PLACEHOLDER` in the destination: + +``` make +# you can use dist-for-goals-with-filenametag function +$(call dist-for-goals-with-filenametag,foo,bar.zip) +# or use FILE_NAME_TAG_PLACEHOLDER manually +$(call dist-for-goals,foo,bar.zip:baz-FILE_NAME_TAG_PLACEHOLDER.zip) +``` + +Which will produce `$DIST_DIR/baz-1234567.zip` on build servers which set +`BUILD_NUMBER=1234567`, or `$DIST_DIR/baz-eng.builder.zip` for local builds. + +If you just want to append `BUILD_NUMBER` at the end of basename, use +`dist-for-goals-with-filenametag` instead of `dist-for-goals`. + #### Renames during copy Instead of specifying just a file, a destination name can be specified, @@ -818,7 +890,7 @@ for this option to exist. ### Stop using clang property -Clang has been deleted from Soong. To fix any build errors, remove the clang +The clang property has been deleted from Soong. To fix any build errors, remove the clang property from affected Android.bp files using bpmodify. diff --git a/CleanSpec.mk b/CleanSpec.mk index 957da92f63..f8c96ffffe 100644 --- a/CleanSpec.mk +++ b/CleanSpec.mk @@ -773,6 +773,20 @@ $(call add-clean-step, rm -rf $(SOONG_HOST_OUT)) # Don't use SOONG_HOST_OUT, it is now an alias for HOST_OUT. $(call add-clean-step, rm -rf $(OUT_DIR)/soong/host) +# Clear out tools/metalava Bazel output dir +$(call add-clean-step, rm -rf $(OUT_DIR)/bazel/output/execroot/__main__/bazel-out/mixed_builds_product-*/bin/tools/metalava) + +# Clear out rustc compiler intermediates after reverting rust compiler/linker split. +$(call add-clean-step, find $(OUT_DIR) -name "*.rsp.whole.a" -print0 | xargs -0 /bin/bash -c 'rm -f $$$${@}; rm -f $$$${@/.rsp.whole.a/.rsp.a}; rm -f $$$${@/.rsp.whole.a/.rsp}') + +# Remove obsolete java compilation artifacts +$(call add-clean-step, rm -rf $(OUT_DIR)/target/common/obj/JAVA_LIBRARIES/) +$(call add-clean-step, find $(OUT_DIR) -type f -name "*.jar" -print0 | xargs -0 rm -f) + +# Remove obsolete java compilation artifacts +$(call add-clean-step, rm -rf $(OUT_DIR)/target/common/obj/JAVA_LIBRARIES/) +$(call add-clean-step, find $(OUT_DIR) -type f -name "*.jar" -print0 | xargs -0 rm -f) + # ************************************************ # NEWER CLEAN STEPS MUST BE AT THE END OF THE LIST # ************************************************ @@ -2,6 +2,6 @@ include platform/build/soong:/OWNERS # Since this file affects all Android developers, lock it down. There is still # round the world timzeone coverage. -per-file envsetup.sh = joeo@google.com, jingwen@google.com, lberki@google.com -per-file shell_utils.sh = joeo@google.com, jingwen@google.com, lberki@google.com +per-file envsetup.sh = joeo@google.com, jingwen@google.com +per-file shell_utils.sh = joeo@google.com, jingwen@google.com diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg index ce7515044e..97ecd33212 100644 --- a/PREUPLOAD.cfg +++ b/PREUPLOAD.cfg @@ -1,2 +1,5 @@ [Hook Scripts] do_not_use_DO_NOT_MERGE = ${REPO_ROOT}/build/soong/scripts/check_do_not_merge.sh ${PREUPLOAD_COMMIT} + +[Builtin Hooks] +ktfmt = true diff --git a/ci/Android.bp b/ci/Android.bp new file mode 100644 index 0000000000..066b83fb2d --- /dev/null +++ b/ci/Android.bp @@ -0,0 +1,85 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +python_test_host { + name: "build_test_suites_test", + main: "build_test_suites_test.py", + pkg_path: "testdata", + srcs: [ + "build_test_suites_test.py", + ], + libs: [ + "build_test_suites", + "pyfakefs", + "ci_test_lib", + ], + test_options: { + unit_test: true, + }, + data: [ + ":py3-cmd", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, +} + +// This test is only intended to be run locally since it's slow, not hermetic, +// and requires a lot of system state. It is therefore not marked as `unit_test` +// and is not part of any test suite. Note that we also don't want to run this +// test with Bazel since that would require disabling sandboxing and explicitly +// passing in all the env vars we depend on via the command-line. The test +// target could be configured to do so but it's not worth doing seeing that +// we're moving away from Bazel. +python_test_host { + name: "build_test_suites_local_test", + main: "build_test_suites_local_test.py", + srcs: [ + "build_test_suites_local_test.py", + ], + libs: [ + "build_test_suites", + "pyfakefs", + "ci_test_lib", + ], + test_config_template: "AndroidTest.xml.template", + test_options: { + unit_test: false, + }, + version: { + py3: { + embedded_launcher: true, + }, + }, +} + +python_library_host { + name: "build_test_suites", + srcs: [ + "build_test_suites.py", + ], +} + +python_library_host { + name: "ci_test_lib", + srcs: [ + "ci_test_lib.py", + ], +} diff --git a/ci/AndroidTest.xml.template b/ci/AndroidTest.xml.template new file mode 100644 index 0000000000..81a3435b68 --- /dev/null +++ b/ci/AndroidTest.xml.template @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2020 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <test class="com.android.tradefed.testtype.python.PythonBinaryHostTest"> + <option name="par-file-name" value="{MODULE}"/> + <option name="use-test-output-file" value="false"/> + <option name="test-timeout" value="5m"/> + </test> +</configuration> diff --git a/target/board/emulator_arm/device.mk b/ci/build_test_suites index af023eb25c..5aaf2f49b7 100644..100755 --- a/target/board/emulator_arm/device.mk +++ b/ci/build_test_suites @@ -1,18 +1,20 @@ +#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B # -# Copyright (C) 2020 The Android Open Source Project +# Copyright 2024, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. +import build_test_suites +import sys + +build_test_suites.main(sys.argv[1:]) diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py new file mode 100644 index 0000000000..29ed50e095 --- /dev/null +++ b/ci/build_test_suites.py @@ -0,0 +1,126 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build script for the CI `test_suites` target.""" + +import argparse +import logging +import os +import pathlib +import subprocess +import sys + + +class Error(Exception): + + def __init__(self, message): + super().__init__(message) + + +class BuildFailureError(Error): + + def __init__(self, return_code): + super().__init__(f'Build command failed with return code: f{return_code}') + self.return_code = return_code + + +REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP']) +SOONG_UI_EXE_REL_PATH = 'build/soong/soong_ui.bash' + + +def get_top() -> pathlib.Path: + return pathlib.Path(os.environ['TOP']) + + +def build_test_suites(argv: list[str]) -> int: + """Builds the general-tests and any other test suites passed in. + + Args: + argv: The command line arguments passed in. + + Returns: + The exit code of the build. + """ + args = parse_args(argv) + check_required_env() + + try: + build_everything(args) + except BuildFailureError as e: + logging.error('Build command failed! Check build_log for details.') + return e.return_code + + return 0 + + +def check_required_env(): + """Check for required env vars. + + Raises: + RuntimeError: If any required env vars are not found. + """ + missing_env_vars = sorted(v for v in REQUIRED_ENV_VARS if v not in os.environ) + + if not missing_env_vars: + return + + t = ','.join(missing_env_vars) + raise Error(f'Missing required environment variables: {t}') + + +def parse_args(argv): + argparser = argparse.ArgumentParser() + + argparser.add_argument( + 'extra_targets', nargs='*', help='Extra test suites to build.' + ) + + return argparser.parse_args(argv) + + +def build_everything(args: argparse.Namespace): + """Builds all tests (regardless of whether they are needed). + + Args: + args: The parsed arguments. + + Raises: + BuildFailure: If the build command fails. + """ + build_command = base_build_command(args, args.extra_targets) + + try: + run_command(build_command) + except subprocess.CalledProcessError as e: + raise BuildFailureError(e.returncode) from e + + +def base_build_command( + args: argparse.Namespace, extra_targets: set[str] +) -> list[str]: + + build_command = [] + build_command.append(get_top().joinpath(SOONG_UI_EXE_REL_PATH)) + build_command.append('--make-mode') + build_command.extend(extra_targets) + + return build_command + + +def run_command(args: list[str], stdout=None): + subprocess.run(args=args, check=True, stdout=stdout) + + +def main(argv): + sys.exit(build_test_suites(argv)) diff --git a/ci/build_test_suites_local_test.py b/ci/build_test_suites_local_test.py new file mode 100644 index 0000000000..78e52d327c --- /dev/null +++ b/ci/build_test_suites_local_test.py @@ -0,0 +1,123 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration tests for build_test_suites that require a local build env.""" + +import os +import pathlib +import shutil +import signal +import subprocess +import tempfile +import time +import ci_test_lib + + +class BuildTestSuitesLocalTest(ci_test_lib.TestCase): + + def setUp(self): + self.top_dir = pathlib.Path(os.environ['ANDROID_BUILD_TOP']).resolve() + self.executable = self.top_dir.joinpath('build/make/ci/build_test_suites') + self.process_session = ci_test_lib.TemporaryProcessSession(self) + self.temp_dir = ci_test_lib.TestTemporaryDirectory.create(self) + + def build_subprocess_args(self, build_args: list[str]): + env = os.environ.copy() + env['TOP'] = str(self.top_dir) + env['OUT_DIR'] = self.temp_dir + + args = ([self.executable] + build_args,) + kwargs = { + 'cwd': self.top_dir, + 'env': env, + 'text': True, + } + + return (args, kwargs) + + def run_build(self, build_args: list[str]) -> subprocess.CompletedProcess: + args, kwargs = self.build_subprocess_args(build_args) + + return subprocess.run( + *args, + **kwargs, + check=True, + capture_output=True, + timeout=5 * 60, + ) + + def assert_children_alive(self, children: list[int]): + for c in children: + self.assertTrue(ci_test_lib.process_alive(c)) + + def assert_children_dead(self, children: list[int]): + for c in children: + self.assertFalse(ci_test_lib.process_alive(c)) + + def test_fails_for_invalid_arg(self): + invalid_arg = '--invalid-arg' + + with self.assertRaises(subprocess.CalledProcessError) as cm: + self.run_build([invalid_arg]) + + self.assertIn(invalid_arg, cm.exception.stderr) + + def test_builds_successfully(self): + self.run_build(['nothing']) + + def test_can_interrupt_build(self): + args, kwargs = self.build_subprocess_args(['general-tests']) + p = self.process_session.create(args, kwargs) + + # TODO(lucafarsi): Replace this (and other instances) with a condition. + time.sleep(5) # Wait for the build to get going. + self.assertIsNone(p.poll()) # Check that the process is still alive. + children = query_child_pids(p.pid) + self.assert_children_alive(children) + + p.send_signal(signal.SIGINT) + p.wait() + + time.sleep(5) # Wait for things to die out. + self.assert_children_dead(children) + + def test_can_kill_build_process_group(self): + args, kwargs = self.build_subprocess_args(['general-tests']) + p = self.process_session.create(args, kwargs) + + time.sleep(5) # Wait for the build to get going. + self.assertIsNone(p.poll()) # Check that the process is still alive. + children = query_child_pids(p.pid) + self.assert_children_alive(children) + + os.killpg(os.getpgid(p.pid), signal.SIGKILL) + p.wait() + + time.sleep(5) # Wait for things to die out. + self.assert_children_dead(children) + + +# TODO(hzalek): Replace this with `psutils` once available in the tree. +def query_child_pids(parent_pid: int) -> set[int]: + p = subprocess.run( + ['pgrep', '-P', str(parent_pid)], + check=True, + capture_output=True, + text=True, + ) + return {int(pid) for pid in p.stdout.splitlines()} + + +if __name__ == '__main__': + ci_test_lib.main() diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py new file mode 100644 index 0000000000..08a79a3294 --- /dev/null +++ b/ci/build_test_suites_test.py @@ -0,0 +1,254 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for build_test_suites.py""" + +from importlib import resources +import multiprocessing +import os +import pathlib +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import textwrap +import time +from typing import Callable +from unittest import mock +import build_test_suites +import ci_test_lib +from pyfakefs import fake_filesystem_unittest + + +class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): + + def setUp(self): + self.setUpPyfakefs() + + os_environ_patcher = mock.patch.dict('os.environ', {}) + self.addCleanup(os_environ_patcher.stop) + self.mock_os_environ = os_environ_patcher.start() + + subprocess_run_patcher = mock.patch('subprocess.run') + self.addCleanup(subprocess_run_patcher.stop) + self.mock_subprocess_run = subprocess_run_patcher.start() + + self._setup_working_build_env() + + def test_missing_target_release_env_var_raises(self): + del os.environ['TARGET_RELEASE'] + + with self.assert_raises_word(build_test_suites.Error, 'TARGET_RELEASE'): + build_test_suites.main([]) + + def test_missing_target_product_env_var_raises(self): + del os.environ['TARGET_PRODUCT'] + + with self.assert_raises_word(build_test_suites.Error, 'TARGET_PRODUCT'): + build_test_suites.main([]) + + def test_missing_top_env_var_raises(self): + del os.environ['TOP'] + + with self.assert_raises_word(build_test_suites.Error, 'TOP'): + build_test_suites.main([]) + + def test_invalid_arg_raises(self): + invalid_args = ['--invalid_arg'] + + with self.assertRaisesRegex(SystemExit, '2'): + build_test_suites.main(invalid_args) + + def test_build_failure_returns(self): + self.mock_subprocess_run.side_effect = subprocess.CalledProcessError( + 42, None + ) + + with self.assertRaisesRegex(SystemExit, '42'): + build_test_suites.main([]) + + def test_build_success_returns(self): + with self.assertRaisesRegex(SystemExit, '0'): + build_test_suites.main([]) + + def assert_raises_word(self, cls, word): + return self.assertRaisesRegex(build_test_suites.Error, rf'\b{word}\b') + + def _setup_working_build_env(self): + self.fake_top = pathlib.Path('/fake/top') + self.fake_top.mkdir(parents=True) + + self.soong_ui_dir = self.fake_top.joinpath('build/soong') + self.soong_ui_dir.mkdir(parents=True, exist_ok=True) + + self.soong_ui = self.soong_ui_dir.joinpath('soong_ui.bash') + self.soong_ui.touch() + + self.mock_os_environ.update({ + 'TARGET_RELEASE': 'release', + 'TARGET_PRODUCT': 'product', + 'TOP': str(self.fake_top), + }) + + self.mock_subprocess_run.return_value = 0 + + +class RunCommandIntegrationTest(ci_test_lib.TestCase): + + def setUp(self): + self.temp_dir = ci_test_lib.TestTemporaryDirectory.create(self) + + # Copy the Python executable from 'non-code' resources and make it + # executable for use by tests that launch a subprocess. Note that we don't + # use Python's native `sys.executable` property since that is not set when + # running via the embedded launcher. + base_name = 'py3-cmd' + dest_file = self.temp_dir.joinpath(base_name) + with resources.as_file( + resources.files('testdata').joinpath(base_name) + ) as p: + shutil.copy(p, dest_file) + dest_file.chmod(dest_file.stat().st_mode | stat.S_IEXEC) + self.python_executable = dest_file + + self._managed_processes = [] + + def tearDown(self): + self._terminate_managed_processes() + + def test_raises_on_nonzero_exit(self): + with self.assertRaises(Exception): + build_test_suites.run_command([ + self.python_executable, + '-c', + textwrap.dedent(f"""\ + import sys + sys.exit(1) + """), + ]) + + def test_streams_stdout(self): + + def run_slow_command(stdout_file, marker): + with open(stdout_file, 'w') as f: + build_test_suites.run_command( + [ + self.python_executable, + '-c', + textwrap.dedent(f"""\ + import time + + print('{marker}', end='', flush=True) + + # Keep process alive until we check stdout. + time.sleep(10) + """), + ], + stdout=f, + ) + + marker = 'Spinach' + stdout_file = self.temp_dir.joinpath('stdout.txt') + + p = self.start_process(target=run_slow_command, args=[stdout_file, marker]) + + self.assert_file_eventually_contains(stdout_file, marker) + + def test_propagates_interruptions(self): + + def run(pid_file): + build_test_suites.run_command([ + self.python_executable, + '-c', + textwrap.dedent(f"""\ + import os + import pathlib + import time + + pathlib.Path('{pid_file}').write_text(str(os.getpid())) + + # Keep the process alive for us to explicitly interrupt it. + time.sleep(10) + """), + ]) + + pid_file = self.temp_dir.joinpath('pid.txt') + p = self.start_process(target=run, args=[pid_file]) + subprocess_pid = int(read_eventual_file_contents(pid_file)) + + os.kill(p.pid, signal.SIGINT) + p.join() + + self.assert_process_eventually_dies(p.pid) + self.assert_process_eventually_dies(subprocess_pid) + + def start_process(self, *args, **kwargs) -> multiprocessing.Process: + p = multiprocessing.Process(*args, **kwargs) + self._managed_processes.append(p) + p.start() + return p + + def assert_process_eventually_dies(self, pid: int): + try: + wait_until(lambda: not ci_test_lib.process_alive(pid)) + except TimeoutError as e: + self.fail(f'Process {pid} did not die after a while: {e}') + + def assert_file_eventually_contains(self, file: pathlib.Path, substring: str): + wait_until(lambda: file.is_file() and file.stat().st_size > 0) + self.assertIn(substring, read_file_contents(file)) + + def _terminate_managed_processes(self): + for p in self._managed_processes: + if not p.is_alive(): + continue + + # We terminate the process with `SIGINT` since using `terminate` or + # `SIGKILL` doesn't kill any grandchild processes and we don't have + # `psutil` available to easily query all children. + os.kill(p.pid, signal.SIGINT) + + +def wait_until( + condition_function: Callable[[], bool], + timeout_secs: float = 3.0, + polling_interval_secs: float = 0.1, +): + """Waits until a condition function returns True.""" + + start_time_secs = time.time() + + while not condition_function(): + if time.time() - start_time_secs > timeout_secs: + raise TimeoutError( + f'Condition not met within timeout: {timeout_secs} seconds' + ) + + time.sleep(polling_interval_secs) + + +def read_file_contents(file: pathlib.Path) -> str: + with open(file, 'r') as f: + return f.read() + + +def read_eventual_file_contents(file: pathlib.Path) -> str: + wait_until(lambda: file.is_file() and file.stat().st_size > 0) + return read_file_contents(file) + + +if __name__ == '__main__': + ci_test_lib.main() diff --git a/ci/ci_test_lib.py b/ci/ci_test_lib.py new file mode 100644 index 0000000000..2d70d3f01e --- /dev/null +++ b/ci/ci_test_lib.py @@ -0,0 +1,86 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Testing utilities for tests in the CI package.""" + +import logging +import os +import unittest +import subprocess +import pathlib +import shutil +import tempfile + + +# Export the TestCase class to reduce the number of imports tests have to list. +TestCase = unittest.TestCase + + +def process_alive(pid): + """Check For the existence of a pid.""" + + try: + os.kill(pid, 0) + except OSError: + return False + + return True + + +class TemporaryProcessSession: + + def __init__(self, test_case: TestCase): + self._created_processes = [] + test_case.addCleanup(self.cleanup) + + def create(self, args, kwargs): + p = subprocess.Popen(*args, **kwargs, start_new_session=True) + self._created_processes.append(p) + return p + + def cleanup(self): + for p in self._created_processes: + if not process_alive(p.pid): + return + os.killpg(os.getpgid(p.pid), signal.SIGKILL) + + +class TestTemporaryDirectory: + + def __init__(self, delete: bool, ): + self._delete = delete + + @classmethod + def create(cls, test_case: TestCase, delete: bool = True): + temp_dir = TestTemporaryDirectory(delete) + temp_dir._dir = pathlib.Path(tempfile.mkdtemp()) + test_case.addCleanup(temp_dir.cleanup) + return temp_dir._dir + + def get_dir(self): + return self._dir + + def cleanup(self): + if not self._delete: + return + shutil.rmtree(self._dir, ignore_errors=True) + + +def main(): + + # Disable logging since it breaks the TF Python test output parser. + # TODO(hzalek): Use TF's `test-output-file` option to re-enable logging. + logging.getLogger().disabled = True + + unittest.main() diff --git a/ci/test_mapping_module_retriever.py b/ci/test_mapping_module_retriever.py new file mode 100644 index 0000000000..d2c13c0e7d --- /dev/null +++ b/ci/test_mapping_module_retriever.py @@ -0,0 +1,125 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Simple parsing code to scan test_mapping files and determine which +modules are needed to build for the given list of changed files. +TODO(lucafarsi): Deduplicate from artifact_helper.py +""" + +from typing import Any, Dict, Set, Text +import json +import os +import re + +# Regex to extra test name from the path of test config file. +TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config' + +# Key name for TEST_MAPPING imports +KEY_IMPORTS = 'imports' +KEY_IMPORT_PATH = 'path' + +# Name of TEST_MAPPING file. +TEST_MAPPING = 'TEST_MAPPING' + +# Pattern used to identify double-quoted strings and '//'-format comments in +# TEST_MAPPING file, but only double-quoted strings are included within the +# matching group. +_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?') + + +def FilterComments(test_mapping_file: Text) -> Text: + """Remove comments in TEST_MAPPING file to valid format. + + Only '//' is regarded as comments. + + Args: + test_mapping_file: Path to a TEST_MAPPING file. + + Returns: + Valid json string without comments. + """ + return re.sub(_COMMENTS_RE, r'\1', test_mapping_file) + +def GetTestMappings(paths: Set[Text], + checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]: + """Get the affected TEST_MAPPING files. + + TEST_MAPPING files in source code are packaged into a build artifact + `test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file + is preserved. From all TEST_MAPPING files in the source code, this method + locates the affected TEST_MAPPING files based on the given paths list. + + A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files + from a different location, e.g., + "imports": [ + { + "path": "../folder2" + } + ] + In that example, TEST_MAPPING files inside ../folder2 (relative to the + TEST_MAPPING file containing that imports section) and its parent directories + will also be included. + + Args: + paths: A set of paths with related TEST_MAPPING files for given changes. + checked_paths: A set of paths that have been checked for TEST_MAPPING file + already. The set is updated after processing each TEST_MAPPING file. It's + used to prevent infinite loop when the method is called recursively. + + Returns: + A dictionary of Test Mapping containing the content of the affected + TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file. + """ + test_mappings = {} + + # Search for TEST_MAPPING files in each modified path and its parent + # directories. + all_paths = set() + for path in paths: + dir_names = path.split(os.path.sep) + all_paths |= set( + [os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))]) + # Add root directory to the paths to search for TEST_MAPPING file. + all_paths.add('') + + all_paths.difference_update(checked_paths) + checked_paths |= all_paths + # Try to load TEST_MAPPING file in each possible path. + for path in all_paths: + try: + test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING') + # Read content of TEST_MAPPING file. + content = FilterComments(open(test_mapping_file, "r").read()) + test_mapping = json.loads(content) + test_mappings[path] = test_mapping + + import_paths = set() + for import_detail in test_mapping.get(KEY_IMPORTS, []): + import_path = import_detail[KEY_IMPORT_PATH] + # Try the import path as absolute path. + import_paths.add(import_path) + # Try the import path as relative path based on the test mapping file + # containing the import. + norm_import_path = os.path.normpath(os.path.join(path, import_path)) + import_paths.add(norm_import_path) + import_paths.difference_update(checked_paths) + if import_paths: + import_test_mappings = GetTestMappings(import_paths, checked_paths) + test_mappings.update(import_test_mappings) + except (KeyError, FileNotFoundError, NotADirectoryError): + # TEST_MAPPING file doesn't exist in path + pass + + return test_mappings diff --git a/cogsetup.sh b/cogsetup.sh new file mode 100644 index 0000000000..ef1485d5f2 --- /dev/null +++ b/cogsetup.sh @@ -0,0 +1,71 @@ +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is executed by build/envsetup.sh, and can use anything +# defined in envsetup.sh. +function _create_out_symlink_for_cog() { + if [[ "${OUT_DIR}" == "" ]]; then + OUT_DIR="out" + fi + + # getoutdir ensures paths are absolute. envsetup could be called from a + # directory other than the root of the source tree + local outdir=$(getoutdir) + if [[ -L "${outdir}" ]]; then + return + fi + if [ -d "${outdir}" ]; then + echo -e "\tOutput directory ${outdir} cannot be present in a Cog workspace." + echo -e "\tDelete \"${outdir}\" or create a symlink from \"${outdir}\" to a directory outside your workspace." + return 1 + fi + + DEFAULT_OUTPUT_DIR="${HOME}/.cog/android-build-out" + mkdir -p ${DEFAULT_OUTPUT_DIR} + ln -s ${DEFAULT_OUTPUT_DIR} ${outdir} +} + +# This function sets up the build environment to be appropriate for Cog. +function _setup_cog_env() { + _create_out_symlink_for_cog + if [ "$?" -eq "1" ]; then + echo -e "\e[0;33mWARNING:\e[00m Cog environment setup failed!" + return 1 + fi + + export ANDROID_BUILD_ENVIRONMENT_CONFIG="googler-cog" + + # Running repo command within Cog workspaces is not supported, so override + # it with this function. If the user is running repo within a Cog workspace, + # we'll fail with an error, otherwise, we run the original repo command with + # the given args. + if ! ORIG_REPO_PATH=`which repo`; then + return 0 + fi + function repo { + if [[ "${PWD}" == /google/cog/* ]]; then + echo "\e[01;31mERROR:\e[0mrepo command is disallowed within Cog workspaces." + return 1 + fi + ${ORIG_REPO_PATH} "$@" + } +} + +if [[ "${PWD}" != /google/cog/* ]]; then + echo -e "\e[01;31mERROR:\e[0m This script must be run from a Cog workspace." +fi + +_setup_cog_env diff --git a/common/math.mk b/common/math.mk index 0271ea8b03..ecee474d14 100644 --- a/common/math.mk +++ b/common/math.mk @@ -25,6 +25,7 @@ __MATH_POS_NUMBERS := 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 2 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 \ 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 __MATH_NUMBERS := 0 $(__MATH_POS_NUMBERS) +__MATH_ONE_NUMBERS := 0 1 2 3 4 5 6 7 8 9 math-error = $(call pretty-error,$(1)) math-expect := @@ -36,6 +37,10 @@ math-expect-error := # make -f ${ANDROID_BUILD_TOP}/build/make/common/math.mk RUN_MATH_TESTS=true # $(get_build_var CKATI) -f ${ANDROID_BUILD_TOP}//build/make/common/math.mk RUN_MATH_TESTS=true ifdef RUN_MATH_TESTS + ifndef empty + empty := + space := $(empty) $(empty) + endif MATH_TEST_FAILURE := MATH_TEST_ERROR := math-error = $(if $(MATH_TEST_ERROR),,$(eval MATH_TEST_ERROR:=$(1))) @@ -61,13 +66,29 @@ ifdef RUN_MATH_TESTS endif # Returns true if $(1) is a non-negative integer <= 100, otherwise returns nothing. -define math_is_number +define math_is_number_in_100 $(strip \ $(if $(1),,$(call math-error,Argument missing)) \ $(if $(word 2,$(1)),$(call math-error,Multiple words in a single argument: $(1))) \ $(if $(filter $(1),$(__MATH_NUMBERS)),true)) endef +# Same with math_is_number_in_100, but no limit. +define _math_ext_is_number +$(strip \ + $(if $(1),,$(call math-error,Argument missing)) \ + $(if $(word 2,$(1)),$(call math-error,Multiple words in a single argument: $(1))) \ + $(eval should_empty:=$(1)) \ + $(foreach num,$(__MATH_ONE_NUMBERS),\ + $(eval should_empty:=$(subst $(num),$(empty),$(should_empty)))) \ + $(if $(should_empty),,true)) +endef + +# Returns true if $(1) is a non-negative integer. +define math_is_number +$(strip $(if $(call math_is_number_in_100,$(1)),true,$(call _math_ext_is_number,$(1)))) +endef + define math_is_zero $(strip \ $(if $(word 2,$(1)),$(call math-error,Multiple words in a single argument: $(1))) \ @@ -76,6 +97,7 @@ endef $(call math-expect-true,(call math_is_number,0)) $(call math-expect-true,(call math_is_number,2)) +$(call math-expect-true,(call math_is_number,202412)) $(call math-expect-false,(call math_is_number,foo)) $(call math-expect-false,(call math_is_number,-1)) $(call math-expect-error,(call math_is_number,1 2),Multiple words in a single argument: 1 2) @@ -88,7 +110,7 @@ $(call math-expect-error,(call math_is_zero,1 2),Multiple words in a single argu $(call math-expect-error,(call math_is_zero,no 2),Multiple words in a single argument: no 2) define _math_check_valid -$(if $(call math_is_number,$(1)),,$(call math-error,Only non-negative integers <= 100 are supported (not $(1)))) +$(if $(call math_is_number_in_100,$(1)),,$(call math-error,Only non-negative integers <= 100 are supported (not $(1)))) endef $(call math-expect,(call _math_check_valid,0)) @@ -113,18 +135,81 @@ $(call math-expect,(call int_range_list,1,2),1 2) $(call math-expect,(call int_range_list,2,1),) $(call math-expect-error,(call int_range_list,1,101),Only non-negative integers <= 100 are supported (not 101)) +# Split an integer into a list of digits +define _math_number_to_list +$(strip \ + $(if $(call _math_ext_is_number,$(1)),,\ + $(call math-error,Only non-negative integers are supported (not $(1)))) \ + $(eval num_list:=$(1)) \ + $(foreach num,$(__MATH_ONE_NUMBERS),\ + $(eval num_list:=$(subst $(num),$(space)$(num),$(num_list)))) \ + $(if $(filter $(words $(num_list)),$(__MATH_ONE_NUMBERS)),,\ + $(call math-error,Only non-negative integers with less than 9 digits are supported (not $(1)))) \ + $(if $(filter 0,$(word 1,$(num_list))),\ + $(call math-error,Only non-negative integers without leading zeros are supported (not $(1)))) \ + $(num_list)) +endef + +$(call math-expect,(call _math_number_to_list,123),1 2 3) +$(call math-expect-error,(call _math_number_to_list,123 456),Multiple words in a single argument: 123 456) +$(call math-expect-error,(call _math_number_to_list,-123),Only non-negative integers are supported (not -123)) +$(call math-expect-error,(call _math_number_to_list,002),Only non-negative integers without leading zeros are supported (not 002)) +$(call math-expect-error,(call _math_number_to_list,1234567890),Only non-negative integers with less than 9 digits are supported (not 1234567890)) + +# Compare 1-digit integer $(1) and $(2). +# Returns 1 if $(1) > $(2), -1 if $(1) < $(2), nothing if equals. +define _math_1digit_comp +$(strip \ + $(if $(filter $(1),$(2)),,\ + $(if $(filter $(1),$(firstword $(filter $(1) $(2),$(__MATH_ONE_NUMBERS)))),-1,1))) +endef + +$(call math-expect,(call _math_1digit_comp,1,1)) +$(call math-expect,(call _math_1digit_comp,0,9),-1) +$(call math-expect,(call _math_1digit_comp,3,1),1) + +# Compare the same $(3)-digit-length integers $(1) and $(2) that are split into a list of digits. +# Returns 1 if $(1) > $(2), -1 if $(1) < $(2), nothing if equals. +define _math_list_comp +$(strip \ + $(eval ans:=) \ + $(foreach num,$(call int_range_list,1,$(3)),\ + $(if $(ans),,$(eval ans:=$(call _math_1digit_comp,$(word $(num),$(1)),$(word $(num),$(2)))))) \ + $(ans)) +endef + +# Compare any two non-negative integers $(1) and $(2). +# Returns 1 if $(1) > $(2), -1 if $(1) < $(2), nothing if equals. +define _math_ext_comp +$(strip \ + $(eval num_list1:=$(call _math_number_to_list,$(1))) \ + $(eval len1:=$(words $(num_list1))) \ + $(eval num_list2:=$(call _math_number_to_list,$(2))) \ + $(eval len2:=$(words $(num_list2))) \ + $(eval comp:=$(call _math_1digit_comp,$(len1),$(len2))) \ + $(if $(comp),$(comp),$(call _math_list_comp,$(num_list1),$(num_list2),$(len1)))) +endef + +$(call math-expect,(call _math_ext_comp,5,10),-1) +$(call math-expect,(call _math_ext_comp,12345,12345)) +$(call math-expect,(call _math_ext_comp,500,5),1) +$(call math-expect,(call _math_ext_comp,202404,202504),-1) # Returns the greater of $1 or $2. -# If $1 or $2 is not a positive integer <= 100, then an error is generated. +# If $1 or $2 is not a positive integer, then an error is generated. define math_max -$(strip $(call _math_check_valid,$(1)) $(call _math_check_valid,$(2)) \ - $(lastword $(filter $(1) $(2),$(__MATH_NUMBERS)))) +$(strip \ + $(if $(filter truetrue,$(call math_is_number_in_100,$(1))$(call math_is_number_in_100,$(2))),\ + $(lastword $(filter $(1) $(2),$(__MATH_NUMBERS))),\ + $(if $(filter 1,$(call _math_ext_comp,$(1),$(2))),$(1),$(2)))) endef # Returns the lesser of $1 or $2. define math_min -$(strip $(call _math_check_valid,$(1)) $(call _math_check_valid,$(2)) \ - $(firstword $(filter $(1) $(2),$(__MATH_NUMBERS)))) +$(strip \ + $(if $(filter truetrue,$(call math_is_number_in_100,$(1))$(call math_is_number_in_100,$(2))),\ + $(firstword $(filter $(1) $(2),$(__MATH_NUMBERS))),\ + $(if $(filter -1,$(call _math_ext_comp,$(1),$(2))),$(1),$(2)))) endef $(call math-expect-error,(call math_max),Argument missing) @@ -142,6 +227,15 @@ $(call math-expect,(call math_min,1,1),1) $(call math-expect,(call math_min,7,32),7) $(call math-expect,(call math_min,32,7),7) +$(call math-expect,(call math_max,32759,7),32759) +$(call math-expect,(call math_max,7,32759),32759) +$(call math-expect,(call math_max,202404,202505),202505) +$(call math-expect,(call math_max,202404,202404),202404) +$(call math-expect,(call math_min,8908527,32),32) +$(call math-expect,(call math_min,32,8908527),32) +$(call math-expect,(call math_min,202404,202505),202404) +$(call math-expect,(call math_min,202404,202404),202404) + define math_gt_or_eq $(if $(filter $(1),$(call math_max,$(1),$(2))),true) endef @@ -150,6 +244,10 @@ define math_gt $(if $(call math_gt_or_eq,$(2),$(1)),,true) endef +define math_lt_or_eq +$(if $(call math_gt_or_eq,$(2),$(1)),true) +endef + define math_lt $(if $(call math_gt_or_eq,$(1),$(2)),,true) endef @@ -160,10 +258,17 @@ $(call math-expect-false,(call math_gt_or_eq, 1, 2)) $(call math-expect-true,(call math_gt, 4, 3)) $(call math-expect-false,(call math_gt, 5, 5)) $(call math-expect-false,(call math_gt, 6, 7)) +$(call math-expect-true,(call math_lt_or_eq, 11, 11)) +$(call math-expect-false,(call math_lt_or_eq, 25, 15)) +$(call math-expect-true,(call math_lt_or_eq, 9, 16)) $(call math-expect-false,(call math_lt, 1, 0)) $(call math-expect-false,(call math_lt, 8, 8)) $(call math-expect-true,(call math_lt, 10, 11)) +$(call math-expect-true,(call math_gt_or_eq, 2573904, 2573900)) +$(call math-expect-true,(call math_gt_or_eq, 12345, 12345)) +$(call math-expect-false,(call math_gt_or_eq, 56, 2780)) + # $1 is the variable name to increment define inc_and_print $(strip $(eval $(1) := $($(1)) .)$(words $($(1)))) @@ -192,6 +297,7 @@ $(call math-expect,(call numbers_less_than,2,0 2 1 3),0 1) $(call math-expect,(call numbers_less_than,3,0 2 1 3),0 2 1) $(call math-expect,(call numbers_less_than,4,0 2 1 3),0 2 1 3) $(call math-expect,(call numbers_less_than,3,0 2 1 3 2),0 2 1 2) +$(call math-expect,(call numbers_less_than,100,0 1000 50 101 100),0 50) # Returns the words in $2 that are numbers and are greater or equal to $1 define numbers_greater_or_equal_to diff --git a/core/BUILD.bazel b/core/BUILD.bazel index 3e69e62702..f4869d4833 100644 --- a/core/BUILD.bazel +++ b/core/BUILD.bazel @@ -1,4 +1,28 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + # Export tradefed templates for tests. exports_files( glob(["*.xml"]), ) + +# Export proguard flag files for r8. +filegroup( + name = "global_proguard_flags", + srcs = [ + "proguard.flags", + "proguard_basic_keeps.flags", + ], + visibility = ["//visibility:public"], +) diff --git a/core/Makefile b/core/Makefile index 5f236771b7..00577edfa5 100644 --- a/core/Makefile +++ b/core/Makefile @@ -16,6 +16,94 @@ VENDOR_DLKM_NOTICE_DEPS := ODM_DLKM_NOTICE_DEPS := SYSTEM_DLKM_NOTICE_DEPS := + +# IMAGES_TO_BUILD is a list of the partition .img files that will be created. +IMAGES_TO_BUILD:= +ifneq ($(BUILDING_BOOT_IMAGE),) + IMAGES_TO_BUILD += boot +endif +ifneq ($(BUILDING_CACHE_IMAGE),) + IMAGES_TO_BUILD += cache +endif +ifneq ($(BUILDING_DEBUG_BOOT_IMAGE),) + IMAGES_TO_BUILD += debug_boot +endif +ifneq ($(BUILDING_DEBUG_VENDOR_BOOT_IMAGE),) + IMAGES_TO_BUILD += debug_vendor_boot +endif +ifneq ($(BUILDING_INIT_BOOT_IMAGE),) + IMAGES_TO_BUILD += init_boot +endif +ifneq ($(BUILDING_ODM_DLKM_IMAGE),) + IMAGES_TO_BUILD += odm_dlkm +endif +ifneq ($(BUILDING_ODM_IMAGE),) + IMAGES_TO_BUILD += odm +endif +ifneq ($(BUILDING_PRODUCT_IMAGE),) + IMAGES_TO_BUILD += product +endif +ifneq ($(BUILDING_RAMDISK_IMAGE),) + IMAGES_TO_BUILD += ramdisk +endif +ifneq ($(BUILDING_RECOVERY_IMAGE),) + IMAGES_TO_BUILD += recovery +endif +ifneq ($(BUILDING_SUPER_EMPTY_IMAGE),) + IMAGES_TO_BUILD += super_empty +endif +ifneq ($(BUILDING_SYSTEM_DLKM_IMAGE),) + IMAGES_TO_BUILD += system_dlkm +endif +ifneq ($(BUILDING_SYSTEM_EXT_IMAGE),) + IMAGES_TO_BUILD += system_ext +endif +ifneq ($(BUILDING_SYSTEM_IMAGE),) + IMAGES_TO_BUILD += system +endif +ifneq ($(BUILDING_SYSTEM_OTHER_IMAGE),) + IMAGES_TO_BUILD += system_other +endif +ifneq ($(BUILDING_USERDATA_IMAGE),) + IMAGES_TO_BUILD += userdata +endif +ifneq ($(BUILDING_VBMETA_IMAGE),) + IMAGES_TO_BUILD += vbmeta +endif +ifneq ($(BUILDING_VENDOR_BOOT_IMAGE),) + IMAGES_TO_BUILD += vendor_boot +endif +ifneq ($(BUILDING_VENDOR_DLKM_IMAGE),) + IMAGES_TO_BUILD += vendor_dlkm +endif +ifneq ($(BUILDING_VENDOR_IMAGE),) + IMAGES_TO_BUILD += vendor +endif +ifneq ($(BUILDING_VENDOR_KERNEL_BOOT_IMAGE),) + IMAGES_TO_BUILD += vendor_kernel_boot +endif + + +########################################################### +# Get the module names suitable for ALL_MODULES.* variables that are installed +# for a given partition +# +# $(1): Partition +########################################################### +define register-names-for-partition +$(sort $(foreach m,$(product_MODULES),\ + $(if $(filter $(PRODUCT_OUT)/$(strip $(1))/%, $(ALL_MODULES.$(m).INSTALLED)), \ + $(m) + ) \ +)) +endef + + +# Release & Aconfig Flags +# ----------------------------------------------------------------- +include $(BUILD_SYSTEM)/packaging/flags.mk + + # ----------------------------------------------------------------- # Define rules to copy PRODUCT_COPY_FILES defined by the product. # PRODUCT_COPY_FILES contains words like <source file>:<dest file>[:<owner>]. @@ -199,6 +287,11 @@ endif endif endif +# Do this early because sysprop.mk depends on BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC. +ifeq (default,$(ENABLE_UFFD_GC)) +BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC := $(OUT_DIR)/soong/dexpreopt/kernel_version_for_uffd_gc.txt +endif # ENABLE_UFFD_GC + include $(BUILD_SYSTEM)/sysprop.mk # ---------------------------------------------------------------- @@ -275,6 +368,10 @@ $(foreach vendor_ramdisk_fragment,$(BOARD_VENDOR_RAMDISK_FRAGMENTS), \ ) INTERNAL_VENDOR_RAMDISK_FRAGMENTS += $(BOARD_VENDOR_RAMDISK_FRAGMENTS) +ifneq ($(BOARD_KERNEL_MODULES_16K),) +INTERNAL_VENDOR_RAMDISK_FRAGMENTS += 16K +endif + # Strip the list in case of any whitespace. INTERNAL_VENDOR_RAMDISK_FRAGMENTS := \ $(strip $(INTERNAL_VENDOR_RAMDISK_FRAGMENTS)) @@ -364,13 +461,11 @@ $(3)/$(DEPMOD_STAGING_SUBDIR)/modules.dep: $(1) $(6) unzip -qoDD -d $$(PRIVATE_MODULE_DIR) $$(PRIVATE_MODULE_ARCHIVE); \ mkdir -p $$(PRIVATE_OUTPUT_DIR)/lib; \ cp -r $(3)/$(DEPMOD_STAGING_SUBDIR)/$(2)/lib/modules $$(PRIVATE_OUTPUT_DIR)/lib/; \ - find $$(PRIVATE_MODULE_DIR) -type f -name *.ko | xargs basename -a > $$(PRIVATE_LOAD_FILE); \ + find $$(PRIVATE_MODULE_DIR) -type f -name '*.ko' | xargs basename -a > $$(PRIVATE_LOAD_FILE); \ ) $(if $(1),\ cp $$(PRIVATE_MODULES) $$(PRIVATE_MODULE_DIR)/; \ - for MODULE in $$(PRIVATE_LOAD_MODULES); do \ - basename $$$$MODULE >> $$(PRIVATE_LOAD_FILE); \ - done; \ + if [ -n "$$(PRIVATE_LOAD_MODULES)" ]; then basename -a $$(PRIVATE_LOAD_MODULES); fi > $$(PRIVATE_LOAD_FILE); \ ) # The ln -sf + find -delete sequence is to remove any modules in # PRIVATE_EXTRA_MODULES which have same basename as MODULES in PRIVATE_MODULES @@ -474,7 +569,10 @@ $(if $(strip $(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver))$(BOARD_$(1)_KERNEL_MODU $(eval BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver) := $(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)))) \ $(if $(filter false,$(BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver))),\ $(eval BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver) :=),) \ - $(call copy-many-files,$(call build-image-kernel-modules,$(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)),$(2),$(3),$(call intermediates-dir-for,PACKAGING,depmod_$(1)$(_sep)$(_kver)),$(BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver)),$(4),$(BOARD_$(1)_KERNEL_MODULES_ARCHIVE$(_sep)$(_kver)),$(_stripped_staging_dir),$(_kver),$(7),$(8)))) \ + $(eval _files := $(call build-image-kernel-modules,$(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)),$(2),$(3),$(call intermediates-dir-for,PACKAGING,depmod_$(1)$(_sep)$(_kver)),$(BOARD_$(1)_KERNEL_MODULES_LOAD$(_sep)$(_kver)),$(4),$(BOARD_$(1)_KERNEL_MODULES_ARCHIVE$(_sep)$(_kver)),$(_stripped_staging_dir),$(_kver),$(7),$(8))) \ + $(call copy-many-files,$(_files)) \ + $(eval _modules := $(BOARD_$(1)_KERNEL_MODULES$(_sep)$(_kver)) ANDROID-GEN ANDROID-GEN ANDROID-GEN ANDROID-GEN) \ + $(eval KERNEL_MODULE_COPY_FILES += $(join $(addsuffix :,$(_modules)),$(_files)))) \ $(if $(_kver), \ $(eval _dir := $(_kver)/), \ $(eval _dir :=)) \ @@ -487,6 +585,7 @@ $(if $(BOARD_$(1)_KERNEL_MODULES_BLOCKLIST_FILE$(_sep)$(_kver)), \ $(eval $(call build-image-kernel-modules-blocklist-file, \ $(BOARD_$(1)_KERNEL_MODULES_BLOCKLIST_FILE$(_sep)$(_kver)), \ $(2)/lib/modules/$(_dir)modules.blocklist)) \ + $(eval ALL_KERNEL_MODULES_BLOCKLIST += $(2)/lib/modules/$(_dir)modules.blocklist) \ $(2)/lib/modules/$(_dir)modules.blocklist) endef @@ -526,6 +625,24 @@ $(if $(filter top,$(1)),\ $(call copy-many-files,$(call module-load-list-copy-paths,$(call intermediates-dir-for,PACKAGING,vendor_charger_module_list$(_sep)$(_kver)),$(BOARD_VENDOR_CHARGER_KERNEL_MODULES$(_sep)$(_kver)),$(BOARD_VENDOR_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),modules.load.charger,$(TARGET_OUT_VENDOR)))) endef +# $(1): kernel module directory name (top is an out of band value for no directory) +define build-vendor-ramdisk-charger-load +$(if $(filter top,$(1)),\ + $(eval _kver :=)$(eval _sep :=),\ + $(eval _kver := $(1))$(eval _sep :=_))\ + $(if $(BOARD_VENDOR_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),\ + $(call copy-many-files,$(call module-load-list-copy-paths,$(call intermediates-dir-for,PACKAGING,vendor_ramdisk_charger_module_list$(_sep)$(_kver)),$(BOARD_VENDOR_RAMDISK_KERNEL_MODULES$(_sep)$(_kver)),$(BOARD_VENDOR_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),modules.load.charger,$(TARGET_VENDOR_RAMDISK_OUT)))) +endef + +# $(1): kernel module directory name (top is an out of band value for no directory) +define build-vendor-kernel-ramdisk-charger-load +$(if $(filter top,$(1)),\ + $(eval _kver :=)$(eval _sep :=),\ + $(eval _kver := $(1))$(eval _sep :=_))\ + $(if $(BOARD_VENDOR_KERNEL_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),\ + $(call copy-many-files,$(call module-load-list-copy-paths,$(call intermediates-dir-for,PACKAGING,vendor_kernel_ramdisk_charger_module_list$(_sep)$(_kver)),$(BOARD_VENDOR_KERNEL_RAMDISK_KERNEL_MODULES$(_sep)$(_kver)),$(BOARD_VENDOR_KERNEL_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),modules.load.charger,$(TARGET_VENDOR_KERNEL_RAMDISK_OUT)))) +endef + ifneq ($(BUILDING_VENDOR_BOOT_IMAGE),true) # If there is no vendor boot partition, store vendor ramdisk kernel modules in the # boot ramdisk. @@ -543,6 +660,12 @@ ifneq ($(strip $(BOARD_GENERIC_RAMDISK_KERNEL_MODULES)),) endif endif +ifneq ($(BOARD_DO_NOT_STRIP_GENERIC_RAMDISK_MODULES),true) + GENERIC_RAMDISK_STRIPPED_MODULE_STAGING_DIR := $(call intermediates-dir-for,PACKAGING,depmod_generic_ramdisk_kernel_stripped) +else + GENERIC_RAMDISK_STRIPPED_MODULE_STAGING_DIR := +endif + ifneq ($(BOARD_DO_NOT_STRIP_RECOVERY_MODULES),true) RECOVERY_STRIPPED_MODULE_STAGING_DIR := $(call intermediates-dir-for,PACKAGING,depmod_recovery_stripped) else @@ -591,11 +714,13 @@ $(foreach kmd,$(BOARD_KERNEL_MODULE_DIRS), \ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-kernel-ramdisk-recovery-load,$(kmd))) \ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,VENDOR,$(if $(filter true,$(BOARD_USES_VENDOR_DLKMIMAGE)),$(TARGET_OUT_VENDOR_DLKM),$(TARGET_OUT_VENDOR)),vendor,modules.load,$(VENDOR_STRIPPED_MODULE_STAGING_DIR),$(kmd),$(BOARD_SYSTEM_KERNEL_MODULES),system)) \ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-charger-load,$(kmd))) \ + $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-ramdisk-charger-load,$(kmd))) \ + $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-kernel-ramdisk-charger-load,$(kmd))) \ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,ODM,$(if $(filter true,$(BOARD_USES_ODM_DLKMIMAGE)),$(TARGET_OUT_ODM_DLKM),$(TARGET_OUT_ODM)),odm,modules.load,,$(kmd))) \ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,SYSTEM,$(if $(filter true,$(BOARD_USES_SYSTEM_DLKMIMAGE)),$(TARGET_OUT_SYSTEM_DLKM),$(TARGET_OUT_SYSTEM)),system,modules.load,,$(kmd))) \ $(if $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)),\ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-recovery-as-boot-load,$(kmd))),\ - $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,GENERIC_RAMDISK,$(TARGET_RAMDISK_OUT),,modules.load,,$(kmd))))) + $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,GENERIC_RAMDISK,$(TARGET_RAMDISK_OUT),,modules.load,$(GENERIC_RAMDISK_STRIPPED_MODULE_STAGING_DIR),$(kmd))))) ifeq ($(BOARD_SYSTEM_KERNEL_MODULES),) ifneq ($(BOARD_SYSTEM_DLKM_SRC),) @@ -641,7 +766,7 @@ name := $(TARGET_PRODUCT) ifeq ($(TARGET_BUILD_TYPE),debug) name := $(name)_debug endif -name := $(name)-apkcerts-$(FILE_NAME_TAG) +name := $(name)-apkcerts intermediates := \ $(call intermediates-dir-for,PACKAGING,apkcerts) APKCERTS_FILE := $(intermediates)/$(name).txt @@ -671,9 +796,25 @@ $(call declare-0p-target,$(APKCERTS_FILE)) .PHONY: apkcerts-list apkcerts-list: $(APKCERTS_FILE) +intermediates := $(call intermediates-dir-for,PACKAGING,apexkeys) +APEX_KEYS_FILE := $(intermediates)/apexkeys.txt + +all_apex_keys_files := $(sort $(foreach m,$(call product-installed-modules,$(INTERNAL_PRODUCT)),$(ALL_MODULES.$(m).APEX_KEYS_FILE))) +$(APEX_KEYS_FILE): $(all_apex_keys_files) + @mkdir -p $(dir $@) + @rm -f $@ + $(hide) touch $@ + $(hide) $(foreach file,$^,cat $(file) >> $@ $(newline)) +all_apex_keys_files := + +$(call declare-0p-target,$(APEX_KEYS_FILE)) + +.PHONY: apexkeys.txt +apexkeys.txt: $(APEX_KEYS_FILE) + ifneq (,$(TARGET_BUILD_APPS)) $(call dist-for-goals, apps_only, $(APKCERTS_FILE):apkcerts.txt) - $(call dist-for-goals, apps_only, $(SOONG_APEX_KEYS_FILE):apexkeys.txt) + $(call dist-for-goals, apps_only, $(APEX_KEYS_FILE):apexkeys.txt) endif @@ -749,6 +890,9 @@ $(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PAC $(call declare-1p-target,$(MK2BP_REMAINING_CSV)) $(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_CSV)) +.PHONY: mk2bp_remaining +mk2bp_remaining: $(MK2BP_REMAINING_HTML) $(MK2BP_REMAINING_CSV) + # ----------------------------------------------------------------- # Modules use -Wno-error, or added default -Wall -Werror WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt @@ -931,14 +1075,131 @@ $(eval $(call declare-0p-target,$(INSTALLED_FILES_JSON_RAMDISK))) BUILT_RAMDISK_TARGET := $(PRODUCT_OUT)/ramdisk.img + +ifneq ($(BOARD_KERNEL_MODULES_16K),) + +TARGET_OUT_RAMDISK_16K := $(PRODUCT_OUT)/ramdisk_16k +BUILT_RAMDISK_16K_TARGET := $(PRODUCT_OUT)/ramdisk_16k.img +RAMDISK_16K_STAGING_DIR := $(call intermediates-dir-for,PACKAGING,depmod_ramdisk_16k) + +ifneq ($(BOARD_SYSTEM_KERNEL_MODULES),) +SYSTEM_DLKM_MODULE_PATTERNS := $(foreach path,$(BOARD_SYSTEM_KERNEL_MODULES),%/$(notdir $(path))) + +endif + +# BOARD_KERNEL_MODULES_16K might contain duplicate modules under different path. +# for example, foo/bar/wifi.ko and foo/wifi.ko . To avoid build issues, de-dup +# module list on basename first. +BOARD_KERNEL_MODULES_16K := $(foreach \ + pattern,\ + $(sort $(foreach \ + path,\ + $(BOARD_KERNEL_MODULES_16K),\ + %/$(notdir $(path)))\ + ),\ + $(firstword $(filter $(pattern),$(BOARD_KERNEL_MODULES_16K))) \ +) +# For non-GKI modules, strip them before install. As debug symbols take up +# significant space. +$(foreach \ + file,\ + $(filter-out $(SYSTEM_DLKM_MODULE_PATTERNS),$(BOARD_KERNEL_MODULES_16K)),\ + $(eval \ + $(call copy-and-strip-kernel-module,\ + $(file),\ + $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/$(notdir $(file)) \ + ) \ + ) \ +) + +# For GKI modules, copy as-is without stripping, because stripping would +# remove the signature of kernel modules, and GKI modules must be signed +# for kernel to load them. +$(foreach \ + file,\ + $(filter $(SYSTEM_DLKM_MODULE_PATTERNS),$(BOARD_KERNEL_MODULES_16K)),\ + $(eval \ + $(call copy-one-file,\ + $(file),\ + $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/$(notdir $(file)) \ + ) \ + ) \ +) + +BOARD_VENDOR_RAMDISK_FRAGMENT.16K.PREBUILT := $(BUILT_RAMDISK_16K_TARGET) + +ifndef BOARD_KERNEL_MODULES_LOAD_16K + BOARD_KERNEL_MODULES_LOAD_16K := $(BOARD_KERNEL_MODULES_16K) +endif + +$(BUILT_RAMDISK_16K_TARGET): $(DEPMOD) $(MKBOOTFS) $(EXTRACT_KERNEL) $(COMPRESSION_COMMAND_DEPS) +$(BUILT_RAMDISK_16K_TARGET): $(foreach file,$(BOARD_KERNEL_MODULES_16K),$(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/$(notdir $(file))) + $(DEPMOD) -b $(RAMDISK_16K_STAGING_DIR) 0.0 + rm -f $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/modules.load + for MODULE in $(BOARD_KERNEL_MODULES_LOAD_16K); do \ + basename $$MODULE >> $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/modules.load ; \ + done; + rm -rf $(TARGET_OUT_RAMDISK_16K)/lib/modules + mkdir -p $(TARGET_OUT_RAMDISK_16K)/lib/modules + KERNEL_RELEASE=`$(EXTRACT_KERNEL) --tools lz4:$(LZ4) --input $(BOARD_KERNEL_PATH_16K) --output-release` ;\ + IS_16K_KERNEL=`$(EXTRACT_KERNEL) --tools lz4:$(LZ4) --input $(BOARD_KERNEL_PATH_16K) --output-config` ;\ + if [[ "$$IS_16K_KERNEL" == *"CONFIG_ARM64_16K_PAGES=y"* ]]; then SUFFIX=_16k; fi ;\ + cp -r $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0 $(TARGET_OUT_RAMDISK_16K)/lib/modules/$$KERNEL_RELEASE$$SUFFIX + $(MKBOOTFS) $(TARGET_OUT_RAMDISK_16K) | $(COMPRESSION_COMMAND) > $@ + +# Builds a ramdisk using modules defined in BOARD_KERNEL_MODULES_16K +ramdisk_16k: $(BUILT_RAMDISK_16K_TARGET) +.PHONY: ramdisk_16k + +endif + +ifneq ($(BOARD_KERNEL_PATH_16K),) +BUILT_KERNEL_16K_TARGET := $(PRODUCT_OUT)/kernel_16k + +$(eval $(call copy-one-file,$(BOARD_KERNEL_PATH_16K),$(BUILT_KERNEL_16K_TARGET))) + +# Copies BOARD_KERNEL_PATH_16K to output directory as is +kernel_16k: $(BUILT_KERNEL_16K_TARGET) +.PHONY: kernel_16k + +BUILT_BOOTIMAGE_16K_TARGET := $(PRODUCT_OUT)/boot_16k.img + +BOARD_KERNEL_16K_BOOTIMAGE_PARTITION_SIZE := $(BOARD_BOOTIMAGE_PARTITION_SIZE) + +$(BUILT_BOOTIMAGE_16K_TARGET): $(MKBOOTIMG) $(AVBTOOL) $(INTERNAL_BOOTIMAGE_FILES) $(BOARD_AVB_BOOT_KEY_PATH) $(BUILT_KERNEL_16K_TARGET) + $(call pretty,"Target boot 16k image: $@") + $(call build_boot_from_kernel_avb_enabled,$@,$(BUILT_KERNEL_16K_TARGET)) + + +bootimage_16k: $(BUILT_BOOTIMAGE_16K_TARGET) +.PHONY: bootimage_16k + +BUILT_BOOT_OTA_PACKAGE_16K := $(PRODUCT_OUT)/boot_ota_16k.zip +$(BUILT_BOOT_OTA_PACKAGE_16K): $(OTA_FROM_RAW_IMG) $(BUILT_BOOTIMAGE_16K_TARGET) $(INSTALLED_BOOTIMAGE_TARGET) $(DEFAULT_SYSTEM_DEV_CERTIFICATE).pk8 + $(OTA_FROM_RAW_IMG) --package_key $(DEFAULT_SYSTEM_DEV_CERTIFICATE) \ + --max_timestamp `cat $(BUILD_DATETIME_FILE)` \ + --path $(HOST_OUT) \ + --partition_name boot \ + --output $@ \ + $(if $(BOARD_16K_OTA_USE_INCREMENTAL),\ + $(INSTALLED_BOOTIMAGE_TARGET):$(BUILT_BOOTIMAGE_16K_TARGET),\ + $(BUILT_BOOTIMAGE_16K_TARGET)\ + ) + +boototapackage_16k: $(BUILT_BOOT_OTA_PACKAGE_16K) +.PHONY: boototapackage_16k + +endif + + ifeq ($(BOARD_RAMDISK_USE_LZ4),true) # -l enables the legacy format used by the Linux kernel COMPRESSION_COMMAND_DEPS := $(LZ4) COMPRESSION_COMMAND := $(LZ4) -l -12 --favor-decSpeed RAMDISK_EXT := .lz4 else -COMPRESSION_COMMAND_DEPS := $(MINIGZIP) -COMPRESSION_COMMAND := $(MINIGZIP) +COMPRESSION_COMMAND_DEPS := $(GZIP) +COMPRESSION_COMMAND := $(GZIP) RAMDISK_EXT := .gz endif @@ -1023,17 +1284,29 @@ define get-partition-size-argument $(if $(1),--partition_size $(1),--dynamic_partition_size) endef +# $1: output boot image target +# $2: input path to kernel binary +define build_boot_from_kernel_avb_enabled + $(eval kernel := $(2)) + $(MKBOOTIMG) --kernel $(kernel) $(INTERNAL_BOOTIMAGE_ARGS) $(INTERNAL_MKBOOTIMG_VERSION_ARGS) $(BOARD_MKBOOTIMG_ARGS) --output $(1) + $(call assert-max-image-size,$(1),$(call get-hash-image-max-size,$(call get-bootimage-partition-size,$(1),boot))) + $(AVBTOOL) add_hash_footer \ + --image $(1) \ + $(call get-partition-size-argument,$(call get-bootimage-partition-size,$(1),boot)) \ + --salt `sha256sum "$(kernel)" | cut -d " " -f 1` \ + --partition_name boot $(INTERNAL_AVB_BOOT_SIGNING_ARGS) \ + $(BOARD_AVB_BOOT_ADD_HASH_FOOTER_ARGS) +endef + + ifndef BOARD_PREBUILT_BOOTIMAGE ifneq ($(strip $(TARGET_NO_KERNEL)),true) INTERNAL_BOOTIMAGE_ARGS := \ $(addprefix --second ,$(INSTALLED_2NDBOOTLOADER_TARGET)) -# TODO(b/229701033): clean up BOARD_BUILD_GKI_BOOT_IMAGE_WITHOUT_RAMDISK. -ifneq ($(BOARD_BUILD_GKI_BOOT_IMAGE_WITHOUT_RAMDISK),true) - ifneq ($(BUILDING_INIT_BOOT_IMAGE),true) - INTERNAL_BOOTIMAGE_ARGS += --ramdisk $(INSTALLED_RAMDISK_TARGET) - endif +ifneq ($(BUILDING_INIT_BOOT_IMAGE),true) + INTERNAL_BOOTIMAGE_ARGS += --ramdisk $(INSTALLED_RAMDISK_TARGET) endif ifndef BUILDING_VENDOR_BOOT_IMAGE @@ -1066,51 +1339,9 @@ else ifndef BUILDING_VENDOR_BOOT_IMAGE # && BOARD_USES_GENERIC_KERNEL_IMAGE != t endif endif # BUILDING_VENDOR_BOOT_IMAGE == "" && BOARD_USES_GENERIC_KERNEL_IMAGE != true -ifdef BOARD_GKI_SIGNING_KEY_PATH - # GKI boot images will not set system version & SPL value in the header. - # They can be set by the device manufacturer in the AVB properties instead. - INTERNAL_MKBOOTIMG_VERSION_ARGS := -else - INTERNAL_MKBOOTIMG_VERSION_ARGS := \ - --os_version $(PLATFORM_VERSION_LAST_STABLE) \ - --os_patch_level $(PLATFORM_SECURITY_PATCH) -endif # BOARD_GKI_SIGNING_KEY_PATH - -# $(1): image target to certify -# $(2): out certificate target -# $(3): image name -# $(4): additional AVB arguments -define generate_generic_boot_image_certificate - rm -rf "$(2)" - mkdir -p "$(dir $(2))" - $(GENERATE_GKI_CERTIFICATE) $(INTERNAL_GKI_CERTIFICATE_ARGS) \ - --additional_avb_args "$(4)" \ - --name "$(3)" --output "$(2)" "$(1)" -endef - -INTERNAL_GKI_CERTIFICATE_ARGS := -INTERNAL_GKI_CERTIFICATE_DEPS := -ifdef BOARD_GKI_SIGNING_KEY_PATH - ifndef BOARD_GKI_SIGNING_ALGORITHM - $(error BOARD_GKI_SIGNING_ALGORITHM should be defined with BOARD_GKI_SIGNING_KEY_PATH) - endif - - INTERNAL_GKI_CERTIFICATE_ARGS := \ - --key "$(BOARD_GKI_SIGNING_KEY_PATH)" \ - --algorithm "$(BOARD_GKI_SIGNING_ALGORITHM)" \ - --avbtool "$(AVBTOOL)" - - # Quote and pass BOARD_GKI_SIGNING_SIGNATURE_ARGS as a single string argument. - ifdef BOARD_GKI_SIGNING_SIGNATURE_ARGS - INTERNAL_GKI_CERTIFICATE_ARGS += --additional_avb_args "$(BOARD_GKI_SIGNING_SIGNATURE_ARGS)" - endif - - INTERNAL_GKI_CERTIFICATE_DEPS := \ - $(GENERATE_GKI_CERTIFICATE) \ - $(BOARD_GKI_SIGNING_KEY_PATH) \ - $(AVBTOOL) - -endif +INTERNAL_MKBOOTIMG_VERSION_ARGS := \ + --os_version $(PLATFORM_VERSION_LAST_STABLE) \ + --os_patch_level $(PLATFORM_SECURITY_PATCH) # Define these only if we are building boot ifdef BUILDING_BOOT_IMAGE @@ -1127,35 +1358,20 @@ ifeq (true,$(BOARD_AVB_ENABLE)) # $1: boot image target define build_boot_board_avb_enabled $(eval kernel := $(call bootimage-to-kernel,$(1))) - $(MKBOOTIMG) --kernel $(kernel) $(INTERNAL_BOOTIMAGE_ARGS) $(INTERNAL_MKBOOTIMG_VERSION_ARGS) $(BOARD_MKBOOTIMG_ARGS) --output $(1) - $(if $(BOARD_GKI_SIGNING_KEY_PATH), \ - $(eval boot_signature := $(call intermediates-dir-for,PACKAGING,generic_boot)/$(notdir $(1)).boot_signature) \ - $(eval kernel_signature := $(call intermediates-dir-for,PACKAGING,generic_kernel)/$(notdir $(kernel)).boot_signature) \ - $(call generate_generic_boot_image_certificate,$(1),$(boot_signature),boot,$(BOARD_AVB_BOOT_ADD_HASH_FOOTER_ARGS)) $(newline) \ - $(call generate_generic_boot_image_certificate,$(kernel),$(kernel_signature),generic_kernel,$(BOARD_AVB_BOOT_ADD_HASH_FOOTER_ARGS)) $(newline) \ - cat $(kernel_signature) >> $(boot_signature) $(newline) \ - $(call assert-max-image-size,$(boot_signature),16 << 10) $(newline) \ - truncate -s $$(( 16 << 10 )) $(boot_signature) $(newline) \ - cat "$(boot_signature)" >> $(1)) - $(call assert-max-image-size,$(1),$(call get-hash-image-max-size,$(call get-bootimage-partition-size,$(1),boot))) - $(AVBTOOL) add_hash_footer \ - --image $(1) \ - $(call get-partition-size-argument,$(call get-bootimage-partition-size,$(1),boot)) \ - --partition_name boot $(INTERNAL_AVB_BOOT_SIGNING_ARGS) \ - $(BOARD_AVB_BOOT_ADD_HASH_FOOTER_ARGS) + $(call build_boot_from_kernel_avb_enabled,$(1),$(kernel)) endef -$(INSTALLED_BOOTIMAGE_TARGET): $(MKBOOTIMG) $(AVBTOOL) $(INTERNAL_BOOTIMAGE_FILES) $(BOARD_AVB_BOOT_KEY_PATH) $(INTERNAL_GKI_CERTIFICATE_DEPS) +$(INSTALLED_BOOTIMAGE_TARGET): $(MKBOOTIMG) $(AVBTOOL) $(INTERNAL_BOOTIMAGE_FILES) $(BOARD_AVB_BOOT_KEY_PATH) $(call pretty,"Target boot image: $@") $(call build_boot_board_avb_enabled,$@) $(call declare-container-license-metadata,$(INSTALLED_BOOTIMAGE_TARGET),SPDX-license-identifier-GPL-2.0-only SPDX-license-identifier-Apache-2.0,restricted notice,$(BUILD_SYSTEM)/LINUX_KERNEL_COPYING build/soong/licenses/LICENSE,"Boot Image",boot) -$(call declare-container-license-deps,$(INSTALLED_BOOTIMAGE_TARGET),$(INTERNAL_BOOTIMAGE_FILES) $(INTERNAL_GKI_CERTIFICATE_DEPS),$(PRODUCT_OUT)/:/) +$(call declare-container-license-deps,$(INSTALLED_BOOTIMAGE_TARGET),$(INTERNAL_BOOTIMAGE_FILES),$(PRODUCT_OUT)/:/) UNMOUNTED_NOTICE_VENDOR_DEPS += $(INSTALLED_BOOTIMAGE_TARGET) .PHONY: bootimage-nodeps -bootimage-nodeps: $(MKBOOTIMG) $(AVBTOOL) $(BOARD_AVB_BOOT_KEY_PATH) $(INTERNAL_GKI_CERTIFICATE_DEPS) +bootimage-nodeps: $(MKBOOTIMG) $(AVBTOOL) $(BOARD_AVB_BOOT_KEY_PATH) @echo "make $@: ignoring dependencies" $(foreach b,$(INSTALLED_BOOTIMAGE_TARGET),$(call build_boot_board_avb_enabled,$(b))) @@ -1216,14 +1432,19 @@ INTERNAL_PREBUILT_BOOTIMAGE := $(BOARD_PREBUILT_BOOTIMAGE) INSTALLED_BOOTIMAGE_TARGET := $(PRODUCT_OUT)/boot.img ifeq ($(BOARD_AVB_ENABLE),true) -$(INSTALLED_BOOTIMAGE_TARGET): $(INTERNAL_PREBUILT_BOOTIMAGE) $(AVBTOOL) $(BOARD_AVB_BOOT_KEY_PATH) +$(INSTALLED_BOOTIMAGE_TARGET): PRIVATE_WORKING_DIR := $(call intermediates-dir-for,PACKAGING,prebuilt_bootimg) +$(INSTALLED_BOOTIMAGE_TARGET): $(INTERNAL_PREBUILT_BOOTIMAGE) $(AVBTOOL) $(BOARD_AVB_BOOT_KEY_PATH) $(UNPACK_BOOTIMG) cp $(INTERNAL_PREBUILT_BOOTIMAGE) $@ + $(UNPACK_BOOTIMG) --boot_img $(INTERNAL_PREBUILT_BOOTIMAGE) --out $(PRIVATE_WORKING_DIR) + chmod +w $@ $(AVBTOOL) add_hash_footer \ --image $@ \ + --salt `sha256sum $(PRIVATE_WORKING_DIR)/kernel | cut -d " " -f 1` \ $(call get-partition-size-argument,$(BOARD_BOOTIMAGE_PARTITION_SIZE)) \ --partition_name boot $(INTERNAL_AVB_BOOT_SIGNING_ARGS) \ $(BOARD_AVB_BOOT_ADD_HASH_FOOTER_ARGS) + $(call declare-container-license-metadata,$(INSTALLED_BOOTIMAGE_TARGET),SPDX-license-identifier-GPL-2.0-only SPDX-license-identifier-Apache-2.0,restricted notice,$(BUILD_SYSTEM)/LINUX_KERNEL_COPYING build/soong/licenses/LICENSE,"Boot Image",bool) $(call declare-container-license-deps,$(INSTALLED_BOOTIMAGE_TARGET),$(INTERNAL_PREBUILT_BOOTIMAGE),$(PRODUCT_OUT)/:/) @@ -1237,6 +1458,37 @@ endif # BOARD_PREBUILT_BOOTIMAGE endif # my_installed_prebuilt_gki_apex not defined +ifneq ($(BOARD_KERNEL_PATH_16K),) +BUILT_BOOT_OTA_PACKAGE_4K := $(PRODUCT_OUT)/boot_ota_4k.zip +$(BUILT_BOOT_OTA_PACKAGE_4K): $(OTA_FROM_RAW_IMG) $(INSTALLED_BOOTIMAGE_TARGET) $(BUILT_BOOTIMAGE_16K_TARGET) $(DEFAULT_SYSTEM_DEV_CERTIFICATE).pk8 + $(OTA_FROM_RAW_IMG) --package_key $(DEFAULT_SYSTEM_DEV_CERTIFICATE) \ + --max_timestamp `cat $(BUILD_DATETIME_FILE)` \ + --path $(HOST_OUT) \ + --partition_name boot \ + --output $@ \ + $(if $(BOARD_16K_OTA_USE_INCREMENTAL),\ + $(BUILT_BOOTIMAGE_16K_TARGET):$(INSTALLED_BOOTIMAGE_TARGET),\ + $(INSTALLED_BOOTIMAGE_TARGET)\ + ) + +boototapackage_4k: $(BUILT_BOOT_OTA_PACKAGE_4K) +.PHONY: boototapackage_4k + +ifeq ($(BOARD_16K_OTA_MOVE_VENDOR),true) +$(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_4K),$(TARGET_OUT_VENDOR)/boot_otas/boot_ota_4k.zip)) +$(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_16K),$(TARGET_OUT_VENDOR)/boot_otas/boot_ota_16k.zip)) +ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT_VENDOR)/boot_otas/boot_ota_4k.zip +ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT_VENDOR)/boot_otas/boot_ota_16k.zip +else +$(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_4K),$(TARGET_OUT)/boot_otas/boot_ota_4k.zip)) +$(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_16K),$(TARGET_OUT)/boot_otas/boot_ota_16k.zip)) +ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT)/boot_otas/boot_ota_4k.zip +ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT)/boot_otas/boot_ota_16k.zip +endif # BOARD_16K_OTA_MOVE_VENDOR == true + + +endif + my_apex_extracted_boot_image := my_installed_prebuilt_gki_apex := @@ -1286,6 +1538,7 @@ INSTALLED_INIT_BOOT_IMAGE_TARGET := $(PRODUCT_OUT)/init_boot.img ifeq ($(BOARD_AVB_ENABLE),true) $(INSTALLED_INIT_BOOT_IMAGE_TARGET): $(INTERNAL_PREBUILT_INIT_BOOT_IMAGE) $(AVBTOOL) $(BOARD_AVB_INIT_BOOT_KEY_PATH) cp $(INTERNAL_PREBUILT_INIT_BOOT_IMAGE) $@ + chmod +w $@ $(AVBTOOL) add_hash_footer \ --image $@ \ $(call get-partition-size-argument,$(BOARD_INIT_BOOT_IMAGE_PARTITION_SIZE)) \ @@ -1635,6 +1888,21 @@ target_system_dlkm_notice_file_txt := $(TARGET_OUT_INTERMEDIATES)/NOTICE_SYSTEM_ target_system_dlkm_notice_file_xml_gz := $(TARGET_OUT_INTERMEDIATES)/NOTICE_SYSTEM_DLKM.xml.gz installed_system_dlkm_notice_xml_gz := $(TARGET_OUT_SYSTEM_DLKM)/etc/NOTICE.xml.gz +ALL_INSTALLED_NOTICE_FILES := \ + $(installed_notice_html_or_xml_gz) \ + $(installed_vendor_notice_xml_gz) \ + $(installed_product_notice_xml_gz) \ + $(installed_system_ext_notice_xml_gz) \ + $(installed_odm_notice_xml_gz) \ + $(installed_vendor_dlkm_notice_xml_gz) \ + $(installed_odm_dlkm_notice_xml_gz) \ + $(installed_system_dlkm_notice_xml_gz) \ + +# $1 installed file path, e.g. out/target/product/vsoc_x86_64/system_ext/etc/NOTICE.xml.gz +define is-notice-file +$(if $(findstring $1,$(ALL_INSTALLED_NOTICE_FILES)),Y) +endef + # Notice files are copied to TARGET_OUT_NOTICE_FILES as a side-effect of their module # being built. A notice xml file must depend on all modules that could potentially # install a license file relevant to it. @@ -1840,11 +2108,6 @@ endif # Get a colon-separated list of search paths. INTERNAL_USERIMAGES_BINARY_PATHS := $(subst $(space),:,$(sort $(dir $(INTERNAL_USERIMAGES_DEPS)))) -# Collects file_contexts files from modules to be installed -$(call merge-fc-files, \ - $(sort $(foreach m,$(product_MODULES),$(ALL_MODULES.$(m).FILE_CONTEXTS))),\ - $(call intermediates-dir-for,ETC,file_contexts.bin)/file_contexts.modules.tmp) - SELINUX_FC := $(call intermediates-dir-for,ETC,file_contexts.bin)/file_contexts.bin INTERNAL_USERIMAGES_DEPS += $(SELINUX_FC) @@ -1864,9 +2127,11 @@ $(eval _var := $(call to-upper,$(1))) $(if $(BOARD_$(_var)IMAGE_EROFS_COMPRESSOR),$(hide) echo "$(1)_erofs_compressor=$(BOARD_$(_var)IMAGE_EROFS_COMPRESSOR)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_EROFS_COMPRESS_HINTS),$(hide) echo "$(1)_erofs_compress_hints=$(BOARD_$(_var)IMAGE_EROFS_COMPRESS_HINTS)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_EROFS_PCLUSTER_SIZE),$(hide) echo "$(1)_erofs_pcluster_size=$(BOARD_$(_var)IMAGE_EROFS_PCLUSTER_SIZE)" >> $(2)) +$(if $(BOARD_$(_var)IMAGE_EROFS_BLOCKSIZE),$(hide) echo "$(1)_erofs_blocksize=$(BOARD_$(_var)IMAGE_EROFS_BLOCKSIZE)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_EXTFS_INODE_COUNT),$(hide) echo "$(1)_extfs_inode_count=$(BOARD_$(_var)IMAGE_EXTFS_INODE_COUNT)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_EXTFS_RSV_PCT),$(hide) echo "$(1)_extfs_rsv_pct=$(BOARD_$(_var)IMAGE_EXTFS_RSV_PCT)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_F2FS_SLOAD_COMPRESS_FLAGS),$(hide) echo "$(1)_f2fs_sldc_flags=$(BOARD_$(_var)IMAGE_F2FS_SLOAD_COMPRESS_FLAGS)" >> $(2)) +$(if $(BOARD_$(_var)IMAGE_F2FS_BLOCKSIZE),$(hide) echo "$(1)_f2fs_blocksize=$(BOARD_$(_var)IMAGE_F2FS_BLOCKSIZE)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_FILE_SYSTEM_COMPRESS),$(hide) echo "$(1)_f2fs_compress=$(BOARD_$(_var)IMAGE_FILE_SYSTEM_COMPRESS)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_FILE_SYSTEM_TYPE),$(hide) echo "$(1)_fs_type=$(BOARD_$(_var)IMAGE_FILE_SYSTEM_TYPE)" >> $(2)) $(if $(BOARD_$(_var)IMAGE_JOURNAL_SIZE),$(hide) echo "$(1)_journal_size=$(BOARD_$(_var)IMAGE_JOURNAL_SIZE)" >> $(2)) @@ -1953,9 +2218,11 @@ $(if $(filter true,$(TARGET_USERIMAGES_SPARSE_F2FS_DISABLED)),,$(hide) echo "f2f $(if $(BOARD_EROFS_COMPRESSOR),$(hide) echo "erofs_default_compressor=$(BOARD_EROFS_COMPRESSOR)" >> $(1)) $(if $(BOARD_EROFS_COMPRESS_HINTS),$(hide) echo "erofs_default_compress_hints=$(BOARD_EROFS_COMPRESS_HINTS)" >> $(1)) $(if $(BOARD_EROFS_PCLUSTER_SIZE),$(hide) echo "erofs_pcluster_size=$(BOARD_EROFS_PCLUSTER_SIZE)" >> $(1)) +$(if $(BOARD_EROFS_BLOCKSIZE),$(hide) echo "erofs_blocksize=$(BOARD_EROFS_BLOCKSIZE)" >> $(1)) $(if $(BOARD_EROFS_SHARE_DUP_BLOCKS),$(hide) echo "erofs_share_dup_blocks=$(BOARD_EROFS_SHARE_DUP_BLOCKS)" >> $(1)) $(if $(BOARD_EROFS_USE_LEGACY_COMPRESSION),$(hide) echo "erofs_use_legacy_compression=$(BOARD_EROFS_USE_LEGACY_COMPRESSION)" >> $(1)) $(if $(BOARD_EXT4_SHARE_DUP_BLOCKS),$(hide) echo "ext4_share_dup_blocks=$(BOARD_EXT4_SHARE_DUP_BLOCKS)" >> $(1)) +$(if $(BOARD_F2FS_BLOCKSIZE),$(hide) echo "f2fs_blocksize=$(BOARD_F2FS_BLOCKSIZE)" >> $(1)) $(if $(BOARD_FLASH_LOGICAL_BLOCK_SIZE), $(hide) echo "flash_logical_block_size=$(BOARD_FLASH_LOGICAL_BLOCK_SIZE)" >> $(1)) $(if $(BOARD_FLASH_ERASE_BLOCK_SIZE), $(hide) echo "flash_erase_block_size=$(BOARD_FLASH_ERASE_BLOCK_SIZE)" >> $(1)) $(if $(filter eng, $(TARGET_BUILD_VARIANT)),$(hide) echo "verity_disable=true" >> $(1)) @@ -1971,81 +2238,80 @@ $(if $(PRODUCT_SUPPORTS_VBOOT),$(hide) echo "vboot_key=$(PRODUCT_VBOOT_SIGNING_K $(if $(PRODUCT_SUPPORTS_VBOOT),$(hide) echo "vboot_subkey=$(PRODUCT_VBOOT_SIGNING_SUBKEY)" >> $(1)) $(if $(PRODUCT_SUPPORTS_VBOOT),$(hide) echo "futility=$(notdir $(FUTILITY))" >> $(1)) $(if $(PRODUCT_SUPPORTS_VBOOT),$(hide) echo "vboot_signer_cmd=$(VBOOT_SIGNER)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_avbtool=$(notdir $(AVBTOOL))" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_system_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_system_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ - $(if $(BOARD_AVB_SYSTEM_KEY_PATH),\ - $(hide) echo "avb_system_key_path=$(BOARD_AVB_SYSTEM_KEY_PATH)" >> $(1) - $(hide) echo "avb_system_algorithm=$(BOARD_AVB_SYSTEM_ALGORITHM)" >> $(1) - $(hide) echo "avb_system_rollback_index_location=$(BOARD_AVB_SYSTEM_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_system_other_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_system_other_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_OTHER_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ +$(if $(BOARD_AVB_ENABLE), \ + $(hide) echo "avb_avbtool=$(notdir $(AVBTOOL))" >> $(1)$(newline) \ + $(if $(filter $(2),system), \ + $(hide) echo "avb_system_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_system_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ + $(if $(BOARD_AVB_SYSTEM_KEY_PATH), \ + $(hide) echo "avb_system_key_path=$(BOARD_AVB_SYSTEM_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_system_algorithm=$(BOARD_AVB_SYSTEM_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_system_rollback_index_location=$(BOARD_AVB_SYSTEM_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),system_other), \ + $(hide) echo "avb_system_other_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_system_other_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_OTHER_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_SYSTEM_OTHER_KEY_PATH),\ - $(hide) echo "avb_system_other_key_path=$(BOARD_AVB_SYSTEM_OTHER_KEY_PATH)" >> $(1) - $(hide) echo "avb_system_other_algorithm=$(BOARD_AVB_SYSTEM_OTHER_ALGORITHM)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_vendor_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_vendor_add_hashtree_footer_args=$(BOARD_AVB_VENDOR_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_system_other_key_path=$(BOARD_AVB_SYSTEM_OTHER_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_system_other_algorithm=$(BOARD_AVB_SYSTEM_OTHER_ALGORITHM)" >> $(1)$(newline))) \ + $(if $(filter $(2),vendor), \ + $(hide) echo "avb_vendor_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_vendor_add_hashtree_footer_args=$(BOARD_AVB_VENDOR_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_VENDOR_KEY_PATH),\ - $(hide) echo "avb_vendor_key_path=$(BOARD_AVB_VENDOR_KEY_PATH)" >> $(1) - $(hide) echo "avb_vendor_algorithm=$(BOARD_AVB_VENDOR_ALGORITHM)" >> $(1) - $(hide) echo "avb_vendor_rollback_index_location=$(BOARD_AVB_VENDOR_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_product_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_product_add_hashtree_footer_args=$(BOARD_AVB_PRODUCT_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_vendor_key_path=$(BOARD_AVB_VENDOR_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_vendor_algorithm=$(BOARD_AVB_VENDOR_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_vendor_rollback_index_location=$(BOARD_AVB_VENDOR_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),product), \ + $(hide) echo "avb_product_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_product_add_hashtree_footer_args=$(BOARD_AVB_PRODUCT_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_PRODUCT_KEY_PATH),\ - $(hide) echo "avb_product_key_path=$(BOARD_AVB_PRODUCT_KEY_PATH)" >> $(1) - $(hide) echo "avb_product_algorithm=$(BOARD_AVB_PRODUCT_ALGORITHM)" >> $(1) - $(hide) echo "avb_product_rollback_index_location=$(BOARD_AVB_PRODUCT_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_system_ext_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ - $(hide) echo "avb_system_ext_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_EXT_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_product_key_path=$(BOARD_AVB_PRODUCT_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_product_algorithm=$(BOARD_AVB_PRODUCT_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_product_rollback_index_location=$(BOARD_AVB_PRODUCT_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),system_ext), \ + $(hide) echo "avb_system_ext_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_system_ext_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_EXT_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_SYSTEM_EXT_KEY_PATH),\ - $(hide) echo "avb_system_ext_key_path=$(BOARD_AVB_SYSTEM_EXT_KEY_PATH)" >> $(1) - $(hide) echo "avb_system_ext_algorithm=$(BOARD_AVB_SYSTEM_EXT_ALGORITHM)" >> $(1) - $(hide) echo "avb_system_ext_rollback_index_location=$(BOARD_AVB_SYSTEM_EXT_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_odm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_odm_add_hashtree_footer_args=$(BOARD_AVB_ODM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_system_ext_key_path=$(BOARD_AVB_SYSTEM_EXT_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_system_ext_algorithm=$(BOARD_AVB_SYSTEM_EXT_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_system_ext_rollback_index_location=$(BOARD_AVB_SYSTEM_EXT_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),odm), \ + $(hide) echo "avb_odm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_odm_add_hashtree_footer_args=$(BOARD_AVB_ODM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_ODM_KEY_PATH),\ - $(hide) echo "avb_odm_key_path=$(BOARD_AVB_ODM_KEY_PATH)" >> $(1) - $(hide) echo "avb_odm_algorithm=$(BOARD_AVB_ODM_ALGORITHM)" >> $(1) - $(hide) echo "avb_odm_rollback_index_location=$(BOARD_AVB_ODM_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_vendor_dlkm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ - $(hide) echo "avb_vendor_dlkm_add_hashtree_footer_args=$(BOARD_AVB_VENDOR_DLKM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_odm_key_path=$(BOARD_AVB_ODM_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_odm_algorithm=$(BOARD_AVB_ODM_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_odm_rollback_index_location=$(BOARD_AVB_ODM_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),vendor_dlkm), \ + $(hide) echo "avb_vendor_dlkm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_vendor_dlkm_add_hashtree_footer_args=$(BOARD_AVB_VENDOR_DLKM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_VENDOR_DLKM_KEY_PATH),\ - $(hide) echo "avb_vendor_dlkm_key_path=$(BOARD_AVB_VENDOR_DLKM_KEY_PATH)" >> $(1) - $(hide) echo "avb_vendor_dlkm_algorithm=$(BOARD_AVB_VENDOR_DLKM_ALGORITHM)" >> $(1) - $(hide) echo "avb_vendor_dlkm_rollback_index_location=$(BOARD_AVB_VENDOR_DLKM_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_odm_dlkm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ - $(hide) echo "avb_odm_dlkm_add_hashtree_footer_args=$(BOARD_AVB_ODM_DLKM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_vendor_dlkm_key_path=$(BOARD_AVB_VENDOR_DLKM_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_vendor_dlkm_algorithm=$(BOARD_AVB_VENDOR_DLKM_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_vendor_dlkm_rollback_index_location=$(BOARD_AVB_VENDOR_DLKM_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),odm_dlkm), \ + $(hide) echo "avb_odm_dlkm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_odm_dlkm_add_hashtree_footer_args=$(BOARD_AVB_ODM_DLKM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_ODM_DLKM_KEY_PATH),\ - $(hide) echo "avb_odm_dlkm_key_path=$(BOARD_AVB_ODM_DLKM_KEY_PATH)" >> $(1) - $(hide) echo "avb_odm_dlkm_algorithm=$(BOARD_AVB_ODM_DLKM_ALGORITHM)" >> $(1) - $(hide) echo "avb_odm_dlkm_rollback_index_location=$(BOARD_AVB_ODM_DLKM_ROLLBACK_INDEX_LOCATION)" >> $(1))) -$(if $(BOARD_AVB_ENABLE),$(hide) echo "avb_system_dlkm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ - $(hide) echo "avb_system_dlkm_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_DLKM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)) -$(if $(BOARD_AVB_ENABLE),\ + $(hide) echo "avb_odm_dlkm_key_path=$(BOARD_AVB_ODM_DLKM_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_odm_dlkm_algorithm=$(BOARD_AVB_ODM_DLKM_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_odm_dlkm_rollback_index_location=$(BOARD_AVB_ODM_DLKM_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ + $(if $(filter $(2),system_dlkm), \ + $(hide) echo "avb_system_dlkm_hashtree_enable=$(BOARD_AVB_ENABLE)" >> $(1)$(newline) \ + $(hide) echo "avb_system_dlkm_add_hashtree_footer_args=$(BOARD_AVB_SYSTEM_DLKM_ADD_HASHTREE_FOOTER_ARGS)" >> $(1)$(newline) \ $(if $(BOARD_AVB_SYSTEM_DLKM_KEY_PATH),\ - $(hide) echo "avb_system_dlkm_key_path=$(BOARD_AVB_SYSTEM_DLKM_KEY_PATH)" >> $(1) - $(hide) echo "avb_system_dlkm_algorithm=$(BOARD_AVB_SYSTEM_DLKM_ALGORITHM)" >> $(1) - $(hide) echo "avb_system_dlkm_rollback_index_location=$(BOARD_SYSTEM_SYSTEM_DLKM_ROLLBACK_INDEX_LOCATION)" >> $(1))) + $(hide) echo "avb_system_dlkm_key_path=$(BOARD_AVB_SYSTEM_DLKM_KEY_PATH)" >> $(1)$(newline) \ + $(hide) echo "avb_system_dlkm_algorithm=$(BOARD_AVB_SYSTEM_DLKM_ALGORITHM)" >> $(1)$(newline) \ + $(hide) echo "avb_system_dlkm_rollback_index_location=$(BOARD_SYSTEM_SYSTEM_DLKM_ROLLBACK_INDEX_LOCATION)" >> $(1)$(newline))) \ +) $(if $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)),\ $(hide) echo "recovery_as_boot=true" >> $(1)) -$(if $(filter true,$(BOARD_BUILD_GKI_BOOT_IMAGE_WITHOUT_RAMDISK)),\ - $(hide) echo "gki_boot_image_without_ramdisk=true" >> $(1)) $(hide) echo "root_dir=$(TARGET_ROOT_OUT)" >> $(1) $(if $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITION_SIZE)),\ $(hide) echo "use_dynamic_partition_size=true" >> $(1)) +$(if $(COPY_IMAGES_FOR_TARGET_FILES_ZIP),\ + $(hide) echo "use_fixed_timestamp=true" >> $(1)) $(if $(3),$(hide) $(foreach kv,$(3),echo "$(kv)" >> $(1);)) +$(hide) sort -o $(1) $(1) endef # $(1): the path of the output dictionary file @@ -2346,7 +2612,11 @@ $(INSTALLED_RECOVERY_BUILD_PROP_TARGET): PRIVATE_RECOVERY_UI_PROPERTIES := \ TARGET_RECOVERY_UI_PROGRESS_BAR_BASELINE:progress_bar_baseline \ TARGET_RECOVERY_UI_TOUCH_LOW_THRESHOLD:touch_low_threshold \ TARGET_RECOVERY_UI_TOUCH_HIGH_THRESHOLD:touch_high_threshold \ - TARGET_RECOVERY_UI_VR_STEREO_OFFSET:vr_stereo_offset + TARGET_RECOVERY_UI_VR_STEREO_OFFSET:vr_stereo_offset \ + TARGET_RECOVERY_UI_BRIGHTNESS_FILE:brightness_file \ + TARGET_RECOVERY_UI_MAX_BRIGHTNESS_FILE:max_brightness_file \ + TARGET_RECOVERY_UI_BRIGHTNESS_NORMAL:brightness_normal_percent \ + TARGET_RECOVERY_UI_BRIGHTNESS_DIMMED:brightness_dimmed_percent # Parses the given list of build variables and writes their values as build properties if defined. # For example, if a target defines `TARGET_RECOVERY_UI_MARGIN_HEIGHT := 100`, @@ -2466,6 +2736,8 @@ $(INTERNAL_RECOVERY_RAMDISK_FILES_TIMESTAMP): $(MKBOOTFS) $(COMPRESSION_COMMAND_ $(if $(strip $(recovery_wipe)), \ cp -f $(recovery_wipe) $(TARGET_RECOVERY_ROOT_OUT)/system/etc/recovery.wipe) ln -sf prop.default $(TARGET_RECOVERY_ROOT_OUT)/default.prop + # Silence warnings in first_stage_console. + touch $(TARGET_RECOVERY_ROOT_OUT)/linkerconfig/ld.config.txt $(BOARD_RECOVERY_IMAGE_PREPARE) $(hide) touch $@ @@ -2960,7 +3232,7 @@ endif # BUILDING_DEBUG_VENDOR_BOOT_IMAGE endif # BUILDING_DEBUG_BOOT_IMAGE || BUILDING_DEBUG_VENDOR_BOOT_IMAGE - +PARTITION_COMPAT_SYMLINKS := # Creates a compatibility symlink between two partitions, e.g. /system/vendor to /vendor # $1: from location (e.g $(TARGET_OUT)/vendor) # $2: destination location (e.g. /vendor) @@ -2976,8 +3248,8 @@ $1: exit 1; \ fi ln -sfn $2 $1 -$1: .KATI_SYMLINK_OUTPUTS := $1 ) +$(eval PARTITION_COMPAT_SYMLINKS += $1) $1 endef @@ -3061,15 +3333,19 @@ $(1): $(HOST_OUT_EXECUTABLES)/fsverity_manifest_generator \ --cert $$(PRIVATE_KEY).x509.pem \ --key $$(PRIVATE_KEY).pk8 -ALL_DEFAULT_INSTALLED_MODULES += $(1) +$(1).idsig: $(1) + +ALL_DEFAULT_INSTALLED_MODULES += $(1) $(1).idsig endef # fsverity-generate-and-install-manifest-apk $(eval $(call fsverity-generate-and-install-manifest-apk, \ $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk,system)) +ALL_FSVERITY_BUILD_MANIFEST_APK += $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk.idsig ifdef BUILDING_SYSTEM_EXT_IMAGE $(eval $(call fsverity-generate-and-install-manifest-apk, \ $(TARGET_OUT_SYSTEM_EXT)/etc/security/fsverity/BuildManifestSystemExt.apk,system_ext)) + ALL_FSVERITY_BUILD_MANIFEST_APK += $(TARGET_OUT_SYSTEM_EXT)/etc/security/fsverity/BuildManifestSystemExt.apk $(TARGET_OUT_SYSTEM_EXT)/etc/security/fsverity/BuildManifestSystemExt.apk.idsig endif endif # PRODUCT_FSVERITY_GENERATE_METADATA @@ -3084,17 +3360,23 @@ INTERNAL_SYSTEMIMAGE_FILES := $(sort $(filter $(TARGET_OUT)/%, \ # Create symlink /system/vendor to /vendor if necessary. ifdef BOARD_USES_VENDORIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/vendor,/vendor,vendor.img) + _vendor_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/vendor,/vendor,vendor.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_vendor_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_vendor_symlink) endif # Create symlink /system/product to /product if necessary. ifdef BOARD_USES_PRODUCTIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/product,/product,product.img) + _product_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/product,/product,product.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_product_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_product_symlink) endif # Create symlink /system/system_ext to /system_ext if necessary. ifdef BOARD_USES_SYSTEM_EXTIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/system_ext,/system_ext,system_ext.img) + _systemext_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/system_ext,/system_ext,system_ext.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_systemext_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_systemext_symlink) endif # ----------------------------------------------------------------- @@ -3107,7 +3389,9 @@ endif # - /system/lib/modules is a symlink to a directory that stores system DLKMs. # - The system_dlkm partition is mounted at /system_dlkm at runtime. ifdef BOARD_USES_SYSTEM_DLKMIMAGE - INTERNAL_SYSTEMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT)/lib/modules,/system_dlkm/lib/modules,system_dlkm.img) + _system_dlkm_lib_modules_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT)/lib/modules,/system_dlkm/lib/modules,system_dlkm.img) + INTERNAL_SYSTEMIMAGE_FILES += $(_system_dlkm_lib_modules_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_system_dlkm_lib_modules_symlink) endif FULL_SYSTEMIMAGE_DEPS := $(INTERNAL_SYSTEMIMAGE_FILES) $(INTERNAL_USERIMAGES_DEPS) @@ -3122,6 +3406,14 @@ endif FULL_SYSTEMIMAGE_DEPS += $(INTERNAL_ROOT_FILES) $(INSTALLED_FILES_FILE_ROOT) +define write-file-lines +$(1): + @echo Writing $$@ + rm -f $$@ + echo -n > $$@ + $$(foreach f,$(2),echo "$$(f)" >> $$@$$(newline)) +endef + # ----------------------------------------------------------------- ifdef BUILDING_SYSTEM_IMAGE @@ -3131,16 +3423,20 @@ ifdef BUILDING_SYSTEM_IMAGE SYSTEM_LINKER_CONFIG := $(TARGET_OUT)/etc/linker.config.pb SYSTEM_LINKER_CONFIG_SOURCE := $(call intermediates-dir-for,ETC,system_linker_config)/system_linker_config $(SYSTEM_LINKER_CONFIG): PRIVATE_SYSTEM_LINKER_CONFIG_SOURCE := $(SYSTEM_LINKER_CONFIG_SOURCE) -$(SYSTEM_LINKER_CONFIG) : $(INTERNAL_SYSTEMIMAGE_FILES) $(SYSTEM_LINKER_CONFIG_SOURCE) | conv_linker_config +$(SYSTEM_LINKER_CONFIG): $(INTERNAL_SYSTEMIMAGE_FILES) $(SYSTEM_LINKER_CONFIG_SOURCE) | conv_linker_config + @echo Creating linker config: $@ + @mkdir -p $(dir $@) + @rm -f $@ $(HOST_OUT_EXECUTABLES)/conv_linker_config systemprovide --source $(PRIVATE_SYSTEM_LINKER_CONFIG_SOURCE) \ - --output $@ --value "$(STUB_LIBRARIES)" --system "$(TARGET_OUT)" + --output $@ --value "$(STUB_LIBRARIES)" --system "$(TARGET_OUT)" $(HOST_OUT_EXECUTABLES)/conv_linker_config append --source $@ --output $@ --key requireLibs \ - --value "$(foreach lib,$(LLNDK_MOVED_TO_APEX_LIBRARIES), $(lib).so)" + --value "$(foreach lib,$(LLNDK_MOVED_TO_APEX_LIBRARIES), $(lib).so)" $(call declare-1p-target,$(SYSTEM_LINKER_CONFIG),) $(call declare-license-deps,$(SYSTEM_LINKER_CONFIG),$(INTERNAL_SYSTEMIMAGE_FILES) $(SYSTEM_LINKER_CONFIG_SOURCE)) FULL_SYSTEMIMAGE_DEPS += $(SYSTEM_LINKER_CONFIG) +ALL_DEFAULT_INSTALLED_MODULES += $(SYSTEM_LINKER_CONFIG) # installed file list # Depending on anything that $(BUILT_SYSTEMIMAGE) depends on. @@ -3163,9 +3459,8 @@ $(eval $(call declare-0p-target,$(INSTALLED_FILES_JSON))) .PHONY: installed-file-list installed-file-list: $(INSTALLED_FILES_FILE) -systemimage_intermediates := \ - $(call intermediates-dir-for,PACKAGING,systemimage) -BUILT_SYSTEMIMAGE := $(systemimage_intermediates)/system.img +systemimage_intermediates :=$= $(call intermediates-dir-for,PACKAGING,system) +BUILT_SYSTEMIMAGE :=$= $(systemimage_intermediates)/system.img # $(1): output file define build-systemimage-target @@ -3175,16 +3470,22 @@ define build-systemimage-target skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(systemimage_intermediates)/file_list.txt) \ $(TARGET_OUT) $(systemimage_intermediates)/system_image_info.txt $(1) $(TARGET_OUT) \ || ( mkdir -p $${DIST_DIR}; \ cp $(INSTALLED_FILES_FILE) $${DIST_DIR}/installed-files-rescued.txt; \ exit 1 ) endef +$(eval $(call write-file-lines,$(systemimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT)/,,$(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS))))) +# Used by soong sandwich to request the staging dir be built +$(systemimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS)) + touch $@ + ifeq ($(BOARD_AVB_ENABLE),true) $(BUILT_SYSTEMIMAGE): $(BOARD_AVB_SYSTEM_KEY_PATH) endif -$(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE) +$(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE) $(systemimage_intermediates)/file_list.txt $(call build-systemimage-target,$@) $(call declare-1p-container,$(BUILT_SYSTEMIMAGE),system/extras) @@ -3240,7 +3541,7 @@ SYSTEM_NOTICE_DEPS += $(INSTALLED_SYSTEMIMAGE_TARGET) .PHONY: systemimage-nodeps snod systemimage-nodeps snod: $(filter-out systemimage-nodeps snod,$(MAKECMDGOALS)) \ - | $(INTERNAL_USERIMAGES_DEPS) + | $(INTERNAL_USERIMAGES_DEPS) $(systemimage_intermediates)/file_list.txt @echo "make $@: ignoring dependencies" $(call build-systemimage-target,$(INSTALLED_SYSTEMIMAGE_TARGET)) $(hide) $(call assert-max-image-size,$(INSTALLED_SYSTEMIMAGE_TARGET),$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)) @@ -3250,8 +3551,8 @@ endif endif # BUILDING_SYSTEM_IMAGE -.PHONY: sync syncsys -sync syncsys: $(INTERNAL_SYSTEMIMAGE_FILES) +.PHONY: sync syncsys sync_system +sync syncsys sync_system: $(INTERNAL_SYSTEMIMAGE_FILES) # ----------------------------------------------------------------- # Old PDK fusion targets @@ -3281,6 +3582,7 @@ define build-userdataimage-target $(call generate-image-prop-dictionary, $(userdataimage_intermediates)/userdata_image_info.txt,userdata,skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(userdataimage_intermediates)/file_list.txt) \ $(TARGET_OUT_DATA) $(userdataimage_intermediates)/userdata_image_info.txt \ $(INSTALLED_USERDATAIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_USERDATAIMAGE_TARGET),$(BOARD_USERDATAIMAGE_PARTITION_SIZE)) @@ -3291,7 +3593,13 @@ INSTALLED_USERDATAIMAGE_TARGET := $(BUILT_USERDATAIMAGE_TARGET) INSTALLED_USERDATAIMAGE_TARGET_DEPS := \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_USERDATAIMAGE_FILES) -$(INSTALLED_USERDATAIMAGE_TARGET): $(INSTALLED_USERDATAIMAGE_TARGET_DEPS) + +$(eval $(call write-file-lines,$(userdataimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_DATA)/,,$(filter $(TARGET_OUT_DATA)/%,$(INSTALLED_USERDATAIMAGE_TARGET_DEPS))))) +# Used by soong sandwich to request the staging dir be built +$(userdataimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_DATA)/%,$(INSTALLED_USERDATAIMAGE_TARGET_DEPS)) + touch $@ + +$(INSTALLED_USERDATAIMAGE_TARGET): $(INSTALLED_USERDATAIMAGE_TARGET_DEPS) $(userdataimage_intermediates)/file_list.txt $(build-userdataimage-target) $(call declare-1p-container,$(INSTALLED_USERDATAIMAGE_TARGET),) @@ -3300,7 +3608,7 @@ $(call declare-container-license-deps,$(INSTALLED_USERDATAIMAGE_TARGET),$(INSTAL UNMOUNTED_NOTICE_VENDOR_DEPS+= $(INSTALLED_USERDATAIMAGE_TARGET) .PHONY: userdataimage-nodeps -userdataimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) +userdataimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) $(userdataimage_intermediates)/file_list.txt $(build-userdataimage-target) endif # BUILDING_USERDATA_IMAGE @@ -3319,45 +3627,6 @@ $(ASAN_IN_SYSTEM_INSTALLED): $(INSTALLED_USERDATAIMAGE_TARGET_DEPS) tar cfj $(ASAN_IN_SYSTEM_INSTALLED) $(ASAN_SYSTEM_INSTALL_OPTIONS) -C $(TARGET_OUT_DATA)/.. $(ASAN_OUT_DIRS_FOR_SYSTEM_INSTALL) >/dev/null # ----------------------------------------------------------------- -# partition table image -ifdef BOARD_BPT_INPUT_FILES - -BUILT_BPTIMAGE_TARGET := $(PRODUCT_OUT)/partition-table.img -BUILT_BPTJSON_TARGET := $(PRODUCT_OUT)/partition-table.bpt - -INTERNAL_BVBTOOL_MAKE_TABLE_ARGS := \ - --output_gpt $(BUILT_BPTIMAGE_TARGET) \ - --output_json $(BUILT_BPTJSON_TARGET) \ - $(foreach file, $(BOARD_BPT_INPUT_FILES), --input $(file)) - -ifdef BOARD_BPT_DISK_SIZE -INTERNAL_BVBTOOL_MAKE_TABLE_ARGS += --disk_size $(BOARD_BPT_DISK_SIZE) -endif - -define build-bptimage-target - $(call pretty,"Target partition table image: $(INSTALLED_BPTIMAGE_TARGET)") - $(hide) $(BPTTOOL) make_table $(INTERNAL_BVBTOOL_MAKE_TABLE_ARGS) $(BOARD_BPT_MAKE_TABLE_ARGS) -endef - -INSTALLED_BPTIMAGE_TARGET := $(BUILT_BPTIMAGE_TARGET) -$(BUILT_BPTJSON_TARGET): $(INSTALLED_BPTIMAGE_TARGET) - $(hide) touch -c $(BUILT_BPTJSON_TARGET) - -$(INSTALLED_BPTIMAGE_TARGET): $(BPTTOOL) $(BOARD_BPT_INPUT_FILES) - $(build-bptimage-target) - -$(call declare-1p-container,$(INSTALLED_BPTIMAGE_TARGET),) -$(call declare-container-license-deps,$(INSTALLED_BPTIMAGE_TARGET),$(BOARD_BPT_INPUT_FILES),$(PRODUCT_OUT)/:/) - -UNMOUNTED_NOTICE_VENDOR_DEPS+= $(INSTALLED_BPTIMAGE_TARGET) - -.PHONY: bptimage-nodeps -bptimage-nodeps: - $(build-bptimage-target) - -endif # BOARD_BPT_INPUT_FILES - -# ----------------------------------------------------------------- # cache partition image INSTALLED_FILES_OUTSIDE_IMAGES := $(filter-out $(TARGET_OUT_CACHE)/%, $(INSTALLED_FILES_OUTSIDE_IMAGES)) ifdef BUILDING_CACHE_IMAGE @@ -3375,14 +3644,20 @@ define build-cacheimage-target $(call generate-image-prop-dictionary, $(cacheimage_intermediates)/cache_image_info.txt,cache,skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(cacheimage_intermediates)/file_list.txt) \ $(TARGET_OUT_CACHE) $(cacheimage_intermediates)/cache_image_info.txt \ $(INSTALLED_CACHEIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_CACHEIMAGE_TARGET),$(BOARD_CACHEIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(cacheimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_CACHE)/,,$(filter $(TARGET_OUT_CACHE)/%,$(INTERNAL_CACHEIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(cacheimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_CACHE)/%,$(INTERNAL_CACHEIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_CACHEIMAGE_TARGET := $(BUILT_CACHEIMAGE_TARGET) -$(INSTALLED_CACHEIMAGE_TARGET): $(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_CACHEIMAGE_FILES) +$(INSTALLED_CACHEIMAGE_TARGET): $(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_CACHEIMAGE_FILES) $(cacheimage_intermediates)/file_list.txt $(build-cacheimage-target) $(call declare-1p-container,$(INSTALLED_CACHEIMAGE_TARGET),) @@ -3391,7 +3666,7 @@ $(call declare-container-license-deps,$(INSTALLED_CACHEIMAGE_TARGET),$(INTERNAL_ UNMOUNTED_NOTICE_VENDOR_DEPS+= $(INSTALLED_CACHEIMAGE_TARGET) .PHONY: cacheimage-nodeps -cacheimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) +cacheimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) $(cacheimage_intermediates)/file_list.txt $(build-cacheimage-target) else # BUILDING_CACHE_IMAGE @@ -3456,16 +3731,22 @@ define build-systemotherimage-target $(call generate-image-prop-dictionary, $(systemotherimage_intermediates)/system_other_image_info.txt,system,skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(systemotherimage_intermediates)/file_list.txt) \ $(TARGET_OUT_SYSTEM_OTHER) $(systemotherimage_intermediates)/system_other_image_info.txt \ $(INSTALLED_SYSTEMOTHERIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_SYSTEMOTHERIMAGE_TARGET),$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(systemotherimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_SYSTEM_OTHER)/,,$(filter $(TARGET_OUT_SYSTEM_OTHER)/%,$(INTERNAL_SYSTEMOTHERIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(systemotherimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_SYSTEM_OTHER)/%,$(INTERNAL_SYSTEMOTHERIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_SYSTEMOTHERIMAGE_TARGET := $(BUILT_SYSTEMOTHERIMAGE_TARGET) ifneq (true,$(SANITIZE_LITE)) # Only create system_other when not building the second stage of a SANITIZE_LITE build. -$(INSTALLED_SYSTEMOTHERIMAGE_TARGET): $(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_SYSTEMOTHERIMAGE_FILES) $(INSTALLED_FILES_FILE_SYSTEMOTHER) +$(INSTALLED_SYSTEMOTHERIMAGE_TARGET): $(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_SYSTEMOTHERIMAGE_FILES) $(INSTALLED_FILES_FILE_SYSTEMOTHER) $(systemotherimage_intermediates)/file_list.txt $(build-systemotherimage-target) $(call declare-1p-container,$(INSTALLED_SYSTEMOTHERIMAGE_TARGET),) @@ -3475,7 +3756,7 @@ UNMOUNTED_NOTICE_DEPS += $(INSTALLED_SYSTEMOTHERIMAGE_TARGET) endif .PHONY: systemotherimage-nodeps -systemotherimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) +systemotherimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) $(systemotherimage_intermediates)/file_list.txt $(build-systemotherimage-target) endif # BUILDING_SYSTEM_OTHER_IMAGE @@ -3492,7 +3773,9 @@ INTERNAL_VENDORIMAGE_FILES := \ # Create symlink /vendor/odm to /odm if necessary. ifdef BOARD_USES_ODMIMAGE - INTERNAL_VENDORIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/odm,/odm,odm.img) + _odm_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/odm,/odm,odm.img) + INTERNAL_VENDORIMAGE_FILES += $(_odm_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_odm_symlink) endif # Create symlinks for vendor_dlkm on devices with a vendor_dlkm partition: @@ -3510,20 +3793,26 @@ endif # The vendor DLKMs and other vendor_dlkm files must not be accessed using other paths because they # are not guaranteed to exist on all devices. ifdef BOARD_USES_VENDOR_DLKMIMAGE - INTERNAL_VENDORIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/lib/modules,/vendor_dlkm/lib/modules,vendor_dlkm.img) + _vendor_dlkm_lib_modules_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT_VENDOR)/lib/modules,/vendor_dlkm/lib/modules,vendor_dlkm.img) + INTERNAL_VENDORIMAGE_FILES += $(_vendor_dlkm_lib_modules_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_vendor_dlkm_lib_modules_symlink) endif -# Install vendor/etc/linker.config.pb when PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS is set -ifneq ($(strip $(PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS)),) +# Install vendor/etc/linker.config.pb with PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS and SOONG_STUB_VENDOR_LIBRARIES vendor_linker_config_file := $(TARGET_OUT_VENDOR)/etc/linker.config.pb $(vendor_linker_config_file): private_linker_config_fragments := $(PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS) -$(vendor_linker_config_file): $(PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS) | $(HOST_OUT_EXECUTABLES)/conv_linker_config +$(vendor_linker_config_file): $(INTERNAL_VENDORIMAGE_FILES) $(PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS) | $(HOST_OUT_EXECUTABLES)/conv_linker_config + @echo Creating linker config: $@ + @mkdir -p $(dir $@) + @rm -f $@ $(HOST_OUT_EXECUTABLES)/conv_linker_config proto \ --source $(call normalize-path-list,$(private_linker_config_fragments)) \ --output $@ + $(HOST_OUT_EXECUTABLES)/conv_linker_config systemprovide --source $@ \ + --output $@ --value "$(SOONG_STUB_VENDOR_LIBRARIES)" --system "$(TARGET_OUT_VENDOR)" $(call define declare-0p-target,$(vendor_linker_config_file),) INTERNAL_VENDORIMAGE_FILES += $(vendor_linker_config_file) -endif +ALL_DEFAULT_INSTALLED_MODULES += $(vendor_linker_config_file) INSTALLED_FILES_FILE_VENDOR := $(PRODUCT_OUT)/installed-files-vendor.txt INSTALLED_FILES_JSON_VENDOR := $(INSTALLED_FILES_FILE_VENDOR:.txt=.json) @@ -3548,18 +3837,25 @@ define build-vendorimage-target $(call generate-image-prop-dictionary, $(vendorimage_intermediates)/vendor_image_info.txt,vendor,skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(vendorimage_intermediates)/file_list.txt) \ $(TARGET_OUT_VENDOR) $(vendorimage_intermediates)/vendor_image_info.txt \ $(INSTALLED_VENDORIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_VENDORIMAGE_TARGET) $(RECOVERY_FROM_BOOT_PATCH),$(BOARD_VENDORIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(vendorimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_VENDOR)/,,$(filter $(TARGET_OUT_VENDOR)/%,$(INTERNAL_VENDORIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(vendorimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_VENDOR)/%,$(INTERNAL_VENDORIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_VENDORIMAGE_TARGET := $(BUILT_VENDORIMAGE_TARGET) $(INSTALLED_VENDORIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_VENDORIMAGE_FILES) \ $(INSTALLED_FILES_FILE_VENDOR) \ - $(RECOVERY_FROM_BOOT_PATCH) + $(RECOVERY_FROM_BOOT_PATCH) \ + $(vendorimage_intermediates)/file_list.txt $(build-vendorimage-target) VENDOR_NOTICE_DEPS += $(INSTALLED_VENDORIMAGE_TARGET) @@ -3568,10 +3864,11 @@ $(call declare-container-license-metadata,$(INSTALLED_VENDORIMAGE_TARGET),legacy $(call declare-container-license-deps,$(INSTALLED_VENDORIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_VENDORIMAGE_FILES) $(RECOVERY_FROM_BOOT_PATH),$(PRODUCT_OUT)/:/) .PHONY: vendorimage-nodeps vnod -vendorimage-nodeps vnod: | $(INTERNAL_USERIMAGES_DEPS) +vendorimage-nodeps vnod: | $(INTERNAL_USERIMAGES_DEPS) $(vendorimage_intermediates)/file_list.txt $(build-vendorimage-target) -sync: $(INTERNAL_VENDORIMAGE_FILES) +.PHONY: sync_vendor +sync sync_vendor: $(INTERNAL_VENDORIMAGE_FILES) else ifdef BOARD_PREBUILT_VENDORIMAGE INSTALLED_VENDORIMAGE_TARGET := $(PRODUCT_OUT)/vendor.img @@ -3613,17 +3910,24 @@ define build-productimage-target $(call generate-image-prop-dictionary, $(productimage_intermediates)/product_image_info.txt,product,skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(productimage_intermediates)/file_list.txt) \ $(TARGET_OUT_PRODUCT) $(productimage_intermediates)/product_image_info.txt \ $(INSTALLED_PRODUCTIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_PRODUCTIMAGE_TARGET),$(BOARD_PRODUCTIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(productimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_PRODUCT)/,,$(filter $(TARGET_OUT_PRODUCT)/%,$(INTERNAL_PRODUCTIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(productimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_PRODUCT)/%,$(INTERNAL_PRODUCTIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_PRODUCTIMAGE_TARGET := $(BUILT_PRODUCTIMAGE_TARGET) $(INSTALLED_PRODUCTIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_PRODUCTIMAGE_FILES) \ - $(INSTALLED_FILES_FILE_PRODUCT) + $(INSTALLED_FILES_FILE_PRODUCT) \ + $(productimage_intermediates)/file_list.txt $(build-productimage-target) PRODUCT_NOTICE_DEPS += $(INSTALLED_PRODUCTIMAGE_TARGET) @@ -3632,10 +3936,11 @@ $(call declare-1p-container,$(INSTALLED_PRODUCTIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_PRODUCTIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_PRODUCTIMAGE_FILES) $(INSTALLED_FILES_FILE_PRODUCT),$(PRODUCT_OUT)/:/) .PHONY: productimage-nodeps pnod -productimage-nodeps pnod: | $(INTERNAL_USERIMAGES_DEPS) +productimage-nodeps pnod: | $(INTERNAL_USERIMAGES_DEPS) $(productimage_intermediates)/file_list.txt $(build-productimage-target) -sync: $(INTERNAL_PRODUCTIMAGE_FILES) +.PHONY: sync_product +sync sync_product: $(INTERNAL_PRODUCTIMAGE_FILES) else ifdef BOARD_PREBUILT_PRODUCTIMAGE INSTALLED_PRODUCTIMAGE_TARGET := $(PRODUCT_OUT)/product.img @@ -3673,6 +3978,7 @@ define build-system_extimage-target $(call generate-image-prop-dictionary, $(system_extimage_intermediates)/system_ext_image_info.txt,system_ext, skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(system_extimage_intermediates)/file_list.txt) \ $(TARGET_OUT_SYSTEM_EXT) \ $(system_extimage_intermediates)/system_ext_image_info.txt \ $(INSTALLED_SYSTEM_EXTIMAGE_TARGET) \ @@ -3680,12 +3986,18 @@ define build-system_extimage-target $(call assert-max-image-size,$(INSTALLED_PRODUCT_SERVICESIMAGE_TARGET),$(BOARD_PRODUCT_SERVICESIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(system_extimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_SYSTEM_EXT)/,,$(filter $(TARGET_OUT_SYSTEM_EXT)/%,$(INTERNAL_SYSTEM_EXTIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(system_extimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_SYSTEM_EXT)/%,$(INTERNAL_SYSTEM_EXTIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_SYSTEM_EXTIMAGE_TARGET := $(BUILT_SYSTEM_EXTIMAGE_TARGET) $(INSTALLED_SYSTEM_EXTIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_SYSTEM_EXTIMAGE_FILES) \ - $(INSTALLED_FILES_FILE_SYSTEM_EXT) + $(INSTALLED_FILES_FILE_SYSTEM_EXT) \ + $(system_extimage_intermediates)/file_list.txt $(build-system_extimage-target) SYSTEM_EXT_NOTICE_DEPS += $(INSTALLED_SYSTEM_EXTIMAGE_TARGET) @@ -3694,10 +4006,11 @@ $(call declare-1p-container,$(INSTALLED_SYSTEM_EXTIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_SYSTEM_EXTIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_SYSTEM_EXTIMAGE_FILES) $(INSTALLED_FILES_FILE_SYSTEM_EXT),$(PRODUCT_OUT)/:/) .PHONY: systemextimage-nodeps senod -systemextimage-nodeps senod: | $(INTERNAL_USERIMAGES_DEPS) +systemextimage-nodeps senod: | $(INTERNAL_USERIMAGES_DEPS) $(system_extimage_intermediates)/file_list.txt $(build-system_extimage-target) -sync: $(INTERNAL_SYSTEM_EXTIMAGE_FILES) +.PHONY: sync_system_ext +sync sync_system_ext: $(INTERNAL_SYSTEM_EXTIMAGE_FILES) else ifdef BOARD_PREBUILT_SYSTEM_EXTIMAGE INSTALLED_SYSTEM_EXTIMAGE_TARGET := $(PRODUCT_OUT)/system_ext.img @@ -3727,7 +4040,9 @@ INTERNAL_ODMIMAGE_FILES := \ # The odm DLKMs and other odm_dlkm files must not be accessed using other paths because they # are not guaranteed to exist on all devices. ifdef BOARD_USES_ODM_DLKMIMAGE - INTERNAL_ODMIMAGE_FILES += $(call create-partition-compat-symlink,$(TARGET_OUT_ODM)/lib/modules,/odm_dlkm/lib/modules,odm_dlkm.img) + _odm_dlkm_lib_modules_symlink := $(call create-partition-compat-symlink,$(TARGET_OUT_ODM)/lib/modules,/odm_dlkm/lib/modules,odm_dlkm.img) + INTERNAL_ODMIMAGE_FILES += $(_odm_dlkm_lib_modules_symlink) + ALL_DEFAULT_INSTALLED_MODULES += $(_odm_dlkm_lib_modules_symlink) endif INSTALLED_FILES_FILE_ODM := $(PRODUCT_OUT)/installed-files-odm.txt @@ -3754,17 +4069,24 @@ define build-odmimage-target skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(odmimage_intermediates)/file_list.txt) \ $(TARGET_OUT_ODM) $(odmimage_intermediates)/odm_image_info.txt \ $(INSTALLED_ODMIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_ODMIMAGE_TARGET),$(BOARD_ODMIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(odmimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_ODM)/,,$(filter $(TARGET_OUT_ODM)/%,$(INTERNAL_ODMIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(odmimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_ODM)/%,$(INTERNAL_ODMIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_ODMIMAGE_TARGET := $(BUILT_ODMIMAGE_TARGET) $(INSTALLED_ODMIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_ODMIMAGE_FILES) \ - $(INSTALLED_FILES_FILE_ODM) + $(INSTALLED_FILES_FILE_ODM) \ + $(odmimage_intermediates)/file_list.txt $(build-odmimage-target) ODM_NOTICE_DEPS += $(INSTALLED_ODMIMAGE_TARGET) @@ -3773,10 +4095,11 @@ $(call declare-1p-container,$(INSTALLED_ODMIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_ODMIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_ODMIMAGE_FILES) $(INSTALLED_FILES_FILE_ODM),$(PRODUCT_OUT)/:/) .PHONY: odmimage-nodeps onod -odmimage-nodeps onod: | $(INTERNAL_USERIMAGES_DEPS) +odmimage-nodeps onod: | $(INTERNAL_USERIMAGES_DEPS) $(odmimage_intermediates)/file_list.txt $(build-odmimage-target) -sync: $(INTERNAL_ODMIMAGE_FILES) +.PHONY: sync_odm +sync sync_odm: $(INTERNAL_ODMIMAGE_FILES) else ifdef BOARD_PREBUILT_ODMIMAGE INSTALLED_ODMIMAGE_TARGET := $(PRODUCT_OUT)/odm.img @@ -3815,17 +4138,24 @@ define build-vendor_dlkmimage-target vendor_dlkm, skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(vendor_dlkmimage_intermediates)/file_list.txt) \ $(TARGET_OUT_VENDOR_DLKM) $(vendor_dlkmimage_intermediates)/vendor_dlkm_image_info.txt \ $(INSTALLED_VENDOR_DLKMIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_VENDOR_DLKMIMAGE_TARGET),$(BOARD_VENDOR_DLKMIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(vendor_dlkmimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_VENDOR_DLKM)/,,$(filter $(TARGET_OUT_VENDOR_DLKM)/%,$(INTERNAL_VENDOR_DLKMIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(vendor_dlkmimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_VENDOR_DLKM)/%,$(INTERNAL_VENDOR_DLKMIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_VENDOR_DLKMIMAGE_TARGET := $(BUILT_VENDOR_DLKMIMAGE_TARGET) $(INSTALLED_VENDOR_DLKMIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_VENDOR_DLKMIMAGE_FILES) \ - $(INSTALLED_FILES_FILE_VENDOR_DLKM) + $(INSTALLED_FILES_FILE_VENDOR_DLKM) \ + $(vendor_dlkmimage_intermediates)/file_list.txt $(build-vendor_dlkmimage-target) VENDOR_DLKM_NOTICE_DEPS += $(INSTALLED_VENDOR_DLKMIMAGE_TARGET) @@ -3834,10 +4164,11 @@ $(call declare-1p-container,$(INSTALLED_VENDOR_DLKMIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_VENDOR_DLKMIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_VENDOR_DLKMIMAGE_FILES) $(INSTALLED_FILES_FILE_VENDOR_DLKM),$(PRODUCT_OUT)/:/) .PHONY: vendor_dlkmimage-nodeps vdnod -vendor_dlkmimage-nodeps vdnod: | $(INTERNAL_USERIMAGES_DEPS) +vendor_dlkmimage-nodeps vdnod: | $(INTERNAL_USERIMAGES_DEPS) $(vendor_dlkmimage_intermediates)/file_list.txt $(build-vendor_dlkmimage-target) -sync: $(INTERNAL_VENDOR_DLKMIMAGE_FILES) +.PHONY: sync_vendor_dlkm +sync sync_vendor_dlkm: $(INTERNAL_VENDOR_DLKMIMAGE_FILES) else ifdef BOARD_PREBUILT_VENDOR_DLKMIMAGE INSTALLED_VENDOR_DLKMIMAGE_TARGET := $(PRODUCT_OUT)/vendor_dlkm.img @@ -3876,17 +4207,24 @@ define build-odm_dlkmimage-target odm_dlkm, skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(odm_dlkmimage_intermediates)/file_list.txt) \ $(TARGET_OUT_ODM_DLKM) $(odm_dlkmimage_intermediates)/odm_dlkm_image_info.txt \ $(INSTALLED_ODM_DLKMIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_ODM_DLKMIMAGE_TARGET),$(BOARD_ODM_DLKMIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(odm_dlkmimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_ODM_DLKM)/,,$(filter $(TARGET_OUT_ODM_DLKM)/%,$(INTERNAL_ODM_DLKMIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(odm_dlkmimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_ODM_DLKM)/%,$(INTERNAL_ODM_DLKMIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_ODM_DLKMIMAGE_TARGET := $(BUILT_ODM_DLKMIMAGE_TARGET) $(INSTALLED_ODM_DLKMIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_ODM_DLKMIMAGE_FILES) \ - $(INSTALLED_FILES_FILE_ODM_DLKM) + $(INSTALLED_FILES_FILE_ODM_DLKM) \ + $(odm_dlkmimage_intermediates)/file_list.txt $(build-odm_dlkmimage-target) ODM_DLKM_NOTICE_DEPS += $(INSTALLED_ODM_DLKMIMAGE_TARGET) @@ -3895,10 +4233,11 @@ $(call declare-1p-container,$(INSTALLED_ODM_DLKMIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_ODM_DLKMIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_ODM_DLKMIMAGE_FILES) $(INSTALLED_FILES_FILE_ODM_DLKM),$(PRODUCT_OUT)/:/) .PHONY: odm_dlkmimage-nodeps odnod -odm_dlkmimage-nodeps odnod: | $(INTERNAL_USERIMAGES_DEPS) +odm_dlkmimage-nodeps odnod: | $(INTERNAL_USERIMAGES_DEPS) $(odm_dlkmimage_intermediates)/file_list.txt $(build-odm_dlkmimage-target) -sync: $(INTERNAL_ODM_DLKMIMAGE_FILES) +.PHONY: sync_odm_dlkm +sync sync_odm_dlkm: $(INTERNAL_ODM_DLKMIMAGE_FILES) else ifdef BOARD_PREBUILT_ODM_DLKMIMAGE INSTALLED_ODM_DLKMIMAGE_TARGET := $(PRODUCT_OUT)/odm_dlkm.img @@ -3939,17 +4278,24 @@ define build-system_dlkmimage-target system_dlkm, skip_fsck=true) PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$$PATH \ $(BUILD_IMAGE) \ + $(if $(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES),,--input-directory-filter-file $(system_dlkmimage_intermediates)/file_list.txt) \ $(TARGET_OUT_SYSTEM_DLKM) $(system_dlkmimage_intermediates)/system_dlkm_image_info.txt \ $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) $(TARGET_OUT) $(call assert-max-image-size,$(INSTALLED_SYSTEM_DLKMIMAGE_TARGET),$(BOARD_SYSTEM_DLKMIMAGE_PARTITION_SIZE)) endef +$(eval $(call write-file-lines,$(system_dlkmimage_intermediates)/file_list.txt,$(subst $(TARGET_OUT_SYSTEM_DLKM)/,,$(filter $(TARGET_OUT_SYSTEM_DLKM)/%,$(INTERNAL_SYSTEM_DLKMIMAGE_FILES))))) +# Used by soong sandwich to request the staging dir be built +$(system_dlkmimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT_SYSTEM_DLKM)/%,$(INTERNAL_SYSTEM_DLKMIMAGE_FILES)) + touch $@ + # We just build this directly to the install location. INSTALLED_SYSTEM_DLKMIMAGE_TARGET := $(BUILT_SYSTEM_DLKMIMAGE_TARGET) $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET): \ $(INTERNAL_USERIMAGES_DEPS) \ $(INTERNAL_SYSTEM_DLKMIMAGE_FILES) \ - $(INSTALLED_FILES_FILE_SYSTEM_DLKM) + $(INSTALLED_FILES_FILE_SYSTEM_DLKM) \ + $(system_dlkmimage_intermediates)/file_list.txt $(build-system_dlkmimage-target) SYSTEM_DLKM_NOTICE_DEPS += $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) @@ -3958,10 +4304,11 @@ $(call declare-1p-container,$(INSTALLED_SYSTEM_DLKMIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_SYSTEM_DLKMIMAGE_TARGET),$(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_SYSTEM_DLKMIMAGE_FILES) $(INSTALLED_FILES_FILE_SYSTEM_DLKM),$(PRODUCT_OUT)/:/) .PHONY: system_dlkmimage-nodeps sdnod -system_dlkmimage-nodeps sdnod: | $(INTERNAL_USERIMAGES_DEPS) +system_dlkmimage-nodeps sdnod: | $(INTERNAL_USERIMAGES_DEPS) $(system_dlkmimage_intermediates)/file_list.txt $(build-system_dlkmimage-target) -sync: $(INTERNAL_SYSTEM_DLKMIMAGE_FILES) +.PHONY: sync_system_dlkm +sync sync_system_dlkm: $(INTERNAL_SYSTEM_DLKMIMAGE_FILES) else ifdef BOARD_PREBUILT_SYSTEM_DLKMIMAGE INSTALLED_SYSTEM_DLKMIMAGE_TARGET := $(PRODUCT_OUT)/system_dlkm.img @@ -3976,6 +4323,7 @@ INSTALLED_DTBOIMAGE_TARGET := $(PRODUCT_OUT)/dtbo.img ifeq ($(BOARD_AVB_ENABLE),true) $(INSTALLED_DTBOIMAGE_TARGET): $(BOARD_PREBUILT_DTBOIMAGE) $(AVBTOOL) $(BOARD_AVB_DTBO_KEY_PATH) cp $(BOARD_PREBUILT_DTBOIMAGE) $@ + chmod +w $@ $(AVBTOOL) add_hash_footer \ --image $@ \ $(call get-partition-size-argument,$(BOARD_DTBOIMG_PARTITION_SIZE)) \ @@ -4002,7 +4350,7 @@ pvmfwimage: $(INSTALLED_PVMFWIMAGE_TARGET) INSTALLED_PVMFWIMAGE_TARGET := $(PRODUCT_OUT)/pvmfw.img INSTALLED_PVMFW_EMBEDDED_AVBKEY_TARGET := $(PRODUCT_OUT)/pvmfw_embedded.avbpubkey -INSTALLED_PVMFW_BINARY_TARGET := $(call module-installed-files,pvmfw_bin) +INSTALLED_PVMFW_BINARY_TARGET := $(call module-target-built-files,pvmfw_bin) INTERNAL_PVMFWIMAGE_FILES := $(call module-target-built-files,pvmfw_img) INTERNAL_PVMFW_EMBEDDED_AVBKEY := $(call module-target-built-files,pvmfw_embedded_key) INTERNAL_PVMFW_SYMBOL := $(TARGET_OUT_EXECUTABLES_UNSTRIPPED)/pvmfw @@ -4040,6 +4388,13 @@ INSTALLED_CUSTOMIMAGES_TARGET := ifneq ($(strip $(BOARD_CUSTOMIMAGES_PARTITION_LIST)),) INTERNAL_AVB_CUSTOMIMAGES_SIGNING_ARGS := +BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST := +# If BOARD_AVB_$(call to-upper,$(partition))_KEY_PATH is set, the image will be included in +# BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST, otherwise the image won't be AVB signed. +$(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ + $(if $(BOARD_AVB_$(call to-upper,$(partition))_KEY_PATH), \ + $(eval BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST += $(partition)) \ + $(eval BOARD_$(call to-upper,$(partition))_IMAGE_LIST := $(BOARD_AVB_$(call to-upper,$(partition))_IMAGE_LIST)))) # Sign custom image. # $(1): the prebuilt custom image. @@ -4064,9 +4419,26 @@ endif INSTALLED_CUSTOMIMAGES_TARGET += $(3) endef -$(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ +# Copy unsigned custom image. +# $(1): the prebuilt custom image. +# $(2): the signed custom image target. +define copy_custom_image +$(2): $(1) $(INTERNAL_USERIMAGES_DEPS) + @echo Target custom image: $(2) + mkdir -p $(dir $(2)) + cp $(1) $(2) +INSTALLED_CUSTOMIMAGES_TARGET += $(2) +endef + +# Add AVB custom image to droid target +$(foreach partition,$(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), \ $(foreach image,$(BOARD_AVB_$(call to-upper,$(partition))_IMAGE_LIST), \ $(eval $(call sign_custom_image,$(image),$(partition),$(PRODUCT_OUT)/$(notdir $(image)))))) + +# Add unsigned custom image to droid target +$(foreach partition,$(filter-out $(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), $(BOARD_CUSTOMIMAGES_PARTITION_LIST)), \ + $(foreach image,$(BOARD_$(call to-upper,$(partition))_IMAGE_LIST), \ + $(eval $(call copy_custom_image,$(image),$(PRODUCT_OUT)/$(notdir $(image)))))) endif # ----------------------------------------------------------------- @@ -4266,6 +4638,20 @@ BOARD_AVB_PVMFW_ADD_HASH_FOOTER_ARGS += \ --prop com.android.build.pvmfw.security_patch:$(PVMFW_SECURITY_PATCH) endif +# Append root digest of microdroid-vendor partition's hashtree descriptor into vendor partition. +ifdef MICRODROID_VENDOR_IMAGE_MODULE +MICRODROID_VENDOR_IMAGE := \ + $(call intermediates-dir-for,ETC,$(MICRODROID_VENDOR_IMAGE_MODULE))/$(MICRODROID_VENDOR_IMAGE_MODULE) +MICRODROID_VENDOR_ROOT_DIGEST := $(PRODUCT_OUT)/microdroid_vendor_root_digest +BOARD_AVB_VENDOR_ADD_HASHTREE_FOOTER_ARGS += \ + --prop_from_file com.android.build.microdroid-vendor.root_digest:$(MICRODROID_VENDOR_ROOT_DIGEST) +$(MICRODROID_VENDOR_ROOT_DIGEST): $(AVBTOOL) $(MICRODROID_VENDOR_IMAGE) + $(AVBTOOL) print_partition_digests \ + --image $(MICRODROID_VENDOR_IMAGE) \ + | tr -d '\n' | sed -E 's/.*: //g' > $@ +$(INSTALLED_VENDORIMAGE_TARGET): $(MICRODROID_VENDOR_ROOT_DIGEST) +endif + BOOT_FOOTER_ARGS := BOARD_AVB_BOOT_ADD_HASH_FOOTER_ARGS INIT_BOOT_FOOTER_ARGS := BOARD_AVB_INIT_BOOT_ADD_HASH_FOOTER_ARGS VENDOR_BOOT_FOOTER_ARGS := BOARD_AVB_VENDOR_BOOT_ADD_HASH_FOOTER_ARGS @@ -4343,7 +4729,9 @@ define check-and-set-custom-avb-chain-args $(eval part := $(1)) $(eval PART=$(call to-upper,$(part))) $(eval _rollback_index_location := BOARD_AVB_$(PART)_ROLLBACK_INDEX_LOCATION) +$(eval _key_path := BOARD_AVB_$(PART)_KEY_PATH) $(if $($(_rollback_index_location)),,$(error $(_rollback_index_location) is not defined)) +$(if $($(_key_path)),,$(error $(_key_path) is not defined)) INTERNAL_AVB_MAKE_VBMETA_IMAGE_ARGS += \ --chain_partition $(part):$($(_rollback_index_location)):$(AVB_CHAIN_KEY_DIR)/$(part).avbpubkey @@ -4423,8 +4811,8 @@ $(foreach partition,$(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS),$(eval $(call check-an $(foreach partition,$(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS),$(eval BOARD_AVB_MAKE_VBMETA_$(call to-upper,$(partition))_IMAGE_ARGS += --padding_size 4096)) endif -ifneq ($(strip $(BOARD_CUSTOMIMAGES_PARTITION_LIST)),) -$(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ +ifneq ($(strip $(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST)),) +$(foreach partition,$(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), \ $(eval $(call check-and-set-custom-avb-chain-args,$(partition)))) endif @@ -4511,8 +4899,8 @@ define extract-avb-chain-public-keys $(if $(BOARD_AVB_VBMETA_VENDOR_KEY_PATH),\ $(hide) $(AVBTOOL) extract_public_key --key $(BOARD_AVB_VBMETA_VENDOR_KEY_PATH) \ --output $(1)/vbmeta_vendor.avbpubkey) - $(if $(BOARD_CUSTOMIMAGES_PARTITION_LIST),\ - $(hide) $(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ + $(if $(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST),\ + $(hide) $(foreach partition,$(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), \ $(AVBTOOL) extract_public_key --key $(BOARD_AVB_$(call to-upper,$(partition))_KEY_PATH) \ --output $(1)/$(partition).avbpubkey;)) \ $(if $(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS),\ @@ -4659,6 +5047,50 @@ INTERNAL_ALLIMAGES_FILES := \ $(INTERNAL_PVMFWIMAGE_FILES) \ # ----------------------------------------------------------------- +# Run apex_sepolicy_tests for all installed APEXes + +ifeq (,$(TARGET_BUILD_UNBUNDLED)) +intermediate := $(call intermediates-dir-for,PACKAGING,apex_sepolicy_tests) +apex_dirs := \ + $(TARGET_OUT)/apex/% \ + $(TARGET_OUT_SYSTEM_EXT)/apex/% \ + $(TARGET_OUT_VENDOR)/apex/% \ + $(TARGET_OUT_PRODUCT)/apex/% \ + +apex_files := $(sort $(filter $(apex_dirs), $(INTERNAL_ALLIMAGES_FILES))) +apex_dirs := + +# $1: apex file +# $2: output file +define _run_apex_sepolicy_tests +$2: $1 \ + $(HOST_OUT_EXECUTABLES)/apex_sepolicy_tests \ + $(HOST_OUT_EXECUTABLES)/deapexer \ + $(HOST_OUT_EXECUTABLES)/debugfs_static + @rm -rf $$@ + @mkdir -p $(dir $$@) + $(HOST_OUT_EXECUTABLES)/apex_sepolicy_tests --all -f <($(HOST_OUT_EXECUTABLES)/deapexer --debugfs_path $(HOST_OUT_EXECUTABLES)/debugfs_static list -Z $$<) + @touch $$@ +endef + +# $1: apex file list +define run_apex_sepolicy_tests +$(foreach apex_file,$1, \ + $(eval passfile := $(patsubst $(PRODUCT_OUT)/%,$(intermediate)/%.pass,$(apex_file))) \ + $(eval $(call _run_apex_sepolicy_tests,$(apex_file),$(passfile))) \ + $(passfile)) +endef + +.PHONY: run_apex_sepolicy_tests +run_apex_sepolicy_tests: $(call run_apex_sepolicy_tests,$(apex_files)) + +droid_targets: run_apex_sepolicy_tests + +apex_files := +intermediate := +endif # TARGET_BUILD_UNBUNDLED + +# ----------------------------------------------------------------- # Check VINTF of build # Note: vendor_dlkm, odm_dlkm, and system_dlkm does not have VINTF files. @@ -4667,39 +5099,47 @@ ifeq (,$(TARGET_BUILD_UNBUNDLED)) intermediates := $(call intermediates-dir-for,PACKAGING,check_vintf_all) check_vintf_all_deps := -APEX_OUT := $(PRODUCT_OUT)/apex # ----------------------------------------------------------------- -# Create apex-info-file.xml +# Activate APEXes for checkvintf apex_dirs := \ $(TARGET_OUT)/apex/% \ + $(TARGET_OUT_PRODUCT)/apex/% \ $(TARGET_OUT_SYSTEM_EXT)/apex/% \ $(TARGET_OUT_VENDOR)/apex/% \ - $(TARGET_OUT_ODM)/apex/% \ - $(TARGET_OUT_PRODUCT)/apex/% \ apex_files := $(sort $(filter $(apex_dirs), $(INTERNAL_ALLIMAGES_FILES))) + +APEX_OUT := $(intermediates)/apex APEX_INFO_FILE := $(APEX_OUT)/apex-info-list.xml -# dump_apex_info scans $(PRODUCT_OUT)/apex and writes apex-info-list.xml there. -# This relies on the fact that rules for .apex files install the contents in $(PRODUCT_OUT)/apex. -$(APEX_INFO_FILE): $(HOST_OUT_EXECUTABLES)/dump_apex_info $(apex_files) - @echo "Creating apex-info-file in $(PRODUCT_OUT) " - $< --root_dir $(PRODUCT_OUT) +# apexd_host scans/activates APEX files and writes /apex/apex-info-list.xml +# Note that `@echo $(PRIVATE_APEX_FILES)` line is added to trigger the rule when the APEX list is changed. +$(APEX_INFO_FILE): PRIVATE_APEX_FILES := $(apex_files) +$(APEX_INFO_FILE): $(HOST_OUT_EXECUTABLES)/apexd_host $(apex_files) + @echo "Extracting apexes..." + @echo $(PRIVATE_APEX_FILES) > /dev/null + @rm -rf $(APEX_OUT) + @mkdir -p $(APEX_OUT) + $< --system_path $(TARGET_OUT) \ + --system_ext_path $(TARGET_OUT_SYSTEM_EXT) \ + --product_path $(TARGET_OUT_PRODUCT) \ + --vendor_path $(TARGET_OUT_VENDOR) \ + --apex_path $(APEX_OUT) apex_files := apex_dirs := # The build system only writes VINTF metadata to */etc/vintf paths. Legacy paths aren't needed here # because they are only used for prebuilt images. -# APEX files in /vendor/apex can have VINTF fragments as well. +# APEX files in /$partition/apex can have VINTF fragments as well. check_vintf_common_srcs_patterns := \ $(TARGET_OUT)/etc/vintf/% \ $(TARGET_OUT_VENDOR)/etc/vintf/% \ $(TARGET_OUT_ODM)/etc/vintf/% \ $(TARGET_OUT_PRODUCT)/etc/vintf/% \ $(TARGET_OUT_SYSTEM_EXT)/etc/vintf/% \ - $(TARGET_OUT_VENDOR)/apex/% \ + $(apex_dirs) check_vintf_common_srcs := $(sort $(filter $(check_vintf_common_srcs_patterns),$(INTERNAL_ALLIMAGES_FILES))) check_vintf_common_srcs_patterns := @@ -4714,15 +5154,20 @@ ifneq (,$(filter EMPTY_VENDOR_SKU_PLACEHOLDER,$(DEVICE_MANIFEST_SKUS))) $(error EMPTY_VENDOR_SKU_PLACEHOLDER is an internal variable and cannot be used for DEIVCE_MANIFEST_SKUS) endif -# -- Check system manifest / matrix including fragments (excluding other framework manifests / matrices, e.g. product); -check_vintf_system_deps := $(filter $(TARGET_OUT)/etc/vintf/%, $(check_vintf_common_srcs)) +# -- Check system and system_ext manifests / matrices including fragments (excluding other framework manifests / matrices, e.g. product); +ifdef BUILDING_SYSTEM_IMAGE +check_vintf_system_deps := $(filter $(TARGET_OUT)/etc/vintf/% \ + $(TARGET_OUT_SYSTEM_EXT)/etc/vintf/% \ + $(TARGET_OUT)/apex/% \ + $(TARGET_OUT_SYSTEM_EXT)/apex/%, \ + $(check_vintf_common_srcs)) ifneq ($(check_vintf_system_deps),) check_vintf_has_system := true check_vintf_system_log := $(intermediates)/check_vintf_system.log check_vintf_all_deps += $(check_vintf_system_log) -$(check_vintf_system_log): $(HOST_OUT_EXECUTABLES)/checkvintf $(check_vintf_system_deps) - @( $< --check-one --dirmap /system:$(TARGET_OUT) > $@ 2>&1 ) || ( cat $@ && exit 1 ) +$(check_vintf_system_log): $(HOST_OUT_EXECUTABLES)/checkvintf $(check_vintf_system_deps) $(APEX_INFO_FILE) + @( $< --check-one --dirmap /system:$(TARGET_OUT) --dirmap /apex:$(APEX_OUT) > $@ 2>&1 ) || ( cat $@ && exit 1 ) $(call declare-1p-target,$(check_vintf_system_log)) check_vintf_system_log := @@ -4731,8 +5176,11 @@ ifneq (true, $(BUILDING_VENDOR_IMAGE)) vintffm_log := $(intermediates)/vintffm.log endif check_vintf_all_deps += $(vintffm_log) -$(vintffm_log): $(HOST_OUT_EXECUTABLES)/vintffm $(check_vintf_system_deps) +$(vintffm_log): $(HOST_OUT_EXECUTABLES)/vintffm $(check_vintf_system_deps) $(APEX_INFO_FILE) @( $< --check --dirmap /system:$(TARGET_OUT) \ + --dirmap /system_ext:$(TARGET_OUT_SYSTEM_EXT) \ + --dirmap /product:$(TARGET_OUT_PRODUCT) \ + --dirmap /apex:$(APEX_OUT) \ $(VINTF_FRAMEWORK_MANIFEST_FROZEN_DIR) > $@ 2>&1 ) || ( cat $@ && exit 1 ) $(call declare-1p-target,$(vintffm_log)) @@ -4740,9 +5188,12 @@ $(call declare-1p-target,$(vintffm_log)) endif # check_vintf_system_deps check_vintf_system_deps := +endif # BUILDING_SYSTEM_IMAGE + # -- Check vendor manifest / matrix including fragments (excluding other device manifests / matrices) -check_vintf_vendor_deps := $(filter $(TARGET_OUT_VENDOR)/etc/vintf/%, $(check_vintf_common_srcs)) -check_vintf_vendor_deps += $(filter $(TARGET_OUT_VENDOR)/apex/%, $(check_vintf_common_srcs)) +check_vintf_vendor_deps := $(filter $(TARGET_OUT_VENDOR)/etc/vintf/% \ + $(TARGET_OUT_VENDOR)/apex/%, \ + $(check_vintf_common_srcs)) ifneq ($(strip $(check_vintf_vendor_deps)),) check_vintf_has_vendor := true check_vintf_vendor_log := $(intermediates)/check_vintf_vendor.log @@ -4772,40 +5223,48 @@ BUILT_KERNEL_VERSION_FILE := $(intermediates)/kernel_version.txt my_board_extracted_kernel := +# Tools for decompression that is not in PATH. +# Check $(EXTRACT_KERNEL) for decompression algorithms supported by the script. +# Algorithms that are in the script but not in this list will be found in PATH. +my_decompress_tools := \ + lz4:$(HOST_OUT_EXECUTABLES)/lz4 \ + + # BOARD_KERNEL_CONFIG_FILE and BOARD_KERNEL_VERSION can be used to override the values extracted # from INSTALLED_KERNEL_TARGET. -ifdef BOARD_KERNEL_CONFIG_FILE ifdef BOARD_KERNEL_VERSION +$(BUILT_KERNEL_VERSION_FILE): PRIVATE_DECOMPRESS_TOOLS := $(my_decompress_tools) +$(BUILT_KERNEL_VERSION_FILE): $(foreach pair,$(my_decompress_tools),$(call word-colon,2,$(pair))) +$(BUILT_KERNEL_VERSION_FILE): $(EXTRACT_KERNEL) $(firstword $(INSTALLED_KERNEL_TARGET)) + KERNEL_RELEASE=`$(EXTRACT_KERNEL) --tools $(PRIVATE_DECOMPRESS_TOOLS) --input $(firstword $(INSTALLED_KERNEL_TARGET)) \ + --output-release` ;\ + if [ "$$KERNEL_RELEASE" != '$(BOARD_KERNEL_VERSION)' ]; then \ + echo "Specified kernel version '$(BOARD_KERNEL_VERSION)' does not match actual kernel version '$$KERNEL_RELEASE' " ; exit 1; fi; + echo '$(BOARD_KERNEL_VERSION)' > $@ + +ifdef BOARD_KERNEL_CONFIG_FILE $(BUILT_KERNEL_CONFIGS_FILE): $(BOARD_KERNEL_CONFIG_FILE) cp $< $@ -$(BUILT_KERNEL_VERSION_FILE): - echo $(BOARD_KERNEL_VERSION) > $@ $(call declare-license-metadata,$(BUILT_KERNEL_CONFIGS_FILE),SPDX-license-identifier-GPL-2.0-only,restricted,$(BUILD_SYSTEM)/LINUX_KERNEL_COPYING,"Kernel",kernel) $(call declare-license-metadata,$(BUILT_KERNEL_VERSION_FILE),SPDX-license-identifier-GPL-2.0-only,restricted,$(BUILD_SYSTEM)/LINUX_KERNEL_COPYING,"Kernel",kernel) my_board_extracted_kernel := true -endif # BOARD_KERNEL_VERSION endif # BOARD_KERNEL_CONFIG_FILE +endif # BOARD_KERNEL_VERSION -ifneq ($(my_board_extracted_kernel),true) -# Tools for decompression that is not in PATH. -# Check $(EXTRACT_KERNEL) for decompression algorithms supported by the script. -# Algorithms that are in the script but not in this list will be found in PATH. -my_decompress_tools := \ - lz4:$(HOST_OUT_EXECUTABLES)/lz4 \ - -endif # my_board_extracted_kernel ifneq ($(my_board_extracted_kernel),true) ifdef INSTALLED_KERNEL_TARGET +ifndef BOARD_KERNEL_VERSION $(BUILT_KERNEL_CONFIGS_FILE): .KATI_IMPLICIT_OUTPUTS := $(BUILT_KERNEL_VERSION_FILE) +endif $(BUILT_KERNEL_CONFIGS_FILE): PRIVATE_DECOMPRESS_TOOLS := $(my_decompress_tools) $(BUILT_KERNEL_CONFIGS_FILE): $(foreach pair,$(my_decompress_tools),$(call word-colon,2,$(pair))) $(BUILT_KERNEL_CONFIGS_FILE): $(EXTRACT_KERNEL) $(firstword $(INSTALLED_KERNEL_TARGET)) $< --tools $(PRIVATE_DECOMPRESS_TOOLS) --input $(firstword $(INSTALLED_KERNEL_TARGET)) \ --output-configs $@ \ - --output-release $(BUILT_KERNEL_VERSION_FILE) + $(if $(BOARD_KERNEL_VERSION),,--output-release $(BUILT_KERNEL_VERSION_FILE)) $(call declare-license-metadata,$(BUILT_KERNEL_CONFIGS_FILE),SPDX-license-identifier-GPL-2.0-only,restricted,$(BUILD_SYSTEM)/LINUX_KERNEL_COPYING,"Kernel",kernel) @@ -4834,7 +5293,9 @@ my_board_extracted_kernel := true endif # INSTALLED_BOOTIMAGE_TARGET endif # my_board_extracted_kernel -ifneq ($(my_board_extracted_kernel),true) +ifeq ($(my_board_extracted_kernel),true) +$(call dist-for-goals, droid_targets, $(BUILT_KERNEL_VERSION_FILE)) +else $(warning Neither INSTALLED_KERNEL_TARGET nor INSTALLED_BOOTIMAGE_TARGET is defined when \ PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS is true. Information about the updated kernel \ cannot be built into OTA update package. You can fix this by: \ @@ -4854,6 +5315,34 @@ my_board_extracted_kernel := endif # PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS +ifeq (default,$(ENABLE_UFFD_GC)) + +ifneq (,$(BUILT_KERNEL_VERSION_FILE)) +$(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC): $(BUILT_KERNEL_VERSION_FILE) +$(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC): + cp $(BUILT_KERNEL_VERSION_FILE) $(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC) +else +# We make this a warning rather than an error to avoid breaking too many builds. When it happens, +# we use a placeholder as the kernel version, which is consumed by uffd_gc_utils.py. +$(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC): + echo $$'\ +Unable to determine UFFD GC flag because the kernel version is not available and\n\ +PRODUCT_ENABLE_UFFD_GC is "default".\n\ +You can fix this by:\n\ + 1. [Recommended] Making the kernel version available.\n\ + (1). Set PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS to "true".\n\ + (2). If you are still getting this message after doing so, see the warning about\n\ + PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS in the build logs.\n\ + or\n\ + 2. Explicitly setting PRODUCT_ENABLE_UFFD_GC to "true" or "false" based on the kernel version.\n\ + (1). Set PRODUCT_ENABLE_UFFD_GC to "true" if the kernel is a GKI kernel and is android12-5.4\n\ + or above, or a non-GKI kernel that supports userfaultfd(2) and MREMAP_DONTUNMAP.\n\ + (2). Set PRODUCT_ENABLE_UFFD_GC to "false" otherwise.'\ + && echo '<unknown-kernel>' > $@ +endif # BUILT_KERNEL_VERSION_FILE + +endif # ENABLE_UFFD_GC == "default" + # -- Check VINTF compatibility of build. # Skip partial builds; only check full builds. Only check if: # - PRODUCT_ENFORCE_VINTF_MANIFEST is true @@ -5073,6 +5562,7 @@ INTERNAL_OTATOOLS_MODULES := \ check_target_files_signatures \ check_target_files_vintf \ checkvintf \ + create_brick_ota \ delta_generator \ e2fsck \ e2fsdroid \ @@ -5081,7 +5571,6 @@ INTERNAL_OTATOOLS_MODULES := \ fsck.erofs \ fsck.f2fs \ fs_config \ - generate_gki_certificate \ generate_verity_key \ host_init_verifier \ img2simg \ @@ -5094,8 +5583,8 @@ INTERNAL_OTATOOLS_MODULES := \ lz4 \ make_f2fs \ make_f2fs_casefold \ + merge_ota \ merge_target_files \ - minigzip \ mk_combined_img \ mkbootfs \ mkbootimg \ @@ -5122,6 +5611,7 @@ INTERNAL_OTATOOLS_MODULES := \ toybox \ tune2fs \ unpack_bootimg \ + update_device \ update_host_simulator \ validate_target_files \ verity_signer \ @@ -5133,13 +5623,13 @@ INTERNAL_OTATOOLS_MODULES := \ # Additional tools to unpack and repack the apex file. INTERNAL_OTATOOLS_MODULES += \ + apexd_host \ apexer \ apex_compression_tool \ - blkid_static \ deapexer \ debugfs_static \ - dump_apex_info \ fsck.erofs \ + make_erofs \ merge_zips \ resize2fs \ soong_zip \ @@ -5214,9 +5704,88 @@ $(call declare-container-license-deps,$(INTERNAL_OTATOOLS_PACKAGE_FILES) $(INTER .PHONY: otatools-package otatools-package: $(BUILT_OTATOOLS_PACKAGE) +$(call dist-for-goals, otatools-package, \ + $(BUILT_OTATOOLS_PACKAGE) \ +) + endif # build_otatools_package # ----------------------------------------------------------------- +# fastboot-info.txt +FASTBOOT_INFO_VERSION = 1 + +INSTALLED_FASTBOOT_INFO_TARGET := $(PRODUCT_OUT)/fastboot-info.txt +ifdef TARGET_BOARD_FASTBOOT_INFO_FILE +$(INSTALLED_FASTBOOT_INFO_TARGET): $(TARGET_BOARD_FASTBOOT_INFO_FILE) + rm -f $@ + $(call pretty,"Target fastboot-info.txt: $@") + $(hide) cp $< $@ +else +$(INSTALLED_FASTBOOT_INFO_TARGET): + rm -f $@ + $(call pretty,"Target fastboot-info.txt: $@") + $(hide) echo "# fastboot-info for $(TARGET_PRODUCT)" >> $@ + $(hide) echo "version $(FASTBOOT_INFO_VERSION)" >> $@ +ifneq ($(INSTALLED_BOOTIMAGE_TARGET),) + $(hide) echo "flash boot" >> $@ +endif +ifneq ($(INSTALLED_INIT_BOOT_IMAGE_TARGET),) + $(hide) echo "flash init_boot" >> $@ +endif +ifdef BOARD_PREBUILT_DTBOIMAGE + $(hide) echo "flash dtbo" >> $@ +endif +ifneq ($(INSTALLED_DTIMAGE_TARGET),) + $(hide) echo "flash dts dt.img" >> $@ +endif +ifneq ($(INSTALLED_VENDOR_KERNEL_BOOTIMAGE_TARGET),) + $(hide) echo "flash vendor_kernel_boot" >> $@ +endif +ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),) + $(hide) echo "flash recovery" >> $@ +endif +ifeq ($(BOARD_USES_PVMFWIMAGE),true) + $(hide) echo "flash pvmfw" >> $@ +endif +ifneq ($(INSTALLED_VENDOR_BOOTIMAGE_TARGET),) + $(hide) echo "flash vendor_boot" >> $@ +endif +ifeq ($(BOARD_AVB_ENABLE),true) +ifeq ($(BUILDING_VBMETA_IMAGE),true) + $(hide) echo "flash --apply-vbmeta vbmeta" >> $@ +endif +ifneq (,$(strip $(BOARD_AVB_VBMETA_SYSTEM))) + $(hide) echo "flash vbmeta_system" >> $@ +endif +ifneq (,$(strip $(BOARD_AVB_VBMETA_VENDOR))) + $(hide) echo "flash vbmeta_vendor" >> $@ +endif +ifneq (,$(strip $(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS))) + $(hide) $(foreach partition,$(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS), \ + echo "flash vbmeta_$(partition)" >> $@;) +endif +endif # BOARD_AVB_ENABLE +ifneq (,$(strip $(BOARD_CUSTOMIMAGES_PARTITION_LIST))) + $(hide) $(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ + echo "flash $(partition)" >> $@;) +endif + $(hide) echo "reboot fastboot" >> $@ + $(hide) echo "update-super" >> $@ + $(hide) $(foreach partition,$(BOARD_SUPER_PARTITION_PARTITION_LIST), \ + echo "flash $(partition)" >> $@;) +ifdef BUILDING_SYSTEM_OTHER_IMAGE + $(hide) echo "flash --slot-other system system_other.img" >> $@ +endif +ifdef BUILDING_CACHE_IMAGE + $(hide) echo "if-wipe erase cache" >> $@ +endif + $(hide) echo "if-wipe erase userdata" >> $@ +ifeq ($(BOARD_USES_METADATA_PARTITION),true) + $(hide) echo "if-wipe erase metadata" >> $@ +endif +endif + +# ----------------------------------------------------------------- # misc_info.txt INSTALLED_MISC_INFO_TARGET := $(PRODUCT_OUT)/misc_info.txt @@ -5296,11 +5865,6 @@ endif $(hide) echo 'recovery_mkbootimg_args=$(BOARD_RECOVERY_MKBOOTIMG_ARGS)' >> $@ $(hide) echo 'mkbootimg_version_args=$(INTERNAL_MKBOOTIMG_VERSION_ARGS)' >> $@ $(hide) echo 'mkbootimg_init_args=$(BOARD_MKBOOTIMG_INIT_ARGS)' >> $@ -ifdef BOARD_GKI_SIGNING_KEY_PATH - $(hide) echo 'gki_signing_key_path=$(BOARD_GKI_SIGNING_KEY_PATH)' >> $@ - $(hide) echo 'gki_signing_algorithm=$(BOARD_GKI_SIGNING_ALGORITHM)' >> $@ - $(hide) echo 'gki_signing_signature_args=$(BOARD_GKI_SIGNING_SIGNATURE_ARGS)' >> $@ -endif $(hide) echo "multistage_support=1" >> $@ $(hide) echo "blockimgdiff_versions=3,4" >> $@ ifeq ($(PRODUCT_BUILD_GENERIC_OTA_PACKAGE),true) @@ -5359,15 +5923,20 @@ ifdef BOARD_AVB_RECOVERY_KEY_PATH $(hide) echo "avb_recovery_rollback_index_location=$(BOARD_AVB_RECOVERY_ROLLBACK_INDEX_LOCATION)" >> $@ endif # BOARD_AVB_RECOVERY_KEY_PATH ifneq (,$(strip $(BOARD_CUSTOMIMAGES_PARTITION_LIST))) - $(hide) echo "avb_custom_images_partition_list=$(BOARD_CUSTOMIMAGES_PARTITION_LIST)" >> $@ - $(hide) $(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ + $(hide) echo "custom_images_partition_list=$(filter-out $(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), $(BOARD_CUSTOMIMAGES_PARTITION_LIST))" >> $@ + $(hide) $(foreach partition,$(filter-out $(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), $(BOARD_CUSTOMIMAGES_PARTITION_LIST)), \ + echo "$(partition)_image_list=$(foreach image,$(BOARD_$(call to-upper,$(partition))_IMAGE_LIST),$(notdir $(image)))" >> $@;) +endif # BOARD_CUSTOMIMAGES_PARTITION_LIST +ifneq (,$(strip $(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST))) + $(hide) echo "avb_custom_images_partition_list=$(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST)" >> $@ + $(hide) $(foreach partition,$(BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST), \ echo "avb_$(partition)_key_path=$(BOARD_AVB_$(call to-upper,$(partition))_KEY_PATH)" >> $@; \ echo "avb_$(partition)_algorithm=$(BOARD_AVB_$(call to-upper,$(partition))_ALGORITHM)" >> $@; \ echo "avb_$(partition)_add_hashtree_footer_args=$(BOARD_AVB_$(call to-upper,$(partition))_ADD_HASHTREE_FOOTER_ARGS)" >> $@; \ echo "avb_$(partition)_rollback_index_location=$(BOARD_AVB_$(call to-upper,$(partition))_ROLLBACK_INDEX_LOCATION)" >> $@; \ echo "avb_$(partition)_partition_size=$(BOARD_AVB_$(call to-upper,$(partition))_PARTITION_SIZE)" >> $@; \ echo "avb_$(partition)_image_list=$(foreach image,$(BOARD_AVB_$(call to-upper,$(partition))_IMAGE_LIST),$(notdir $(image)))" >> $@;) -endif # BOARD_CUSTOMIMAGES_PARTITION_LIST +endif # BOARD_AVB_CUSTOMIMAGES_PARTITION_LIST ifneq (,$(strip $(BOARD_AVB_VBMETA_SYSTEM))) $(hide) echo "avb_vbmeta_system=$(BOARD_AVB_VBMETA_SYSTEM)" >> $@ $(hide) echo "avb_vbmeta_system_args=$(BOARD_AVB_MAKE_VBMETA_SYSTEM_IMAGE_ARGS)" >> $@ @@ -5392,14 +5961,6 @@ ifneq (,$(strip $(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS))) echo "avb_vbmeta_$(partition)_rollback_index_location=$(BOARD_AVB_VBMETA_$(call to-upper,$(partition))_ROLLBACK_INDEX_LOCATION)" >> $@ ;) endif # BOARD_AVB_VBMETA_CUSTOM_PARTITIONS endif # BOARD_AVB_ENABLE -ifdef BOARD_BPT_INPUT_FILES - $(hide) echo "board_bpt_enable=true" >> $@ - $(hide) echo "board_bpt_make_table_args=$(BOARD_BPT_MAKE_TABLE_ARGS)" >> $@ - $(hide) echo "board_bpt_input_files=$(BOARD_BPT_INPUT_FILES)" >> $@ -endif -ifdef BOARD_BPT_DISK_SIZE - $(hide) echo "board_bpt_disk_size=$(BOARD_BPT_DISK_SIZE)" >> $@ -endif $(call generate-userimage-prop-dictionary, $@) ifeq ($(AB_OTA_UPDATER),true) @# Include the build type in META/misc_info.txt so the server can easily differentiate production builds. @@ -5409,6 +5970,9 @@ endif ifeq ($(TARGET_OTA_ALLOW_NON_AB),true) $(hide) echo "allow_non_ab=true" >> $@ endif +ifeq ($(BOARD_NON_AB_OTA_DISABLE_COMPRESSION),true) + $(hide) echo "board_non_ab_ota_disable_compression=true" >> $@ +endif ifdef BOARD_PREBUILT_DTBOIMAGE $(hide) echo "has_dtbo=true" >> $@ ifeq ($(BOARD_AVB_ENABLE),true) @@ -5462,9 +6026,13 @@ endif ifeq ($(BUILDING_WITH_VSDK),true) $(hide) echo "building_with_vsdk=true" >> $@ endif -ifeq ($(TARGET_FLATTEN_APEX),false) - $(hide) echo "target_flatten_apex=false" >> $@ -endif + +$(call declare-0p-target,$(INSTALLED_FASTBOOT_INFO_TARGET)) + +.PHONY: fastboot_info +fastboot_info: $(INSTALLED_FASTBOOT_INFO_TARGET) + +droidcore-unbundled: $(INSTALLED_FASTBOOT_INFO_TARGET) $(call declare-0p-target,$(INSTALLED_MISC_INFO_TARGET)) @@ -5482,21 +6050,23 @@ name := $(TARGET_PRODUCT) ifeq ($(TARGET_BUILD_TYPE),debug) name := $(name)_debug endif -name := $(name)-target_files-$(FILE_NAME_TAG) +name := $(name)-target_files intermediates := $(call intermediates-dir-for,PACKAGING,target_files) +BUILT_TARGET_FILES_DIR := $(intermediates)/$(name).zip.list BUILT_TARGET_FILES_PACKAGE := $(intermediates)/$(name).zip -$(BUILT_TARGET_FILES_PACKAGE): intermediates := $(intermediates) -$(BUILT_TARGET_FILES_PACKAGE): \ - zip_root := $(intermediates)/$(name) +$(BUILT_TARGET_FILES_PACKAGE): zip_root := $(intermediates)/$(name) +$(BUILT_TARGET_FILES_DIR): zip_root := $(intermediates)/$(name) +$(BUILT_TARGET_FILES_DIR): intermediates := $(intermediates) + # $(1): Directory to copy # $(2): Location to copy it to -# The "ls -A" is to prevent "acp s/* d" from failing if s is empty. +# The "ls -A" is to skip if $(1) is empty. define package_files-copy-root if [ -d "$(strip $(1))" -a "$$(ls -A $(1))" ]; then \ mkdir -p $(2) && \ - $(ACP) -rd $(strip $(1))/* $(2); \ + $(ACP) -rd $(strip $(1))/. $(strip $(2))/; \ fi endef @@ -5508,10 +6078,10 @@ built_ota_tools += \ $(call intermediates-dir-for,EXECUTABLES,updater)/updater endif -$(BUILT_TARGET_FILES_PACKAGE): PRIVATE_OTA_TOOLS := $(built_ota_tools) +$(BUILT_TARGET_FILES_DIR): PRIVATE_OTA_TOOLS := $(built_ota_tools) tool_extension := $(wildcard $(tool_extensions)/releasetools.py) -$(BUILT_TARGET_FILES_PACKAGE): PRIVATE_TOOL_EXTENSION := $(tool_extension) +$(BUILT_TARGET_FILES_DIR): PRIVATE_TOOL_EXTENSION := $(tool_extension) updater_dep := ifeq ($(AB_OTA_UPDATER),true) @@ -5527,23 +6097,23 @@ ifeq ($(TARGET_OTA_ALLOW_NON_AB),true) updater_dep += $(built_ota_tools) endif -$(BUILT_TARGET_FILES_PACKAGE): $(updater_dep) +$(BUILT_TARGET_FILES_DIR): $(updater_dep) # If we are using recovery as boot, output recovery files to BOOT/. # If we are moving recovery resources to vendor_boot, output recovery files to VENDOR_BOOT/. ifeq ($(BOARD_USES_RECOVERY_AS_BOOT),true) -$(BUILT_TARGET_FILES_PACKAGE): PRIVATE_RECOVERY_OUT := BOOT +$(BUILT_TARGET_FILES_DIR): PRIVATE_RECOVERY_OUT := BOOT else ifeq ($(BOARD_MOVE_RECOVERY_RESOURCES_TO_VENDOR_BOOT),true) -$(BUILT_TARGET_FILES_PACKAGE): PRIVATE_RECOVERY_OUT := VENDOR_BOOT +$(BUILT_TARGET_FILES_DIR): PRIVATE_RECOVERY_OUT := VENDOR_BOOT else -$(BUILT_TARGET_FILES_PACKAGE): PRIVATE_RECOVERY_OUT := RECOVERY +$(BUILT_TARGET_FILES_DIR): PRIVATE_RECOVERY_OUT := RECOVERY endif ifeq ($(AB_OTA_UPDATER),true) ifdef OSRELEASED_DIRECTORY - $(BUILT_TARGET_FILES_PACKAGE): $(TARGET_OUT_OEM)/$(OSRELEASED_DIRECTORY)/product_id - $(BUILT_TARGET_FILES_PACKAGE): $(TARGET_OUT_OEM)/$(OSRELEASED_DIRECTORY)/product_version - $(BUILT_TARGET_FILES_PACKAGE): $(TARGET_OUT_ETC)/$(OSRELEASED_DIRECTORY)/system_version + $(BUILT_TARGET_FILES_DIR): $(TARGET_OUT_OEM)/$(OSRELEASED_DIRECTORY)/product_id + $(BUILT_TARGET_FILES_DIR): $(TARGET_OUT_OEM)/$(OSRELEASED_DIRECTORY)/product_version + $(BUILT_TARGET_FILES_DIR): $(TARGET_OUT_ETC)/$(OSRELEASED_DIRECTORY)/system_version endif # Not checking in board_config.mk, since AB_OTA_PARTITIONS may be updated in Android.mk (e.g. to @@ -5563,7 +6133,7 @@ endif # $1: root directory # $2: add prefix define fs_config -(cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) -R "$(2)" +(cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -R "$(2)" endef define filter-out-missing-vendor @@ -5645,34 +6215,67 @@ define dump-dynamic-partitions-info echo "virtual_ab_compression_method=$(PRODUCT_VIRTUAL_AB_COMPRESSION_METHOD)" >> $(1)) $(if $(filter true,$(PRODUCT_VIRTUAL_AB_OTA_RETROFIT)), \ echo "virtual_ab_retrofit=true" >> $(1)) + $(if $(PRODUCT_VIRTUAL_AB_COW_VERSION), \ + echo "virtual_ab_cow_version=$(PRODUCT_VIRTUAL_AB_COW_VERSION)" >> $(1)) + $(if $(PRODUCT_VIRTUAL_AB_COMPRESSION_FACTOR), \ + echo "virtual_ab_compression_factor=$(PRODUCT_VIRTUAL_AB_COMPRESSION_FACTOR)" >> $(1)) +endef + +# Copy an image file to a directory and generate a block list map file from the image, +# only if the map_file_generator supports the file system. +# Otherwise, skip generating map files as well as copying images. The image will be +# generated from the $(ADD_IMG_TO_TARGET_FILES) to generate the map file with it. +# $(1): path of the image file +# $(2): target out directory +# $(3): image name to generate a map file. skip generating map file if empty +define copy-image-and-generate-map + $(if $(COPY_IMAGES_FOR_TARGET_FILES_ZIP), \ + $(eval _supported_fs_for_map_file_generator := erofs ext%) \ + $(eval _img := $(call to-upper,$(3))) \ + $(if $(3),$(eval _map_fs_type := $(BOARD_$(_img)IMAGE_FILE_SYSTEM_TYPE)),\ + $(eval _no_map_file := "true")) \ + $(if $(filter $(_supported_fs_for_map_file_generator),$(_map_fs_type))$(_no_map_file),\ + mkdir -p $(2); \ + cp $(1) $(2); \ + $(if $(3),$(HOST_OUT_EXECUTABLES)/map_file_generator $(1) $(2)/$(3).map)) \ + $(eval _img :=) \ + $(eval _map_fs_type :=) \ + $(eval _no_map_file :=) \ + ) endef # By conditionally including the dependency of the target files package on the # full system image deps, we speed up builds that do not build the system # image. ifdef BUILDING_SYSTEM_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(FULL_SYSTEMIMAGE_DEPS) + $(BUILT_TARGET_FILES_DIR): $(FULL_SYSTEMIMAGE_DEPS) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_SYSTEMIMAGE) + endif else # releasetools may need the system build.prop even when building a # system-image-less product. - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_BUILD_PROP_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_BUILD_PROP_TARGET) endif ifdef BUILDING_USERDATA_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_USERDATAIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_USERDATAIMAGE_FILES) endif ifdef BUILDING_SYSTEM_OTHER_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_SYSTEMOTHERIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_SYSTEMOTHERIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_SYSTEMOTHERIMAGE_TARGET) + endif endif ifdef BUILDING_VENDOR_BOOT_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_VENDOR_RAMDISK_FILES) - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_VENDOR_RAMDISK_FRAGMENT_TARGETS) - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_VENDOR_BOOTCONFIG_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_VENDOR_RAMDISK_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_VENDOR_RAMDISK_FRAGMENT_TARGETS) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_VENDOR_BOOTCONFIG_TARGET) # The vendor ramdisk may be built from the recovery ramdisk. ifeq (true,$(BOARD_MOVE_RECOVERY_RESOURCES_TO_VENDOR_BOOT)) - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_RECOVERY_RAMDISK_FILES_TIMESTAMP) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_RECOVERY_RAMDISK_FILES_TIMESTAMP) endif endif @@ -5682,11 +6285,11 @@ ifdef BUILDING_RECOVERY_IMAGE # commands in build-recoveryimage-target, which would touch the files under # TARGET_RECOVERY_OUT and race with packaging target-files.zip. ifeq ($(BOARD_USES_RECOVERY_AS_BOOT),true) - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_BOOTIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_BOOTIMAGE_TARGET) else - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_RECOVERYIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_RECOVERYIMAGE_TARGET) endif - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_RECOVERYIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_RECOVERYIMAGE_FILES) endif # Conditionally depend on the image files if the image is being built so the @@ -5694,68 +6297,89 @@ endif # if it is coming from a prebuilt. ifdef BUILDING_VENDOR_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_VENDORIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_VENDORIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_VENDORIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_VENDORIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_VENDORIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_VENDORIMAGE_TARGET) endif ifdef BUILDING_PRODUCT_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_PRODUCTIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_PRODUCTIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_PRODUCTIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_PRODUCTIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_PRODUCTIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_PRODUCTIMAGE_TARGET) endif ifdef BUILDING_SYSTEM_EXT_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_SYSTEM_EXTIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_SYSTEM_EXTIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_SYSTEM_EXTIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_SYSTEM_EXTIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_SYSTEM_EXTIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_SYSTEM_EXTIMAGE_TARGET) endif ifneq (,$(BUILDING_BOOT_IMAGE)$(BUILDING_INIT_BOOT_IMAGE)) - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_RAMDISK_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_RAMDISK_FILES) endif # BUILDING_BOOT_IMAGE != "" || BUILDING_INIT_BOOT_IMAGE != "" ifneq (,$(INTERNAL_PREBUILT_BOOTIMAGE) $(filter true,$(BOARD_COPY_BOOT_IMAGE_TO_TARGET_FILES))) - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_BOOTIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_BOOTIMAGE_TARGET) endif ifdef BUILDING_ODM_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_ODMIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_ODMIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_ODMIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_ODMIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_ODMIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_ODMIMAGE_TARGET) endif ifdef BUILDING_VENDOR_DLKM_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_VENDOR_DLKMIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_VENDOR_DLKMIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_VENDOR_DLKMIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_VENDOR_DLKMIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_VENDOR_DLKMIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_VENDOR_DLKMIMAGE_TARGET) endif ifdef BUILDING_ODM_DLKM_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_ODM_DLKMIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_ODM_DLKMIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_ODM_DLKMIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_ODM_DLKMIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_ODM_DLKMIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_ODM_DLKMIMAGE_TARGET) endif ifdef BUILDING_SYSTEM_DLKM_IMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INTERNAL_SYSTEM_DLKMIMAGE_FILES) + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_SYSTEM_DLKMIMAGE_FILES) + ifdef COPY_IMAGES_FOR_TARGET_FILES_ZIP + $(BUILT_TARGET_FILES_DIR): $(BUILT_SYSTEM_DLKMIMAGE_TARGET) + endif else ifdef BOARD_PREBUILT_SYSTEM_DLKMIMAGE - $(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) + $(BUILT_TARGET_FILES_DIR): $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) endif ifeq ($(BUILD_QEMU_IMAGES),true) MK_VBMETA_BOOT_KERNEL_CMDLINE_SH := device/generic/goldfish/tools/mk_vbmeta_boot_params.sh - $(BUILT_TARGET_FILES_PACKAGE): $(MK_VBMETA_BOOT_KERNEL_CMDLINE_SH) + $(BUILT_TARGET_FILES_DIR): $(MK_VBMETA_BOOT_KERNEL_CMDLINE_SH) endif ifdef BOARD_PREBUILT_BOOTLOADER -$(BUILT_TARGET_FILES_PACKAGE): $(INSTALLED_BOOTLOADER_MODULE) +$(BUILT_TARGET_FILES_DIR): $(INSTALLED_BOOTLOADER_MODULE) droidcore-unbundled: $(INSTALLED_BOOTLOADER_MODULE) endif # Depending on the various images guarantees that the underlying # directories are up-to-date. -$(BUILT_TARGET_FILES_PACKAGE): \ +$(BUILT_TARGET_FILES_DIR): \ $(INSTALLED_RADIOIMAGE_TARGET) \ $(INSTALLED_RECOVERYIMAGE_TARGET) \ $(INSTALLED_CACHEIMAGE_TARGET) \ @@ -5769,6 +6393,8 @@ $(BUILT_TARGET_FILES_PACKAGE): \ $(INSTALLED_RAMDISK_TARGET) \ $(INSTALLED_DTBIMAGE_TARGET) \ $(INSTALLED_2NDBOOTLOADER_TARGET) \ + $(BUILT_RAMDISK_16K_TARGET) \ + $(BUILT_KERNEL_16K_TARGET) \ $(BOARD_PREBUILT_DTBOIMAGE) \ $(BOARD_PREBUILT_RECOVERY_DTBOIMAGE) \ $(BOARD_RECOVERY_ACPIO) \ @@ -5783,16 +6409,18 @@ $(BUILT_TARGET_FILES_PACKAGE): \ $(LPMAKE) \ $(SELINUX_FC) \ $(INSTALLED_MISC_INFO_TARGET) \ + $(INSTALLED_FASTBOOT_INFO_TARGET) \ $(APKCERTS_FILE) \ - $(SOONG_APEX_KEYS_FILE) \ + $(APEX_KEYS_FILE) \ $(SOONG_ZIP) \ $(HOST_OUT_EXECUTABLES)/fs_config \ + $(HOST_OUT_EXECUTABLES)/map_file_generator \ $(ADD_IMG_TO_TARGET_FILES) \ $(MAKE_RECOVERY_PATCH) \ $(BUILT_KERNEL_CONFIGS_FILE) \ $(BUILT_KERNEL_VERSION_FILE) \ | $(ACP) - @echo "Package target files: $@" + @echo "Building target files: $@" $(hide) rm -rf $@ $@.list $(zip_root) $(hide) mkdir -p $(dir $@) $(zip_root) ifneq (,$(INSTALLED_RECOVERYIMAGE_TARGET)$(filter true,$(BOARD_USES_RECOVERY_AS_BOOT))$(filter true,$(BOARD_MOVE_RECOVERY_RESOURCES_TO_VENDOR_BOOT))) @@ -5994,13 +6622,16 @@ endif @# build them. $(hide) mkdir -p $(zip_root)/META $(hide) cp $(APKCERTS_FILE) $(zip_root)/META/apkcerts.txt - $(hide) cp $(SOONG_APEX_KEYS_FILE) $(zip_root)/META/apexkeys.txt + $(hide) cp $(APEX_KEYS_FILE) $(zip_root)/META/apexkeys.txt ifneq ($(tool_extension),) $(hide) cp $(PRIVATE_TOOL_EXTENSION) $(zip_root)/META/ endif $(hide) echo "$(PRODUCT_OTA_PUBLIC_KEYS)" > $(zip_root)/META/otakeys.txt $(hide) cp $(SELINUX_FC) $(zip_root)/META/file_contexts.bin $(hide) cp $(INSTALLED_MISC_INFO_TARGET) $(zip_root)/META/misc_info.txt +ifneq ($(INSTALLED_FASTBOOT_INFO_TARGET),) + $(hide) cp $(INSTALLED_FASTBOOT_INFO_TARGET) $(zip_root)/META/fastboot-info.txt +endif ifneq ($(PRODUCT_SYSTEM_BASE_FS_PATH),) $(hide) cp $(PRODUCT_SYSTEM_BASE_FS_PATH) \ $(zip_root)/META/$(notdir $(PRODUCT_SYSTEM_BASE_FS_PATH)) @@ -6108,6 +6739,14 @@ ifdef BOARD_PREBUILT_DTBOIMAGE $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES $(hide) cp $(INSTALLED_DTBOIMAGE_TARGET) $(zip_root)/PREBUILT_IMAGES/ endif # BOARD_PREBUILT_DTBOIMAGE +ifdef BUILT_KERNEL_16K_TARGET + $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES + $(hide) cp $(BUILT_KERNEL_16K_TARGET) $(zip_root)/PREBUILT_IMAGES/ +endif # BUILT_KERNEL_16K_TARGET +ifdef BUILT_RAMDISK_16K_TARGET + $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES + $(hide) cp $(BUILT_RAMDISK_16K_TARGET) $(zip_root)/PREBUILT_IMAGES/ +endif # BUILT_RAMDISK_16K_TARGET ifeq ($(BOARD_USES_PVMFWIMAGE),true) $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES $(hide) cp $(INSTALLED_PVMFWIMAGE_TARGET) $(zip_root)/PREBUILT_IMAGES/ @@ -6122,7 +6761,7 @@ endif ifneq ($(strip $(BOARD_CUSTOMIMAGES_PARTITION_LIST)),) $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES $(hide) $(foreach partition,$(BOARD_CUSTOMIMAGES_PARTITION_LIST), \ - $(foreach image,$(BOARD_AVB_$(call to-upper,$(partition))_IMAGE_LIST),cp $(image) $(zip_root)/PREBUILT_IMAGES/;)) + $(foreach image,$(BOARD_$(call to-upper,$(partition))_IMAGE_LIST),cp $(image) $(zip_root)/PREBUILT_IMAGES/;)) endif # BOARD_CUSTOMIMAGES_PARTITION_LIST @# The radio images in BOARD_PACK_RADIOIMAGES will be additionally copied from RADIO/ into @# IMAGES/, which then will be added into <product>-img.zip. Such images must be listed in @@ -6132,27 +6771,35 @@ endif # BOARD_CUSTOMIMAGES_PARTITION_LIST @# Run fs_config on all the system, vendor, boot ramdisk, @# and recovery ramdisk files in the zip, and save the output ifdef BUILDING_SYSTEM_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_SYSTEMIMAGE),$(zip_root)/IMAGES,system) $(hide) $(call fs_config,$(zip_root)/SYSTEM,system/) > $(zip_root)/META/filesystem_config.txt endif ifdef BUILDING_VENDOR_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_VENDORIMAGE_TARGET),$(zip_root)/IMAGES,vendor) $(hide) $(call fs_config,$(zip_root)/VENDOR,vendor/) > $(zip_root)/META/vendor_filesystem_config.txt endif ifdef BUILDING_PRODUCT_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_PRODUCTIMAGE_TARGET),$(zip_root)/IMAGES,product) $(hide) $(call fs_config,$(zip_root)/PRODUCT,product/) > $(zip_root)/META/product_filesystem_config.txt endif ifdef BUILDING_SYSTEM_EXT_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_SYSTEM_EXTIMAGE_TARGET),$(zip_root)/IMAGES,system_ext) $(hide) $(call fs_config,$(zip_root)/SYSTEM_EXT,system_ext/) > $(zip_root)/META/system_ext_filesystem_config.txt endif ifdef BUILDING_ODM_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_ODMIMAGE_TARGET),$(zip_root)/IMAGES,odm) $(hide) $(call fs_config,$(zip_root)/ODM,odm/) > $(zip_root)/META/odm_filesystem_config.txt endif ifdef BUILDING_VENDOR_DLKM_IMAGE + $(hide)$(call copy-image-and-generate-map,$(BUILT_VENDOR_DLKMIMAGE_TARGET),$(zip_root)/IMAGES,vendor_dlkm) $(hide) $(call fs_config,$(zip_root)/VENDOR_DLKM,vendor_dlkm/) > $(zip_root)/META/vendor_dlkm_filesystem_config.txt endif ifdef BUILDING_ODM_DLKM_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_ODM_DLKMIMAGE_TARGET),$(zip_root)/IMAGES,odm_dlkm) $(hide) $(call fs_config,$(zip_root)/ODM_DLKM,odm_dlkm/) > $(zip_root)/META/odm_dlkm_filesystem_config.txt endif ifdef BUILDING_SYSTEM_DLKM_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_SYSTEM_DLKMIMAGE_TARGET),$(zip_root)/IMAGES,system_dlkm) $(hide) $(call fs_config,$(zip_root)/SYSTEM_DLKM,system_dlkm/) > $(zip_root)/META/system_dlkm_filesystem_config.txt endif @# ROOT always contains the files for the root under normal boot. @@ -6174,6 +6821,7 @@ ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),) $(hide) $(call fs_config,$(zip_root)/RECOVERY/RAMDISK,) > $(zip_root)/META/recovery_filesystem_config.txt endif ifdef BUILDING_SYSTEM_OTHER_IMAGE + $(hide) $(call copy-image-and-generate-map,$(BUILT_SYSTEMOTHERIMAGE_TARGET),$(zip_root)/IMAGES) $(hide) $(call fs_config,$(zip_root)/SYSTEM_OTHER,system/) > $(zip_root)/META/system_other_filesystem_config.txt endif @# Metadata for compatibility verification. @@ -6195,13 +6843,19 @@ ifeq ($(BUILD_QEMU_IMAGES),true) endif @# Zip everything up, preserving symlinks and placing META/ files first to @# help early validation of the .zip file while uploading it. - $(hide) find $(zip_root)/META | sort >$@.list - $(hide) find $(zip_root) -path $(zip_root)/META -prune -o -print | sort >>$@.list - $(hide) $(SOONG_ZIP) -d -o $@ -C $(zip_root) -r $@.list + $(hide) find $(zip_root)/META | sort >$@ + $(hide) find $(zip_root) -path $(zip_root)/META -prune -o -print | sort >>$@ + +$(BUILT_TARGET_FILES_PACKAGE): $(BUILT_TARGET_FILES_DIR) + @echo "Packaging target files: $@" + $(hide) $(SOONG_ZIP) -d -o $@ -C $(zip_root) -r $@.list -sha256 .PHONY: target-files-package target-files-package: $(BUILT_TARGET_FILES_PACKAGE) +.PHONY: target-files-dir +target-files-dir: $(BUILT_TARGET_FILES_DIR) + $(call declare-1p-container,$(BUILT_TARGET_FILES_PACKAGE),) $(call declare-container-license-deps,$(BUILT_TARGET_FILES_PACKAGE), $(INSTALLED_RADIOIMAGE_TARGET) \ $(INSTALLED_RECOVERYIMAGE_TARGET) \ @@ -6229,15 +6883,17 @@ $(call declare-container-license-deps,$(BUILT_TARGET_FILES_PACKAGE), $(INSTALLED $(LPMAKE) \ $(SELINUX_FC) \ $(INSTALLED_MISC_INFO_TARGET) \ + $(INSTALLED_FASTBOOT_INFO_TARGET) \ $(APKCERTS_FILE) \ - $(SOONG_APEX_KEYS_FILE) \ + $(APEX_KEYS_FILE) \ $(HOST_OUT_EXECUTABLES)/fs_config \ + $(HOST_OUT_EXECUTABLES)/map_file_generator \ $(ADD_IMG_TO_TARGET_FILES) \ $(MAKE_RECOVERY_PATCH) \ $(BUILT_KERNEL_CONFIGS_FILE) \ $(BUILT_KERNEL_VERSION_FILE),$(BUILT_TARGET_FILES_PACKAGE):) -$(call dist-for-goals, target-files-package, $(BUILT_TARGET_FILES_PACKAGE)) +$(call dist-for-goals-with-filenametag, target-files-package, $(BUILT_TARGET_FILES_PACKAGE)) # ----------------------------------------------------------------- # NDK Sysroot Package @@ -6262,18 +6918,18 @@ define build-ota-package-target PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$(dir $(ZIP2ZIP)):$$PATH \ $(OTA_FROM_TARGET_FILES) \ --verbose \ - --extracted_input_target_files $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE)) \ --path $(HOST_OUT) \ $(if $(OEM_OTA_CONFIG), --oem_settings $(OEM_OTA_CONFIG)) \ + $(if $(BOOT_VAR_OTA_CONFIG), --boot_variable_file $(BOOT_VAR_OTA_CONFIG)) \ $(2) \ - $(BUILT_TARGET_FILES_PACKAGE) $(1) + $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE)) $(1) endef product_name := $(TARGET_PRODUCT) ifeq ($(TARGET_BUILD_TYPE),debug) product_name := $(product_name)_debug endif -name := $(product_name)-ota-$(FILE_NAME_TAG) +name := $(product_name)-ota INTERNAL_OTA_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip INTERNAL_OTA_METADATA := $(PRODUCT_OUT)/ota_metadata @@ -6282,7 +6938,7 @@ $(call declare-0p-target,$(INTERNAL_OTA_METADATA)) $(INTERNAL_OTA_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR) $(INTERNAL_OTA_PACKAGE_TARGET): .KATI_IMPLICIT_OUTPUTS := $(INTERNAL_OTA_METADATA) -$(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(OTA_FROM_TARGET_FILES) $(INTERNAL_OTATOOLS_FILES) +$(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_DIR) $(OTA_FROM_TARGET_FILES) $(INTERNAL_OTATOOLS_FILES) @echo "Package OTA: $@" $(call build-ota-package-target,$@,-k $(KEY_CERT_PAIR) --output_metadata_path $(INTERNAL_OTA_METADATA)) @@ -6293,7 +6949,7 @@ $(call declare-container-license-deps,$(INTERNAL_OTA_PACKAGE_TARGET),$(BUILT_TAR otapackage: $(INTERNAL_OTA_PACKAGE_TARGET) ifeq ($(BOARD_BUILD_RETROFIT_DYNAMIC_PARTITIONS_OTA_PACKAGE),true) -name := $(product_name)-ota-retrofit-$(FILE_NAME_TAG) +name := $(product_name)-ota-retrofit INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip $(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR) @@ -6314,11 +6970,11 @@ otapackage otardppackage: $(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TAR endif # BOARD_BUILD_RETROFIT_DYNAMIC_PARTITIONS_OTA_PACKAGE ifneq ($(BOARD_PARTIAL_OTA_UPDATE_PARTITIONS_LIST),) -name := $(product_name)-partial-ota-$(FILE_NAME_TAG) +name := $(product_name)-partial-ota INTERNAL_OTA_PARTIAL_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip $(INTERNAL_OTA_PARTIAL_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR) -$(INTERNAL_OTA_PARTIAL_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(OTA_FROM_TARGET_FILES) $(INTERNAL_OTATOOLS_FILES) +$(INTERNAL_OTA_PARTIAL_PACKAGE_TARGET): $(BUILT_TARGET_FILES_DIR) $(OTA_FROM_TARGET_FILES) $(INTERNAL_OTATOOLS_FILES) @echo "Package partial OTA: $@" $(call build-ota-package-target,$@,-k $(KEY_CERT_PAIR) --partial "$(BOARD_PARTIAL_OTA_UPDATE_PARTITIONS_LIST)") @@ -6414,9 +7070,9 @@ ifeq ($(TARGET_BUILD_TYPE),debug) endif # The path to the zip file containing binaries with symbols. -SYMBOLS_ZIP := $(PRODUCT_OUT)/$(name)-symbols-$(FILE_NAME_TAG).zip +SYMBOLS_ZIP := $(PRODUCT_OUT)/$(name)-symbols.zip # The path to a file containing mappings from elf IDs to filenames. -SYMBOLS_MAPPING := $(PRODUCT_OUT)/$(name)-symbols-mapping-$(FILE_NAME_TAG).textproto +SYMBOLS_MAPPING := $(PRODUCT_OUT)/$(name)-symbols-mapping.textproto .KATI_READONLY := SYMBOLS_ZIP SYMBOLS_MAPPING # For apps_only build we'll establish the dependency later in build/make/core/main.mk. ifeq (,$(TARGET_BUILD_UNBUNDLED)) @@ -6473,7 +7129,7 @@ SYSTEM_NOTICE_DEPS += $(COVERAGE_ZIP) ifeq (true,$(CLANG_COVERAGE)) LLVM_PROFDATA := $(LLVM_PREBUILTS_BASE)/linux-x86/$(LLVM_PREBUILTS_VERSION)/bin/llvm-profdata LLVM_COV := $(LLVM_PREBUILTS_BASE)/linux-x86/$(LLVM_PREBUILTS_VERSION)/bin/llvm-cov - LIBCXX := $(LLVM_PREBUILTS_BASE)/linux-x86/$(LLVM_PREBUILTS_VERSION)/lib/x86_64-unknown-linux-gnu/libc++.so.1 + LIBCXX := $(LLVM_PREBUILTS_BASE)/linux-x86/$(LLVM_PREBUILTS_VERSION)/lib/x86_64-unknown-linux-gnu/libc++.so # Use llvm-profdata.zip for backwards compatibility with tradefed code. LLVM_COVERAGE_TOOLS_ZIP := $(PRODUCT_OUT)/llvm-profdata.zip @@ -6491,7 +7147,7 @@ name := $(TARGET_PRODUCT) ifeq ($(TARGET_BUILD_TYPE),debug) name := $(name)_debug endif -name := $(name)-apps-$(FILE_NAME_TAG) +name := $(name)-apps APPS_ZIP := $(PRODUCT_OUT)/$(name).zip $(APPS_ZIP): $(FULL_SYSTEMIMAGE_DEPS) @@ -6530,6 +7186,14 @@ $(JACOCO_REPORT_CLASSES_ALL) : ifeq (,$(TARGET_BUILD_UNBUNDLED)) $(JACOCO_REPORT_CLASSES_ALL): $(INTERNAL_ALLIMAGES_FILES) endif + +# This is not ideal, but it is difficult to correctly figure out the actual jacoco report +# jars we need to add here as dependencies, so we add the device-tests as a dependency when +# the env variable is set and this should guarantee thaat all the jacoco report jars are ready +# when we package the final report jar here. +ifeq ($(JACOCO_PACKAGING_INCLUDE_DEVICE_TESTS),true) + $(JACOCO_REPORT_CLASSES_ALL): $(COMPATIBILITY.device-tests.FILES) +endif endif # EMMA_INSTRUMENT=true @@ -6541,29 +7205,34 @@ endif # EMMA_INSTRUMENT=true # finding the appropriate dictionary to deobfuscate a stack trace frame. # -# The path to the zip file containing proguard dictionaries. -PROGUARD_DICT_ZIP := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict-$(FILE_NAME_TAG).zip -# The path to the zip file containing mappings from dictionary hashes to filenames. -PROGUARD_DICT_MAPPING := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict-mapping-$(FILE_NAME_TAG).textproto -.KATI_READONLY := PROGUARD_DICT_ZIP PROGUARD_DICT_MAPPING -# For apps_only build we'll establish the dependency later in build/make/core/main.mk. ifeq (,$(TARGET_BUILD_UNBUNDLED)) -$(PROGUARD_DICT_ZIP): $(INTERNAL_ALLIMAGES_FILES) $(updater_dep) + _proguard_dict_zip_modules := $(call product-installed-modules,$(INTERNAL_PRODUCT)) +else + _proguard_dict_zip_modules := $(unbundled_build_modules) endif -$(PROGUARD_DICT_ZIP): PRIVATE_PACKAGING_DIR := $(call intermediates-dir-for,PACKAGING,proguard_dictionary) -$(PROGUARD_DICT_ZIP): PRIVATE_MAPPING_PACKAGING_DIR := $(call intermediates-dir-for,PACKAGING,proguard_dictionary_mapping) -$(PROGUARD_DICT_ZIP): PRIVATE_LIST_FILE := $(call intermediates-dir-for,PACKAGING,proguard_dictionary_filelist)/filelist -$(PROGUARD_DICT_ZIP): $(SOONG_ZIP) $(SYMBOLS_MAP) + +# The path to the zip file containing proguard dictionaries. +PROGUARD_DICT_ZIP :=$= $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict.zip +$(PROGUARD_DICT_ZIP): PRIVATE_SOONG_ZIP_ARGUMENTS := $(foreach m,$(_proguard_dict_zip_modules),$(ALL_MODULES.$(m).PROGUARD_DICTIONARY_SOONG_ZIP_ARGUMENTS)) +$(PROGUARD_DICT_ZIP): $(SOONG_ZIP) $(foreach m,$(_proguard_dict_zip_modules),$(ALL_MODULES.$(m).PROGUARD_DICTIONARY_FILES)) @echo "Packaging Proguard obfuscation dictionary files." - rm -rf $@ $(PRIVATE_LIST_FILE) - mkdir -p $(PRIVATE_PACKAGING_DIR) $(PRIVATE_MAPPING_PACKAGING_DIR) $(dir $(PRIVATE_LIST_FILE)) - # Zip all of the files in the proguard dictionary directory. - $(SOONG_ZIP) --ignore_missing_files -d -o $@ -C $(PRIVATE_PACKAGING_DIR) -D $(PRIVATE_PACKAGING_DIR) - # Find all of the files in the proguard dictionary mapping directory and merge them into the mapping textproto. - # Strip the PRIVATE_PACKAGING_DIR off the filenames to match soong_zip's -C argument. - $(hide) find -L $(PRIVATE_MAPPING_PACKAGING_DIR) -type f | sort >$(PRIVATE_LIST_FILE) - $(SYMBOLS_MAP) -merge $(PROGUARD_DICT_MAPPING) -strip_prefix $(PRIVATE_PACKAGING_DIR)/ -ignore_missing_files @$(PRIVATE_LIST_FILE) -$(PROGUARD_DICT_ZIP): .KATI_IMPLICIT_OUTPUTS := $(PROGUARD_DICT_MAPPING) + # Zip all of the files in PROGUARD_DICTIONARY_FILES. + echo -n > $@.tmparglist + $(foreach arg,$(PRIVATE_SOONG_ZIP_ARGUMENTS),printf "%s\n" "$(arg)" >> $@.tmparglist$(newline)) + $(SOONG_ZIP) -d -o $@ @$@.tmparglist + rm -f $@.tmparglist + +# The path to the zip file containing mappings from dictionary hashes to filenames. +PROGUARD_DICT_MAPPING :=$= $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict-mapping.textproto +_proguard_dict_mapping_files := $(foreach m,$(_proguard_dict_zip_modules),$(ALL_MODULES.$(m).PROGUARD_DICTIONARY_MAPPING)) +$(PROGUARD_DICT_MAPPING): PRIVATE_MAPPING_FILES := $(_proguard_dict_mapping_files) +$(PROGUARD_DICT_MAPPING): $(SYMBOLS_MAP) $(_proguard_dict_mapping_files) + @echo "Packaging Proguard obfuscation dictionary mapping files." + # Merge all the mapping files together + echo -n > $@.tmparglist + $(foreach mf,$(PRIVATE_MAPPING_FILES),echo "$(mf)" >> $@.tmparglist$(newline)) + $(SYMBOLS_MAP) -merge $(PROGUARD_DICT_MAPPING) @$@.tmparglist + rm -f $@.tmparglist $(call declare-1p-container,$(PROGUARD_DICT_ZIP),) ifeq (,$(TARGET_BUILD_UNBUNDLED)) @@ -6573,31 +7242,19 @@ endif #------------------------------------------------------------------ # A zip of Proguard usage files. # -PROGUARD_USAGE_ZIP := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-usage-$(FILE_NAME_TAG).zip -# For apps_only build we'll establish the dependency later in build/make/core/main.mk. -ifeq (,$(TARGET_BUILD_UNBUNDLED)) -$(PROGUARD_USAGE_ZIP): \ - $(INSTALLED_SYSTEMIMAGE_TARGET) \ - $(INSTALLED_RAMDISK_TARGET) \ - $(INSTALLED_BOOTIMAGE_TARGET) \ - $(INSTALLED_INIT_BOOT_IMAGE_TARGET) \ - $(INSTALLED_USERDATAIMAGE_TARGET) \ - $(INSTALLED_VENDORIMAGE_TARGET) \ - $(INSTALLED_PRODUCTIMAGE_TARGET) \ - $(INSTALLED_SYSTEM_EXTIMAGE_TARGET) \ - $(INSTALLED_ODMIMAGE_TARGET) \ - $(INSTALLED_VENDOR_DLKMIMAGE_TARGET) \ - $(INSTALLED_ODM_DLKMIMAGE_TARGET) \ - $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) \ - $(updater_dep) -endif -$(PROGUARD_USAGE_ZIP): PRIVATE_LIST_FILE := $(call intermediates-dir-for,PACKAGING,proguard_usage.zip)/filelist -$(PROGUARD_USAGE_ZIP): PRIVATE_PACKAGING_DIR := $(call intermediates-dir-for,PACKAGING,proguard_usage) -$(PROGUARD_USAGE_ZIP): $(MERGE_ZIPS) +PROGUARD_USAGE_ZIP :=$= $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-usage.zip +_proguard_usage_zips := $(foreach m,$(_proguard_dict_zip_modules),$(ALL_MODULES.$(m).PROGUARD_USAGE_ZIP)) +$(PROGUARD_USAGE_ZIP): PRIVATE_ZIPS := $(_proguard_usage_zips) +$(PROGUARD_USAGE_ZIP): $(MERGE_ZIPS) $(_proguard_usage_zips) @echo "Packaging Proguard usage files." - mkdir -p $(dir $@) $(PRIVATE_PACKAGING_DIR) $(dir $(PRIVATE_LIST_FILE)) - find $(PRIVATE_PACKAGING_DIR) -name proguard_usage.zip > $(PRIVATE_LIST_FILE) - $(MERGE_ZIPS) $@ @$(PRIVATE_LIST_FILE) + echo -n > $@.tmparglist + $(foreach z,$(PRIVATE_ZIPS),echo "$(z)" >> $@.tmparglist$(newline)) + $(MERGE_ZIPS) $@ @$@.tmparglist + rm -rf $@.tmparglist + +_proguard_dict_mapping_files := +_proguard_usage_zips := +_proguard_dict_zip_modules := $(call declare-1p-container,$(PROGUARD_USAGE_ZIP),) ifeq (,$(TARGET_BUILD_UNBUNDLED)) @@ -6641,7 +7298,7 @@ ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)) # For real devices and for dist builds, build super image from target files to an intermediate directory. INTERNAL_SUPERIMAGE_DIST_TARGET := $(call intermediates-dir-for,PACKAGING,super.img)/super.img $(INTERNAL_SUPERIMAGE_DIST_TARGET): extracted_input_target_files := $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE)) -$(INTERNAL_SUPERIMAGE_DIST_TARGET): $(LPMAKE) $(BUILT_TARGET_FILES_PACKAGE) $(BUILD_SUPER_IMAGE) +$(INTERNAL_SUPERIMAGE_DIST_TARGET): $(LPMAKE) $(BUILT_TARGET_FILES_DIR) $(BUILD_SUPER_IMAGE) $(call pretty,"Target super fs image from target files: $@") PATH=$(dir $(LPMAKE)):$$PATH \ $(BUILD_SUPER_IMAGE) -v $(extracted_input_target_files) $@ @@ -6750,7 +7407,7 @@ name := $(TARGET_PRODUCT) ifeq ($(TARGET_BUILD_TYPE),debug) name := $(name)_debug endif -name := $(name)-img-$(FILE_NAME_TAG) +name := $(name)-img INTERNAL_UPDATE_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip @@ -6759,6 +7416,7 @@ $(INTERNAL_UPDATE_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(IMG_FROM_TARG PATH=$(INTERNAL_USERIMAGES_BINARY_PATHS):$(dir $(ZIP2ZIP)):$$PATH \ $(IMG_FROM_TARGET_FILES) \ --additional IMAGES/VerifiedBootParams.textproto:VerifiedBootParams.textproto \ + --build_super_image $(BUILD_SUPER_IMAGE) \ $(BUILT_TARGET_FILES_PACKAGE) $@ $(call declare-1p-container,$(INTERNAL_UPDATE_PACKAGE_TARGET),) @@ -6766,7 +7424,7 @@ $(call declare-container-license-deps,$(INTERNAL_UPDATE_PACKAGE_TARGET),$(BUILT_ .PHONY: updatepackage updatepackage: $(INTERNAL_UPDATE_PACKAGE_TARGET) -$(call dist-for-goals,updatepackage,$(INTERNAL_UPDATE_PACKAGE_TARGET)) +$(call dist-for-goals-with-filenametag,updatepackage,$(INTERNAL_UPDATE_PACKAGE_TARGET)) # ----------------------------------------------------------------- @@ -6895,7 +7553,7 @@ INTERNAL_EMULATOR_PACKAGE_FILES += \ $(INSTALLED_SYSTEMIMAGE_TARGET) \ $(INSTALLED_USERDATAIMAGE_TARGET) -name := $(TARGET_PRODUCT)-emulator-$(FILE_NAME_TAG) +name := $(TARGET_PRODUCT)-emulator INTERNAL_EMULATOR_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip @@ -6923,7 +7581,7 @@ sdk_dir := $(HOST_OUT)/sdk/$(TARGET_PRODUCT) ifneq ($(HOST_OS),linux) $(error Building the monolithic SDK is only supported on Linux) endif -sdk_name := android-sdk_$(FILE_NAME_TAG) +sdk_name := android-sdk INTERNAL_SDK_HOST_OS_NAME := linux-$(SDK_HOST_ARCH) sdk_name := $(sdk_name)_$(INTERNAL_SDK_HOST_OS_NAME) @@ -6958,6 +7616,24 @@ else sdk_atree_files += $(atree_dir)/sdk.atree endif +SDK_METADATA_DIR :=$= $(call intermediates-dir-for,PACKAGING,framework-doc-stubs-metadata,,COMMON) +SDK_METADATA_FILES :=$= $(addprefix $(SDK_METADATA_DIR)/,\ + activity_actions.txt \ + broadcast_actions.txt \ + categories.txt \ + features.txt \ + service_actions.txt \ + widgets.txt) +SDK_METADATA :=$= $(firstword $(SDK_METADATA_FILES)) +$(SDK_METADATA): .KATI_IMPLICIT_OUTPUTS := $(filter-out $(SDK_METADATA),$(SDK_METADATA_FILES)) +$(SDK_METADATA): $(TARGET_OUT_COMMON_INTERMEDIATES)/PACKAGING/framework-doc-stubs-metadata.zip + rm -rf $(SDK_METADATA_DIR) + mkdir -p $(SDK_METADATA_DIR) + unzip -DDqo $< -d $(SDK_METADATA_DIR) + +.PHONY: framework-doc-stubs +framework-doc-stubs: $(SDK_METADATA) + deps := \ $(OUT_DOCS)/offline-sdk-timestamp \ $(SDK_METADATA_FILES) \ @@ -7097,25 +7773,20 @@ FUZZ_SHARED_DEPS := $(call copy-many-files,$(strip $(FUZZ_TARGET_SHARED_DEPS_INS haiku: $(SOONG_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_FUZZ_TARGETS) $(call dist-for-goals,haiku,$(SOONG_FUZZ_PACKAGING_ARCH_MODULES)) $(call dist-for-goals,haiku,$(PRODUCT_OUT)/module-info.json) - -.PHONY: haiku-java-device -haiku-java-device: $(SOONG_JAVA_FUZZ_DEVICE_PACKAGING_ARCH_MODULES) $(ALL_JAVA_FUZZ_DEVICE_TARGETS) -$(call dist-for-goals,haiku-java-device,$(SOONG_JAVA_FUZZ_DEVICE_PACKAGING_ARCH_MODULES)) -$(call dist-for-goals,haiku-java-device,$(PRODUCT_OUT)/module-info.json) - -.PHONY: haiku-java-host -haiku-java-host: $(SOONG_JAVA_FUZZ_HOST_PACKAGING_ARCH_MODULES) $(ALL_JAVA_FUZZ_HOST_TARGETS) -$(call dist-for-goals,haiku-java-host,$(SOONG_JAVA_FUZZ_HOST_PACKAGING_ARCH_MODULES)) -$(call dist-for-goals,haiku-java-host,$(PRODUCT_OUT)/module-info.json) - +.PHONY: haiku-java +haiku-java: $(SOONG_JAVA_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_JAVA_FUZZ_TARGETS) +$(call dist-for-goals,haiku-java,$(SOONG_JAVA_FUZZ_PACKAGING_ARCH_MODULES)) .PHONY: haiku-rust haiku-rust: $(SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_RUST_FUZZ_TARGETS) $(call dist-for-goals,haiku-rust,$(SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES)) $(call dist-for-goals,haiku-rust,$(PRODUCT_OUT)/module-info.json) +.PHONY: haiku-presubmit +haiku-presubmit: $(SOONG_PRESUBMIT_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_PRESUBMIT_FUZZ_TARGETS) +$(call dist-for-goals,haiku-presubmit,$(SOONG_PRESUBMIT_FUZZ_PACKAGING_ARCH_MODULES)) # ----------------------------------------------------------------- # Extract platform fonts used in Layoutlib -include $(BUILD_SYSTEM)/layoutlib_fonts.mk +include $(BUILD_SYSTEM)/layoutlib_data.mk # ----------------------------------------------------------------- diff --git a/core/OWNERS b/core/OWNERS index eb1d5c3c46..35ea83d2fe 100644 --- a/core/OWNERS +++ b/core/OWNERS @@ -3,5 +3,11 @@ per-file proguard*.flags = jdduke@google.com # For version updates -per-file version_defaults.mk = aseaton@google.com,lubomir@google.com,pscovanner@google.com,bkhalife@google.com,jainne@google.com +per-file version_defaults.mk = ankurbakshi@google.com,bkhalife@google.com,jainne@google.com,lokeshgoel@google.com,lubomir@google.com,pscovanner@google.com + +# For sdk extensions version updates +per-file version_defaults.mk = amhk@google.com,gurpreetgs@google.com,mkhokhlova@google.com,robertogil@google.com + +# For Ravenwood test configs +per-file ravenwood_test_config_template.xml = jsharkey@google.com,omakoto@google.com diff --git a/core/aapt2.mk b/core/aapt2.mk index 7b17df4a9c..0e23477967 100644 --- a/core/aapt2.mk +++ b/core/aapt2.mk @@ -68,6 +68,8 @@ $(my_res_resources_flat) $(my_overlay_resources_flat) $(my_resources_flata) $(my # support for it. my_static_library_resources := $(foreach l, $(call reverse-list,$(LOCAL_STATIC_ANDROID_LIBRARIES) $(LOCAL_STATIC_JAVA_AAR_LIBRARIES)),\ $(call intermediates-dir-for,JAVA_LIBRARIES,$(l),,COMMON)/package-res.apk) +my_static_library_transitive_resource_packages_lists := $(foreach l, $(call reverse-list,$(LOCAL_STATIC_ANDROID_LIBRARIES) $(LOCAL_STATIC_JAVA_AAR_LIBRARIES)),\ + $(call intermediates-dir-for,JAVA_LIBRARIES,$(l),,COMMON)/transitive-res-packages) my_static_library_extra_packages := $(foreach l, $(call reverse-list,$(LOCAL_STATIC_ANDROID_LIBRARIES) $(LOCAL_STATIC_JAVA_AAR_LIBRARIES)),\ $(call intermediates-dir-for,JAVA_LIBRARIES,$(l),,COMMON)/extra_packages) my_shared_library_resources := $(foreach l, $(LOCAL_SHARED_ANDROID_LIBRARIES),\ @@ -95,6 +97,7 @@ $(my_res_package): PRIVATE_ASSET_DIRS := $(my_asset_dirs) $(my_res_package): PRIVATE_JAVA_GEN_DIR := $(intermediates.COMMON)/aapt2 $(my_res_package): PRIVATE_SRCJAR := $(my_srcjar) $(my_res_package): PRIVATE_STATIC_LIBRARY_EXTRA_PACKAGES := $(my_static_library_extra_packages) +$(my_res_package): PRIVATE_STATIC_LIBRARY_TRANSITIVE_RES_PACKAGES_LISTS := $(my_static_library_transitive_resource_packages_lists) $(my_res_package): PRIVATE_AAPT_EXTRA_PACKAGES := $(aapt_extra_packages) $(my_res_package): .KATI_IMPLICIT_OUTPUTS := $(my_srcjar) $(aapt_extra_packages) @@ -117,7 +120,7 @@ ifdef proguard_options_file $(my_res_package): .KATI_IMPLICIT_OUTPUTS += $(proguard_options_file) endif -$(my_res_package): $(full_android_manifest) $(my_static_library_resources) $(my_shared_library_resources) +$(my_res_package): $(full_android_manifest) $(my_static_library_resources) $(my_static_library_transitive_resource_packages_lists) $(my_shared_library_resources) $(my_res_package): $(my_full_asset_paths) $(my_res_package): $(my_res_resources_flat) $(my_overlay_resources_flat) \ $(my_resources_flata) $(my_static_library_resources) $(my_static_library_extra_packages) \ diff --git a/core/android_manifest.mk b/core/android_manifest.mk index ff4926285f..7f469033c2 100644 --- a/core/android_manifest.mk +++ b/core/android_manifest.mk @@ -51,6 +51,9 @@ ifdef TARGET_BUILD_APPS my_target_sdk_version := $(my_target_sdk_version).$$(cat $(API_FINGERPRINT)) my_min_sdk_version := $(my_min_sdk_version).$$(cat $(API_FINGERPRINT)) $(fixed_android_manifest): $(API_FINGERPRINT) + else ifdef UNBUNDLED_BUILD_TARGET_SDK_WITH_DESSERT_SHA + my_target_sdk_version := $(UNBUNDLED_BUILD_TARGET_SDK_WITH_DESSERT_SHA) + my_min_sdk_version := $(UNBUNDLED_BUILD_TARGET_SDK_WITH_DESSERT_SHA) endif endif endif diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk index bf113eef1d..ed72fc3a0d 100644 --- a/core/android_soong_config_vars.mk +++ b/core/android_soong_config_vars.mk @@ -26,63 +26,25 @@ $(call add_soong_config_namespace,ANDROID) # Add variables to the namespace below: -$(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_MEDIASERVER) -$(call add_soong_config_var,ANDROID,TARGET_ENABLE_MEDIADRM_64) -$(call add_soong_config_var,ANDROID,IS_TARGET_MIXED_SEPOLICY) -ifeq ($(IS_TARGET_MIXED_SEPOLICY),true) -$(call add_soong_config_var_value,ANDROID,MIXED_SEPOLICY_VERSION,$(BOARD_SEPOLICY_VERS)) -endif $(call add_soong_config_var,ANDROID,BOARD_USES_ODMIMAGE) $(call add_soong_config_var,ANDROID,BOARD_USES_RECOVERY_AS_BOOT) +$(call add_soong_config_var,ANDROID,CHECK_DEV_TYPE_VIOLATIONS) +$(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_COMPAT_VERSIONS) $(call add_soong_config_var,ANDROID,PRODUCT_INSTALL_DEBUG_POLICY_TO_SYSTEM_EXT) +$(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_DRMSERVER) +$(call add_soong_config_var,ANDROID,TARGET_ENABLE_MEDIADRM_64) +$(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_MEDIASERVER) -# Default behavior for the tree wrt building modules or using prebuilts. This -# can always be overridden by setting the environment variable -# MODULE_BUILD_FROM_SOURCE. -BRANCH_DEFAULT_MODULE_BUILD_FROM_SOURCE := true - -ifneq (,$(MODULE_BUILD_FROM_SOURCE)) - # Keep an explicit setting. -else ifeq (,$(filter docs sdk win_sdk sdk_addon,$(MAKECMDGOALS))$(findstring com.google.android.conscrypt,$(PRODUCT_PACKAGES))) - # Prebuilt module SDKs require prebuilt modules to work, and currently - # prebuilt modules are only provided for com.google.android.xxx. If we can't - # find one of them in PRODUCT_PACKAGES then assume com.android.xxx are in use, - # and disable prebuilt SDKs. In particular this applies to AOSP builds. - # - # However, docs/sdk/win_sdk/sdk_addon builds might not include com.google.android.xxx - # packages, so for those we respect the default behavior. - MODULE_BUILD_FROM_SOURCE := true -else ifneq (,$(PRODUCT_MODULE_BUILD_FROM_SOURCE)) - # Let products override the branch default. - MODULE_BUILD_FROM_SOURCE := $(PRODUCT_MODULE_BUILD_FROM_SOURCE) -else - MODULE_BUILD_FROM_SOURCE := $(BRANCH_DEFAULT_MODULE_BUILD_FROM_SOURCE) -endif +# PRODUCT_PRECOMPILED_SEPOLICY defaults to true. Explicitly check if it's "false" or not. +$(call add_soong_config_var_value,ANDROID,PRODUCT_PRECOMPILED_SEPOLICY,$(if $(filter false,$(PRODUCT_PRECOMPILED_SEPOLICY)),false,true)) -ifneq (,$(ART_MODULE_BUILD_FROM_SOURCE)) - # Keep an explicit setting. -else ifneq (,$(findstring .android.art,$(TARGET_BUILD_APPS))) - # Build ART modules from source if they are listed in TARGET_BUILD_APPS. - ART_MODULE_BUILD_FROM_SOURCE := true -else - # Do the same as other modules by default. - ART_MODULE_BUILD_FROM_SOURCE := $(MODULE_BUILD_FROM_SOURCE) +ifdef ART_DEBUG_OPT_FLAG +$(call soong_config_set,art_module,art_debug_opt_flag,$(ART_DEBUG_OPT_FLAG)) endif -$(call soong_config_set,art_module,source_build,$(ART_MODULE_BUILD_FROM_SOURCE)) - -# Ensure that those mainline modules who have individually toggleable prebuilts -# are controlled by the MODULE_BUILD_FROM_SOURCE environment variable by -# default. -INDIVIDUALLY_TOGGLEABLE_PREBUILT_MODULES := \ - bluetooth \ - permission \ - uwb \ - wifi \ - -$(foreach m, $(INDIVIDUALLY_TOGGLEABLE_PREBUILT_MODULES),\ - $(if $(call soong_config_get,$(m)_module,source_build),,\ - $(call soong_config_set,$(m)_module,source_build,$(MODULE_BUILD_FROM_SOURCE)))) +ifdef TARGET_BOARD_AUTO + $(call add_soong_config_var_value, ANDROID, target_board_auto, $(TARGET_BOARD_AUTO)) +endif # Apex build mode variables ifdef APEX_BUILD_FOR_PRE_S_DEVICES @@ -93,9 +55,10 @@ $(call add_soong_config_var_value,ANDROID,library_linking_strategy,prefer_static endif endif -ifeq (true,$(MODULE_BUILD_FROM_SOURCE)) +# TODO(b/308187800): some internal modules set `prefer` to true on the prebuilt apex module, +# and set that to false when `ANDROID.module_build_from_source` is true. +# Set this soong config variable to true for now, and cleanup `prefer` as part of b/308187800 $(call add_soong_config_var_value,ANDROID,module_build_from_source,true) -endif # Messaging app vars ifeq (eng,$(TARGET_BUILD_VARIANT)) @@ -106,10 +69,45 @@ endif SYSTEMUI_OPTIMIZE_JAVA ?= true $(call add_soong_config_var,ANDROID,SYSTEMUI_OPTIMIZE_JAVA) +# Enable Compose in SystemUI by default. +SYSTEMUI_USE_COMPOSE ?= true +$(call add_soong_config_var,ANDROID,SYSTEMUI_USE_COMPOSE) + ifdef PRODUCT_AVF_ENABLED $(call add_soong_config_var_value,ANDROID,avf_enabled,$(PRODUCT_AVF_ENABLED)) endif +ifdef PRODUCT_AVF_MICRODROID_GUEST_GKI_VERSION +$(call add_soong_config_var_value,ANDROID,avf_microdroid_guest_gki_version,$(PRODUCT_AVF_MICRODROID_GUEST_GKI_VERSION)) +endif + +ifdef PRODUCT_MEMCG_V2_FORCE_ENABLED +$(call add_soong_config_var_value,ANDROID,memcg_v2_force_enabled,$(PRODUCT_MEMCG_V2_FORCE_ENABLED)) +endif + +ifdef PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED +$(call add_soong_config_var_value,ANDROID,cgroup_v2_sys_app_isolation,$(PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED)) +endif + +$(call add_soong_config_var_value,ANDROID,release_avf_allow_preinstalled_apps,$(RELEASE_AVF_ALLOW_PREINSTALLED_APPS)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_device_assignment,$(RELEASE_AVF_ENABLE_DEVICE_ASSIGNMENT)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_dice_changes,$(RELEASE_AVF_ENABLE_DICE_CHANGES)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_llpvm_changes,$(RELEASE_AVF_ENABLE_LLPVM_CHANGES)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_multi_tenant_microdroid_vm,$(RELEASE_AVF_ENABLE_MULTI_TENANT_MICRODROID_VM)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_remote_attestation,$(RELEASE_AVF_ENABLE_REMOTE_ATTESTATION)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_vendor_modules,$(RELEASE_AVF_ENABLE_VENDOR_MODULES)) +$(call add_soong_config_var_value,ANDROID,release_avf_enable_virt_cpufreq,$(RELEASE_AVF_ENABLE_VIRT_CPUFREQ)) +$(call add_soong_config_var_value,ANDROID,release_avf_microdroid_kernel_version,$(RELEASE_AVF_MICRODROID_KERNEL_VERSION)) +$(call add_soong_config_var_value,ANDROID,release_avf_support_custom_vm_with_paravirtualized_devices,$(RELEASE_AVF_SUPPORT_CUSTOM_VM_WITH_PARAVIRTUALIZED_DEVICES)) + +$(call add_soong_config_var_value,ANDROID,release_binder_death_recipient_weak_from_jni,$(RELEASE_BINDER_DEATH_RECIPIENT_WEAK_FROM_JNI)) + +$(call add_soong_config_var_value,ANDROID,release_package_libandroid_runtime_punch_holes,$(RELEASE_PACKAGE_LIBANDROID_RUNTIME_PUNCH_HOLES)) + +$(call add_soong_config_var_value,ANDROID,release_selinux_data_data_ignore,$(RELEASE_SELINUX_DATA_DATA_IGNORE)) + +$(call add_soong_config_var_value,ANDROID,release_write_appcompat_override_system_properties,$(RELEASE_WRITE_APPCOMPAT_OVERRIDE_SYSTEM_PROPERTIES)) + # Enable system_server optimizations by default unless explicitly set or if # there may be dependent runtime jars. # TODO(b/240588226): Remove the off-by-default exceptions after handling @@ -125,7 +123,16 @@ else ifneq (platform:services,$(lastword $(PRODUCT_SYSTEM_SERVER_JARS))) else SYSTEM_OPTIMIZE_JAVA ?= true endif + +ifeq (true,$(FULL_SYSTEM_OPTIMIZE_JAVA)) + SYSTEM_OPTIMIZE_JAVA := true +endif + $(call add_soong_config_var,ANDROID,SYSTEM_OPTIMIZE_JAVA) +$(call add_soong_config_var,ANDROID,FULL_SYSTEM_OPTIMIZE_JAVA) + +# TODO(b/319697968): Remove this build flag support when metalava fully supports flagged api +$(call soong_config_set,ANDROID,release_hidden_api_exportable_stubs,$(RELEASE_HIDDEN_API_EXPORTABLE_STUBS)) # Check for SupplementalApi module. ifeq ($(wildcard packages/modules/SupplementalApi),) @@ -134,3 +141,14 @@ else $(call add_soong_config_var_value,ANDROID,include_nonpublic_framework_api,true) endif +# Add crashrecovery build flag to soong +$(call soong_config_set,ANDROID,release_crashrecovery_module,$(RELEASE_CRASHRECOVERY_MODULE)) +ifeq (true,$(RELEASE_CRASHRECOVERY_FILE_MOVE)) + $(call soong_config_set,ANDROID,crashrecovery_files_in_module,true) + $(call soong_config_set,ANDROID,crashrecovery_files_in_platform,false) +else + $(call soong_config_set,ANDROID,crashrecovery_files_in_module,false) + $(call soong_config_set,ANDROID,crashrecovery_files_in_platform,true) +endif +# Weirdly required because platform_bootclasspath is using AUTO namespace +$(call soong_config_set,AUTO,release_crashrecovery_module,$(RELEASE_CRASHRECOVERY_MODULE)) diff --git a/core/app_prebuilt_internal.mk b/core/app_prebuilt_internal.mk index eb429cdd5c..2671956c7a 100644 --- a/core/app_prebuilt_internal.mk +++ b/core/app_prebuilt_internal.mk @@ -85,11 +85,6 @@ $(my_extracted_apk): $(my_prebuilt_src_file) my_prebuilt_src_file := $(my_extracted_apk) my_extracted_apk := my_extract_apk := -ifeq ($(PRODUCT_ALWAYS_PREOPT_EXTRACTED_APK),true) -# If the product property is set, always preopt for extracted modules to prevent executing out of -# the APK. -my_preopt_for_extracted_apk := true -endif endif rs_compatibility_jni_libs := @@ -227,7 +222,7 @@ endif $(built_module): PRIVATE_EMBEDDED_JNI_LIBS := $(embedded_prebuilt_jni_libs) ifdef LOCAL_COMPRESSED_MODULE -$(built_module) : $(MINIGZIP) +$(built_module) : $(GZIP) endif ifeq ($(module_run_appcompat),true) @@ -302,3 +297,7 @@ $(my_all_targets): $(installed_apk_splits) endif # LOCAL_PACKAGE_SPLITS +########################################################### +## SBOM generation +########################################################### +include $(BUILD_SBOM_GEN) diff --git a/core/art_config.mk b/core/art_config.mk index 1ea05db2c3..47b4bcfce6 100644 --- a/core/art_config.mk +++ b/core/art_config.mk @@ -12,35 +12,29 @@ # ENABLE_UFFD_GC: Whether to use userfaultfd GC. config_enable_uffd_gc := \ - $(firstword $(OVERRIDE_ENABLE_UFFD_GC) $(PRODUCT_ENABLE_UFFD_GC)) + $(firstword $(OVERRIDE_ENABLE_UFFD_GC) $(PRODUCT_ENABLE_UFFD_GC) default) -ifeq (,$(filter-out default,$(config_enable_uffd_gc))) - ENABLE_UFFD_GC := true - - # Disable userfaultfd GC if the device doesn't support it (i.e., if - # `min(ro.board.api_level ?? ro.board.first_api_level ?? MAX_VALUE, - # ro.product.first_api_level ?? ro.build.version.sdk ?? MAX_VALUE) < 31`) - # This logic aligns with how `ro.vendor.api_level` is calculated in - # `system/core/init/property_service.cpp`. - # We omit the check on `ro.build.version.sdk` here because we are on the latest build system. - board_api_level := $(firstword $(BOARD_API_LEVEL) $(BOARD_SHIPPING_API_LEVEL)) - ifneq (,$(board_api_level)) - ifeq (true,$(call math_lt,$(board_api_level),31)) - ENABLE_UFFD_GC := false - endif - endif - - ifneq (,$(PRODUCT_SHIPPING_API_LEVEL)) - ifeq (true,$(call math_lt,$(PRODUCT_SHIPPING_API_LEVEL),31)) - ENABLE_UFFD_GC := false - endif - endif -else ifeq (true,$(config_enable_uffd_gc)) - ENABLE_UFFD_GC := true -else ifeq (false,$(config_enable_uffd_gc)) - ENABLE_UFFD_GC := false -else +ifeq (,$(filter default true false,$(config_enable_uffd_gc))) $(error Unknown PRODUCT_ENABLE_UFFD_GC value: $(config_enable_uffd_gc)) endif -ADDITIONAL_PRODUCT_PROPERTIES += ro.dalvik.vm.enable_uffd_gc=$(ENABLE_UFFD_GC) +ENABLE_UFFD_GC := $(config_enable_uffd_gc) +# If the value is "default", it will be mangled by post_process_props.py. +ADDITIONAL_PRODUCT_PROPERTIES += ro.dalvik.vm.enable_uffd_gc=$(config_enable_uffd_gc) + +# Create APEX_BOOT_JARS_EXCLUDED which is a list of jars to be removed from +# ApexBoorJars when built from mainline prebuilts. +# soong variables indicate whether the prebuilt is enabled: +# - $(m)_module/source_build for art and TOGGLEABLE_PREBUILT_MODULES +# - ANDROID/module_build_from_source for other mainline modules +# Note that RELEASE_APEX_BOOT_JARS_PREBUILT_EXCLUDED_LIST is the list of module names +# and library names of jars that need to be removed. We have to keep separated list per +# release config due to possibility of different prebuilt content. +APEX_BOOT_JARS_EXCLUDED := +$(foreach pair, $(RELEASE_APEX_BOOT_JARS_PREBUILT_EXCLUDED_LIST),\ + $(eval m := $(subst com.android.,,$(call word-colon,1,$(pair)))) \ + $(if $(call soong_config_get,$(m)_module,source_build), \ + $(if $(filter true,$(call soong_config_get,$(m)_module,source_build)),, \ + $(eval APEX_BOOT_JARS_EXCLUDED += $(pair))), \ + $(if $(filter true,$(call soong_config_get,ANDROID,module_build_from_source)),, \ + $(eval APEX_BOOT_JARS_EXCLUDED += $(pair))))) diff --git a/core/artifact_path_requirements.mk b/core/artifact_path_requirements.mk index 566b9f7446..c949cc4d2b 100644 --- a/core/artifact_path_requirements.mk +++ b/core/artifact_path_requirements.mk @@ -4,6 +4,7 @@ # Fakes don't get installed, and NDK stubs aren't installed to device. static_allowed_patterns := $(TARGET_OUT_FAKE)/% $(SOONG_OUT_DIR)/ndk/% # RROs become REQUIRED by the source module, but are always placed on the vendor partition. +static_allowed_patterns += %__auto_generated_characteristics_rro.apk static_allowed_patterns += %__auto_generated_rro_product.apk static_allowed_patterns += %__auto_generated_rro_vendor.apk # Auto-included targets are not considered diff --git a/core/autogen_test_config.mk b/core/autogen_test_config.mk index 137b1185ab..b69f6945c1 100644 --- a/core/autogen_test_config.mk +++ b/core/autogen_test_config.mk @@ -29,7 +29,7 @@ ifeq (true,$(is_native)) ifeq (true,$(LOCAL_VENDOR_MODULE)) autogen_test_install_base = /data/local/tests/vendor endif - ifeq (true,$(LOCAL_USE_VNDK)) + ifeq (true,$(call module-in-vendor-or-product)) autogen_test_install_base = /data/local/tests/vendor endif endif diff --git a/core/base_rules.mk b/core/base_rules.mk index ec5a21e917..b5941933e3 100644 --- a/core/base_rules.mk +++ b/core/base_rules.mk @@ -120,13 +120,16 @@ non_system_module := $(filter true, \ $(LOCAL_VENDOR_MODULE) \ $(LOCAL_PROPRIETARY_MODULE)) -include $(BUILD_SYSTEM)/local_vndk.mk -include $(BUILD_SYSTEM)/local_systemsdk.mk +include $(BUILD_SYSTEM)/local_vendor_product.mk + +# local_current_sdk needs to run before local_systemsdk because the former may override +# LOCAL_SDK_VERSION which is used by the latter. include $(BUILD_SYSTEM)/local_current_sdk.mk -my_module_tags := $(LOCAL_MODULE_TAGS) -ifeq ($(my_host_cross),true) - my_module_tags := +# Check if the use of System SDK is correct. Note that, for Soong modules, the system sdk version +# check is done in Soong. No need to do it twice. +ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) +include $(BUILD_SYSTEM)/local_systemsdk.mk endif # Ninja has an implicit dependency on the command being run, and kati will @@ -148,58 +151,13 @@ endif ## Validate and define fallbacks for input LOCAL_* variables. ########################################################### -## Dump a .csv file of all modules and their tags -#ifneq ($(tag-list-first-time),false) -#$(shell rm -f tag-list.csv) -#tag-list-first-time := false -#endif -#$(shell echo $(lastword $(filter-out config/% out/%,$(MAKEFILE_LIST))),$(LOCAL_MODULE),$(strip $(LOCAL_MODULE_CLASS)),$(subst $(space),$(comma),$(sort $(my_module_tags))) >> tag-list.csv) - LOCAL_UNINSTALLABLE_MODULE := $(strip $(LOCAL_UNINSTALLABLE_MODULE)) -my_module_tags := $(sort $(my_module_tags)) -ifeq (,$(my_module_tags)) - my_module_tags := optional -endif - -# User tags are not allowed anymore. Fail early because it will not be installed -# like it used to be. -ifneq ($(filter $(my_module_tags),user),) - $(warning *** Module name: $(LOCAL_MODULE)) - $(warning *** Makefile location: $(LOCAL_MODULE_MAKEFILE)) - $(warning * ) - $(warning * Module is attempting to use the 'user' tag. This) - $(warning * used to cause the module to be installed automatically.) - $(warning * Now, the module must be listed in the PRODUCT_PACKAGES) - $(warning * section of a product makefile to have it installed.) - $(warning * ) - $(error user tag detected on module.) -endif - -my_bad_module_tags := $(filter eng debug,$(my_module_tags)) -ifdef my_bad_module_tags - ifeq (true,$(LOCAL_UNINSTALLABLE_MODULE)) - $(call pretty-warning,LOCAL_MODULE_TAGS := $(my_bad_module_tags) does not do anything for uninstallable modules) - endif - $(call pretty-error,LOCAL_MODULE_TAGS := $(my_bad_module_tags) is obsolete. See $(CHANGES_URL)#LOCAL_MODULE_TAGS) -endif # Only the tags mentioned in this test are expected to be set by module # makefiles. Anything else is either a typo or a source of unexpected # behaviors. -ifneq ($(filter-out tests optional samples,$(my_module_tags)),) -$(call pretty-error,unusual tags: $(filter-out tests optional samples,$(my_module_tags))) -endif - -# Add implicit tags. -# -# If the local directory or one of its parents contains a MODULE_LICENSE_GPL -# file, tag the module as "gnu". Search for "*_GPL*", "*_LGPL*" and "*_MPL*" -# so that we can also find files like MODULE_LICENSE_GPL_AND_AFL -# -gpl_license_file := $(call find-parent-file,$(LOCAL_PATH),MODULE_LICENSE*_GPL* MODULE_LICENSE*_MPL* MODULE_LICENSE*_LGPL*) -ifneq ($(gpl_license_file),) - my_module_tags += gnu - ALL_GPL_MODULE_LICENSE_FILES += $(gpl_license_file) +ifneq ($(filter-out tests optional samples,$(LOCAL_MODULE_TAGS)),) +$(call pretty-error,unusual tags: $(filter-out tests optional samples,$(LOCAL_MODULE_TAGS))) endif LOCAL_MODULE_CLASS := $(strip $(LOCAL_MODULE_CLASS)) @@ -252,7 +210,7 @@ else ifeq (NATIVE_TESTS,$(LOCAL_MODULE_CLASS)) else # The definition of should-install-to-system will be different depending # on which goal (e.g., sdk or just droid) is being built. - partition_tag := $(if $(call should-install-to-system,$(my_module_tags)),,_DATA) + partition_tag := $(if $(call should-install-to-system,$(LOCAL_MODULE_TAGS)),,_DATA) actual_partition_tag := $(if $(partition_tag),data,system) endif endif @@ -264,7 +222,7 @@ ifndef LOCAL_COMPATIBILITY_SUITE LOCAL_COMPATIBILITY_SUITE := null-suite endif ifneq ($(filter APPS, $(LOCAL_MODULE_CLASS)),) - ifneq ($(filter $(my_module_tags),tests),) + ifneq ($(filter $(LOCAL_MODULE_TAGS),tests),) LOCAL_COMPATIBILITY_SUITE := null-suite endif endif @@ -435,8 +393,8 @@ endif logtags_sources := $(filter %.logtags,$(LOCAL_SRC_FILES)) $(LOCAL_LOGTAGS_FILES) -ifneq ($(strip $(logtags_sources)),) -event_log_tags := $(foreach f,$(addprefix $(LOCAL_PATH)/,$(logtags_sources)),$(call clean-path,$(f))) +ifneq ($(strip $(logtags_sources) $(LOCAL_SOONG_LOGTAGS_FILES)),) +event_log_tags := $(foreach f,$(LOCAL_SOONG_LOGTAGS_FILES) $(addprefix $(LOCAL_PATH)/,$(logtags_sources)),$(call clean-path,$(f))) else event_log_tags := endif @@ -462,6 +420,12 @@ $(LOCAL_INTERMEDIATE_TARGETS) : PRIVATE_PATH:=$(LOCAL_PATH) $(LOCAL_INTERMEDIATE_TARGETS) : PRIVATE_IS_HOST_MODULE := $(LOCAL_IS_HOST_MODULE) $(LOCAL_INTERMEDIATE_TARGETS) : PRIVATE_HOST:= $(my_host) $(LOCAL_INTERMEDIATE_TARGETS) : PRIVATE_PREFIX := $(my_prefix) +$(LOCAL_INTERMEDIATE_TARGETS) : .KATI_TAGS += ;module_name=$(LOCAL_MODULE) +ifeq ($(LOCAL_MODULE_CLASS),) +$(error "$(LOCAL_MODULE) in $(LOCAL_PATH) does not set $(LOCAL_MODULE_CLASS)") +else +$(LOCAL_INTERMEDIATE_TARGETS) : .KATI_TAGS += ;module_type=$(LOCAL_MODULE_CLASS) +endif $(LOCAL_INTERMEDIATE_TARGETS) : PRIVATE_INTERMEDIATES_DIR:= $(intermediates) $(LOCAL_INTERMEDIATE_TARGETS) : PRIVATE_2ND_ARCH_VAR_PREFIX := $(LOCAL_2ND_ARCH_VAR_PREFIX) @@ -527,10 +491,6 @@ ifneq (,$(LOCAL_SOONG_INSTALLED_MODULE)) # copy of the intermediates for now, as some rules that collect intermediates may expect # them to exist. $(LOCAL_INSTALLED_MODULE): $(LOCAL_BUILT_MODULE) - - $(foreach symlink, $(LOCAL_SOONG_INSTALL_SYMLINKS), \ - $(call declare-0p-target,$(symlink))) - $(my_all_targets) : | $(LOCAL_SOONG_INSTALL_SYMLINKS) else ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) $(LOCAL_INSTALLED_MODULE): PRIVATE_POST_INSTALL_CMD := $(LOCAL_POST_INSTALL_CMD) $(LOCAL_INSTALLED_MODULE): $(LOCAL_BUILT_MODULE) @@ -552,6 +512,15 @@ else ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) endif # !LOCAL_UNINSTALLABLE_MODULE +# Add dependencies on LOCAL_SOONG_INSTALL_SYMLINKS if we're installing any kind of module, not just +# ones that set LOCAL_SOONG_INSTALLED_MODULE. This is so we can have a soong module that only +# installs symlinks (e.g. install_symlink). We can't set LOCAL_SOONG_INSTALLED_MODULE to a symlink +# because cp commands will fail on symlinks. +ifneq (,$(or $(LOCAL_SOONG_INSTALLED_MODULE),$(call boolean-not,$(LOCAL_UNINSTALLABLE_MODULE)))) + $(foreach symlink, $(LOCAL_SOONG_INSTALL_SYMLINKS), $(call declare-0p-target,$(symlink))) + $(my_all_targets) : | $(LOCAL_SOONG_INSTALL_SYMLINKS) +endif + ########################################################### ## VINTF manifest fragment and init.rc goals ########################################################### @@ -580,11 +549,14 @@ ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) # Only set up copy rules once, even if another arch variant shares it my_vintf_new_pairs := $(filter-out $(ALL_VINTF_MANIFEST_FRAGMENTS_LIST),$(my_vintf_pairs)) - my_vintf_new_installed := $(call copy-many-vintf-manifest-files-checked,$(my_vintf_new_pairs)) - ALL_VINTF_MANIFEST_FRAGMENTS_LIST += $(my_vintf_new_pairs) - $(my_all_targets) : $(my_vintf_new_installed) + ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) + $(call copy-many-vintf-manifest-files-checked,$(my_vintf_new_pairs)) + $(my_all_targets) : $(my_vintf_installed) + # Install fragments together with the target + $(LOCAL_INSTALLED_MODULE) : | $(my_vintf_installed) + endif endif # my_vintf_fragments # Rule to install the module's companion init.rc. @@ -616,11 +588,14 @@ ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) # Make sure we only set up the copy rules once, even if another arch variant # shares a common LOCAL_INIT_RC. my_init_rc_new_pairs := $(filter-out $(ALL_INIT_RC_INSTALLED_PAIRS),$(my_init_rc_pairs)) - my_init_rc_new_installed := $(call copy-many-init-script-files-checked,$(my_init_rc_new_pairs)) - ALL_INIT_RC_INSTALLED_PAIRS += $(my_init_rc_new_pairs) - $(my_all_targets) : $(my_init_rc_installed) + ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) + $(call copy-many-init-script-files-checked,$(my_init_rc_new_pairs)) + $(my_all_targets) : $(my_init_rc_installed) + # Install init_rc together with the target + $(LOCAL_INSTALLED_MODULE) : | $(my_init_rc_installed) + endif endif # my_init_rc endif # !LOCAL_IS_HOST_MODULE @@ -705,14 +680,30 @@ $(foreach td,$(LOCAL_TEST_DATA),$(eval $(copy_test_data_pairs))) copy_test_data_pairs := -my_installed_test_data := $(call copy-many-files,$(my_test_data_pairs)) -$(LOCAL_INSTALLED_MODULE): $(my_installed_test_data) +ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) + my_installed_test_data := $(call copy-many-files,$(my_test_data_pairs)) + $(LOCAL_INSTALLED_MODULE): $(my_installed_test_data) +else + # Skip installing test data for Soong modules, it's already been handled. + # Just compute my_installed_test_data. + my_installed_test_data := $(foreach f, $(my_test_data_pairs), $(call word-colon,2,$(f))) +endif endif endif endif ########################################################### +## SOONG INSTALL PAIRS +########################################################### +# Declare dependencies for LOCAL_SOONG_INSTALL_PAIRS in soong to the module it relies on. +ifneq (,$(LOCAL_SOONG_INSTALLED_MODULE)) +$(my_all_targets): \ + $(foreach f, $(LOCAL_SOONG_INSTALL_PAIRS),\ + $(word 2,$(subst :,$(space),$(f)))) +endif + +########################################################### ## Compatibility suite files. ########################################################### ifdef LOCAL_COMPATIBILITY_SUITE @@ -825,7 +816,7 @@ else ifneq (,$(test_config)) $(foreach suite, $(LOCAL_COMPATIBILITY_SUITE), \ $(eval my_compat_dist_config_$(suite) += $(foreach dir, $(call compatibility_suite_dirs,$(suite)), \ - $(test_config):$(dir)/$(LOCAL_MODULE).config))) + $(test_config):$(dir)/$(LOCAL_MODULE).config$(LOCAL_TEST_CONFIG_SUFFIX)))) endif ifneq (,$(LOCAL_EXTRA_FULL_TEST_CONFIGS)) @@ -944,11 +935,13 @@ ALL_MODULES.$(my_register_name).CLASS := \ ALL_MODULES.$(my_register_name).PATH := \ $(ALL_MODULES.$(my_register_name).PATH) $(LOCAL_PATH) ALL_MODULES.$(my_register_name).TAGS := \ - $(ALL_MODULES.$(my_register_name).TAGS) $(my_module_tags) + $(ALL_MODULES.$(my_register_name).TAGS) $(LOCAL_MODULE_TAGS) ALL_MODULES.$(my_register_name).CHECKED := \ $(ALL_MODULES.$(my_register_name).CHECKED) $(my_checked_module) ALL_MODULES.$(my_register_name).BUILT := \ $(ALL_MODULES.$(my_register_name).BUILT) $(LOCAL_BUILT_MODULE) +ALL_MODULES.$(my_register_name).SOONG_MODULE_TYPE := \ + $(ALL_MODULES.$(my_register_name).SOONG_MODULE_TYPE) $(LOCAL_SOONG_MODULE_TYPE) ifndef LOCAL_IS_HOST_MODULE ALL_MODULES.$(my_register_name).TARGET_BUILT := \ $(ALL_MODULES.$(my_register_name).TARGET_BUILT) $(LOCAL_BUILT_MODULE) @@ -965,6 +958,9 @@ ifneq (,$(LOCAL_SOONG_INSTALLED_MODULE)) $(my_init_rc_installed) \ $(my_installed_test_data) \ $(my_vintf_installed)) + + ALL_MODULES.$(my_register_name).INSTALLED_SYMLINKS := $(LOCAL_SOONG_INSTALL_SYMLINKS) + # Store the list of colon-separated pairs of the built and installed locations # of files provided by this module. Used by custom packaging rules like # package-modules.mk that need to copy the built files to a custom install @@ -979,6 +975,11 @@ ifneq (,$(LOCAL_SOONG_INSTALLED_MODULE)) $(my_init_rc_pairs) \ $(my_test_data_pairs) \ $(my_vintf_pairs)) + # Store the list of vintf/init_rc as order-only dependencies + ALL_MODULES.$(my_register_name).ORDERONLY_INSTALLED := \ + $(strip $(ALL_MODULES.$(my_register_name).ORDERONLY_INSTALLED) \ + $(my_init_rc_installed) \ + $(my_vintf_installed)) else ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) ALL_MODULES.$(my_register_name).INSTALLED := \ $(strip $(ALL_MODULES.$(my_register_name).INSTALLED) \ @@ -988,7 +989,21 @@ else ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) $(strip $(ALL_MODULES.$(my_register_name).BUILT_INSTALLED) \ $(LOCAL_BUILT_MODULE):$(LOCAL_INSTALLED_MODULE) \ $(my_init_rc_pairs) $(my_test_data_pairs) $(my_vintf_pairs)) + ALL_MODULES.$(my_register_name).ORDERONLY_INSTALLED := \ + $(strip $(ALL_MODULES.$(my_register_name).ORDERONLY_INSTALLED) \ + $(my_init_rc_installed) \ + $(my_vintf_installed)) endif + +# Mark LOCAL_SOONG_INSTALL_SYMLINKS as installed if we're installing any kind of module, not just +# ones that set LOCAL_SOONG_INSTALLED_MODULE. This is so we can have a soong module that only +# installs symlinks (e.g. installed_symlink). We can't set LOCAL_SOONG_INSTALLED_MODULE to a symlink +# because cp commands will fail on symlinks. +ifneq (,$(or $(LOCAL_SOONG_INSTALLED_MODULE),$(call boolean-not,$(LOCAL_UNINSTALLABLE_MODULE)))) + ALL_MODULES.$(my_register_name).INSTALLED += $(LOCAL_SOONG_INSTALL_SYMLINKS) + ALL_MODULES.$(my_register_name).INSTALLED_SYMLINKS := $(LOCAL_SOONG_INSTALL_SYMLINKS) +endif + ifdef LOCAL_PICKUP_FILES # Files or directories ready to pick up by the build system # when $(LOCAL_BUILT_MODULE) is done. @@ -1012,48 +1027,88 @@ ifdef LOCAL_IS_HOST_MODULE my_required_modules += $(LOCAL_REQUIRED_MODULES_$($(my_prefix)OS)) endif -ALL_MODULES.$(my_register_name).SHARED_LIBS := \ - $(ALL_MODULES.$(my_register_name).SHARED_LIBS) $(LOCAL_SHARED_LIBRARIES) +ifdef LOCAL_ACONFIG_FILES + ALL_MODULES.$(my_register_name).ACONFIG_FILES := \ + $(ALL_MODULES.$(my_register_name).ACONFIG_FILES) $(LOCAL_ACONFIG_FILES) +endif -ALL_MODULES.$(my_register_name).SYSTEM_SHARED_LIBS := \ - $(ALL_MODULES.$(my_register_name).SYSTEM_SHARED_LIBS) $(LOCAL_SYSTEM_SHARED_LIBRARIES) +ifndef LOCAL_SOONG_MODULE_INFO_JSON + ALL_MAKE_MODULE_INFO_JSON_MODULES += $(my_register_name) + ALL_MODULES.$(my_register_name).SHARED_LIBS := \ + $(ALL_MODULES.$(my_register_name).SHARED_LIBS) $(LOCAL_SHARED_LIBRARIES) -ALL_MODULES.$(my_register_name).LOCAL_RUNTIME_LIBRARIES := \ - $(ALL_MODULES.$(my_register_name).LOCAL_RUNTIME_LIBRARIES) $(LOCAL_RUNTIME_LIBRARIES) \ - $(LOCAL_JAVA_LIBRARIES) + ALL_MODULES.$(my_register_name).STATIC_LIBS := \ + $(ALL_MODULES.$(my_register_name).STATIC_LIBS) $(LOCAL_STATIC_LIBRARIES) -ALL_MODULES.$(my_register_name).LOCAL_STATIC_LIBRARIES := \ - $(ALL_MODULES.$(my_register_name).LOCAL_STATIC_LIBRARIES) $(LOCAL_STATIC_JAVA_LIBRARIES) + ALL_MODULES.$(my_register_name).SYSTEM_SHARED_LIBS := \ + $(ALL_MODULES.$(my_register_name).SYSTEM_SHARED_LIBS) $(LOCAL_SYSTEM_SHARED_LIBRARIES) -ifdef LOCAL_TEST_DATA - # Export the list of targets that are handled as data inputs and required - # by tests at runtime. The LOCAL_TEST_DATA format is generated from below - # https://cs.android.com/android/platform/superproject/+/master:build/soong/android/androidmk.go;l=925-944;drc=master - # which format is like $(path):$(relative_file) but for module-info, only - # the string after ":" is needed. - ALL_MODULES.$(my_register_name).TEST_DATA := \ - $(strip $(ALL_MODULES.$(my_register_name).TEST_DATA) \ - $(foreach f, $(LOCAL_TEST_DATA),\ - $(call word-colon,2,$(f)))) -endif + ALL_MODULES.$(my_register_name).LOCAL_RUNTIME_LIBRARIES := \ + $(ALL_MODULES.$(my_register_name).LOCAL_RUNTIME_LIBRARIES) $(LOCAL_RUNTIME_LIBRARIES) \ + $(LOCAL_JAVA_LIBRARIES) -ifdef LOCAL_TEST_DATA_BINS - ALL_MODULES.$(my_register_name).TEST_DATA_BINS := \ - $(ALL_MODULES.$(my_register_name).TEST_DATA_BINS) $(LOCAL_TEST_DATA_BINS) -endif + ALL_MODULES.$(my_register_name).LOCAL_STATIC_LIBRARIES := \ + $(ALL_MODULES.$(my_register_name).LOCAL_STATIC_LIBRARIES) $(LOCAL_STATIC_JAVA_LIBRARIES) -ALL_MODULES.$(my_register_name).SUPPORTED_VARIANTS := \ - $(ALL_MODULES.$(my_register_name).SUPPORTED_VARIANTS) \ - $(filter-out $(ALL_MODULES.$(my_register_name).SUPPORTED_VARIANTS),$(my_supported_variant)) + ifneq ($(my_test_data_file_pairs),) + # Export the list of targets that are handled as data inputs and required + # by tests at runtime. The format of my_test_data_file_pairs is + # is $(path):$(relative_file) but for module-info, only the string after + # ":" is needed. + ALL_MODULES.$(my_register_name).TEST_DATA := \ + $(strip $(ALL_MODULES.$(my_register_name).TEST_DATA) \ + $(foreach f, $(my_test_data_file_pairs),\ + $(call word-colon,2,$(f)))) + endif + + ifdef LOCAL_TEST_DATA_BINS + ALL_MODULES.$(my_register_name).TEST_DATA_BINS := \ + $(ALL_MODULES.$(my_register_name).TEST_DATA_BINS) $(LOCAL_TEST_DATA_BINS) + endif + + ALL_MODULES.$(my_register_name).SUPPORTED_VARIANTS := \ + $(ALL_MODULES.$(my_register_name).SUPPORTED_VARIANTS) \ + $(filter-out $(ALL_MODULES.$(my_register_name).SUPPORTED_VARIANTS),$(my_supported_variant)) + + ALL_MODULES.$(my_register_name).COMPATIBILITY_SUITES := \ + $(ALL_MODULES.$(my_register_name).COMPATIBILITY_SUITES) $(LOCAL_COMPATIBILITY_SUITE) + ALL_MODULES.$(my_register_name).MODULE_NAME := $(LOCAL_MODULE) + ALL_MODULES.$(my_register_name).TEST_CONFIG := $(test_config) + ALL_MODULES.$(my_register_name).EXTRA_TEST_CONFIGS := $(LOCAL_EXTRA_FULL_TEST_CONFIGS) + ALL_MODULES.$(my_register_name).TEST_MAINLINE_MODULES := $(LOCAL_TEST_MAINLINE_MODULES) + ifdef LOCAL_IS_UNIT_TEST + ALL_MODULES.$(my_register_name).IS_UNIT_TEST := $(LOCAL_IS_UNIT_TEST) + endif + ifdef LOCAL_TEST_OPTIONS_TAGS + ALL_MODULES.$(my_register_name).TEST_OPTIONS_TAGS := $(LOCAL_TEST_OPTIONS_TAGS) + endif + + ########################################################## + # Track module-level dependencies. + # (b/204397180) Unlock RECORD_ALL_DEPS was acknowledged reasonable for better Atest performance. + ALL_MODULES.$(my_register_name).ALL_DEPS := \ + $(ALL_MODULES.$(my_register_name).ALL_DEPS) \ + $(LOCAL_STATIC_LIBRARIES) \ + $(LOCAL_WHOLE_STATIC_LIBRARIES) \ + $(LOCAL_SHARED_LIBRARIES) \ + $(LOCAL_DYLIB_LIBRARIES) \ + $(LOCAL_RLIB_LIBRARIES) \ + $(LOCAL_PROC_MACRO_LIBRARIES) \ + $(LOCAL_HEADER_LIBRARIES) \ + $(LOCAL_STATIC_JAVA_LIBRARIES) \ + $(LOCAL_JAVA_LIBRARIES) \ + $(LOCAL_JNI_SHARED_LIBRARIES) + +endif ########################################################################## ## When compiling against API imported module, use API import stub ## libraries. ########################################################################## -ifneq ($(LOCAL_USE_VNDK),) +ifneq ($(call module-in-vendor-or-product),) ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) apiimport_postfix := .apiimport - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) + ifeq ($(LOCAL_IN_PRODUCT),true) apiimport_postfix := .apiimport.product else apiimport_postfix := .apiimport.vendor @@ -1068,7 +1123,7 @@ endif ## When compiling against the VNDK, add the .vendor or .product suffix to ## required modules. ########################################################################## -ifneq ($(LOCAL_USE_VNDK),) +ifneq ($(call module-in-vendor-or-product),) ##################################################### ## Soong modules may be built three times, once for ## /system, once for /vendor and once for /product. @@ -1079,7 +1134,7 @@ ifneq ($(LOCAL_USE_VNDK),) # We don't do this renaming for soong-defined modules since they already # have correct names (with .vendor or .product suffix when necessary) in # their LOCAL_*_LIBRARIES. - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) + ifeq ($(LOCAL_IN_PRODUCT),true) my_required_modules := $(foreach l,$(my_required_modules),\ $(if $(SPLIT_PRODUCT.SHARED_LIBRARIES.$(l)),$(l).product,$(l))) else @@ -1125,70 +1180,31 @@ else $(call pretty-error,LOCAL_TARGET_REQUIRED_MODULES may not be used from target modules. Use LOCAL_REQUIRED_MODULES instead) endif endif -ALL_MODULES.$(my_register_name).EVENT_LOG_TAGS := \ - $(ALL_MODULES.$(my_register_name).EVENT_LOG_TAGS) $(event_log_tags) + +ifdef event_log_tags + ALL_MODULES.$(my_register_name).EVENT_LOG_TAGS := \ + $(ALL_MODULES.$(my_register_name).EVENT_LOG_TAGS) $(event_log_tags) +endif + ALL_MODULES.$(my_register_name).MAKEFILE := \ $(ALL_MODULES.$(my_register_name).MAKEFILE) $(LOCAL_MODULE_MAKEFILE) + ifdef LOCAL_MODULE_OWNER -ALL_MODULES.$(my_register_name).OWNER := \ - $(sort $(ALL_MODULES.$(my_register_name).OWNER) $(LOCAL_MODULE_OWNER)) + ALL_MODULES.$(my_register_name).OWNER := \ + $(sort $(ALL_MODULES.$(my_register_name).OWNER) $(LOCAL_MODULE_OWNER)) endif + ifdef LOCAL_2ND_ARCH_VAR_PREFIX ALL_MODULES.$(my_register_name).FOR_2ND_ARCH := true endif ALL_MODULES.$(my_register_name).FOR_HOST_CROSS := $(my_host_cross) -ALL_MODULES.$(my_register_name).MODULE_NAME := $(LOCAL_MODULE) -ALL_MODULES.$(my_register_name).COMPATIBILITY_SUITES := \ - $(ALL_MODULES.$(my_register_name).COMPATIBILITY_SUITES) \ - $(filter-out $(ALL_MODULES.$(my_register_name).COMPATIBILITY_SUITES),$(LOCAL_COMPATIBILITY_SUITE)) -ALL_MODULES.$(my_register_name).TEST_CONFIG := $(test_config) -ALL_MODULES.$(my_register_name).EXTRA_TEST_CONFIGS := $(LOCAL_EXTRA_FULL_TEST_CONFIGS) -ALL_MODULES.$(my_register_name).TEST_MAINLINE_MODULES := $(LOCAL_TEST_MAINLINE_MODULES) ifndef LOCAL_IS_HOST_MODULE -ALL_MODULES.$(my_register_name).FILE_CONTEXTS := $(LOCAL_FILE_CONTEXTS) -endif -ifdef LOCAL_IS_UNIT_TEST -ALL_MODULES.$(my_register_name).IS_UNIT_TEST := $(LOCAL_IS_UNIT_TEST) -endif -ifdef LOCAL_TEST_OPTIONS_TAGS -ALL_MODULES.$(my_register_name).TEST_OPTIONS_TAGS := $(LOCAL_TEST_OPTIONS_TAGS) +ALL_MODULES.$(my_register_name).APEX_KEYS_FILE := $(LOCAL_APEX_KEY_PATH) endif test_config := INSTALLABLE_FILES.$(LOCAL_INSTALLED_MODULE).MODULE := $(my_register_name) -########################################################## -# Track module-level dependencies. -# Use $(LOCAL_MODULE) instead of $(my_register_name) to ignore module's bitness. -# (b/204397180) Unlock RECORD_ALL_DEPS was acknowledged reasonable for better Atest performance. -ALL_DEPS.MODULES += $(LOCAL_MODULE) -ALL_DEPS.$(LOCAL_MODULE).ALL_DEPS := $(sort \ - $(ALL_DEPS.$(LOCAL_MODULE).ALL_DEPS) \ - $(LOCAL_STATIC_LIBRARIES) \ - $(LOCAL_WHOLE_STATIC_LIBRARIES) \ - $(LOCAL_SHARED_LIBRARIES) \ - $(LOCAL_DYLIB_LIBRARIES) \ - $(LOCAL_RLIB_LIBRARIES) \ - $(LOCAL_PROC_MACRO_LIBRARIES) \ - $(LOCAL_HEADER_LIBRARIES) \ - $(LOCAL_STATIC_JAVA_LIBRARIES) \ - $(LOCAL_JAVA_LIBRARIES) \ - $(LOCAL_JNI_SHARED_LIBRARIES)) - -license_files := $(call find-parent-file,$(LOCAL_PATH),MODULE_LICENSE*) -ALL_DEPS.$(LOCAL_MODULE).LICENSE := $(sort $(ALL_DEPS.$(LOCAL_MODULE).LICENSE) $(license_files)) - -########################################################### -## Take care of my_module_tags -########################################################### - -# Keep track of all the tags we've seen. -ALL_MODULE_TAGS := $(sort $(ALL_MODULE_TAGS) $(my_module_tags)) - -# Add this module name to the tag list of each specified tag. -$(foreach tag,$(filter-out optional,$(my_module_tags)),\ - $(eval ALL_MODULE_NAME_TAGS.$(tag) := $$(ALL_MODULE_NAME_TAGS.$(tag)) $(my_register_name))) - ########################################################### ## umbrella targets used to verify builds ########################################################### @@ -1215,7 +1231,7 @@ endif ifdef j_or_n $(j_or_n) $(h_or_t) $(j_or_n)-$(h_or_hc_or_t) : $(my_checked_module) -ifneq (,$(filter $(my_module_tags),tests)) +ifneq (,$(filter $(LOCAL_MODULE_TAGS),tests)) $(j_or_n)-$(h_or_t)-tests $(j_or_n)-tests $(h_or_t)-tests : $(my_checked_module) endif $(LOCAL_MODULE)-$(h_or_hc_or_t) : $(my_all_targets) @@ -1240,3 +1256,8 @@ endif ########################################################### include $(BUILD_NOTICE_FILE) + +########################################################### +## SBOM generation +########################################################### +include $(BUILD_SBOM_GEN) diff --git a/core/binary.mk b/core/binary.mk index 6f1d814a6e..f86b5a464e 100644 --- a/core/binary.mk +++ b/core/binary.mk @@ -145,15 +145,21 @@ endif ifneq (,$(strip $(foreach dir,$(NATIVE_COVERAGE_PATHS),$(filter $(dir)%,$(LOCAL_PATH))))) ifeq (,$(strip $(foreach dir,$(NATIVE_COVERAGE_EXCLUDE_PATHS),$(filter $(dir)%,$(LOCAL_PATH))))) my_native_coverage := true + my_clang_coverage := true else my_native_coverage := false + my_clang_coverage := false endif else my_native_coverage := false + my_clang_coverage := false endif ifneq ($(NATIVE_COVERAGE),true) my_native_coverage := false endif +ifneq ($(CLANG_COVERAGE),true) + my_clang_coverage := false +endif # Exclude directories from checking allowed manual binder interface lists. # TODO(b/145621474): Move this check into IInterface.h when clang-tidy no longer uses absolute paths. @@ -168,7 +174,6 @@ my_allow_undefined_symbols := true endif endif -my_ndk_sysroot := my_ndk_sysroot_include := my_ndk_sysroot_lib := my_api_level := 10000 @@ -183,11 +188,7 @@ ifneq ($(LOCAL_SDK_VERSION),) # Make sure we've built the NDK. my_additional_dependencies += $(SOONG_OUT_DIR)/ndk_base.timestamp - ifneq (,$(filter arm64 x86_64,$(my_arch))) - my_min_sdk_version := 21 - else - my_min_sdk_version := $(MIN_SUPPORTED_SDK_VERSION) - endif + my_min_sdk_version := $(MIN_SUPPORTED_SDK_VERSION) # Historically we've just set up a bunch of symlinks in prebuilts/ndk to map # missing API levels to existing ones where necessary, but we're not doing @@ -200,38 +201,19 @@ ifneq ($(LOCAL_SDK_VERSION),) my_ndk_crt_version := $(my_ndk_api) - my_ndk_hist_api := $(my_ndk_api) - ifeq ($(my_ndk_api),current) - # The last API level supported by the old prebuilt NDKs. - my_ndk_hist_api := 24 - else + ifneq ($(my_ndk_api),current) my_api_level := $(my_ndk_api) endif my_ndk_source_root := \ $(HISTORICAL_NDK_VERSIONS_ROOT)/$(LOCAL_NDK_VERSION)/sources - my_ndk_sysroot := \ - $(HISTORICAL_NDK_VERSIONS_ROOT)/$(LOCAL_NDK_VERSION)/platforms/android-$(my_ndk_hist_api)/arch-$(my_arch) my_built_ndk := $(SOONG_OUT_DIR)/ndk my_ndk_triple := $($(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_NDK_TRIPLE) my_ndk_sysroot_include := \ $(my_built_ndk)/sysroot/usr/include \ $(my_built_ndk)/sysroot/usr/include/$(my_ndk_triple) \ - $(my_ndk_sysroot)/usr/include \ - - # x86_64 is a multilib toolchain, so their libraries are - # installed in /usr/lib64. Aarch64, on the other hand, is not a multilib - # compiler, so its libraries are in /usr/lib. - ifneq (,$(filter x86_64,$(my_arch))) - my_ndk_libdir_name := lib64 - else - my_ndk_libdir_name := lib - endif - my_ndk_platform_dir := \ - $(my_built_ndk)/platforms/android-$(my_ndk_api)/arch-$(my_arch) - my_built_ndk_libs := $(my_ndk_platform_dir)/usr/$(my_ndk_libdir_name) - my_ndk_sysroot_lib := $(my_ndk_sysroot)/usr/$(my_ndk_libdir_name) + my_ndk_sysroot_lib := $(my_built_ndk)/sysroot/usr/lib/$(my_ndk_triple)/$(my_ndk_api) # The bionic linker now has support for packed relocations and gnu style # hashes (which are much faster!), but shipping to older devices requires @@ -294,10 +276,18 @@ ifneq ($(LOCAL_SDK_VERSION),) ifneq ($(my_ndk_api),current) ifeq ($(call math_lt, $(my_ndk_api),23),true) my_native_coverage := false + my_clang_coverage := false endif endif endif +ifneq ($(LOCAL_MIN_SDK_VERSION),) + ifdef LOCAL_IS_HOST_MODULE + $(error $(LOCAL_PATH): LOCAL_MIN_SDK_VERSION cannot be used in host module) + endif + my_api_level := $(LOCAL_MIN_SDK_VERSION) +endif + ifeq ($(NATIVE_COVERAGE),true) ifndef LOCAL_IS_HOST_MODULE my_ldflags += -Wl,--wrap,getenv @@ -312,26 +302,51 @@ ifeq ($(NATIVE_COVERAGE),true) endif endif -ifneq ($(LOCAL_USE_VNDK),) - # Required VNDK version for vendor modules is BOARD_VNDK_VERSION. - my_api_level := $(BOARD_VNDK_VERSION) - ifeq ($(my_api_level),current) - # Build with current PLATFORM_VNDK_VERSION. - # If PLATFORM_VNDK_VERSION has a CODENAME, it will return - # __ANDROID_API_FUTURE__. - my_api_level := $(call codename-or-sdk-to-sdk,$(PLATFORM_VNDK_VERSION)) - else - # Build with current BOARD_VNDK_VERSION. - my_api_level := $(call codename-or-sdk-to-sdk,$(BOARD_VNDK_VERSION)) +ifeq ($(CLANG_COVERAGE),true) + ifndef LOCAL_IS_HOST_MODULE + my_ldflags += $(CLANG_COVERAGE_HOST_LDFLAGS) + ifneq ($(LOCAL_MODULE_CLASS),STATIC_LIBRARIES) + my_whole_static_libraries += libclang_rt.profile + ifeq ($(LOCAL_SDK_VERSION),) + my_whole_static_libraries += libprofile-clang-extras + else + my_whole_static_libraries += libprofile-clang-extras_ndk + endif + endif + endif + ifeq ($(my_clang_coverage),true) + my_profile_instr_generate := $(CLANG_COVERAGE_INSTR_PROFILE) + ifeq ($(CLANG_COVERAGE_CONTINUOUS_MODE),true) + my_cflags += $(CLANG_COVERAGE_CONTINUOUS_FLAGS) + my_ldflags += $(CLANG_COVERAGE_CONTINUOUS_FLAGS) + endif + + my_profile_instr_generate += $(CLANG_COVERAGE_CONFIG_COMMFLAGS) + my_cflags += $(CLANG_COVERAGE_INSTR_PROFILE) $(CLANG_COVERAGE_CONFIG_CFLAGS) $(CLANG_COVERAGE_CONFIG_COMMFLAGS) + my_ldflags += $(CLANG_COVERAGE_CONFIG_COMMFLAGS) + + ifneq ($(filter hwaddress,$(my_sanitize)),) + my_cflags += $(CLANG_COVERAGE_HWASAN_FLAGS) + my_ldflags += $(CLANG_COVERAGE_HWASAN_FLAGS) + endif endif +endif + +ifneq ($(call module-in-vendor-or-product),) my_cflags += -D__ANDROID_VNDK__ - ifneq ($(LOCAL_USE_VNDK_VENDOR),) - # Vendor modules have LOCAL_USE_VNDK_VENDOR when - # BOARD_VNDK_VERSION is defined. + ifneq ($(LOCAL_IN_VENDOR),) + # Vendor modules have LOCAL_IN_VENDOR my_cflags += -D__ANDROID_VENDOR__ - else ifneq ($(LOCAL_USE_VNDK_PRODUCT),) - # Product modules have LOCAL_USE_VNDK_PRODUCT when - # PRODUCT_PRODUCT_VNDK_VERSION is defined. + + ifeq ($(BOARD_API_LEVEL),) + # TODO(b/314036847): This is a fallback for UDC targets. + # This must be a build failure when UDC is no longer built from this source tree. + my_cflags += -D__ANDROID_VENDOR_API__=$(PLATFORM_SDK_VERSION) + else + my_cflags += -D__ANDROID_VENDOR_API__=$(BOARD_API_LEVEL) + endif + else ifneq ($(LOCAL_IN_PRODUCT),) + # Product modules have LOCAL_IN_PRODUCT my_cflags += -D__ANDROID_PRODUCT__ endif endif @@ -378,17 +393,15 @@ endif # MinGW spits out warnings about -fPIC even for -fpie?!) being ignored because # all code is position independent, and then those warnings get promoted to # errors. -ifneq ($(LOCAL_NO_PIC),true) - ifneq ($(filter EXECUTABLES NATIVE_TESTS,$(LOCAL_MODULE_CLASS)),) - my_cflags += -fPIE - ifndef BUILD_HOST_static - ifneq ($(LOCAL_FORCE_STATIC_EXECUTABLE),true) - my_ldflags += -pie - endif +ifneq ($(filter EXECUTABLES NATIVE_TESTS,$(LOCAL_MODULE_CLASS)),) + my_cflags += -fPIE + ifndef BUILD_HOST_static + ifneq ($(LOCAL_FORCE_STATIC_EXECUTABLE),true) + my_ldflags += -pie endif - else - my_cflags += -fPIC endif +else + my_cflags += -fPIC endif ifdef LOCAL_IS_HOST_MODULE @@ -468,6 +481,34 @@ ifneq ($(filter external/%,$(LOCAL_PATH)),) my_cflags += $(CLANG_EXTERNAL_CFLAGS) endif +# Extra cflags for projects under hardware/ directory. +# This should match the definition of `thirdPartyDirPrefixExceptions` +# in build/soong/android/paths.go. +# Get the second element of LOCAL_PATH +ifneq ($(filter hardware/%,$(LOCAL_PATH)),) + my_subdir := $(word 2,$(subst /,$(space),$(LOCAL_PATH))) + must_compile_hardware_subdirs := \ + hardware/google/% \ + hardware/interfaces/% \ + hardware/libhardware/% \ + hardware/libhardware_legacy/% \ + hardware/ril/% + ifeq ($(filter $(must_compile_hardware_subdirs),$(my_subdir)),) + my_cflags += $(CLANG_EXTERNAL_CFLAGS) + endif +endif + +# Extra cflags for projects under vendor/ directory. +# This should match the definition of `thirdPartyDirPrefixExceptions` +# in build/soong/android/paths.go. +ifneq ($(filter vendor/%,$(LOCAL_PATH)),) + my_subdir := $(word 2,$(subst /,$(space),$(LOCAL_PATH))) + # Do not add the flags for any subdir that contains the string "google". + ifneq ($(findstring google,$(my_subdir)),) + my_cflags += $(CLANG_EXTERNAL_CFLAGS) + endif +endif + # arch-specific static libraries go first so that generic ones can depend on them my_static_libraries := $(LOCAL_STATIC_LIBRARIES_$($(my_prefix)$(LOCAL_2ND_ARCH_VAR_PREFIX)ARCH)) $(LOCAL_STATIC_LIBRARIES_$(my_32_64_bit_suffix)) $(my_static_libraries) my_whole_static_libraries := $(LOCAL_WHOLE_STATIC_LIBRARIES_$($(my_prefix)$(LOCAL_2ND_ARCH_VAR_PREFIX)ARCH)) $(LOCAL_WHOLE_STATIC_LIBRARIES_$(my_32_64_bit_suffix)) $(my_whole_static_libraries) @@ -1155,14 +1196,25 @@ ifneq ($(filter hwaddress,$(my_sanitize)),) endif ################################################################### +## When compiling a memtag_stack enabled target, use the .memtag_stack variant +## of any static dependencies (where they exist). +################################################################## +ifneq ($(filter memtag_stack,$(my_sanitize)),) + my_whole_static_libraries := $(call use_soong_sanitized_static_libraries,\ + $(my_whole_static_libraries),memtag_stack) + my_static_libraries := $(call use_soong_sanitized_static_libraries,\ + $(my_static_libraries),memtag_stack) +endif + +################################################################### ## When compiling against API imported module, use API import stub ## libraries. ################################################################## apiimport_postfix := .apiimport -ifneq ($(LOCAL_USE_VNDK),) - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) +ifneq ($(call module-in-vendor-or-product),) + ifeq ($(LOCAL_IN_PRODUCT),true) apiimport_postfix := .apiimport.product else apiimport_postfix := .apiimport.vendor @@ -1179,14 +1231,14 @@ my_header_libraries := $(foreach l,$(my_header_libraries), \ ########################################################### ## When compiling against the VNDK, use LL-NDK libraries ########################################################### -ifneq ($(LOCAL_USE_VNDK),) +ifneq ($(call module-in-vendor-or-product),) ##################################################### ## Soong modules may be built three times, once for ## /system, once for /vendor and once for /product. ## If we're using the VNDK, switch all soong ## libraries over to the /vendor or /product variant. ##################################################### - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) + ifeq ($(LOCAL_IN_PRODUCT),true) my_whole_static_libraries := $(foreach l,$(my_whole_static_libraries),\ $(if $(SPLIT_PRODUCT.STATIC_LIBRARIES.$(l)),$(l).product,$(l))) my_static_libraries := $(foreach l,$(my_static_libraries),\ @@ -1213,7 +1265,7 @@ endif # Platform can use vendor public libraries. If a required shared lib is one of # the vendor public libraries, the lib is switched to the stub version of the lib. -ifeq ($(LOCAL_USE_VNDK),) +ifeq ($(call module-in-vendor-or-product),) my_shared_libraries := $(foreach l,$(my_shared_libraries),\ $(if $(filter $(l),$(VENDOR_PUBLIC_LIBRARIES)),$(l).vendorpublic,$(l))) endif @@ -1265,7 +1317,7 @@ ifdef LOCAL_SDK_VERSION my_link_type := native:ndk:$(my_ndk_stl_family):$(my_ndk_stl_link_type) my_warn_types := $(my_warn_ndk_types) my_allowed_types := $(my_allowed_ndk_types) -else ifdef LOCAL_USE_VNDK +else ifeq ($(call module-in-vendor-or-product),true) _name := $(patsubst %.vendor,%,$(LOCAL_MODULE)) _name := $(patsubst %.product,%,$(LOCAL_MODULE)) ifneq ($(filter $(_name),$(VNDK_CORE_LIBRARIES) $(VNDK_SAMEPROCESS_LIBRARIES) $(LLNDK_LIBRARIES)),) @@ -1276,7 +1328,7 @@ else ifdef LOCAL_USE_VNDK endif my_warn_types := my_allowed_types := native:vndk native:vndk_private - else ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) + else ifeq ($(LOCAL_IN_PRODUCT),true) # Modules installed to /product cannot directly depend on modules marked # with vendor_available: false my_link_type := native:product @@ -1428,7 +1480,6 @@ my_system_shared_libraries_fullpath := \ my_ndk_shared_libraries_fullpath := \ $(foreach _lib,$(my_ndk_shared_libraries),\ $(if $(filter $(NDK_KNOWN_LIBS),$(_lib)),\ - $(my_built_ndk_libs)/$(_lib)$(so_suffix),\ $(my_ndk_sysroot_lib)/$(_lib)$(so_suffix))) built_shared_libraries += \ @@ -1462,17 +1513,6 @@ built_whole_libraries := \ $(call intermediates-dir-for, \ STATIC_LIBRARIES,$(lib),$(my_kind),,$(LOCAL_2ND_ARCH_VAR_PREFIX),$(my_host_cross))/$(lib)$(a_suffix)) -# We don't care about installed static libraries, since the -# libraries have already been linked into the module at that point. -# We do, however, care about the NOTICE files for any static -# libraries that we use. (see notice_files.mk) -installed_static_library_notice_file_targets := \ - $(foreach lib,$(my_static_libraries) $(my_whole_static_libraries), \ - NOTICE-$(if $(LOCAL_IS_HOST_MODULE),HOST$(if $(my_host_cross),_CROSS,),TARGET)-STATIC_LIBRARIES-$(lib)) - -$(notice_target): | $(installed_static_library_notice_file_targets) -$(LOCAL_INSTALLED_MODULE): | $(notice_target) - # Default is -fno-rtti. ifeq ($(strip $(LOCAL_RTTI_FLAG)),) LOCAL_RTTI_FLAG := -fno-rtti @@ -1591,21 +1631,16 @@ my_ldlibs += $(my_cxx_ldlibs) ########################################################### ifndef LOCAL_IS_HOST_MODULE -ifdef LOCAL_USE_VNDK +ifeq ($(call module-in-vendor-or-product),true) my_target_global_c_includes := my_target_global_c_system_includes := $(TARGET_OUT_HEADERS) else ifdef LOCAL_SDK_VERSION my_target_global_c_includes := my_target_global_c_system_includes := $(my_ndk_stl_include_path) $(my_ndk_sysroot_include) -else ifdef BOARD_VNDK_VERSION - my_target_global_c_includes := $(SRC_HEADERS) \ - $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_INCLUDES) - my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) \ - $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_SYSTEM_INCLUDES) else my_target_global_c_includes := $(SRC_HEADERS) \ $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_INCLUDES) - my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) $(TARGET_OUT_HEADERS) \ + my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) \ $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_SYSTEM_INCLUDES) endif @@ -1690,16 +1725,10 @@ endif #################################################### imported_includes := -ifdef LOCAL_USE_VNDK +ifeq (true,$(call module-in-vendor-or-product)) imported_includes += $(call intermediates-dir-for,HEADER_LIBRARIES,device_kernel_headers,$(my_kind),,$(LOCAL_2ND_ARCH_VAR_PREFIX),$(my_host_cross)) -else ifdef LOCAL_SDK_VERSION - # Apps shouldn't need device-specific kernel headers -else ifdef BOARD_VNDK_VERSION - # For devices building with the VNDK, only the VNDK gets device-specific kernel headers by default - # In soong, it's entirely opt-in else - # For older non-VNDK builds, continue adding in kernel headers to everything like we used to - imported_includes += $(call intermediates-dir-for,HEADER_LIBRARIES,device_kernel_headers,$(my_kind),,$(LOCAL_2ND_ARCH_VAR_PREFIX),$(my_host_cross)) + # everything else should manually specify headers endif imported_includes := $(strip \ diff --git a/core/board_config.mk b/core/board_config.mk index 70c91a80b3..e184601008 100644 --- a/core/board_config.mk +++ b/core/board_config.mk @@ -144,9 +144,6 @@ _board_strip_list += BOARD_AVB_PVMFW_KEY_PATH _board_strip_list += BOARD_AVB_PVMFW_ALGORITHM _board_strip_list += BOARD_AVB_PVMFW_ROLLBACK_INDEX_LOCATION _board_strip_list += BOARD_PARTIAL_OTA_UPDATE_PARTITIONS_LIST -_board_strip_list += BOARD_BPT_DISK_SIZE -_board_strip_list += BOARD_BPT_INPUT_FILES -_board_strip_list += BOARD_BPT_MAKE_TABLE_ARGS _board_strip_list += BOARD_AVB_VBMETA_VENDOR_ROLLBACK_INDEX_LOCATION _board_strip_list += BOARD_AVB_VBMETA_VENDOR_ALGORITHM _board_strip_list += BOARD_AVB_VBMETA_VENDOR_KEY_PATH @@ -164,9 +161,6 @@ _board_strip_list += BOARD_AVB_VENDOR_BOOT_ROLLBACK_INDEX_LOCATION _board_strip_list += BOARD_AVB_VENDOR_KERNEL_BOOT_KEY_PATH _board_strip_list += BOARD_AVB_VENDOR_KERNEL_BOOT_ALGORITHM _board_strip_list += BOARD_AVB_VENDOR_KERNEL_BOOT_ROLLBACK_INDEX_LOCATION -_board_strip_list += BOARD_GKI_SIGNING_SIGNATURE_ARGS -_board_strip_list += BOARD_GKI_SIGNING_ALGORITHM -_board_strip_list += BOARD_GKI_SIGNING_KEY_PATH _board_strip_list += BOARD_MKBOOTIMG_ARGS _board_strip_list += BOARD_VENDOR_BOOTIMAGE_PARTITION_SIZE _board_strip_list += BOARD_VENDOR_KERNEL_BOOTIMAGE_PARTITION_SIZE @@ -190,6 +184,9 @@ _build_broken_var_list := \ BUILD_BROKEN_USES_NETWORK \ BUILD_BROKEN_VENDOR_PROPERTY_NAMESPACE \ BUILD_BROKEN_VINTF_PRODUCT_COPY_FILES \ + BUILD_BROKEN_INCORRECT_PARTITION_IMAGES \ + BUILD_BROKEN_GENRULE_SANDBOXING \ + BUILD_BROKEN_DONT_CHECK_SYSTEMSDK \ _build_broken_var_list += \ $(foreach m,$(AVAILABLE_BUILD_MODULE_TYPES) \ @@ -203,7 +200,7 @@ _board_strip_readonly_list += $(_build_broken_var_list) \ # Conditional to building on linux, as dex2oat currently does not work on darwin. ifeq ($(HOST_OS),linux) - WITH_DEXPREOPT := true + WITH_DEXPREOPT ?= true endif # ############################################################### @@ -225,6 +222,8 @@ else board_config_mk := \ $(strip $(sort $(wildcard \ $(SRC_TARGET_DIR)/board/$(TARGET_DEVICE)/BoardConfig.mk \ + device/generic/goldfish/board/$(TARGET_DEVICE)/BoardConfig.mk \ + device/google/cuttlefish/board/$(TARGET_DEVICE)/BoardConfig.mk \ $(shell test -d device && find -L device -maxdepth 4 -path '*/$(TARGET_DEVICE)/BoardConfig.mk') \ $(shell test -d vendor && find -L vendor -maxdepth 4 -path '*/$(TARGET_DEVICE)/BoardConfig.mk') \ ))) @@ -255,7 +254,7 @@ else endif $(shell build/soong/scripts/update_out $(OUT_DIR)/rbc/rbc_board_config_results.mk \ - $(OUT_DIR)/rbcrun RBC_OUT="make" $(OUT_DIR)/rbc/boardlauncher.rbc) + $(OUT_DIR)/rbcrun --mode=rbc $(OUT_DIR)/rbc/boardlauncher.rbc) ifneq ($(.SHELLSTATUS),0) $(error board configuration runner failed: $(.SHELLSTATUS)) endif @@ -275,7 +274,7 @@ endif ifneq ($(MALLOC_IMPL),) $(warning *** Unsupported option MALLOC_IMPL defined by board config: $(board_config_mk).) - $(error Use `MALLOC_SVELTE := true` to configure jemalloc for low-memory) + $(error Use `MALLOC_LOW_MEMORY := true` to use low-memory allocator config) endif board_config_mk := @@ -288,6 +287,9 @@ $(foreach var,$(_board_true_false_vars), \ include $(BUILD_SYSTEM)/board_config_wifi.mk +# Set up soong config for "soong_config_value_variable". +-include vendor/google/build/soong/soong_config_namespace/camera.mk + # Default *_CPU_VARIANT_RUNTIME to CPU_VARIANT if unspecified. TARGET_CPU_VARIANT_RUNTIME := $(or $(TARGET_CPU_VARIANT_RUNTIME),$(TARGET_CPU_VARIANT)) TARGET_2ND_CPU_VARIANT_RUNTIME := $(or $(TARGET_2ND_CPU_VARIANT_RUNTIME),$(TARGET_2ND_CPU_VARIANT)) @@ -918,12 +920,6 @@ ifeq ($(PRODUCT_BUILD_PVMFW_IMAGE),true) endif .KATI_READONLY := BOARD_USES_PVMFWIMAGE -BUILDING_PVMFW_IMAGE := -ifeq ($(PRODUCT_BUILD_PVMFW_IMAGE),true) - BUILDING_PVMFW_IMAGE := true -endif -.KATI_READONLY := BUILDING_PVMFW_IMAGE - ########################################### # Ensure consistency among TARGET_RECOVERY_UPDATER_LIBS, AB_OTA_UPDATER, and PRODUCT_OTA_FORCE_NON_AB_PACKAGE. TARGET_RECOVERY_UPDATER_LIBS ?= @@ -953,6 +949,11 @@ ifneq ($(TARGET_OTA_ALLOW_NON_AB),true) endif endif +# For Non A/B full OTA, disable brotli compression. +ifeq ($(TARGET_OTA_ALLOW_NON_AB),true) + BOARD_NON_AB_OTA_DISABLE_COMPRESSION := true +endif + # Quick check for building generic OTA packages. Currently it only supports A/B OTAs. ifeq ($(PRODUCT_BUILD_GENERIC_OTA_PACKAGE),true) ifneq ($(AB_OTA_UPDATER),true) @@ -972,43 +973,7 @@ define check_vndk_version $(if $(wildcard $(vndk_path)/*/Android.bp),,$(error VNDK version $(1) not found)) endef -ifdef BOARD_VNDK_VERSION - ifeq ($(BOARD_VNDK_VERSION),$(PLATFORM_VNDK_VERSION)) - $(error BOARD_VNDK_VERSION is equal to PLATFORM_VNDK_VERSION; use BOARD_VNDK_VERSION := current) - endif - ifneq ($(BOARD_VNDK_VERSION),current) - $(call check_vndk_version,$(BOARD_VNDK_VERSION)) - endif - TARGET_VENDOR_TEST_SUFFIX := /vendor -else - TARGET_VENDOR_TEST_SUFFIX := -endif - -# If PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY is set, -# BOARD_VNDK_VERSION must be set because PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY -# is a enforcement of inter-partition dependency, and it doesn't have any meaning -# when BOARD_VNDK_VERSION isn't set. -ifeq ($(PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY),true) - ifeq ($(BOARD_VNDK_VERSION),) - $(error BOARD_VNDK_VERSION must be set when PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY is true) - endif -endif - -########################################### -# APEXes are by default flattened, i.e. non-updatable, if not building unbundled -# apps. It can be unflattened (and updatable) by inheriting from -# updatable_apex.mk -# -# APEX flattening can also be forcibly enabled (resp. disabled) by -# setting OVERRIDE_TARGET_FLATTEN_APEX to true (resp. false), e.g. by -# setting the OVERRIDE_TARGET_FLATTEN_APEX environment variable. -ifdef OVERRIDE_TARGET_FLATTEN_APEX - TARGET_FLATTEN_APEX := $(OVERRIDE_TARGET_FLATTEN_APEX) -else - ifeq (,$(TARGET_BUILD_APPS)$(TARGET_FLATTEN_APEX)) - TARGET_FLATTEN_APEX := true - endif -endif +TARGET_VENDOR_TEST_SUFFIX := /vendor ifeq (,$(TARGET_BUILD_UNBUNDLED)) ifdef PRODUCT_EXTRA_VNDK_VERSIONS @@ -1024,6 +989,16 @@ ifneq (,$(_unsupported_systemsdk_versions)) endif ########################################### +# BOARD_API_LEVEL for vendor API surface +ifdef RELEASE_BOARD_API_LEVEL + ifdef BOARD_API_LEVEL + $(error BOARD_API_LEVEL must not be set manually. The build system automatically sets this value.) + endif + BOARD_API_LEVEL := $(RELEASE_BOARD_API_LEVEL) + .KATI_READONLY := BOARD_API_LEVEL +endif + +########################################### # Handle BUILD_BROKEN_USES_BUILD_* $(foreach m,$(DEFAULT_WARNING_BUILD_MODULE_TYPES),\ diff --git a/core/board_config_wifi.mk b/core/board_config_wifi.mk index ddeb0d7d6d..8289bf2c5d 100644 --- a/core/board_config_wifi.mk +++ b/core/board_config_wifi.mk @@ -74,4 +74,10 @@ ifdef WIFI_HIDL_FEATURE_DISABLE_AP_MAC_RANDOMIZATION endif ifdef WIFI_AVOID_IFACE_RESET_MAC_CHANGE $(call soong_config_set,wifi,avoid_iface_reset_mac_change,true) +endif +ifdef WIFI_SKIP_STATE_TOGGLE_OFF_ON_FOR_NAN + $(call soong_config_set,wifi,wifi_skip_state_toggle_off_on_for_nan,true) +endif +ifeq ($(strip $(TARGET_USES_AOSP_FOR_WLAN)),true) + $(call soong_config_set,wifi,target_uses_aosp_for_wlan,true) endif
\ No newline at end of file diff --git a/core/build_id.mk b/core/build_id.mk index ba5ca42dc7..bed839f57c 100644 --- a/core/build_id.mk +++ b/core/build_id.mk @@ -18,4 +18,4 @@ # (like "CRB01"). It must be a single word, and is # capitalized by convention. -BUILD_ID=AOSP.MASTER +BUILD_ID=MAIN diff --git a/core/cc_prebuilt_internal.mk b/core/cc_prebuilt_internal.mk index 2de4115fb4..e34e110953 100644 --- a/core/cc_prebuilt_internal.mk +++ b/core/cc_prebuilt_internal.mk @@ -56,9 +56,6 @@ ifneq ($(filter true keep_symbols no_debuglink mini-debug-info,$(my_strip_module ifeq ($(filter SHARED_LIBRARIES EXECUTABLES NATIVE_TESTS,$(LOCAL_MODULE_CLASS)),) $(call pretty-error,Can strip/pack only shared libraries or executables) endif - ifneq ($(LOCAL_PREBUILT_STRIP_COMMENTS),) - $(call pretty-error,Cannot strip/pack scripts) - endif # Set the arch-specific variables to set up the strip rules LOCAL_STRIP_MODULE_$($(my_prefix)$(LOCAL_2ND_ARCH_VAR_PREFIX)ARCH) := $(my_strip_module) include $(BUILD_SYSTEM)/dynamic_binary.mk @@ -83,7 +80,7 @@ include $(BUILD_SYSTEM)/allowed_ndk_types.mk ifdef LOCAL_SDK_VERSION my_link_type := native:ndk:$(my_ndk_stl_family):$(my_ndk_stl_link_type) -else ifdef LOCAL_USE_VNDK +else ifeq ($(call module-in-vendor-or-product),true) _name := $(patsubst %.vendor,%,$(LOCAL_MODULE)) _name := $(patsubst %.product,%,$(LOCAL_MODULE)) ifneq ($(filter $(_name),$(VNDK_CORE_LIBRARIES) $(VNDK_SAMEPROCESS_LIBRARIES) $(LLNDK_LIBRARIES)),) @@ -93,7 +90,7 @@ else ifdef LOCAL_USE_VNDK my_link_type := native:vndk_private endif else - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) + ifeq ($(LOCAL_IN_PRODUCT),true) my_link_type := native:product else my_link_type := native:vendor @@ -142,8 +139,8 @@ include $(BUILD_SYSTEM)/cxx_stl_setup.mk # When compiling against API imported module, use API import stub libraries. apiimport_postfix := .apiimport -ifneq ($(LOCAL_USE_VNDK),) - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) +ifeq ($(call module-in-vendor-or-product),true) + ifeq ($(LOCAL_IN_PRODUCT),true) apiimport_postfix := .apiimport.product else apiimport_postfix := .apiimport.vendor @@ -161,8 +158,8 @@ my_system_shared_libraries := $(foreach l,$(my_system_shared_libraries), \ endif #my_system_shared_libraries ifdef my_shared_libraries -ifdef LOCAL_USE_VNDK - ifeq ($(LOCAL_USE_VNDK_PRODUCT),true) +ifeq ($(call module-in-vendor-or-product),true) + ifeq ($(LOCAL_IN_PRODUCT),true) my_shared_libraries := $(foreach l,$(my_shared_libraries),\ $(if $(SPLIT_PRODUCT.SHARED_LIBRARIES.$(l)),$(l).product,$(l))) else diff --git a/core/cleanbuild.mk b/core/cleanbuild.mk index 5576785eca..f41f1b7c41 100644 --- a/core/cleanbuild.mk +++ b/core/cleanbuild.mk @@ -33,8 +33,6 @@ include $(BUILD_SYSTEM)/clang/config.mk # CTS-specific config. -include cts/build/config.mk -# VTS-specific config. --include test/vts/tools/vts-tradefed/build/config.mk # device-tests-specific-config. -include tools/tradefederation/build/suites/device-tests/config.mk # general-tests-specific-config. diff --git a/core/clear_vars.mk b/core/clear_vars.mk index e32576067d..fb42878584 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -5,21 +5,18 @@ # '',true LOCAL_2ND_ARCH_VAR_PREFIX:= LOCAL_32_BIT_ONLY:= -LOCAL_AAPT2_ONLY:= LOCAL_AAPT_FLAGS:= LOCAL_AAPT_INCLUDE_ALL_RESOURCES:= LOCAL_AAPT_NAMESPACES:= +LOCAL_ACONFIG_FILES:= LOCAL_ADDITIONAL_CERTIFICATES:= LOCAL_ADDITIONAL_CHECKED_MODULE:= LOCAL_ADDITIONAL_DEPENDENCIES:= -LOCAL_ADDITIONAL_HTML_DIR:= -LOCAL_ADDITIONAL_JAVA_DIR:= LOCAL_AIDL_INCLUDES:= LOCAL_ALLOW_UNDEFINED_SYMBOLS:= LOCAL_ANNOTATION_PROCESSORS:= LOCAL_ANNOTATION_PROCESSOR_CLASSES:= -LOCAL_APIDIFF_NEWAPI:= -LOCAL_APIDIFF_OLDAPI:= +LOCAL_APEX_KEY_PATH:= LOCAL_APK_LIBRARIES:= LOCAL_APK_SET_INSTALL_FILE:= LOCAL_APKCERTS_FILE:= @@ -52,8 +49,6 @@ LOCAL_CPP_EXTENSION:= LOCAL_CPPFLAGS:= LOCAL_CPP_STD:= LOCAL_C_STD:= -LOCAL_CTS_TEST_PACKAGE:= -LOCAL_CTS_TEST_RUNNER:= LOCAL_CXX:= LOCAL_CXX_STL := default LOCAL_DEX_PREOPT_APP_IMAGE:= @@ -72,21 +67,11 @@ LOCAL_DPI_FILE_STEM:= LOCAL_DPI_VARIANTS:= LOCAL_DROIDDOC_ANNOTATIONS_ZIP := LOCAL_DROIDDOC_API_VERSIONS_XML := -LOCAL_DROIDDOC_ASSET_DIR:= -LOCAL_DROIDDOC_CUSTOM_ASSET_DIR:= -LOCAL_DROIDDOC_CUSTOM_TEMPLATE_DIR:= LOCAL_DROIDDOC_DOC_ZIP := -LOCAL_DROIDDOC_HTML_DIR:= LOCAL_DROIDDOC_METADATA_ZIP:= -LOCAL_DROIDDOC_OPTIONS:= -LOCAL_DROIDDOC_SOURCE_PATH:= -LOCAL_DROIDDOC_STUB_OUT_DIR:= LOCAL_DROIDDOC_STUBS_SRCJAR := -LOCAL_DROIDDOC_TEMPLATE_DIR:= -LOCAL_DROIDDOC_USE_STANDARD_DOCLET:= LOCAL_DX_FLAGS:= LOCAL_DYLIB_LIBRARIES:= -LOCAL_EMMA_COVERAGE_FILTER:= LOCAL_EMMA_INSTRUMENT:= LOCAL_ENFORCE_USES_LIBRARIES:= LOCAL_ERROR_PRONE_FLAGS:= @@ -102,7 +87,6 @@ LOCAL_EXPORT_STATIC_LIBRARY_HEADERS:= LOCAL_EXTRA_FULL_TEST_CONFIGS:= LOCAL_EXTRACT_APK:= LOCAL_EXTRACT_DPI_APK:= -LOCAL_FILE_CONTEXTS:= LOCAL_FINDBUGS_FLAGS:= LOCAL_FORCE_STATIC_EXECUTABLE:= LOCAL_FULL_CLASSES_JACOCO_JAR:= @@ -127,7 +111,6 @@ LOCAL_INJECT_BSSL_HASH:= LOCAL_INSTALLED_MODULE:= LOCAL_INSTALLED_MODULE_STEM:= LOCAL_INSTRUMENTATION_FOR:= -LOCAL_INTERMEDIATE_SOURCE_DIR:= LOCAL_INTERMEDIATE_SOURCES:= LOCAL_INTERMEDIATE_TARGETS:= LOCAL_IS_FUZZ_TARGET:= @@ -135,15 +118,8 @@ LOCAL_IS_HOST_MODULE:= LOCAL_IS_RUNTIME_RESOURCE_OVERLAY:= LOCAL_IS_UNIT_TEST:= LOCAL_TEST_OPTIONS_TAGS:= -LOCAL_JACK_CLASSPATH:= LOCAL_JACK_COVERAGE_EXCLUDE_FILTER:= LOCAL_JACK_COVERAGE_INCLUDE_FILTER:= -# '' (ie disabled), disabled, full, incremental, javac_frontend -LOCAL_JACK_ENABLED:=$(DEFAULT_JACK_ENABLED) -LOCAL_JACK_FLAGS:= -LOCAL_JACK_PLUGIN:= -LOCAL_JACK_PLUGIN_PATH:= -LOCAL_JACK_PROGUARD_FLAGS:= LOCAL_JAR_EXCLUDE_FILES:= LOCAL_JAR_EXCLUDE_PACKAGES:= LOCAL_JARJAR_RULES:= @@ -156,7 +132,6 @@ LOCAL_JAVA_LANGUAGE_VERSION:= LOCAL_JAVA_LIBRARIES:= LOCAL_JAVA_RESOURCE_DIRS:= LOCAL_JAVA_RESOURCE_FILES:= -LOCAL_JETIFIER_ENABLED:= LOCAL_JNI_SHARED_LIBRARIES:= LOCAL_JNI_SHARED_LIBRARIES_ABI:= LOCAL_CERTIFICATE_LINEAGE:= @@ -198,10 +173,8 @@ LOCAL_NDK_STL_VARIANT:= LOCAL_NDK_VERSION:=current LOCAL_NO_CRT:= LOCAL_NO_DEFAULT_COMPILER_FLAGS:= -LOCAL_NO_FPIE := LOCAL_NO_LIBCRT_BUILTINS:= LOCAL_NO_NOTICE_FILE:= -LOCAL_NO_PIC:= LOCAL_NOSANITIZE:= LOCAL_NO_STANDARD_LIBRARIES:= LOCAL_NO_STATIC_ANALYZER:= @@ -227,7 +200,6 @@ LOCAL_PREBUILT_LIBS:= LOCAL_PREBUILT_MODULE_FILE:= LOCAL_PREBUILT_OBJ_FILES:= LOCAL_PREBUILT_STATIC_JAVA_LIBRARIES:= -LOCAL_PREBUILT_STRIP_COMMENTS:= LOCAL_USE_EMBEDDED_DEX:= LOCAL_USE_EMBEDDED_NATIVE_LIBS:= LOCAL_PRESUBMIT_DISABLED:= @@ -263,7 +235,6 @@ LOCAL_REQUIRED_MODULES:= LOCAL_RES_LIBRARIES:= LOCAL_RESOURCE_DIR:= LOCAL_RLIB_LIBRARIES:= -LOCAL_RMTYPEDEFS:= LOCAL_ROTATION_MIN_SDK_VERSION:= LOCAL_RUNTIME_LIBRARIES:= LOCAL_RRO_THEME:= @@ -293,9 +264,13 @@ LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR := LOCAL_SOONG_LICENSE_METADATA := LOCAL_SOONG_LINK_TYPE := LOCAL_SOONG_LINT_REPORTS := +LOCAL_SOONG_LOGTAGS_FILES := +LOCAL_SOONG_MODULE_INFO_JSON := +LOCAL_SOONG_MODULE_TYPE := LOCAL_SOONG_PROGUARD_DICT := LOCAL_SOONG_PROGUARD_USAGE_ZIP := LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE := +LOCAL_SOONG_TRANSITIVE_RES_PACKAGES := LOCAL_SOONG_DEVICE_RRO_DIRS := LOCAL_SOONG_PRODUCT_RRO_DIRS := LOCAL_SOONG_STATIC_LIBRARY_EXTRA_PACKAGES := @@ -317,6 +292,7 @@ LOCAL_STRIP_MODULE:= LOCAL_SYSTEM_SHARED_LIBRARIES:=none LOCAL_TARGET_REQUIRED_MODULES:= LOCAL_TEST_CONFIG:= +LOCAL_TEST_CONFIG_SUFFIX:= LOCAL_TEST_DATA:= LOCAL_TEST_DATA_BINS:= LOCAL_TEST_MAINLINE_MODULES:= @@ -330,21 +306,16 @@ LOCAL_UNSTRIPPED_PATH:= LOCAL_USE_AAPT2:= LOCAL_USE_CLANG_LLD:= LOCAL_USE_VNDK:= -LOCAL_USE_VNDK_PRODUCT:= +LOCAL_IN_VENDOR:= +LOCAL_IN_PRODUCT:= LOCAL_USES_LIBRARIES:= LOCAL_VENDOR_MODULE:= LOCAL_VINTF_FRAGMENTS:= LOCAL_VNDK_DEPEND_ON_CORE_VARIANT:= -LOCAL_VTSC_FLAGS:= -LOCAL_VTS_INCLUDES:= -LOCAL_VTS_MODE:= LOCAL_WARNINGS_ENABLE:= LOCAL_WHOLE_STATIC_LIBRARIES:= LOCAL_YACCFLAGS:= LOCAL_CHECK_ELF_FILES:= -# TODO: deprecate, it does nothing -OVERRIDE_BUILT_MODULE_PATH:= - # arch specific variables LOCAL_ASFLAGS_$(TARGET_ARCH):= LOCAL_CFLAGS_$(TARGET_ARCH):= @@ -502,6 +473,7 @@ LOCAL_WHOLE_STATIC_LIBRARIES_64:= # Robolectric variables LOCAL_INSTRUMENT_SOURCE_DIRS := +LOCAL_INSTRUMENT_SRCJARS := LOCAL_ROBOTEST_FAILURE_FATAL := LOCAL_ROBOTEST_FILES := LOCAL_ROBOTEST_TIMEOUT := diff --git a/core/combo/HOST_darwin-x86_64.mk b/core/combo/HOST_darwin.mk index dac3bbf5cd..11063e6d25 100644 --- a/core/combo/HOST_darwin-x86_64.mk +++ b/core/combo/HOST_darwin.mk @@ -14,7 +14,7 @@ # limitations under the License. # -# Configuration for Darwin (Mac OS X) on x86_64. +# Configuration for Darwin (Mac OS X). # Included by combo/select.mk define $(combo_var_prefix)transform-shared-lib-to-toc diff --git a/core/combo/HOST_linux-x86.mk b/core/combo/HOST_linux.mk index 3f4ec0aed1..bfdd3eb701 100644 --- a/core/combo/HOST_linux-x86.mk +++ b/core/combo/HOST_linux.mk @@ -14,7 +14,7 @@ # limitations under the License. # -# Configuration for builds hosted on linux-x86. +# Configuration for builds hosted on linux. # Included by combo/select.mk define $(combo_var_prefix)transform-shared-lib-to-toc diff --git a/target/board/emulator_x86_64_arm64/device.mk b/core/combo/arch/arm64/armv9-a.mk index af023eb25c..de0760ae23 100755..100644 --- a/target/board/emulator_x86_64_arm64/device.mk +++ b/core/combo/arch/arm64/armv9-a.mk @@ -1,5 +1,5 @@ # -# Copyright (C) 2020 The Android Open Source Project +# Copyright (C) 2023 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,5 +14,6 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. +# .mk file required to support build for the new armv9-a Arm64 arch +# variant. The file just needs to be present but does not require to contain +# anything diff --git a/core/combo/arch/x86/goldmont-without-sha-xsaves.mk b/core/combo/arch/x86/goldmont-without-sha-xsaves.mk new file mode 100644 index 0000000000..1b93c17c69 --- /dev/null +++ b/core/combo/arch/x86/goldmont-without-sha-xsaves.mk @@ -0,0 +1,7 @@ +# This file contains feature macro definitions specific to the +# goldmont-without-xsaves arch variant. +# +# See build/make/core/combo/arch/x86/x86-atom.mk for differences. +# + +ARCH_X86_HAVE_SSE4_1 := true diff --git a/core/combo/arch/x86_64/goldmont-without-sha-xsaves.mk b/core/combo/arch/x86_64/goldmont-without-sha-xsaves.mk new file mode 100644 index 0000000000..1b93c17c69 --- /dev/null +++ b/core/combo/arch/x86_64/goldmont-without-sha-xsaves.mk @@ -0,0 +1,7 @@ +# This file contains feature macro definitions specific to the +# goldmont-without-xsaves arch variant. +# +# See build/make/core/combo/arch/x86/x86-atom.mk for differences. +# + +ARCH_X86_HAVE_SSE4_1 := true diff --git a/core/combo/select.mk b/core/combo/select.mk index 9c7e69e439..a836ed8d68 100644 --- a/core/combo/select.mk +++ b/core/combo/select.mk @@ -21,8 +21,12 @@ # combo_2nd_arch_prefix -- it's defined if this is loaded for the 2nd arch. # -# Build a target string like "linux-arm" or "darwin-x86". -combo_os_arch := $($(combo_target)OS)-$($(combo_target)$(combo_2nd_arch_prefix)ARCH) +ifeq ($(combo_target),HOST_) + combo_os_arch := $(HOST_OS) +else + # Build a target string like "linux-arm" or "darwin-x86". + combo_os_arch := $($(combo_target)OS)-$($(combo_target)$(combo_2nd_arch_prefix)ARCH) +endif combo_var_prefix := $(combo_2nd_arch_prefix)$(combo_target) diff --git a/core/config.mk b/core/config.mk index 025a3a1de5..aaf81177dd 100644 --- a/core/config.mk +++ b/core/config.mk @@ -42,6 +42,9 @@ endif # Mark variables deprecated/obsolete CHANGES_URL := https://android.googlesource.com/platform/build/+/master/Changes.md .KATI_READONLY := CHANGES_URL +$(KATI_deprecated_var TARGET_USES_64_BIT_BINDER,All devices use 64-bit binder by default now. Uses of TARGET_USES_64_BIT_BINDER should be removed.) +$(KATI_deprecated_var PRODUCT_SEPOLICY_SPLIT,All devices are built with split sepolicy.) +$(KATI_deprecated_var PRODUCT_SEPOLICY_SPLIT_OVERRIDE,All devices are built with split sepolicy.) $(KATI_obsolete_var PATH,Do not use PATH directly. See $(CHANGES_URL)#PATH) $(KATI_obsolete_var PYTHONPATH,Do not use PYTHONPATH directly. See $(CHANGES_URL)#PYTHONPATH) $(KATI_obsolete_var OUT,Use OUT_DIR instead. See $(CHANGES_URL)#OUT) @@ -107,6 +110,7 @@ $(KATI_obsolete_var BUILD_BROKEN_DUP_COPY_HEADERS) $(KATI_obsolete_var BUILD_BROKEN_ENG_DEBUG_TAGS) $(KATI_obsolete_export It is a global setting. See $(CHANGES_URL)#export_keyword) $(KATI_obsolete_var BUILD_BROKEN_ANDROIDMK_EXPORTS) +$(KATI_obsolete_var PRODUCT_NOTICE_SPLIT_OVERRIDE,Stop using this, keep calm, and carry on.) $(KATI_obsolete_var PRODUCT_STATIC_BOOT_CONTROL_HAL,Use shared library module instead. See $(CHANGES_URL)#PRODUCT_STATIC_BOOT_CONTROL_HAL) $(KATI_obsolete_var \ ARCH_ARM_HAVE_ARMV7A \ @@ -166,7 +170,9 @@ $(KATI_obsolete_var PRODUCT_SUPPORTS_VERITY_FEC,VB 1.0 and related variables are $(KATI_obsolete_var PRODUCT_SUPPORTS_BOOT_SIGNER,VB 1.0 and related variables are no longer supported) $(KATI_obsolete_var PRODUCT_VERITY_SIGNING_KEY,VB 1.0 and related variables are no longer supported) $(KATI_obsolete_var BOARD_PREBUILT_PVMFWIMAGE,pvmfw.bin is now built in AOSP and custom versions are no longer supported) +$(KATI_obsolete_var BUILDING_PVMFW_IMAGE,BUILDING_PVMFW_IMAGE is no longer used) $(KATI_obsolete_var BOARD_BUILD_SYSTEM_ROOT_IMAGE) +$(KATI_obsolete_var FS_GET_STATS) # Used to force goals to build. Only use for conditionally defined goals. .PHONY: FORCE @@ -232,6 +238,7 @@ BUILD_NATIVE_TEST :=$= $(BUILD_SYSTEM)/native_test.mk BUILD_FUZZ_TEST :=$= $(BUILD_SYSTEM)/fuzz_test.mk BUILD_NOTICE_FILE :=$= $(BUILD_SYSTEM)/notice_files.mk +BUILD_SBOM_GEN :=$= $(BUILD_SYSTEM)/sbom.mk include $(BUILD_SYSTEM)/deprecation.mk @@ -268,7 +275,7 @@ SOONG_CONFIG_NAMESPACES := # Ex: $(call add_soong_config_namespace,acme) define add_soong_config_namespace -$(eval SOONG_CONFIG_NAMESPACES += $1) \ +$(eval SOONG_CONFIG_NAMESPACES += $(strip $1)) \ $(eval SOONG_CONFIG_$(strip $1) :=) endef @@ -278,8 +285,8 @@ endef # $1 is the namespace. $2 is the list of variables. # Ex: $(call add_soong_config_var,acme,COOL_FEATURE_A COOL_FEATURE_B) define add_soong_config_var -$(eval SOONG_CONFIG_$(strip $1) += $2) \ -$(foreach v,$(strip $2),$(eval SOONG_CONFIG_$(strip $1)_$v := $($v))) +$(eval SOONG_CONFIG_$(strip $1) += $(strip $2)) \ +$(foreach v,$(strip $2),$(eval SOONG_CONFIG_$(strip $1)_$v := $(strip $($v)))) endef # The add_soong_config_var_value function defines a make variable and also adds @@ -288,7 +295,7 @@ endef # Ex: $(call add_soong_config_var_value,acme,COOL_FEATURE,true) define add_soong_config_var_value -$(eval $2 := $3) \ +$(eval $(strip $2) := $(strip $3)) \ $(call add_soong_config_var,$1,$2) endef @@ -296,8 +303,8 @@ endef # # internal utility to define a namespace and a variable in it. define soong_config_define_internal -$(if $(filter $1,$(SOONG_CONFIG_NAMESPACES)),,$(eval SOONG_CONFIG_NAMESPACES:=$(SOONG_CONFIG_NAMESPACES) $1)) \ -$(if $(filter $2,$(SOONG_CONFIG_$(strip $1))),,$(eval SOONG_CONFIG_$(strip $1):=$(SOONG_CONFIG_$(strip $1)) $2)) +$(if $(filter $1,$(SOONG_CONFIG_NAMESPACES)),,$(eval SOONG_CONFIG_NAMESPACES:=$(SOONG_CONFIG_NAMESPACES) $(strip $1))) \ +$(if $(filter $2,$(SOONG_CONFIG_$(strip $1))),,$(eval SOONG_CONFIG_$(strip $1):=$(SOONG_CONFIG_$(strip $1)) $(strip $2))) endef # soong_config_set defines the variable in the given Soong config namespace @@ -306,7 +313,7 @@ endef # Ex: $(call soong_config_set,acme,COOL_FEATURE,true) define soong_config_set $(call soong_config_define_internal,$1,$2) \ -$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$3) +$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(strip $3)) endef # soong_config_append appends to the value of the variable in the given Soong @@ -315,7 +322,7 @@ endef # $1 is the namespace, $2 is the variable name, $3 is the value define soong_config_append $(call soong_config_define_internal,$1,$2) \ -$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(SOONG_CONFIG_$(strip $1)_$(strip $2)) $3) +$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(SOONG_CONFIG_$(strip $1)_$(strip $2)) $(strip $3)) endef # soong_config_append gets to the value of the variable in the given Soong @@ -337,6 +344,23 @@ else JAVA_TMPDIR_ARG := endif +# These build broken variables are intended to be set in a buildspec file, +# while other build broken flags are expected to be set in a board config. +# These are build broken variables that are expected to apply across board +# configs, generally for cross-cutting features. + +# Build broken variables that should be treated as booleans +_build_broken_bool_vars := \ + BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES \ + +# Build broken variables that should be treated as lists +_build_broken_list_vars := \ + BUILD_BROKEN_PLUGIN_VALIDATION \ + +_build_broken_var_names := $(_build_broken_bool_vars) +_build_broken_var_names += $(_build_broken_list_vars) +$(foreach v,$(_build_broken_var_names),$(eval $(v) :=)) + # ############################################################### # Include sub-configuration files # ############################################################### @@ -356,6 +380,55 @@ endif # are specific to the user's build configuration. include $(BUILD_SYSTEM)/envsetup.mk + +$(foreach var,$(_build_broken_bool_vars), \ + $(if $(filter-out true false,$($(var))), \ + $(error Valid values of $(var) are "true", "false", and "". Not "$($(var))"))) + +.KATI_READONLY := $(_build_broken_var_names) + +# Returns true if it is a low memory device, otherwise it returns false. +define is-low-mem-device +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_PROPERTY_OVERRIDES)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_DEFAULT_PROPERTY_OVERRIDES)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_COMPATIBLE_PROPERTY_OVERRIDE)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_COMPATIBLE_PROPERTY)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_SYSTEM_DEFAULT_PROPERTIES)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_SYSTEM_EXT_PROPERTIES)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_PRODUCT_PROPERTIES)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_VENDOR_PROPERTIES)),true,\ +$(if $(findstring ro.config.low_ram=true,$(PRODUCT_ODM_PROPERTIES)),true,false))))))))) +endef + +# Set TARGET_MAX_PAGE_SIZE_SUPPORTED. +# TARGET_MAX_PAGE_SIZE_SUPPORTED indicates the alignment of the ELF segments. +ifdef PRODUCT_MAX_PAGE_SIZE_SUPPORTED + TARGET_MAX_PAGE_SIZE_SUPPORTED := $(PRODUCT_MAX_PAGE_SIZE_SUPPORTED) +else ifeq ($(strip $(call is-low-mem-device)),true) + # Low memory device will have 4096 binary alignment. + TARGET_MAX_PAGE_SIZE_SUPPORTED := 4096 +else ifeq ($(call math_lt,$(VSR_VENDOR_API_LEVEL),34),true) + TARGET_MAX_PAGE_SIZE_SUPPORTED := 4096 +else ifeq (,$(filter arm64 x86_64,$(TARGET_ARCH))) + # TARGET_MAX_PAGE_SIZE_SUPPORTED > 4096 is only supported in arm64 and + # x86_64 targets. + TARGET_MAX_PAGE_SIZE_SUPPORTED := 4096 +else + # The default binary alignment for userspace is 16384. + TARGET_MAX_PAGE_SIZE_SUPPORTED := 16384 +endif +.KATI_READONLY := TARGET_MAX_PAGE_SIZE_SUPPORTED + +# Boolean variable determining if AOSP relies on bionic's PAGE_SIZE macro. +ifdef PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO + TARGET_NO_BIONIC_PAGE_SIZE_MACRO := $(PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO) +else ifeq ($(call math_lt,$(VSR_VENDOR_API_LEVEL),35),true) + TARGET_NO_BIONIC_PAGE_SIZE_MACRO := false +else + TARGET_NO_BIONIC_PAGE_SIZE_MACRO := true +endif +.KATI_READONLY := TARGET_NO_BIONIC_PAGE_SIZE_MACRO + # Pruned directory options used when using findleaves.py # See envsetup.mk for a description of SCAN_EXCLUDE_DIRS FIND_LEAVES_EXCLUDES := $(addprefix --prune=, $(SCAN_EXCLUDE_DIRS) .repo .git) @@ -436,7 +509,6 @@ include $(BUILD_SYSTEM)/combo/javac.mk ifeq ($(CALLED_FROM_SETUP),true) include $(BUILD_SYSTEM)/ccache.mk -include $(BUILD_SYSTEM)/goma.mk include $(BUILD_SYSTEM)/rbe.mk endif @@ -498,8 +570,13 @@ endif TARGET_BUILD_USE_PREBUILT_SDKS := DISABLE_PREOPT := +DISABLE_PREOPT_BOOT_IMAGES := ifneq (,$(TARGET_BUILD_APPS)$(TARGET_BUILD_UNBUNDLED_IMAGE)) DISABLE_PREOPT := true + # VSDK builds perform dexpreopt during merge_target_files build step. + ifneq (true,$(BUILDING_WITH_VSDK)) + DISABLE_PREOPT_BOOT_IMAGES := true + endif endif ifeq (true,$(TARGET_BUILD_UNBUNDLED)) ifneq (true,$(UNBUNDLED_BUILD_SDKS_FROM_SOURCE)) @@ -510,6 +587,7 @@ endif .KATI_READONLY := \ TARGET_BUILD_USE_PREBUILT_SDKS \ DISABLE_PREOPT \ + DISABLE_PREOPT_BOOT_IMAGES \ prebuilt_sdk_tools := prebuilts/sdk/tools prebuilt_sdk_tools_bin := $(prebuilt_sdk_tools)/$(HOST_OS)/bin @@ -524,8 +602,6 @@ else prebuilt_build_tools_bin := $(prebuilt_build_tools)/$(HOST_PREBUILT_TAG)/asan/bin endif -USE_PREBUILT_SDK_TOOLS_IN_PLACE := true - # Work around for b/68406220 # This should match the soong version. USE_D8 := true @@ -559,6 +635,7 @@ FILESLIST := $(HOST_OUT_EXECUTABLES)/fileslist FILESLIST_UTIL :=$= build/make/tools/fileslist_util.py HOST_INIT_VERIFIER := $(HOST_OUT_EXECUTABLES)/host_init_verifier XMLLINT := $(HOST_OUT_EXECUTABLES)/xmllint +ACONFIG := $(HOST_OUT_EXECUTABLES)/aconfig # SOONG_ZIP is exported by Soong, but needs to be defined early for # $OUT/dexpreopt.global. It will be verified against the Soong version. @@ -579,30 +656,23 @@ else # For non-supported hosts, do not generate breakpad symbols. BREAKPAD_GENERATE_SYMBOLS := false endif +GZIP := prebuilts/build-tools/path/$(BUILD_OS)-$(HOST_PREBUILT_ARCH)/gzip PROTOC := $(HOST_OUT_EXECUTABLES)/aprotoc$(HOST_EXECUTABLE_SUFFIX) NANOPB_SRCS := $(HOST_OUT_EXECUTABLES)/protoc-gen-nanopb -VTSC := $(HOST_OUT_EXECUTABLES)/vtsc$(HOST_EXECUTABLE_SUFFIX) MKBOOTFS := $(HOST_OUT_EXECUTABLES)/mkbootfs$(HOST_EXECUTABLE_SUFFIX) -MINIGZIP := $(HOST_OUT_EXECUTABLES)/minigzip$(HOST_EXECUTABLE_SUFFIX) +MINIGZIP := $(GZIP) LZ4 := $(HOST_OUT_EXECUTABLES)/lz4$(HOST_EXECUTABLE_SUFFIX) -GENERATE_GKI_CERTIFICATE := $(HOST_OUT_EXECUTABLES)/generate_gki_certificate$(HOST_EXECUTABLE_SUFFIX) ifeq (,$(strip $(BOARD_CUSTOM_MKBOOTIMG))) MKBOOTIMG := $(HOST_OUT_EXECUTABLES)/mkbootimg$(HOST_EXECUTABLE_SUFFIX) else MKBOOTIMG := $(BOARD_CUSTOM_MKBOOTIMG) endif -ifeq (,$(strip $(BOARD_CUSTOM_BPTTOOL))) -BPTTOOL := $(HOST_OUT_EXECUTABLES)/bpttool$(HOST_EXECUTABLE_SUFFIX) -else -BPTTOOL := $(BOARD_CUSTOM_BPTTOOL) -endif ifeq (,$(strip $(BOARD_CUSTOM_AVBTOOL))) AVBTOOL := $(HOST_OUT_EXECUTABLES)/avbtool$(HOST_EXECUTABLE_SUFFIX) else AVBTOOL := $(BOARD_CUSTOM_AVBTOOL) endif APICHECK := $(HOST_OUT_JAVA_LIBRARIES)/metalava$(COMMON_JAVA_PACKAGE_SUFFIX) -FS_GET_STATS := $(HOST_OUT_EXECUTABLES)/fs_get_stats$(HOST_EXECUTABLE_SUFFIX) MKEXTUSERIMG := $(HOST_OUT_EXECUTABLES)/mkuserimg_mke2fs MKE2FS_CONF := system/extras/ext4_utils/mke2fs.conf MKEROFS := $(HOST_OUT_EXECUTABLES)/mkfs.erofs @@ -618,10 +688,16 @@ CHECK_ELF_FILE := $(HOST_OUT_EXECUTABLES)/check_elf_file$(HOST_EXECUTABLE_SUFFIX LPMAKE := $(HOST_OUT_EXECUTABLES)/lpmake$(HOST_EXECUTABLE_SUFFIX) ADD_IMG_TO_TARGET_FILES := $(HOST_OUT_EXECUTABLES)/add_img_to_target_files$(HOST_EXECUTABLE_SUFFIX) BUILD_IMAGE := $(HOST_OUT_EXECUTABLES)/build_image$(HOST_EXECUTABLE_SUFFIX) +ifeq (,$(strip $(BOARD_CUSTOM_BUILD_SUPER_IMAGE))) BUILD_SUPER_IMAGE := $(HOST_OUT_EXECUTABLES)/build_super_image$(HOST_EXECUTABLE_SUFFIX) +else +BUILD_SUPER_IMAGE := $(BOARD_CUSTOM_BUILD_SUPER_IMAGE) +endif IMG_FROM_TARGET_FILES := $(HOST_OUT_EXECUTABLES)/img_from_target_files$(HOST_EXECUTABLE_SUFFIX) +UNPACK_BOOTIMG := $(HOST_OUT_EXECUTABLES)/unpack_bootimg MAKE_RECOVERY_PATCH := $(HOST_OUT_EXECUTABLES)/make_recovery_patch$(HOST_EXECUTABLE_SUFFIX) OTA_FROM_TARGET_FILES := $(HOST_OUT_EXECUTABLES)/ota_from_target_files$(HOST_EXECUTABLE_SUFFIX) +OTA_FROM_RAW_IMG := $(HOST_OUT_EXECUTABLES)/ota_from_raw_img$(HOST_EXECUTABLE_SUFFIX) SPARSE_IMG := $(HOST_OUT_EXECUTABLES)/sparse_img$(HOST_EXECUTABLE_SUFFIX) CHECK_PARTITION_SIZES := $(HOST_OUT_EXECUTABLES)/check_partition_sizes$(HOST_EXECUTABLE_SUFFIX) SYMBOLS_MAP := $(HOST_OUT_EXECUTABLES)/symbols_map @@ -641,6 +717,8 @@ VBOOT_SIGNER := $(HOST_OUT_EXECUTABLES)/vboot_signer DEXDUMP := $(HOST_OUT_EXECUTABLES)/dexdump$(BUILD_EXECUTABLE_SUFFIX) PROFMAN := $(HOST_OUT_EXECUTABLES)/profman +GEN_SBOM := $(HOST_OUT_EXECUTABLES)/generate-sbom + FINDBUGS_DIR := external/owasp/sanitizer/tools/findbugs/bin FINDBUGS := $(FINDBUGS_DIR)/findbugs @@ -651,7 +729,7 @@ EXTRACT_KERNEL := build/make/tools/extract_kernel.py # Path to tools.jar HOST_JDK_TOOLS_JAR := $(ANDROID_JAVA8_HOME)/lib/tools.jar -APICHECK_COMMAND := $(JAVA) -Xmx4g -jar $(APICHECK) --no-banner +APICHECK_COMMAND := $(JAVA) -Xmx4g -jar $(APICHECK) # Boolean variable determining if the allow list for compatible properties is enabled PRODUCT_COMPATIBLE_PROPERTY := true @@ -672,17 +750,9 @@ else ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),26),) PRODUCT_FULL_TREBLE := true endif -# TODO(b/69865032): Make PRODUCT_NOTICE_SPLIT the default behavior and remove -# references to it here and below. -ifdef PRODUCT_NOTICE_SPLIT_OVERRIDE - $(error PRODUCT_NOTICE_SPLIT_OVERRIDE cannot be set.) -endif - requirements := \ PRODUCT_TREBLE_LINKER_NAMESPACES \ - PRODUCT_SEPOLICY_SPLIT \ - PRODUCT_ENFORCE_VINTF_MANIFEST \ - PRODUCT_NOTICE_SPLIT + PRODUCT_ENFORCE_VINTF_MANIFEST # If it is overriden, then the requirement override is taken, otherwise it's # PRODUCT_FULL_TREBLE @@ -695,13 +765,8 @@ $(foreach req,$(requirements),$(eval \ PRODUCT_FULL_TREBLE_OVERRIDE ?= $(foreach req,$(requirements),$(eval $(req)_OVERRIDE ?=)) -ifneq ($(PRODUCT_SEPOLICY_SPLIT),true) -# WARNING: DO NOT CHANGE: if you are downstream of AOSP, and you change this, without -# letting upstream know it's important to you, we may do cleanup which breaks this -# significantly. Please let us know if you are changing this. -# TODO(b/257176017) - unsplit sepolicy is no longer supported -PRODUCT_SEPOLICY_SPLIT := true -endif +# used to be a part of PRODUCT_FULL_TREBLE, but now always set it +PRODUCT_NOTICE_SPLIT := true # TODO(b/114488870): disallow PRODUCT_FULL_TREBLE_OVERRIDE from being used. .KATI_READONLY := \ @@ -709,6 +774,11 @@ endif $(foreach req,$(requirements),$(req)_OVERRIDE) \ $(requirements) \ PRODUCT_FULL_TREBLE \ + PRODUCT_NOTICE_SPLIT \ + +ifneq ($(PRODUCT_FULL_TREBLE),true) + $(warning This device does not have Treble enabled. This is unsafe.) +endif $(KATI_obsolete_var $(foreach req,$(requirements),$(req)_OVERRIDE) \ ,This should be referenced without the _OVERRIDE suffix.) @@ -723,35 +793,13 @@ ifeq ($(PRODUCT_FULL_TREBLE),true) BOARD_PROPERTY_OVERRIDES_SPLIT_ENABLED ?= true endif -# Starting in Android U, non-VNDK devices not supported -# WARNING: DO NOT CHANGE: if you are downstream of AOSP, and you change this, without -# letting upstream know it's important to you, we may do cleanup which breaks this -# significantly. Please let us know if you are changing this. -ifndef BOARD_VNDK_VERSION -# READ WARNING - DO NOT CHANGE -BOARD_VNDK_VERSION := current -# READ WARNING - DO NOT CHANGE -endif - -ifdef PRODUCT_PRODUCT_VNDK_VERSION - ifndef BOARD_VNDK_VERSION - # VNDK for product partition is not available unless BOARD_VNDK_VERSION - # defined. - $(error PRODUCT_PRODUCT_VNDK_VERSION cannot be defined without defining BOARD_VNDK_VERSION) - endif -endif - # Set BOARD_SYSTEMSDK_VERSIONS to the latest SystemSDK version starting from P-launching # devices if unset. ifndef BOARD_SYSTEMSDK_VERSIONS - ifdef PRODUCT_SHIPPING_API_LEVEL - ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),28),) - ifeq (REL,$(PLATFORM_VERSION_CODENAME)) - BOARD_SYSTEMSDK_VERSIONS := $(PLATFORM_SDK_VERSION) - else - BOARD_SYSTEMSDK_VERSIONS := $(PLATFORM_VERSION_CODENAME) - endif - endif + ifeq (REL,$(PLATFORM_VERSION_CODENAME)) + BOARD_SYSTEMSDK_VERSIONS := $(PLATFORM_SDK_VERSION) + else + BOARD_SYSTEMSDK_VERSIONS := $(PLATFORM_VERSION_CODENAME) endif endif @@ -776,13 +824,6 @@ ifdef PRODUCT_SHIPPING_API_LEVEL ifneq ($(call numbers_less_than,$(min_systemsdk_version),$(BOARD_SYSTEMSDK_VERSIONS)),) $(error BOARD_SYSTEMSDK_VERSIONS ($(BOARD_SYSTEMSDK_VERSIONS)) must all be greater than or equal to BOARD_API_LEVEL, BOARD_SHIPPING_API_LEVEL or PRODUCT_SHIPPING_API_LEVEL ($(min_systemsdk_version))) endif - ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),28),) - ifneq ($(TARGET_IS_64_BIT), true) - ifneq ($(TARGET_USES_64_BIT_BINDER), true) - $(error When PRODUCT_SHIPPING_API_LEVEL >= 28, TARGET_USES_64_BIT_BINDER must be true) - endif - endif - endif ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),29),) ifneq ($(BOARD_OTA_FRAMEWORK_VBMETA_VERSION_OVERRIDE),) $(error When PRODUCT_SHIPPING_API_LEVEL >= 29, BOARD_OTA_FRAMEWORK_VBMETA_VERSION_OVERRIDE cannot be set) @@ -807,66 +848,31 @@ endif .KATI_READONLY := MAINLINE_SEPOLICY_DEV_CERTIFICATES BUILD_NUMBER_FROM_FILE := $$(cat $(SOONG_OUT_DIR)/build_number.txt) +BUILD_HOSTNAME_FROM_FILE := $$(cat $(SOONG_OUT_DIR)/build_hostname.txt) BUILD_DATETIME_FROM_FILE := $$(cat $(BUILD_DATETIME_FILE)) # SEPolicy versions -# PLATFORM_SEPOLICY_VERSION is a number of the form "NN.m" with "NN" mapping to -# PLATFORM_SDK_VERSION and "m" as a minor number which allows for SELinux -# changes independent of PLATFORM_SDK_VERSION. This value will be set to -# 10000.0 to represent tip-of-tree development that is inherently unstable and -# thus designed not to work with any shipping vendor policy. This is similar in -# spirit to how DEFAULT_APP_TARGET_SDK is set. -# The minor version ('m' component) must be updated every time a platform release -# is made which breaks compatibility with the previous platform sepolicy version, -# not just on every increase in PLATFORM_SDK_VERSION. The minor version should -# be reset to 0 on every bump of the PLATFORM_SDK_VERSION. -sepolicy_major_vers := 33 -sepolicy_minor_vers := 0 - -ifneq ($(sepolicy_major_vers), $(PLATFORM_SDK_VERSION)) -$(error sepolicy_major_version does not match PLATFORM_SDK_VERSION, please update.) -endif - -TOT_SEPOLICY_VERSION := 10000.0 -ifneq (REL,$(PLATFORM_VERSION_CODENAME)) - PLATFORM_SEPOLICY_VERSION := $(TOT_SEPOLICY_VERSION) -else - PLATFORM_SEPOLICY_VERSION := $(join $(addsuffix .,$(sepolicy_major_vers)), $(sepolicy_minor_vers)) -endif -sepolicy_major_vers := -sepolicy_minor_vers := - -# BOARD_SEPOLICY_VERS must take the format "NN.m" and contain the sepolicy -# version identifier corresponding to the sepolicy on which the non-platform -# policy is to be based. If unspecified, this will build against the current -# public platform policy in tree -ifndef BOARD_SEPOLICY_VERS -# The default platform policy version. +# PLATFORM_SEPOLICY_VERSION is a number of the form "YYYYMM" with "YYYYMM" +# mapping to vFRC version. +PLATFORM_SEPOLICY_VERSION := $(BOARD_API_LEVEL) BOARD_SEPOLICY_VERS := $(PLATFORM_SEPOLICY_VERSION) -endif - -ifeq ($(BOARD_SEPOLICY_VERS),$(PLATFORM_SEPOLICY_VERSION)) -IS_TARGET_MIXED_SEPOLICY := -else -IS_TARGET_MIXED_SEPOLICY := true -endif - -.KATI_READONLY := IS_TARGET_MIXED_SEPOLICY +.KATI_READONLY := PLATFORM_SEPOLICY_VERSION BOARD_SEPOLICY_VERS # A list of SEPolicy versions, besides PLATFORM_SEPOLICY_VERSION, that the framework supports. -PLATFORM_SEPOLICY_COMPAT_VERSIONS := \ - 28.0 \ +PLATFORM_SEPOLICY_COMPAT_VERSIONS := $(filter-out $(PLATFORM_SEPOLICY_VERSION), \ 29.0 \ 30.0 \ 31.0 \ 32.0 \ 33.0 \ + 34.0 \ + 202404 \ + ) .KATI_READONLY := \ PLATFORM_SEPOLICY_COMPAT_VERSIONS \ PLATFORM_SEPOLICY_VERSION \ - TOT_SEPOLICY_VERSION \ ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true) ifneq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true) @@ -1156,15 +1162,11 @@ TARGET_AVAILABLE_SDK_VERSIONS := $(patsubst %/test,test_%,$(TARGET_AVAILABLE_SDK TARGET_AVAILABLE_SDK_VERSIONS := $(filter-out %/module-lib %/system-server,$(TARGET_AVAILABLE_SDK_VERSIONS)) TARGET_AVAIALBLE_SDK_VERSIONS := $(call numerically_sort,$(TARGET_AVAILABLE_SDK_VERSIONS)) -TARGET_SDK_VERSIONS_WITHOUT_JAVA_18_SUPPORT := $(call numbers_less_than,24,$(TARGET_AVAILABLE_SDK_VERSIONS)) -TARGET_SDK_VERSIONS_WITHOUT_JAVA_19_SUPPORT := $(call numbers_less_than,30,$(TARGET_AVAILABLE_SDK_VERSIONS)) +TARGET_SDK_VERSIONS_WITHOUT_JAVA_1_9_SUPPORT := $(call numbers_less_than,30,$(TARGET_AVAILABLE_SDK_VERSIONS)) +TARGET_SDK_VERSIONS_WITHOUT_JAVA_11_SUPPORT := $(call numbers_less_than,32,$(TARGET_AVAILABLE_SDK_VERSIONS)) +TARGET_SDK_VERSIONS_WITHOUT_JAVA_17_SUPPORT := $(call numbers_less_than,34,$(TARGET_AVAILABLE_SDK_VERSIONS)) -# Missing optional uses-libraries so that the platform doesn't create build rules that depend on -# them. -INTERNAL_PLATFORM_MISSING_USES_LIBRARIES := \ - com.google.android.ble \ - com.google.android.media.effects \ - com.google.android.wearable \ +JAVA_LANGUAGE_VERSIONS_WITHOUT_SYSTEM_MODULES := 1.7 1.8 # This is the standard way to name a directory containing prebuilt target # objects. E.g., prebuilt/$(TARGET_PREBUILT_TAG)/libc.so @@ -1182,16 +1184,7 @@ RS_PREBUILT_COMPILER_RT := prebuilts/sdk/renderscript/lib/$(TARGET_ARCH)/libcomp RSCOMPAT_32BIT_ONLY_API_LEVELS := 8 9 10 11 12 13 14 15 16 17 18 19 20 RSCOMPAT_NO_USAGEIO_API_LEVELS := 8 9 10 11 12 13 -# Add BUILD_NUMBER to apps default version name if it's unbundled build. -ifdef TARGET_BUILD_APPS -TARGET_BUILD_WITH_APPS_VERSION_NAME := true -endif - -ifdef TARGET_BUILD_WITH_APPS_VERSION_NAME -APPS_DEFAULT_VERSION_NAME := $(PLATFORM_VERSION)-$(BUILD_NUMBER_FROM_FILE) -else APPS_DEFAULT_VERSION_NAME := $(PLATFORM_VERSION) -endif # ANDROID_WARNING_ALLOWED_PROJECTS is generated by build/soong. define find_warning_allowed_projects @@ -1231,6 +1224,9 @@ endif .KATI_READONLY := JAVAC_NINJA_POOL R8_NINJA_POOL D8_NINJA_POOL +# Soong modules that are known to have broken optional_uses_libs dependencies. +BUILD_WARNING_BAD_OPTIONAL_USES_LIBS_ALLOWLIST := LegacyCamera Gallery2 + # These goals don't need to collect and include Android.mks/CleanSpec.mks # in the source tree. dont_bother_goals := out product-graph @@ -1244,9 +1240,23 @@ include $(BUILD_SYSTEM)/ninja_config.mk include $(BUILD_SYSTEM)/soong_config.mk endif --include external/linux-kselftest/android/kselftest_test_list.mk -include external/ltp/android/ltp_package_list.mk -DEFAULT_DATA_OUT_MODULES := ltp $(ltp_packages) $(kselftest_modules) +DEFAULT_DATA_OUT_MODULES := ltp $(ltp_packages) .KATI_READONLY := DEFAULT_DATA_OUT_MODULES include $(BUILD_SYSTEM)/dumpvar.mk + +ifneq ($(KEEP_VNDK),true) +ifdef BOARD_VNDK_VERSION +BOARD_VNDK_VERSION= +endif +ifdef PLATFORM_VNDK_VERSION +PLATFORM_VNDK_VERSION= +endif +endif + +ifeq (true,$(FULL_SYSTEM_OPTIMIZE_JAVA)) +ifeq (false,$(SYSTEM_OPTIMIZE_JAVA)) +$(error SYSTEM_OPTIMIZE_JAVA must be enabled when FULL_SYSTEM_OPTIMIZE_JAVA is enabled) +endif +endif diff --git a/core/config_sanitizers.mk b/core/config_sanitizers.mk index 35c632c039..ab2d5c1ddf 100644 --- a/core/config_sanitizers.mk +++ b/core/config_sanitizers.mk @@ -65,6 +65,18 @@ ifneq ($(filter memtag_heap, $(my_global_sanitize)),) endif endif +# Disable global HWASan in excluded paths +ifneq ($(filter hwaddress, $(my_global_sanitize)),) + combined_exclude_paths := $(HWASAN_EXCLUDE_PATHS) \ + $(PRODUCT_HWASAN_EXCLUDE_PATHS) + + ifneq ($(strip $(foreach dir,$(subst $(comma),$(space),$(combined_exclude_paths)),\ + $(filter $(dir)%,$(LOCAL_PATH)))),) + my_global_sanitize := $(filter-out hwaddress,$(my_global_sanitize)) + my_global_sanitize_diag := $(filter-out hwaddress,$(my_global_sanitize_diag)) + endif +endif + ifneq ($(my_global_sanitize),) my_sanitize := $(my_global_sanitize) $(my_sanitize) endif @@ -140,6 +152,10 @@ ifeq ($(filter memtag_heap, $(my_sanitize)),) $(PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) combined_exclude_paths := $(MEMTAG_HEAP_EXCLUDE_PATHS) \ $(PRODUCT_MEMTAG_HEAP_EXCLUDE_PATHS) + ifneq ($(PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS),true) + combined_sync_include_paths += $(PRODUCT_MEMTAG_HEAP_SYNC_DEFAULT_INCLUDE_PATHS) + combined_async_include_paths += $(PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS) + endif ifeq ($(strip $(foreach dir,$(subst $(comma),$(space),$(combined_exclude_paths)),\ $(filter $(dir)%,$(LOCAL_PATH)))),) @@ -155,6 +171,17 @@ ifeq ($(filter memtag_heap, $(my_sanitize)),) endif endif +# Enable HWASan in included paths. +ifeq ($(filter hwaddress, $(my_sanitize)),) + combined_include_paths := $(HWASAN_INCLUDE_PATHS) \ + $(PRODUCT_HWASAN_INCLUDE_PATHS) + + ifneq ($(strip $(foreach dir,$(subst $(comma),$(space),$(combined_include_paths)),\ + $(filter $(dir)%,$(LOCAL_PATH)))),) + my_sanitize := hwaddress $(my_sanitize) + endif +endif + # If CFI is disabled globally, remove it from my_sanitize. ifeq ($(strip $(ENABLE_CFI)),false) my_sanitize := $(filter-out cfi,$(my_sanitize)) @@ -165,6 +192,7 @@ endif ifneq ($(filter address,$(my_sanitize)),) my_sanitize := $(filter-out cfi,$(my_sanitize)) my_sanitize := $(filter-out memtag_stack,$(my_sanitize)) + my_sanitize := $(filter-out memtag_globals,$(my_sanitize)) my_sanitize := $(filter-out memtag_heap,$(my_sanitize)) my_sanitize_diag := $(filter-out cfi,$(my_sanitize_diag)) endif @@ -172,8 +200,8 @@ endif # Disable memtag for host targets. Host executables in AndroidMk files are # deprecated, but some partners still have them floating around. ifdef LOCAL_IS_HOST_MODULE - my_sanitize := $(filter-out memtag_heap memtag_stack,$(my_sanitize)) - my_sanitize_diag := $(filter-out memtag_heap memtag_stack,$(my_sanitize_diag)) + my_sanitize := $(filter-out memtag_heap memtag_stack memtag_globals,$(my_sanitize)) + my_sanitize_diag := $(filter-out memtag_heap memtag_stack memtag_globals,$(my_sanitize_diag)) endif # Disable sanitizers which need the UBSan runtime for host targets. @@ -208,11 +236,13 @@ ifneq ($(filter arm x86 x86_64,$(TARGET_$(LOCAL_2ND_ARCH_VAR_PREFIX)ARCH)),) my_sanitize := $(filter-out hwaddress,$(my_sanitize)) my_sanitize := $(filter-out memtag_heap,$(my_sanitize)) my_sanitize := $(filter-out memtag_stack,$(my_sanitize)) + my_sanitize := $(filter-out memtag_globals,$(my_sanitize)) endif ifneq ($(filter hwaddress,$(my_sanitize)),) my_sanitize := $(filter-out address,$(my_sanitize)) my_sanitize := $(filter-out memtag_stack,$(my_sanitize)) + my_sanitize := $(filter-out memtag_globals,$(my_sanitize)) my_sanitize := $(filter-out memtag_heap,$(my_sanitize)) my_sanitize := $(filter-out thread,$(my_sanitize)) my_sanitize := $(filter-out cfi,$(my_sanitize)) @@ -229,28 +259,46 @@ ifneq ($(filter hwaddress,$(my_sanitize)),) endif endif -ifneq ($(filter memtag_heap memtag_stack,$(my_sanitize)),) +ifneq ($(filter memtag_heap memtag_stack memtag_globals,$(my_sanitize)),) ifneq ($(filter memtag_heap,$(my_sanitize_diag)),) - my_cflags += -fsanitize-memtag-mode=sync + my_ldflags += -fsanitize-memtag-mode=sync my_sanitize_diag := $(filter-out memtag_heap,$(my_sanitize_diag)) else - my_cflags += -fsanitize-memtag-mode=async + my_ldflags += -fsanitize-memtag-mode=async endif endif +# Ignore SANITIZE_TARGET_DIAG=memtag_heap without SANITIZE_TARGET=memtag_heap +# This can happen if a condition above filters out memtag_heap from +# my_sanitize. It is easier to handle all of these cases here centrally. +ifneq ($(filter memtag_heap,$(my_sanitize_diag)),) + my_sanitize_diag := $(filter-out memtag_heap,$(my_sanitize_diag)) +endif + ifneq ($(filter memtag_heap,$(my_sanitize)),) my_cflags += -fsanitize=memtag-heap + my_ldflags += -fsanitize=memtag-heap my_sanitize := $(filter-out memtag_heap,$(my_sanitize)) endif ifneq ($(filter memtag_stack,$(my_sanitize)),) my_cflags += -fsanitize=memtag-stack + my_ldflags += -fsanitize=memtag-stack my_cflags += -march=armv8a+memtag my_ldflags += -march=armv8a+memtag my_asflags += -march=armv8a+memtag my_sanitize := $(filter-out memtag_stack,$(my_sanitize)) endif +ifneq ($(filter memtag_globals,$(my_sanitize)),) + my_cflags += -fsanitize=memtag-globals + my_ldflags += -fsanitize=memtag-globals + # TODO(mitchp): For now, enable memtag-heap with memtag-globals because the + # linker isn't new enough + # (https://reviews.llvm.org/differential/changeset/?ref=4243566). + my_sanitize := $(filter-out memtag_globals,$(my_sanitize)) +endif + # TSAN is not supported on 32-bit architectures. For non-multilib cases, make # its use an error. For multilib cases, don't use it for the 32-bit case. ifneq ($(filter thread,$(my_sanitize)),) @@ -310,6 +358,12 @@ ifneq ($(filter fuzzer,$(my_sanitize)),) my_sanitize := $(filter-out cfi,$(my_sanitize)) my_cflags += -fno-lto my_ldflags += -fno-lto + + # TODO(b/142430592): Upstream linker scripts for sanitizer runtime libraries + # discard the sancov_lowest_stack symbol, because it's emulated TLS (and thus + # doesn't match the linker script due to the "__emutls_v." prefix). + my_cflags += -fno-sanitize-coverage=stack-depth + my_ldflags += -fno-sanitize-coverage=stack-depth endif ifneq ($(filter integer_overflow,$(my_sanitize)),) @@ -359,7 +413,6 @@ ifneq ($(my_sanitize),) my_ldflags += -fsanitize=$(fsanitize_arg) else my_cflags += -fsanitize-trap=all - my_cflags += -ftrap-function=abort ifneq ($(filter address thread,$(my_sanitize)),) my_cflags += -fno-sanitize-trap=address,thread my_shared_libraries += libdl @@ -380,7 +433,6 @@ ifneq ($(filter cfi,$(my_sanitize)),) my_cflags += -fvisibility=default endif my_ldflags += $(CFI_EXTRA_LDFLAGS) - my_arflags += --plugin $(LLVM_PREBUILTS_PATH)/../lib64/LLVMgold.so ifeq ($(LOCAL_FORCE_STATIC_EXECUTABLE),true) my_ldflags := $(filter-out -fsanitize-cfi-cross-dso,$(my_ldflags)) @@ -435,6 +487,13 @@ endif # If local module needs HWASAN, add compiler flags. ifneq ($(filter hwaddress,$(my_sanitize)),) my_cflags += $(HWADDRESS_SANITIZER_CONFIG_EXTRA_CFLAGS) + + ifneq ($(filter EXECUTABLES NATIVE_TESTS,$(LOCAL_MODULE_CLASS)),) + ifneq ($(LOCAL_FORCE_STATIC_EXECUTABLE),true) + my_linker := /system/bin/linker_hwasan64 + endif + endif + endif # Use minimal diagnostics when integer overflow is enabled; never do it for HOST modules diff --git a/core/copy_headers.mk b/core/copy_headers.mk index 054d271d8d..397ea629a9 100644 --- a/core/copy_headers.mk +++ b/core/copy_headers.mk @@ -13,15 +13,12 @@ ifdef LOCAL_SDK_VERSION $(call pretty-error,Modules using LOCAL_SDK_VERSION may not use LOCAL_COPY_HEADERS) endif -include $(BUILD_SYSTEM)/local_vndk.mk +include $(BUILD_SYSTEM)/local_vendor_product.mk -# If we're using the VNDK, only vendor modules using the VNDK may use -# LOCAL_COPY_HEADERS. Platform libraries will not have the include path -# present. -ifdef BOARD_VNDK_VERSION -ifndef LOCAL_USE_VNDK - $(call pretty-error,Only vendor modules using LOCAL_USE_VNDK may use LOCAL_COPY_HEADERS) -endif +# Modules in vendor or product may use LOCAL_COPY_HEADERS. +# Platform libraries will not have the include path present. +ifeq ($(call module-in-vendor-or-product),) + $(call pretty-error,Only modules in vendor or product may use LOCAL_COPY_HEADERS) endif # Clean up LOCAL_COPY_HEADERS_TO, since soong_ui will be comparing cleaned diff --git a/core/definitions.mk b/core/definitions.mk index a0337c26f2..40b798013d 100644 --- a/core/definitions.mk +++ b/core/definitions.mk @@ -37,6 +37,8 @@ ALL_DOCS:= # sub-variables. ALL_MODULES:= +ALL_MAKE_MODULE_INFO_JSON_MODULES:= + # The relative paths of the non-module targets in the system. ALL_NON_MODULES:= NON_MODULES_WITHOUT_LICENSE_METADATA:= @@ -48,20 +50,6 @@ ALL_COPIED_TARGETS:= # set of installed targets. ALL_DEFAULT_INSTALLED_MODULES:= -# The list of tags that have been defined by -# LOCAL_MODULE_TAGS. Each word in this variable maps -# to a corresponding ALL_MODULE_TAGS.<tagname> variable -# that contains all of the INSTALLED_MODULEs with that tag. -ALL_MODULE_TAGS:= - -# Similar to ALL_MODULE_TAGS, but contains the short names -# of all targets for a particular tag. The top-level variable -# won't have the list of tags; ust ALL_MODULE_TAGS to get -# the list of all known tags. (This means that this variable -# will always be empty; it's just here as a placeholder for -# its sub-variables.) -ALL_MODULE_NAME_TAGS:= - # Full path to all asm, C, C++, lex and yacc generated C files. # These all have an order-only dependency on the copied headers ALL_C_CPP_ETC_OBJECTS:= @@ -75,9 +63,6 @@ INTERNAL_DALVIK_MODULES:= # All findbugs xml files ALL_FINDBUGS_FILES:= -# GPL module license files -ALL_GPL_MODULE_LICENSE_FILES:= - # Packages with certificate violation CERTIFICATE_VIOLATION_MODULES := @@ -123,9 +108,6 @@ EXPORTS_LIST := # All modules already converted to Soong SOONG_ALREADY_CONV := -# ALL_DEPS.*.ALL_DEPS keys -ALL_DEPS.MODULES := - ########################################################### ## Debugging; prints a variable list to stdout ########################################################### @@ -150,6 +132,10 @@ define true-or-empty $(filter true, $(1)) endef +define boolean-not +$(if $(filter true,$(1)),,true) +endef + ########################################################### ## Rule for touching GCNO files. ########################################################### @@ -597,7 +583,7 @@ endef define declare-copy-target-license-metadata $(strip $(if $(filter $(OUT_DIR)%,$(2)),\ $(eval _tgt:=$(strip $(1)))\ - $(eval ALL_COPIED_TARGETS.$(_tgt).SOURCES := $(ALL_COPIED_TARGETS.$(_tgt).SOURCES) $(filter $(OUT_DIR)%,$(2)))\ + $(eval ALL_COPIED_TARGETS.$(_tgt).SOURCES := $(sort $(ALL_COPIED_TARGETS.$(_tgt).SOURCES) $(filter $(OUT_DIR)%,$(2))))\ $(eval ALL_COPIED_TARGETS += $(_tgt)))) endef @@ -897,7 +883,8 @@ $(call declare-container-license-metadata,$(1),SPDX-license-identifier-Apache-2. endef ########################################################### -## Declare license dependencies $(2) for non-module target $(1) +## Declare license dependencies $(2) with optional colon-separated +## annotations for non-module target $(1) ########################################################### define declare-license-deps $(strip \ @@ -909,7 +896,8 @@ $(strip \ endef ########################################################### -## Declare license dependencies $(2) for non-module container-type target $(1) +## Declare license dependencies $(2) with optional colon-separated +## annotations for non-module container-type target $(1) ## ## Container-type targets are targets like .zip files that ## merely aggregate other files. @@ -1292,38 +1280,6 @@ $(foreach lib,$(1),$(call intermediates-dir-for,JAVA_LIBRARIES,$(lib),,COMMON)/e endef ########################################################### -## MODULE_TAG set operations -########################################################### - -# Given a list of tags, return the targets that specify -# any of those tags. -# $(1): tag list -define modules-for-tag-list -$(sort $(foreach tag,$(1),$(foreach m,$(ALL_MODULE_NAME_TAGS.$(tag)),$(ALL_MODULES.$(m).INSTALLED)))) -endef - -# Same as modules-for-tag-list, but operates on -# ALL_MODULE_NAME_TAGS. -# $(1): tag list -define module-names-for-tag-list -$(sort $(foreach tag,$(1),$(ALL_MODULE_NAME_TAGS.$(tag)))) -endef - -# Given an accept and reject list, find the matching -# set of targets. If a target has multiple tags and -# any of them are rejected, the target is rejected. -# Reject overrides accept. -# $(1): list of tags to accept -# $(2): list of tags to reject -#TODO(dbort): do $(if $(strip $(1)),$(1),$(ALL_MODULE_TAGS)) -#TODO(jbq): as of 20100106 nobody uses the second parameter -define get-tagged-modules -$(filter-out \ - $(call modules-for-tag-list,$(2)), \ - $(call modules-for-tag-list,$(1))) -endef - -########################################################### ## Append a leaf to a base path. Properly deals with ## base paths ending in /. ## @@ -1548,10 +1504,10 @@ endef # # You must call this with $(eval). define define-aidl-java-rule -define-aidl-java-rule-src := $(patsubst %.aidl,%.java,$(subst ../,dotdot/,$(addprefix $(2)/,$(1)))) -$$(define-aidl-java-rule-src) : $(call clean-path,$(LOCAL_PATH)/$(1)) $(AIDL) +define_aidl_java_rule_src := $(patsubst %.aidl,%.java,$(subst ../,dotdot/,$(addprefix $(2)/,$(1)))) +$$(define_aidl_java_rule_src) : $(call clean-path,$(LOCAL_PATH)/$(1)) $(AIDL) $$(transform-aidl-to-java) -$(3) += $$(define-aidl-java-rule-src) +$(3) += $$(define_aidl_java_rule_src) endef ## Given a .aidl file path generate the rule to compile it a .cpp file. @@ -1561,10 +1517,10 @@ endef # # You must call this with $(eval). define define-aidl-cpp-rule -define-aidl-cpp-rule-src := $(patsubst %.aidl,%$(LOCAL_CPP_EXTENSION),$(subst ../,dotdot/,$(addprefix $(2)/,$(1)))) -$$(define-aidl-cpp-rule-src) : $(call clean-path,$(LOCAL_PATH)/$(1)) $(AIDL_CPP) +define_aidl_cpp_rule_src := $(patsubst %.aidl,%$(LOCAL_CPP_EXTENSION),$(subst ../,dotdot/,$(addprefix $(2)/,$(1)))) +$$(define_aidl_cpp_rule_src) : $(call clean-path,$(LOCAL_PATH)/$(1)) $(AIDL_CPP) $$(transform-aidl-to-cpp) -$(3) += $$(define-aidl-cpp-rule-src) +$(3) += $$(define_aidl_cpp_rule_src) endef ########################################################### @@ -1586,10 +1542,10 @@ endef # # You must call this with $(eval). define define-vts-cpp-rule -define-vts-cpp-rule-src := $(patsubst %.vts,%$(LOCAL_CPP_EXTENSION),$(subst ../,dotdot/,$(addprefix $(2)/,$(1)))) -$$(define-vts-cpp-rule-src) : $(LOCAL_PATH)/$(1) $(VTSC) +define_vts_cpp_rule_src := $(patsubst %.vts,%$(LOCAL_CPP_EXTENSION),$(subst ../,dotdot/,$(addprefix $(2)/,$(1)))) +$$(define_vts_cpp_rule_src) : $(LOCAL_PATH)/$(1) $(VTSC) $$(transform-vts-to-cpp) -$(3) += $$(define-vts-cpp-rule-src) +$(3) += $$(define_vts_cpp_rule_src) endef ########################################################### @@ -2378,6 +2334,7 @@ rm -rf $(PRIVATE_JAVA_GEN_DIR) mkdir -p $(PRIVATE_JAVA_GEN_DIR) $(call dump-words-to-file,$(PRIVATE_RES_FLAT),$(dir $@)aapt2-flat-list) $(call dump-words-to-file,$(PRIVATE_OVERLAY_FLAT),$(dir $@)aapt2-flat-overlay-list) +cat $(PRIVATE_STATIC_LIBRARY_TRANSITIVE_RES_PACKAGES_LISTS) | sort -u | tr '\n' ' ' > $(dir $@)aapt2-transitive-overlay-list $(hide) $(AAPT2) link -o $@ \ $(PRIVATE_AAPT_FLAGS) \ $(if $(PRIVATE_STATIC_LIBRARY_EXTRA_PACKAGES),$$(cat $(PRIVATE_STATIC_LIBRARY_EXTRA_PACKAGES))) \ @@ -2397,6 +2354,7 @@ $(hide) $(AAPT2) link -o $@ \ $(addprefix --rename-manifest-package ,$(PRIVATE_MANIFEST_PACKAGE_NAME)) \ $(addprefix --rename-instrumentation-target-package ,$(PRIVATE_MANIFEST_INSTRUMENTATION_FOR)) \ -R \@$(dir $@)aapt2-flat-overlay-list \ + -R \@$(dir $@)aapt2-transitive-overlay-list \ \@$(dir $@)aapt2-flat-list $(SOONG_ZIP) -o $(PRIVATE_SRCJAR) -C $(PRIVATE_JAVA_GEN_DIR) -D $(PRIVATE_JAVA_GEN_DIR) $(EXTRACT_JAR_PACKAGES) -i $(PRIVATE_SRCJAR) -o $(PRIVATE_AAPT_EXTRA_PACKAGES) --prefix '--extra-packages ' @@ -2546,7 +2504,87 @@ define dump-words-to-file @$(call emit-line,$(wordlist 58001,58500,$(1)),$(2)) @$(call emit-line,$(wordlist 58501,59000,$(1)),$(2)) @$(call emit-line,$(wordlist 59001,59500,$(1)),$(2)) - @$(if $(wordlist 59501,59502,$(1)),$(error Too many words ($(words $(1))))) + @$(call emit-line,$(wordlist 59501,60000,$(1)),$(2)) + @$(call emit-line,$(wordlist 60001,60500,$(1)),$(2)) + @$(call emit-line,$(wordlist 60501,61000,$(1)),$(2)) + @$(call emit-line,$(wordlist 61001,61500,$(1)),$(2)) + @$(call emit-line,$(wordlist 61501,62000,$(1)),$(2)) + @$(call emit-line,$(wordlist 62001,62500,$(1)),$(2)) + @$(call emit-line,$(wordlist 62501,63000,$(1)),$(2)) + @$(call emit-line,$(wordlist 63001,63500,$(1)),$(2)) + @$(call emit-line,$(wordlist 63501,64000,$(1)),$(2)) + @$(call emit-line,$(wordlist 64001,64500,$(1)),$(2)) + @$(call emit-line,$(wordlist 64501,65000,$(1)),$(2)) + @$(call emit-line,$(wordlist 65001,65500,$(1)),$(2)) + @$(call emit-line,$(wordlist 65501,66000,$(1)),$(2)) + @$(call emit-line,$(wordlist 66001,66500,$(1)),$(2)) + @$(call emit-line,$(wordlist 66501,67000,$(1)),$(2)) + @$(call emit-line,$(wordlist 67001,67500,$(1)),$(2)) + @$(call emit-line,$(wordlist 67501,68000,$(1)),$(2)) + @$(call emit-line,$(wordlist 68001,68500,$(1)),$(2)) + @$(call emit-line,$(wordlist 68501,69000,$(1)),$(2)) + @$(call emit-line,$(wordlist 69001,69500,$(1)),$(2)) + @$(call emit-line,$(wordlist 69501,70000,$(1)),$(2)) + @$(call emit-line,$(wordlist 70001,70500,$(1)),$(2)) + @$(call emit-line,$(wordlist 70501,71000,$(1)),$(2)) + @$(call emit-line,$(wordlist 71001,71500,$(1)),$(2)) + @$(call emit-line,$(wordlist 71501,72000,$(1)),$(2)) + @$(call emit-line,$(wordlist 72001,72500,$(1)),$(2)) + @$(call emit-line,$(wordlist 72501,73000,$(1)),$(2)) + @$(call emit-line,$(wordlist 73001,73500,$(1)),$(2)) + @$(call emit-line,$(wordlist 73501,74000,$(1)),$(2)) + @$(call emit-line,$(wordlist 74001,74500,$(1)),$(2)) + @$(call emit-line,$(wordlist 74501,75000,$(1)),$(2)) + @$(call emit-line,$(wordlist 75001,75500,$(1)),$(2)) + @$(call emit-line,$(wordlist 75501,76000,$(1)),$(2)) + @$(call emit-line,$(wordlist 76001,76500,$(1)),$(2)) + @$(call emit-line,$(wordlist 76501,77000,$(1)),$(2)) + @$(call emit-line,$(wordlist 77001,77500,$(1)),$(2)) + @$(call emit-line,$(wordlist 77501,78000,$(1)),$(2)) + @$(call emit-line,$(wordlist 78001,78500,$(1)),$(2)) + @$(call emit-line,$(wordlist 78501,79000,$(1)),$(2)) + @$(call emit-line,$(wordlist 79001,79500,$(1)),$(2)) + @$(call emit-line,$(wordlist 79501,80000,$(1)),$(2)) + @$(call emit-line,$(wordlist 80001,80500,$(1)),$(2)) + @$(call emit-line,$(wordlist 80501,81000,$(1)),$(2)) + @$(call emit-line,$(wordlist 81001,81500,$(1)),$(2)) + @$(call emit-line,$(wordlist 81501,82000,$(1)),$(2)) + @$(call emit-line,$(wordlist 82001,82500,$(1)),$(2)) + @$(call emit-line,$(wordlist 82501,83000,$(1)),$(2)) + @$(call emit-line,$(wordlist 83001,83500,$(1)),$(2)) + @$(call emit-line,$(wordlist 83501,84000,$(1)),$(2)) + @$(call emit-line,$(wordlist 84001,84500,$(1)),$(2)) + @$(call emit-line,$(wordlist 84501,85000,$(1)),$(2)) + @$(call emit-line,$(wordlist 85001,85500,$(1)),$(2)) + @$(call emit-line,$(wordlist 85501,86000,$(1)),$(2)) + @$(call emit-line,$(wordlist 86001,86500,$(1)),$(2)) + @$(call emit-line,$(wordlist 86501,87000,$(1)),$(2)) + @$(call emit-line,$(wordlist 87001,87500,$(1)),$(2)) + @$(call emit-line,$(wordlist 87501,88000,$(1)),$(2)) + @$(call emit-line,$(wordlist 88001,88500,$(1)),$(2)) + @$(call emit-line,$(wordlist 88501,89000,$(1)),$(2)) + @$(call emit-line,$(wordlist 89001,89500,$(1)),$(2)) + @$(call emit-line,$(wordlist 89501,90000,$(1)),$(2)) + @$(call emit-line,$(wordlist 90001,90500,$(1)),$(2)) + @$(call emit-line,$(wordlist 90501,91000,$(1)),$(2)) + @$(call emit-line,$(wordlist 91001,91500,$(1)),$(2)) + @$(call emit-line,$(wordlist 91501,92000,$(1)),$(2)) + @$(call emit-line,$(wordlist 92001,92500,$(1)),$(2)) + @$(call emit-line,$(wordlist 92501,93000,$(1)),$(2)) + @$(call emit-line,$(wordlist 93001,93500,$(1)),$(2)) + @$(call emit-line,$(wordlist 93501,94000,$(1)),$(2)) + @$(call emit-line,$(wordlist 94001,94500,$(1)),$(2)) + @$(call emit-line,$(wordlist 94501,95000,$(1)),$(2)) + @$(call emit-line,$(wordlist 95001,95500,$(1)),$(2)) + @$(call emit-line,$(wordlist 95501,96000,$(1)),$(2)) + @$(call emit-line,$(wordlist 96001,96500,$(1)),$(2)) + @$(call emit-line,$(wordlist 96501,97000,$(1)),$(2)) + @$(call emit-line,$(wordlist 97001,97500,$(1)),$(2)) + @$(call emit-line,$(wordlist 97501,98000,$(1)),$(2)) + @$(call emit-line,$(wordlist 98001,98500,$(1)),$(2)) + @$(call emit-line,$(wordlist 98501,99000,$(1)),$(2)) + @$(call emit-line,$(wordlist 99001,99500,$(1)),$(2)) + @$(if $(wordlist 99501,99502,$(1)),$(error dump-words-to-file: Too many words ($(words $(1))))) endef # Return jar arguments to compress files in a given directory # $(1): directory @@ -2736,6 +2774,10 @@ define module-min-sdk-version $(if $(LOCAL_MIN_SDK_VERSION),$(LOCAL_MIN_SDK_VERSION),$(call module-target-sdk-version)) endef +# Checks if module is in vendor or product +define module-in-vendor-or-product +$(if $(filter true,$(LOCAL_IN_VENDOR) $(LOCAL_IN_PRODUCT)),true) +endef define transform-classes.jar-to-dex @echo "target Dex: $(PRIVATE_MODULE)" @@ -2862,7 +2904,7 @@ endef define compress-package $(hide) \ mv $@ $@.uncompressed; \ - $(MINIGZIP) -c $@.uncompressed > $@.compressed; \ + $(GZIP) -9 -c $@.uncompressed > $@.compressed; \ rm -f $@.uncompressed; \ mv $@.compressed $@; endef @@ -2870,7 +2912,7 @@ endef ifeq ($(HOST_OS),linux) # Runs appcompat and store logs in $(PRODUCT_OUT)/appcompat define extract-package -$(AAPT2) dump resources $@ | awk -F ' |=' '/^Package/{print $$3}' >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log && +$(AAPT2) dump resources $@ | awk -F ' |=' '/^Package/{print $$3; exit}' >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log && endef define appcompat-header $(hide) \ @@ -2880,7 +2922,7 @@ $(hide) \ $(extract-package) \ echo "Module name in Android tree: $(PRIVATE_MODULE)" >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log && \ echo "Local path in Android tree: $(PRIVATE_PATH)" >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log && \ - echo "Install path on $(TARGET_PRODUCT)-$(TARGET_BUILD_VARIANT): $(PRIVATE_INSTALLED_MODULE)" >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log && \ + echo "Install path: $(patsubst $(PRODUCT_OUT)/%,%,$(PRIVATE_INSTALLED_MODULE))" >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log && \ echo >> $(PRODUCT_OUT)/appcompat/$(PRIVATE_MODULE).log endef ART_VERIDEX_APPCOMPAT_SCRIPT:=$(HOST_OUT)/bin/appcompat.sh @@ -3084,14 +3126,12 @@ endef # Copies many init script files and check they are well-formed. # $(1): The init script files to copy. Each entry is a ':' separated src:dst pair. -# Evaluates to the list of the dst files. (ie suitable for a dependency list.) define copy-many-init-script-files-checked $(foreach f, $(1), $(strip \ $(eval _cmf_tuple := $(subst :, ,$(f))) \ $(eval _cmf_src := $(word 1,$(_cmf_tuple))) \ $(eval _cmf_dest := $(word 2,$(_cmf_tuple))) \ - $(eval $(call copy-init-script-file-checked,$(_cmf_src),$(_cmf_dest))) \ - $(_cmf_dest))) + $(eval $(call copy-init-script-file-checked,$(_cmf_src),$(_cmf_dest))))) endef # Copy the file only if it's a well-formed xml file. For use via $(eval). @@ -3122,20 +3162,19 @@ endef define copy-vintf-manifest-checked $(2): $(1) $(HOST_OUT_EXECUTABLES)/assemble_vintf @echo "Copy xml: $$@" - $(hide) $(HOST_OUT_EXECUTABLES)/assemble_vintf -i $$< >/dev/null # Don't print the xml file to stdout. - $$(copy-file-to-target) + $(hide) mkdir -p "$$(dir $$@)" + $(hide) VINTF_IGNORE_TARGET_FCM_VERSION=true\ + $(HOST_OUT_EXECUTABLES)/assemble_vintf -i $$< -o $$@ endef # Copies many vintf manifest files checked. # $(1): The files to copy. Each entry is a ':' separated src:dst pair -# Evaluates to the list of the dst files (ie suitable for a dependency list) define copy-many-vintf-manifest-files-checked $(foreach f, $(1), $(strip \ $(eval _cmf_tuple := $(subst :, ,$(f))) \ $(eval _cmf_src := $(word 1,$(_cmf_tuple))) \ $(eval _cmf_dest := $(word 2,$(_cmf_tuple))) \ - $(eval $(call copy-vintf-manifest-checked,$(_cmf_src),$(_cmf_dest))) \ - $(_cmf_dest))) + $(eval $(call copy-vintf-manifest-checked,$(_cmf_src),$(_cmf_dest))))) endef # Copy the file only if it's not an ELF file. For use via $(eval). @@ -3203,14 +3242,6 @@ $(hide) rm -f $@ $(hide) cp -p "$<" "$@" endef -# The same as copy-file-to-target, but strip out "# comment"-style -# comments (for config files and such). -define copy-file-to-target-strip-comments -@mkdir -p $(dir $@) -$(hide) rm -f $@ -$(hide) sed -e 's/#.*$$//' -e 's/[ \t]*$$//' -e '/^$$/d' < $< > $@ -endef - # The same as copy-file-to-target, but don't preserve # the old modification time. define copy-file-to-new-target @@ -3245,12 +3276,6 @@ define transform-prebuilt-to-target $(copy-file-to-target) endef -# Copy a prebuilt file to a target location, stripping "# comment" comments. -define transform-prebuilt-to-target-strip-comments -@echo "$($(PRIVATE_PREFIX)DISPLAY) Prebuilt: $(PRIVATE_MODULE) ($@)" -$(copy-file-to-target-strip-comments) -endef - # Copy a prebuilt file to a target location, but preserve symlinks rather than # dereference them. define copy-or-link-prebuilt-to-target @@ -3285,7 +3310,6 @@ $(3): $(1) @mkdir -p $$(dir $$@) @rm -rf $$@ $(hide) ln -sf $(2) $$@ -$(3): .KATI_SYMLINK_OUTPUTS := $(3) endef # Copy an apk to a target location while removing classes*.dex @@ -3336,6 +3360,10 @@ endef # $(2): path in symbols directory # $(3): file type (elf or r8) # $(4): path in the mappings directory +# +# Regarding the restats at the end: I think you should only need to use KATI_RESTAT on $(2), but +# there appears to be a bug in kati where it was not adding restat=true in the ninja file unless we +# also added 4 to KATI_RESTAT. define _copy-symbols-file-with-mapping $(2): .KATI_IMPLICIT_OUTPUTS := $(4) $(2): $(SYMBOLS_MAP) @@ -3344,16 +3372,7 @@ $(2): $(1) $$(copy-file-to-target) $(SYMBOLS_MAP) -$(strip $(3)) $(2) -write_if_changed $(4) .KATI_RESTAT: $(2) -endef - -# Returns the directory to copy proguard dictionaries into -define local-proguard-dictionary-directory -$(call intermediates-dir-for,PACKAGING,proguard_dictionary)/out/target/common/obj/$(LOCAL_MODULE_CLASS)/$(LOCAL_MODULE)_intermediates -endef - -# Returns the directory to copy proguard dictionary mappings into -define local-proguard-dictionary-mapping-directory -$(call intermediates-dir-for,PACKAGING,proguard_dictionary_mapping)/out/target/common/obj/$(LOCAL_MODULE_CLASS)/$(LOCAL_MODULE)_intermediates +.KATI_RESTAT: $(4) endef diff --git a/core/device.mk b/core/device.mk deleted file mode 100644 index 20ff447caa..0000000000 --- a/core/device.mk +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (C) 2007 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -_device_var_list := \ - DEVICE_NAME \ - DEVICE_BOARD \ - DEVICE_REGION - -define dump-device -$(info ==== $(1) ====)\ -$(foreach v,$(_device_var_list),\ -$(info DEVICES.$(1).$(v) := $(DEVICES.$(1).$(v))))\ -$(info --------) -endef - -define dump-devices -$(foreach p,$(DEVICES),$(call dump-device,$(p))) -endef - -# -# $(1): device to inherit -# -define inherit-device - $(foreach v,$(_device_var_list), \ - $(eval $(v) := $($(v)) $(INHERIT_TAG)$(strip $(1)))) -endef - -# -# $(1): device makefile list -# -#TODO: check to make sure that devices have all the necessary vars defined -define import-devices -$(call import-nodes,DEVICES,$(1),$(_device_var_list)) -endef - - -# -# $(1): short device name like "sooner" -# -define _resolve-short-device-name - $(eval dn := $(strip $(1))) - $(eval d := \ - $(foreach d,$(DEVICES), \ - $(if $(filter $(dn),$(DEVICES.$(d).DEVICE_NAME)), \ - $(d) \ - )) \ - ) - $(eval d := $(sort $(d))) - $(if $(filter 1,$(words $(d))), \ - $(d), \ - $(if $(filter 0,$(words $(d))), \ - $(error No matches for device "$(dn)"), \ - $(error Device "$(dn)" ambiguous: matches $(d)) \ - ) \ - ) -endef - -# -# $(1): short device name like "sooner" -# -define resolve-short-device-name -$(strip $(call _resolve-short-device-name,$(1))) -endef diff --git a/core/dex_preopt.mk b/core/dex_preopt.mk index 88ec47ffc4..26b8b17a49 100644 --- a/core/dex_preopt.mk +++ b/core/dex_preopt.mk @@ -57,7 +57,9 @@ my_boot_image_module := # Build the boot.zip which contains the boot jars and their compilation output # We can do this only if preopt is enabled and if the product uses libart config (which sets the # default properties for preopting). +# At the time of writing, this is only for ART Cloud. ifeq ($(WITH_DEXPREOPT), true) +ifneq ($(WITH_DEXPREOPT_ART_BOOT_IMG_ONLY), true) ifeq ($(PRODUCT_USES_DEFAULT_ART_CONFIG), true) boot_zip := $(PRODUCT_OUT)/boot.zip @@ -80,34 +82,69 @@ system_server_jars += \ $(foreach m,$(other_system_server_jars),\ $(PRODUCT_OUT)/$(call word-colon,1,$(m))/framework/$(call word-colon,2,$(m)).jar) +# Infix can be 'art' (ART image for testing), 'boot' (primary), or 'mainline' (mainline extension). +# Soong creates a set of variables for Make, one or each boot image. The only reason why the ART +# image is exposed to Make is testing (art gtests) and benchmarking (art golem benchmarks). Install +# rules that use those variables are in dex_preopt_libart.mk. Here for dexpreopt purposes the infix +# is always 'boot' or 'mainline'. +DEXPREOPT_INFIX := $(if $(filter true,$(DEX_PREOPT_WITH_UPDATABLE_BCP)),mainline,boot) + +# The input variables are written by build/soong/java/dexpreopt_bootjars.go. Examples can be found +# at the bottom of build/soong/java/dexpreopt_config_testing.go. +dexpreopt_root_dir := $(dir $(patsubst %/,%,$(dir $(firstword $(bootclasspath_jars))))) +bootclasspath_arg := $(subst $(space),:,$(patsubst $(dexpreopt_root_dir)%,%,$(DEXPREOPT_BOOTCLASSPATH_DEX_FILES))) +bootclasspath_locations_arg := $(subst $(space),:,$(DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS)) +boot_images := $(subst :,$(space),$(DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICE$(DEXPREOPT_INFIX))) +boot_image_arg := $(subst $(space),:,$(patsubst /%,%,$(boot_images))) +uffd_gc_flag_txt := $(OUT_DIR)/soong/dexpreopt/uffd_gc_flag.txt + +boot_zip_metadata_txt := $(dir $(boot_zip))boot_zip/METADATA.txt +$(boot_zip_metadata_txt): $(uffd_gc_flag_txt) +$(boot_zip_metadata_txt): + rm -f $@ + echo "bootclasspath = $(bootclasspath_arg)" >> $@ + echo "bootclasspath-locations = $(bootclasspath_locations_arg)" >> $@ + echo "boot-image = $(boot_image_arg)" >> $@ + echo "extra-args = `cat $(uffd_gc_flag_txt)`" >> $@ + +$(call dist-for-goals, droidcore, $(boot_zip_metadata_txt)) + $(boot_zip): PRIVATE_BOOTCLASSPATH_JARS := $(bootclasspath_jars) $(boot_zip): PRIVATE_SYSTEM_SERVER_JARS := $(system_server_jars) -$(boot_zip): $(bootclasspath_jars) $(system_server_jars) $(SOONG_ZIP) $(MERGE_ZIPS) $(DEXPREOPT_IMAGE_ZIP_boot) $(DEXPREOPT_IMAGE_ZIP_art) +$(boot_zip): $(bootclasspath_jars) $(system_server_jars) $(SOONG_ZIP) $(MERGE_ZIPS) $(DEXPREOPT_IMAGE_ZIP_boot) $(DEXPREOPT_IMAGE_ZIP_art) $(DEXPREOPT_IMAGE_ZIP_mainline) $(boot_zip_metadata_txt) @echo "Create boot package: $@" rm -f $@ $(SOONG_ZIP) -o $@.tmp \ -C $(dir $(firstword $(PRIVATE_BOOTCLASSPATH_JARS)))/.. $(addprefix -f ,$(PRIVATE_BOOTCLASSPATH_JARS)) \ - -C $(PRODUCT_OUT) $(addprefix -f ,$(PRIVATE_SYSTEM_SERVER_JARS)) - $(MERGE_ZIPS) $@ $@.tmp $(DEXPREOPT_IMAGE_ZIP_boot) $(DEXPREOPT_IMAGE_ZIP_art) + -C $(PRODUCT_OUT) $(addprefix -f ,$(PRIVATE_SYSTEM_SERVER_JARS)) \ + -j -f $(boot_zip_metadata_txt) + $(MERGE_ZIPS) $@ $@.tmp $(DEXPREOPT_IMAGE_ZIP_boot) $(DEXPREOPT_IMAGE_ZIP_art) $(DEXPREOPT_IMAGE_ZIP_mainline) rm -f $@.tmp $(call dist-for-goals, droidcore, $(boot_zip)) # Build the system_server.zip which contains the Apex system server jars and standalone system server jars +system_server_dex2oat_dir := $(SOONG_OUT_DIR)/system_server_dexjars system_server_zip := $(PRODUCT_OUT)/system_server.zip +# non_updatable_system_server_jars contains jars in /system and /system_ext that are not part of an apex. +non_updatable_system_server_jars := \ + $(foreach m,$(PRODUCT_SYSTEM_SERVER_JARS),\ + $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) + apex_system_server_jars := \ $(foreach m,$(PRODUCT_APEX_SYSTEM_SERVER_JARS),\ - $(PRODUCT_OUT)/apex/$(call word-colon,1,$(m))/javalib/$(call word-colon,2,$(m)).jar) + $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) apex_standalone_system_server_jars := \ $(foreach m,$(PRODUCT_APEX_STANDALONE_SYSTEM_SERVER_JARS),\ - $(PRODUCT_OUT)/apex/$(call word-colon,1,$(m))/javalib/$(call word-colon,2,$(m)).jar) + $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) standalone_system_server_jars := \ $(foreach m,$(PRODUCT_STANDALONE_SYSTEM_SERVER_JARS),\ - $(PRODUCT_OUT)/apex/$(call word-colon,1,$(m))/javalib/$(call word-colon,2,$(m)).jar) + $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) -$(system_server_zip): PRIVATE_SYSTEM_SERVER_JARS := $(system_server_jars) +$(system_server_zip): PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR := $(system_server_dex2oat_dir) +$(system_server_zip): PRIVATE_SYSTEM_SERVER_JARS := $(non_updatable_system_server_jars) $(system_server_zip): PRIVATE_APEX_SYSTEM_SERVER_JARS := $(apex_system_server_jars) $(system_server_zip): PRIVATE_APEX_STANDALONE_SYSTEM_SERVER_JARS := $(apex_standalone_system_server_jars) $(system_server_zip): PRIVATE_STANDALONE_SYSTEM_SERVER_JARS := $(standalone_system_server_jars) @@ -115,12 +152,13 @@ $(system_server_zip): $(system_server_jars) $(apex_system_server_jars) $(apex_st @echo "Create system server package: $@" rm -f $@ $(SOONG_ZIP) -o $@ \ - -C $(PRODUCT_OUT) $(addprefix -f ,$(PRIVATE_SYSTEM_SERVER_JARS)) \ - -C $(PRODUCT_OUT) $(addprefix -f ,$(PRIVATE_APEX_SYSTEM_SERVER_JARS)) \ - -C $(PRODUCT_OUT) $(addprefix -f ,$(PRIVATE_APEX_STANDALONE_SYSTEM_SERVER_JARS)) \ - -C $(PRODUCT_OUT) $(addprefix -f ,$(PRIVATE_STANDALONE_SYSTEM_SERVER_JARS)) + -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_SYSTEM_SERVER_JARS)) \ + -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_APEX_SYSTEM_SERVER_JARS)) \ + -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_APEX_STANDALONE_SYSTEM_SERVER_JARS)) \ + -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_STANDALONE_SYSTEM_SERVER_JARS)) $(call dist-for-goals, droidcore, $(system_server_zip)) endif #PRODUCT_USES_DEFAULT_ART_CONFIG +endif #WITH_DEXPREOPT_ART_BOOT_IMG_ONLY endif #WITH_DEXPREOPT diff --git a/core/dex_preopt_config.mk b/core/dex_preopt_config.mk index e36e2ebb6f..d51de33273 100644 --- a/core/dex_preopt_config.mk +++ b/core/dex_preopt_config.mk @@ -12,9 +12,15 @@ else ifneq (true,$(filter true,$(PRODUCT_USES_DEFAULT_ART_CONFIG))) # would result in passing bad arguments to dex2oat and failing the build. ENABLE_PREOPT := ENABLE_PREOPT_BOOT_IMAGES := -else ifeq (true,$(DISABLE_PREOPT)) - # Disable dexpreopt for libraries/apps, but do compile boot images. - ENABLE_PREOPT := +else + ifeq (true,$(DISABLE_PREOPT)) + # Disable dexpreopt for libraries/apps, but may compile boot images. + ENABLE_PREOPT := + endif + ifeq (true,$(DISABLE_PREOPT_BOOT_IMAGES)) + # Disable dexpreopt for boot images, but may compile libraries/apps. + ENABLE_PREOPT_BOOT_IMAGES := + endif endif # The default value for LOCAL_DEX_PREOPT @@ -52,25 +58,10 @@ DEX_PREOPT_WITH_UPDATABLE_BCP := true # Conditional to building on linux, as dex2oat currently does not work on darwin. ifeq ($(HOST_OS),linux) - ifeq (eng,$(TARGET_BUILD_VARIANT)) - # For an eng build only pre-opt the boot image and system server. This gives reasonable performance - # and still allows a simple workflow: building in frameworks/base and syncing. - WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY ?= true - endif # Add mini-debug-info to the boot classpath unless explicitly asked not to. ifneq (false,$(WITH_DEXPREOPT_DEBUG_INFO)) PRODUCT_DEX_PREOPT_BOOT_FLAGS += --generate-mini-debug-info endif - - # Non eng linux builds must have preopt enabled so that system server doesn't run as interpreter - # only. b/74209329 - ifeq (,$(filter eng, $(TARGET_BUILD_VARIANT))) - ifneq (true,$(WITH_DEXPREOPT)) - ifneq (true,$(WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY)) - $(call pretty-error, DEXPREOPT must be enabled for user and userdebug builds) - endif - endif - endif endif # Get value of a property. It is first searched from PRODUCT_VENDOR_PROPERTIES @@ -94,7 +85,7 @@ ifeq ($(WRITE_SOONG_VARIABLES),true) $(call add_json_bool, DisablePreopt, $(call invert_bool,$(ENABLE_PREOPT))) $(call add_json_bool, DisablePreoptBootImages, $(call invert_bool,$(ENABLE_PREOPT_BOOT_IMAGES))) $(call add_json_list, DisablePreoptModules, $(DEXPREOPT_DISABLED_MODULES)) - $(call add_json_bool, OnlyPreoptBootImageAndSystemServer, $(filter true,$(WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY))) + $(call add_json_bool, OnlyPreoptArtBootImage , $(filter true,$(WITH_DEXPREOPT_ART_BOOT_IMG_ONLY))) $(call add_json_bool, PreoptWithUpdatableBcp, $(filter true,$(DEX_PREOPT_WITH_UPDATABLE_BCP))) $(call add_json_bool, DontUncompressPrivAppsDex, $(filter true,$(DONT_UNCOMPRESS_PRIV_APPS_DEXS))) $(call add_json_list, ModulesLoadedByPrivilegedModules, $(PRODUCT_LOADED_BY_PRIVILEGED_MODULES)) @@ -103,8 +94,9 @@ ifeq ($(WRITE_SOONG_VARIABLES),true) $(call add_json_bool, DisableGenerateProfile, $(filter false,$(WITH_DEX_PREOPT_GENERATE_PROFILE))) $(call add_json_str, ProfileDir, $(PRODUCT_DEX_PREOPT_PROFILE_DIR)) $(call add_json_list, BootJars, $(PRODUCT_BOOT_JARS)) - $(call add_json_list, ApexBootJars, $(PRODUCT_APEX_BOOT_JARS)) + $(call add_json_list, ApexBootJars, $(filter-out $(APEX_BOOT_JARS_EXCLUDED), $(PRODUCT_APEX_BOOT_JARS))) $(call add_json_list, ArtApexJars, $(filter $(PRODUCT_BOOT_JARS),$(ART_APEX_JARS))) + $(call add_json_list, TestOnlyArtBootImageJars, $(PRODUCT_TEST_ONLY_ART_BOOT_IMAGE_JARS)) $(call add_json_list, SystemServerJars, $(PRODUCT_SYSTEM_SERVER_JARS)) $(call add_json_list, SystemServerApps, $(PRODUCT_SYSTEM_SERVER_APPS)) $(call add_json_list, ApexSystemServerJars, $(PRODUCT_APEX_SYSTEM_SERVER_JARS)) @@ -130,7 +122,7 @@ ifeq ($(WRITE_SOONG_VARIABLES),true) $(call add_json_str, Dex2oatXmx, $(DEX2OAT_XMX)) $(call add_json_str, Dex2oatXms, $(DEX2OAT_XMS)) $(call add_json_str, EmptyDirectory, $(OUT_DIR)/empty) - $(call add_json_bool, EnableUffdGc, $(filter true,$(ENABLE_UFFD_GC))) + $(call add_json_str, EnableUffdGc, $(ENABLE_UFFD_GC)) ifdef TARGET_ARCH $(call add_json_map, CpuVariant) diff --git a/core/dex_preopt_odex_install.mk b/core/dex_preopt_odex_install.mk index b303b52f12..151591e130 100644 --- a/core/dex_preopt_odex_install.mk +++ b/core/dex_preopt_odex_install.mk @@ -60,18 +60,8 @@ ifeq (,$(strip $(built_dex)$(my_prebuilt_src_file)$(LOCAL_SOONG_DEX_JAR))) LOCAL_DEX_PREOPT := endif -# if WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY=true and module is not in boot class path skip -# Also preopt system server jars since selinux prevents system server from loading anything from -# /data. If we don't do this they will need to be extracted which is not favorable for RAM usage -# or performance. If my_preopt_for_extracted_apk is true, we ignore the only preopt boot image -# options. -system_server_jars := $(foreach m,$(PRODUCT_SYSTEM_SERVER_JARS),$(call word-colon,2,$(m))) -ifneq (true,$(my_preopt_for_extracted_apk)) - ifeq (true,$(WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY)) - ifeq ($(filter $(system_server_jars) $(DEXPREOPT_BOOT_JARS_MODULES),$(LOCAL_MODULE)),) - LOCAL_DEX_PREOPT := - endif - endif +ifeq (true,$(WITH_DEXPREOPT_ART_BOOT_IMG_ONLY)) + LOCAL_DEX_PREOPT := endif my_process_profile := @@ -84,12 +74,13 @@ endif ifndef LOCAL_DEX_PREOPT_GENERATE_PROFILE # If LOCAL_DEX_PREOPT_GENERATE_PROFILE is not defined, default it based on the existence of the # profile class listing. TODO: Use product specific directory here. - my_classes_directory := $(PRODUCT_DEX_PREOPT_PROFILE_DIR) - LOCAL_DEX_PREOPT_PROFILE := $(my_classes_directory)/$(LOCAL_MODULE).prof + ifdef PRODUCT_DEX_PREOPT_PROFILE_DIR + LOCAL_DEX_PREOPT_PROFILE := $(PRODUCT_DEX_PREOPT_PROFILE_DIR)/$(LOCAL_MODULE).prof - ifneq (,$(wildcard $(LOCAL_DEX_PREOPT_PROFILE))) - my_process_profile := true - my_profile_is_text_listing := + ifneq (,$(wildcard $(LOCAL_DEX_PREOPT_PROFILE))) + my_process_profile := true + my_profile_is_text_listing := + endif endif else my_process_profile := $(LOCAL_DEX_PREOPT_GENERATE_PROFILE) @@ -110,18 +101,19 @@ endif # Local module variables and functions used in dexpreopt and manifest_check. ################################################################################ -my_filtered_optional_uses_libraries := $(filter-out $(INTERNAL_PLATFORM_MISSING_USES_LIBRARIES), \ - $(LOCAL_OPTIONAL_USES_LIBRARIES)) - # TODO(b/132357300): This may filter out too much, as PRODUCT_PACKAGES doesn't # include all packages (the full list is unknown until reading all Android.mk # makefiles). As a consequence, a library may be present but not included in # dexpreopt, which will result in class loader context mismatch and a failure -# to load dexpreopt code on device. We should fix this, either by deferring -# dependency computation until the full list of product packages is known, or -# by adding product-specific lists of missing libraries. +# to load dexpreopt code on device. +# However, we have to do filtering here. Otherwise, we may include extra +# libraries that Soong and Make don't generate build rules for (e.g., a library +# that exists in the source tree but not installable), and therefore get Ninja +# errors. +# We have deferred CLC computation to the Ninja phase, but the dependency +# computation still needs to be done early. For now, this is the best we can do. my_filtered_optional_uses_libraries := $(filter $(PRODUCT_PACKAGES), \ - $(my_filtered_optional_uses_libraries)) + $(LOCAL_OPTIONAL_USES_LIBRARIES)) ifeq ($(LOCAL_MODULE_CLASS),APPS) # compatibility libraries are added to class loader context of an app only if @@ -150,6 +142,9 @@ my_dexpreopt_libs := \ $(LOCAL_USES_LIBRARIES) \ $(my_filtered_optional_uses_libraries) +# The order needs to be deterministic. +my_dexpreopt_libs_all := $(sort $(my_dexpreopt_libs) $(my_dexpreopt_libs_compat)) + # Module dexpreopt.config depends on dexpreopt.config files of each # <uses-library> dependency, because these libraries may be processed after # the current module by Make (there's no topological order), so the dependency @@ -221,7 +216,7 @@ endif # as a failure to get manifest from an APK). ifneq (true,$(WITH_DEXPREOPT)) LOCAL_ENFORCE_USES_LIBRARIES := false -else ifeq (true,$(WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY)) +else ifeq (true,$(WITH_DEXPREOPT_ART_BOOT_IMG_ONLY)) LOCAL_ENFORCE_USES_LIBRARIES := false endif @@ -240,7 +235,7 @@ ifeq (true,$(LOCAL_ENFORCE_USES_LIBRARIES)) --enforce-uses-libraries-relax,) my_dexpreopt_config_args := $(patsubst %,--dexpreopt-config %,$(my_dexpreopt_dep_configs)) - my_enforced_uses_libraries := $(intermediates.COMMON)/enforce_uses_libraries.status + my_enforced_uses_libraries := $(intermediates)/enforce_uses_libraries.status $(my_enforced_uses_libraries): PRIVATE_USES_LIBRARIES := $(my_uses_libs_args) $(my_enforced_uses_libraries): PRIVATE_OPTIONAL_USES_LIBRARIES := $(my_optional_uses_libs_args) $(my_enforced_uses_libraries): PRIVATE_DEXPREOPT_CONFIGS := $(my_dexpreopt_config_args) @@ -272,12 +267,7 @@ my_dexpreopt_images := my_dexpreopt_images_deps := my_dexpreopt_image_locations_on_host := my_dexpreopt_image_locations_on_device := -# Infix can be 'boot' or 'art'. Soong creates a set of variables for Make, one -# for each boot image (primary and the framework extension). The only reason why -# the primary image is exposed to Make is testing (art gtests) and benchmarking -# (art golem benchmarks). Install rules that use those variables are in -# dex_preopt_libart.mk. Here for dexpreopt purposes the infix is always 'boot'. -my_dexpreopt_infix := boot +my_dexpreopt_infix := $(DEXPREOPT_INFIX) my_create_dexpreopt_config := ifdef LOCAL_DEX_PREOPT @@ -400,7 +390,6 @@ ifeq ($(my_create_dexpreopt_config), true) $(call add_json_list, DexPreoptImageLocationsOnDevice,$(my_dexpreopt_image_locations_on_device)) $(call add_json_list, PreoptBootClassPathDexFiles, $(DEXPREOPT_BOOTCLASSPATH_DEX_FILES)) $(call add_json_list, PreoptBootClassPathDexLocations,$(DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS)) - $(call add_json_bool, PreoptExtractedApk, $(my_preopt_for_extracted_apk)) $(call add_json_bool, NoCreateAppImage, $(filter false,$(LOCAL_DEX_PREOPT_APP_IMAGE))) $(call add_json_bool, ForceCreateAppImage, $(filter true,$(LOCAL_DEX_PREOPT_APP_IMAGE))) $(call add_json_bool, PresignedPrebuilt, $(filter PRESIGNED,$(LOCAL_CERTIFICATE))) @@ -445,65 +434,124 @@ ifdef LOCAL_DEX_PREOPT @cp $(PRIVATE_BUILT_MODULE) $@ endif + # The root "product_packages.txt" is generated by `build/make/core/Makefile`. It contains a list + # of all packages that are installed on the device. We use `grep` to filter the list by the app's + # dependencies to create a per-app list, and use `rsync --checksum` to prevent the file's mtime + # from being changed if the contents don't change. This avoids unnecessary dexpreopt reruns. + my_dexpreopt_product_packages := $(intermediates)/product_packages.txt + .KATI_RESTAT: $(my_dexpreopt_product_packages) + $(my_dexpreopt_product_packages): PRIVATE_MODULE := $(LOCAL_MODULE) + $(my_dexpreopt_product_packages): PRIVATE_LIBS := $(my_dexpreopt_libs_all) + $(my_dexpreopt_product_packages): PRIVATE_STAGING := $(my_dexpreopt_product_packages).tmp + $(my_dexpreopt_product_packages): $(PRODUCT_OUT)/product_packages.txt + @echo "$(PRIVATE_MODULE) dexpreopt product_packages" + ifneq (,$(my_dexpreopt_libs_all)) + grep -F -x \ + $(addprefix -e ,$(PRIVATE_LIBS)) \ + $(PRODUCT_OUT)/product_packages.txt \ + > $(PRIVATE_STAGING) \ + || true + else + rm -f $(PRIVATE_STAGING) && touch $(PRIVATE_STAGING) + endif + rsync --checksum $(PRIVATE_STAGING) $@ + my_dexpreopt_script := $(intermediates)/dexpreopt.sh - my_dexpreopt_zip := $(intermediates)/dexpreopt.zip .KATI_RESTAT: $(my_dexpreopt_script) $(my_dexpreopt_script): PRIVATE_MODULE := $(LOCAL_MODULE) $(my_dexpreopt_script): PRIVATE_GLOBAL_SOONG_CONFIG := $(DEX_PREOPT_SOONG_CONFIG_FOR_MAKE) $(my_dexpreopt_script): PRIVATE_GLOBAL_CONFIG := $(DEX_PREOPT_CONFIG_FOR_MAKE) $(my_dexpreopt_script): PRIVATE_MODULE_CONFIG := $(my_dexpreopt_config) + $(my_dexpreopt_script): PRIVATE_PRODUCT_PACKAGES := $(my_dexpreopt_product_packages) $(my_dexpreopt_script): $(DEXPREOPT_GEN) $(my_dexpreopt_script): $(my_dexpreopt_jar_copy) - $(my_dexpreopt_script): $(my_dexpreopt_config) $(DEX_PREOPT_SOONG_CONFIG_FOR_MAKE) $(DEX_PREOPT_CONFIG_FOR_MAKE) + $(my_dexpreopt_script): $(my_dexpreopt_config) $(DEX_PREOPT_SOONG_CONFIG_FOR_MAKE) $(DEX_PREOPT_CONFIG_FOR_MAKE) $(my_dexpreopt_product_packages) @echo "$(PRIVATE_MODULE) dexpreopt gen" $(DEXPREOPT_GEN) \ -global_soong $(PRIVATE_GLOBAL_SOONG_CONFIG) \ -global $(PRIVATE_GLOBAL_CONFIG) \ -module $(PRIVATE_MODULE_CONFIG) \ -dexpreopt_script $@ \ - -out_dir $(OUT_DIR) + -out_dir $(OUT_DIR) \ + -product_packages $(PRIVATE_PRODUCT_PACKAGES) my_dexpreopt_deps := $(my_dex_jar) my_dexpreopt_deps += $(if $(my_process_profile),$(LOCAL_DEX_PREOPT_PROFILE)) my_dexpreopt_deps += \ - $(foreach lib, $(my_dexpreopt_libs) $(my_dexpreopt_libs_compat), \ + $(foreach lib, $(my_dexpreopt_libs_all), \ $(call intermediates-dir-for,JAVA_LIBRARIES,$(lib),,COMMON)/javalib.jar) my_dexpreopt_deps += $(my_dexpreopt_images_deps) my_dexpreopt_deps += $(DEXPREOPT_BOOTCLASSPATH_DEX_FILES) ifeq ($(LOCAL_ENFORCE_USES_LIBRARIES),true) - my_dexpreopt_deps += $(intermediates.COMMON)/enforce_uses_libraries.status + my_dexpreopt_deps += $(intermediates)/enforce_uses_libraries.status endif + # We need to add all the installed files to ALL_MODULES.$(my_register_name).INSTALLED in order + # for the build system to properly track installed files. (for sbom, installclean, etc) + # We install all the files in a zip file generated at execution time, which means we have to guess + # what's going to be in that zip file before it's created. We then check at executation time that + # our guess is correct. + # _system_other corresponds to OdexOnSystemOtherByName() in soong. + # The other paths correspond to dexpreoptCommand() + _dexlocation := $(patsubst $(PRODUCT_OUT)/%,%,$(LOCAL_INSTALLED_MODULE)) + _dexname := $(basename $(notdir $(_dexlocation))) + _system_other := $(strip $(if $(strip $(BOARD_USES_SYSTEM_OTHER_ODEX)), \ + $(if $(strip $(SANITIZE_LITE)),, \ + $(if $(filter $(_dexname),$(PRODUCT_DEXPREOPT_SPEED_APPS))$(filter $(_dexname),$(PRODUCT_SYSTEM_SERVER_APPS)),, \ + $(if $(strip $(foreach myfilter,$(SYSTEM_OTHER_ODEX_FILTER),$(filter system/$(myfilter),$(_dexlocation)))), \ + system_other/))))) + # _dexdir has a trailing / + _dexdir := $(_system_other)$(dir $(_dexlocation)) + my_dexpreopt_zip_contents := $(sort \ + $(foreach arch,$(my_dexpreopt_archs), \ + $(_dexdir)oat/$(arch)/$(_dexname).odex \ + $(_dexdir)oat/$(arch)/$(_dexname).vdex \ + $(if $(filter false,$(LOCAL_DEX_PREOPT_APP_IMAGE)),, \ + $(if $(my_process_profile)$(filter true,$(LOCAL_DEX_PREOPT_APP_IMAGE)), \ + $(_dexdir)oat/$(arch)/$(_dexname).art))) \ + $(if $(my_process_profile),$(_dexlocation).prof)) + _dexlocation := + _dexdir := + _dexname := + _system_other := + + my_dexpreopt_zip := $(intermediates)/dexpreopt.zip $(my_dexpreopt_zip): PRIVATE_MODULE := $(LOCAL_MODULE) $(my_dexpreopt_zip): $(my_dexpreopt_deps) $(my_dexpreopt_zip): | $(DEXPREOPT_GEN_DEPS) $(my_dexpreopt_zip): .KATI_DEPFILE := $(my_dexpreopt_zip).d $(my_dexpreopt_zip): PRIVATE_DEX := $(my_dex_jar) $(my_dexpreopt_zip): PRIVATE_SCRIPT := $(my_dexpreopt_script) + $(my_dexpreopt_zip): PRIVATE_ZIP_CONTENTS := $(my_dexpreopt_zip_contents) $(my_dexpreopt_zip): $(my_dexpreopt_script) @echo "$(PRIVATE_MODULE) dexpreopt" + rm -f $@ + echo -n > $@.contents + $(foreach f,$(PRIVATE_ZIP_CONTENTS),echo "$(f)" >> $@.contents$(newline)) bash $(PRIVATE_SCRIPT) $(PRIVATE_DEX) $@ + if ! diff <(zipinfo -1 $@ | sort) $@.contents >&2; then \ + echo "Contents of $@ did not match what make was expecting." >&2 && exit 1; \ + fi - ifdef LOCAL_POST_INSTALL_CMD - # Add a shell command separator - LOCAL_POST_INSTALL_CMD += && - endif - - LOCAL_POST_INSTALL_CMD += \ - for i in $$(zipinfo -1 $(my_dexpreopt_zip)); \ - do mkdir -p $(PRODUCT_OUT)/$$(dirname $$i); \ - done && \ - ( unzip -qoDD -d $(PRODUCT_OUT) $(my_dexpreopt_zip) 2>&1 | grep -v "zipfile is empty"; exit $${PIPESTATUS[0]} ) || \ - ( code=$$?; if [ $$code -ne 0 -a $$code -ne 1 ]; then exit $$code; fi ) + $(foreach installed_dex_file,$(my_dexpreopt_zip_contents),\ + $(eval $(PRODUCT_OUT)/$(installed_dex_file): $(my_dexpreopt_zip) \ +$(newline) unzip -qoDD -d $(PRODUCT_OUT) $(my_dexpreopt_zip) $(installed_dex_file))) - $(LOCAL_INSTALLED_MODULE): PRIVATE_POST_INSTALL_CMD := $(LOCAL_POST_INSTALL_CMD) - $(LOCAL_INSTALLED_MODULE): $(my_dexpreopt_zip) + ALL_MODULES.$(my_register_name).INSTALLED += $(addprefix $(PRODUCT_OUT)/,$(my_dexpreopt_zip_contents)) - $(my_all_targets): $(my_dexpreopt_zip) + # Normally this happens in sbom.mk, which is included from base_rules.mk. But since + # dex_preopt_odex_install.mk is included after base_rules.mk, it misses these odex files. + $(foreach installed_file,$(addprefix $(PRODUCT_OUT)/,$(my_dexpreopt_zip_contents)), \ + $(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))) my_dexpreopt_config := + my_dexpreopt_config_for_postprocessing := + my_dexpreopt_jar_copy := + my_dexpreopt_product_packages := my_dexpreopt_script := my_dexpreopt_zip := - my_dexpreopt_config_for_postprocessing := + my_dexpreopt_zip_contents := endif # LOCAL_DEX_PREOPT -endif # my_create_dexpreopt_config
\ No newline at end of file +endif # my_create_dexpreopt_config + +my_dexpreopt_libs_all := diff --git a/core/distdir.mk b/core/distdir.mk index bce8e7fd3c..032d1b7ca9 100644 --- a/core/distdir.mk +++ b/core/distdir.mk @@ -45,6 +45,18 @@ $(foreach file,$(2), \ $(eval _all_dist_goal_output_pairs += $$(goal):$$(dst)))) endef +define add_file_name_tag_suffix +$(basename $(notdir $1))-FILE_NAME_TAG_PLACEHOLDER$(suffix $1) +endef + +# This function appends suffix FILE_NAME_TAG_PLACEHOLDER from the input file +# $(1): a list of goals (e.g. droid, sdk, ndk). These must be PHONY +# $(2): the dist files to add to those goals. +define dist-for-goals-with-filenametag +$(if $(strip $(2)), \ + $(foreach file,$(2), \ + $(call dist-for-goals,$(1),$(file):$(call add_file_name_tag_suffix,$(file))))) +endef .PHONY: shareprojects define __share-projects-rule @@ -209,4 +221,4 @@ $(strip \ fi)) endef -.KATI_READONLY := dist-for-goals dist-write-file +.KATI_READONLY := dist-for-goals dist-write-file dist-for-goals-with-filenametag diff --git a/core/dupcheck.sh b/core/dupcheck.sh new file mode 100755 index 0000000000..13ab78211e --- /dev/null +++ b/core/dupcheck.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Find duplicate shared libraries by md5 checksum and possible duplicates by size. +# Results will be available in the out directory of the build. +# Usage: +# ./dupcheck.sh <out_dir> <image> + +OUT_DIR="$1" +IMG="$2" +TMP_MD5="${OUT_DIR}/_dup_md5" +TMP_SIZE="${OUT_DIR}/_dup_size" +TMP_CHECK="${OUT_DIR}/_dup_tmp_check" +TMP_SIZE_REAL="${OUT_DIR}/_dup_size_real" +TMP_FILE1="${OUT_DIR}/_dup_f1" +TMP_FILE2="${OUT_DIR}/_dup_f2" +MD5_DUPLICATES="${OUT_DIR}/duplicate-libs-md5-${IMG}.txt" +SIZE_DUPLICATES="${OUT_DIR}/duplicate-libs-size-${IMG}.txt" + +# Check arguments +if [ "$#" -ne 2 ]; then + echo "Usage: ./dupcheck.sh <out_dir> <image>" + exit 1 +fi + +# Check host and toolchain version +CHECK_HOST=$(uname) +if [ "${CHECK_HOST}" == "Linux" ]; then + ARCH="linux-x86" +else + ARCH="darwin-x86" +fi +BINUTILS_PATH="./prebuilts/clang/host/${ARCH}/llvm-binutils-stable" + +# Remove any old files if they exist. +if [ -f "${MD5_DUPLICATES}" ]; then + rm "${MD5_DUPLICATES}" +fi + +if [ -f "${SIZE_DUPLICATES}" ]; then + rm "${SIZE_DUPLICATES}" +fi + +# Find all .so files and calculate their md5. +find ./"${OUT_DIR}"/${IMG}/ -name "lib*.so" -type f -print0 | xargs -0 md5sum | sed -e "s# .*/# #" | sort | uniq -c | sort -g | sed "/^.*1 /d" | sed "s/^. *[0-9] //" > "${TMP_MD5}" 2>&1 + +if [ -s "${TMP_MD5}" ]; then + while read -r list; do + checksum=$(echo "${list}" | cut -f1 -d ' ') + filename=$(echo "${list}" | cut -f2 -d ' ') + # For each md5, list the file paths that match. + { + echo "MD5: ${checksum}"; \ + find ./"${OUT_DIR}"/${IMG}/ -name "${filename}" -type f -print0 | xargs -0 md5sum | grep "${checksum}" | sed 's/^.* //'; \ + echo ""; \ + } >> "${MD5_DUPLICATES}" + done <"${TMP_MD5}" +else + echo "No duplicate files by md5 found." >> "${MD5_DUPLICATES}" +fi + +# Cleanup +rm "${TMP_MD5}" + +# Find possible duplicate .so files by size. +find ./"${OUT_DIR}"/${IMG}/ -name "*.so" -type f -print0 | xargs -0 stat --format="%s %n" 2>/dev/null | sed -e "s# .*/# #" | sort | uniq -c | sort -g | sed "/^.*1 /d" > "${TMP_SIZE}" 2>&1 +if [ -s "${TMP_SIZE}" ]; then + while read -r list; do + size=$(echo "${list}" | cut -f2 -d ' ') + filename=$(echo "${list}" | cut -f3 -d ' ') + # Check if the files are not in the md5sum list and do nothing if that is the case. + find ./"${OUT_DIR}"/${IMG}/ -name "${filename}" -type f -print0 | xargs -0 stat --format="%s %n" 2>/dev/null | grep "${size}" | sed "s/^.* //" | sort > "${TMP_CHECK}" 2>&1 + while read -r filepath; do + found=$(grep -F "${filepath}" "${MD5_DUPLICATES}") + if [ -z "${found}" ]; then + echo "${filepath}" >> "${TMP_SIZE_REAL}" + fi + done<"${TMP_CHECK}" + # For every duplication found, diff the .note and .text sections. + if [ -s "${TMP_SIZE_REAL}" ]; then + { + echo "File: ${filename}, Size: ${size}"; \ + cat "${TMP_SIZE_REAL}"; \ + echo ""; \ + } >> "${SIZE_DUPLICATES}" + count=$(wc -l "${TMP_SIZE_REAL}" | cut -f1 -d ' ') + # Limitation: this only works for file pairs. If more than two possible duplications are found, the user need to check manually + # all the possible combinations using the llvm-readelf and llvm-objdump commands below. + if [ "${count}" = 2 ]; then + file1=$(head -n 1 "${TMP_SIZE_REAL}") + file2=$(tail -n 1 "${TMP_SIZE_REAL}") + # Check .note section + ${BINUTILS_PATH}/llvm-readelf --wide --notes "${file1}" > "${TMP_FILE1}" 2>&1 + ${BINUTILS_PATH}/llvm-readelf --wide --notes "${file2}" > "${TMP_FILE2}" 2>&1 + { + diff -u "${TMP_FILE1}" "${TMP_FILE2}" | sed "1d;2d;3d"; \ + echo ""; + } >> "${SIZE_DUPLICATES}" + # Check .text section + ${BINUTILS_PATH}/llvm-objdump --line-numbers --disassemble --demangle --reloc --no-show-raw-insn --section=.text "${file1}" | sed "1d;2d"> "${TMP_FILE1}" 2>&1 + ${BINUTILS_PATH}/llvm-objdump --line-numbers --disassemble --demangle --reloc --no-show-raw-insn --section=.text "${file2}" | sed "1d;2d"> "${TMP_FILE2}" 2>&1 + { + diff -u "${TMP_FILE1}" "${TMP_FILE2}" | sed "1d;2d;3d"; \ + echo ""; + } >> "${SIZE_DUPLICATES}" + # Cleanup + rm "${TMP_FILE1}" "${TMP_FILE2}" + else + echo "*Note: more than one duplicate. Manually verify all possible combinations." >> "${SIZE_DUPLICATES}" + fi + rm "${TMP_SIZE_REAL}" + echo "" >> "${SIZE_DUPLICATES}" + fi + done <"${TMP_SIZE}" + # Cleanup + rm "${TMP_SIZE}" "${TMP_CHECK}" +else + echo "No duplicate files by size found." >> "${SIZE_DUPLICATES}" +fi diff --git a/core/envsetup.mk b/core/envsetup.mk index 7dd9b124c5..1c3a1b3b6b 100644 --- a/core/envsetup.mk +++ b/core/envsetup.mk @@ -24,50 +24,49 @@ endef #$(warning $(call find_and_earlier,A B C,C)) #$(warning $(call find_and_earlier,A B C,D)) -define version-list -$(1)P1A $(1)P1B $(1)P2A $(1)P2B $(1)D1A $(1)D1B $(1)D2A $(1)D2B $(1)Q1A $(1)Q1B $(1)Q2A $(1)Q2B $(1)Q3A $(1)Q3B +# Runs a starlark file, and sets all the variables in its top-level +# variables_to_export_to_make variable as make variables. +# +# In order to avoid running starlark every time the stamp file is checked, we use +# $(KATI_shell_no_rerun). Then, to make sure that we actually do rerun kati when +# modifying the starlark files, we add the starlark files to the kati stamp file with +# $(KATI_extra_file_deps). +# +# Arguments: +# $(1): A single starlark file to use as the entrypoint +# $(2): An optional list of starlark files to NOT include as kati dependencies. +# $(3): An optional list of extra flags to pass to rbcrun +define run-starlark +$(eval _starlark_results := $(OUT_DIR)/starlark_results/$(subst /,_,$(1)).mk) +$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(3) $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results)) +$(if $(filter-out 0,$(.SHELLSTATUS)),$(error Starlark failed to run)) +$(eval include $(_starlark_results)) +$(KATI_extra_file_deps $(filter-out $(2),$(LOADED_STARLARK_FILES))) +$(eval LOADED_STARLARK_FILES :=) +$(eval _starlark_results :=) endef -PREV_VERSIONS := OPR1 OPD1 OPD2 OPM1 OPM2 PPR1 PPD1 PPD2 PPM1 PPM2 QPR1 -ALL_VERSIONS := Q R S T U V W X Y Z -ALL_VERSIONS := $(PREV_VERSIONS) $(foreach v,$(ALL_VERSIONS),$(call version-list,$(v))) -PREV_VERSIONS := - -# Filters ALL_VERSIONS down to the range [$1, $2], and errors if $1 > $2 or $3 is -# not in [$1, $2] -# $(1): min platform version -# $(2): max platform version -# $(3): default platform version -define allowed-platform-versions -$(strip \ - $(if $(filter $(ALL_VERSIONS),$(1)),, - $(error Invalid MIN_PLATFORM_VERSION '$(1)')) - $(if $(filter $(ALL_VERSIONS),$(2)),, - $(error Invalid MAX_PLATFORM_VERSION '$(2)')) - $(if $(filter $(ALL_VERSIONS),$(3)),, - $(error Invalid DEFAULT_PLATFORM_VERSION '$(3)')) - - $(eval allowed_versions_ := $(call find_and_earlier,$(ALL_VERSIONS),$(2))) - - $(if $(filter $(allowed_versions_),$(1)),, - $(error MIN_PLATFORM_VERSION '$(1)' must be before MAX_PLATFORM_VERSION '$(2)')) - - $(eval allowed_versions_ := $(1) \ - $(filter-out $(call find_and_earlier,$(allowed_versions_),$(1)),$(allowed_versions_))) - - $(if $(filter $(allowed_versions_),$(3)),, - $(error DEFAULT_PLATFORM_VERSION '$(3)' must be between MIN_PLATFORM_VERSION '$(1)' and MAX_PLATFORM_VERSION '$(2)')) - - $(allowed_versions_)) -endef +# --------------------------------------------------------------- +# Release config +include $(BUILD_SYSTEM)/release_config.mk + +# Set default value of KEEP_VNDK. +ifeq ($(RELEASE_DEPRECATE_VNDK),true) + KEEP_VNDK ?= false +else + KEEP_VNDK ?= true +endif -#$(warning $(call allowed-platform-versions,OPR1,PPR1,OPR1)) -#$(warning $(call allowed-platform-versions,OPM1,PPR1,OPR1)) +# --------------------------------------------------------------- +# Set up version information +include $(BUILD_SYSTEM)/version_util.mk -# Set up version information. -include $(BUILD_SYSTEM)/version_defaults.mk +# This used to be calculated, but is now fixed and not expected +# to change over time anymore. New code attempting to use a +# variable like IS_AT_LAST_* should instead use a +# build system flag. -ENABLED_VERSIONS := $(call find_and_earlier,$(ALL_VERSIONS),$(TARGET_PLATFORM_VERSION)) +ENABLED_VERSIONS := "OPR1 OPD1 OPD2 OPM1 OPM2 PPR1 PPD1 PPD2 PPM1 PPM2 QPR1 QP1A QP1B QP2A QP2B QD1A QD1B QD2A QD2B QQ1A QQ1B QQ2A QQ2B QQ3A QQ3B RP1A RP1B RP2A RP2B RD1A RD1B RD2A RD2B RQ1A RQ1B RQ2A RQ2B RQ3A RQ3B SP1A SP1B SP2A SP2B SD1A SD1B SD2A SD2B SQ1A SQ1B SQ2A SQ2B SQ3A SQ3B TP1A TP1B TP2A TP2B TD1A TD1B TD2A TD2B TQ1A TQ1B TQ2A TQ2B TQ3A TQ3B UP1A UP1B UP2A UP2B UD1A UD1B UD2A UD2B UQ1A UQ1B UQ2A UQ2B UQ3A UQ3B" $(foreach v,$(ENABLED_VERSIONS), \ $(eval IS_AT_LEAST_$(v) := true)) @@ -339,6 +338,7 @@ $(eval _dump_variables_rbc_excluded := \ RBC_PRODUCT_CONFIG \ RBC_BOARD_CONFIG \ SOONG_% \ + TARGET_RELEASE \ TOPDIR \ TRACE_BEGIN_SOONG \ USER) @@ -378,6 +378,8 @@ ifneq ($(TARGET_BUILD_TYPE),debug) TARGET_BUILD_TYPE := release endif +include $(BUILD_SYSTEM)/product_validation_checks.mk + # --------------------------------------------------------------- # figure out the output directories @@ -553,6 +555,8 @@ TARGET_OUT_ETC := $(TARGET_OUT)/etc TARGET_OUT_NOTICE_FILES := $(TARGET_OUT_INTERMEDIATES)/NOTICE_FILES TARGET_OUT_FAKE := $(PRODUCT_OUT)/fake_packages TARGET_OUT_TESTCASES := $(PRODUCT_OUT)/testcases +TARGET_OUT_FLAGS := $(TARGET_OUT_INTERMEDIATES)/FLAGS + .KATI_READONLY := \ TARGET_OUT_EXECUTABLES \ TARGET_OUT_OPTIONAL_EXECUTABLES \ @@ -566,7 +570,8 @@ TARGET_OUT_TESTCASES := $(PRODUCT_OUT)/testcases TARGET_OUT_ETC \ TARGET_OUT_NOTICE_FILES \ TARGET_OUT_FAKE \ - TARGET_OUT_TESTCASES + TARGET_OUT_TESTCASES \ + TARGET_OUT_FLAGS ifeq ($(SANITIZE_LITE),true) # When using SANITIZE_LITE, APKs must not be packaged with sanitized libraries, as they will not @@ -695,6 +700,7 @@ TARGET_OUT_VENDOR_JAVA_LIBRARIES := $(TARGET_OUT_VENDOR)/framework TARGET_OUT_VENDOR_APPS := $(target_out_vendor_app_base)/app TARGET_OUT_VENDOR_APPS_PRIVILEGED := $(target_out_vendor_app_base)/priv-app TARGET_OUT_VENDOR_ETC := $(TARGET_OUT_VENDOR)/etc +TARGET_OUT_VENDOR_FAKE := $(PRODUCT_OUT)/vendor_fake_packages .KATI_READONLY := \ TARGET_OUT_VENDOR_EXECUTABLES \ TARGET_OUT_VENDOR_OPTIONAL_EXECUTABLES \ @@ -703,7 +709,8 @@ TARGET_OUT_VENDOR_ETC := $(TARGET_OUT_VENDOR)/etc TARGET_OUT_VENDOR_JAVA_LIBRARIES \ TARGET_OUT_VENDOR_APPS \ TARGET_OUT_VENDOR_APPS_PRIVILEGED \ - TARGET_OUT_VENDOR_ETC + TARGET_OUT_VENDOR_ETC \ + TARGET_OUT_VENDOR_FAKE $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_VENDOR_EXECUTABLES := $(TARGET_OUT_VENDOR_EXECUTABLES) $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_VENDOR_SHARED_LIBRARIES := $(target_out_vendor_shared_libraries_base)/lib @@ -770,6 +777,7 @@ TARGET_OUT_ODM_JAVA_LIBRARIES := $(TARGET_OUT_ODM)/framework TARGET_OUT_ODM_APPS := $(target_out_odm_app_base)/app TARGET_OUT_ODM_APPS_PRIVILEGED := $(target_out_odm_app_base)/priv-app TARGET_OUT_ODM_ETC := $(TARGET_OUT_ODM)/etc +TARGET_OUT_ODM_FAKE := $(PRODUCT_OUT)/odm_fake_packages .KATI_READONLY := \ TARGET_OUT_ODM \ TARGET_OUT_ODM_EXECUTABLES \ @@ -779,7 +787,8 @@ TARGET_OUT_ODM_ETC := $(TARGET_OUT_ODM)/etc TARGET_OUT_ODM_JAVA_LIBRARIES \ TARGET_OUT_ODM_APPS \ TARGET_OUT_ODM_APPS_PRIVILEGED \ - TARGET_OUT_ODM_ETC + TARGET_OUT_ODM_ETC \ + TARGET_OUT_ODM_FAKE $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_ODM_EXECUTABLES := $(TARGET_OUT_ODM_EXECUTABLES) $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_ODM_SHARED_LIBRARIES := $(target_out_odm_shared_libraries_base)/lib @@ -917,13 +926,15 @@ TARGET_OUT_PRODUCT_JAVA_LIBRARIES := $(TARGET_OUT_PRODUCT)/framework TARGET_OUT_PRODUCT_APPS := $(target_out_product_app_base)/app TARGET_OUT_PRODUCT_APPS_PRIVILEGED := $(target_out_product_app_base)/priv-app TARGET_OUT_PRODUCT_ETC := $(TARGET_OUT_PRODUCT)/etc +TARGET_OUT_PRODUCT_FAKE := $(TARGET_OUT_PRODUCT)/product_fake_packages .KATI_READONLY := \ TARGET_OUT_PRODUCT_EXECUTABLES \ TARGET_OUT_PRODUCT_SHARED_LIBRARIES \ TARGET_OUT_PRODUCT_JAVA_LIBRARIES \ TARGET_OUT_PRODUCT_APPS \ TARGET_OUT_PRODUCT_APPS_PRIVILEGED \ - TARGET_OUT_PRODUCT_ETC + TARGET_OUT_PRODUCT_ETC \ + TARGET_OUT_PRODUCT_FAKE $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_PRODUCT_EXECUTABLES := $(TARGET_OUT_PRODUCT_EXECUTABLES) $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_PRODUCT_SHARED_LIBRARIES := $(target_out_product_shared_libraries_base)/lib @@ -960,13 +971,15 @@ TARGET_OUT_SYSTEM_EXT_APPS := $(target_out_system_ext_app_base)/app TARGET_OUT_SYSTEM_EXT_APPS_PRIVILEGED := $(target_out_system_ext_app_base)/priv-app TARGET_OUT_SYSTEM_EXT_ETC := $(TARGET_OUT_SYSTEM_EXT)/etc TARGET_OUT_SYSTEM_EXT_EXECUTABLES := $(TARGET_OUT_SYSTEM_EXT)/bin +TARGET_OUT_SYSTEM_EXT_FAKE := $(PRODUCT_OUT)/system_ext_fake_packages .KATI_READONLY := \ TARGET_OUT_SYSTEM_EXT_EXECUTABLES \ TARGET_OUT_SYSTEM_EXT_SHARED_LIBRARIES \ TARGET_OUT_SYSTEM_EXT_JAVA_LIBRARIES \ TARGET_OUT_SYSTEM_EXT_APPS \ TARGET_OUT_SYSTEM_EXT_APPS_PRIVILEGED \ - TARGET_OUT_SYSTEM_EXT_ETC + TARGET_OUT_SYSTEM_EXT_ETC \ + TARGET_OUT_SYSTEM_EXT_FAKE $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_SYSTEM_EXT_EXECUTABLES := $(TARGET_OUT_SYSTEM_EXT_EXECUTABLES) $(TARGET_2ND_ARCH_VAR_PREFIX)TARGET_OUT_SYSTEM_EXT_SHARED_LIBRARIES := $(target_out_system_ext_shared_libraries_base)/lib diff --git a/core/executable_internal.mk b/core/executable_internal.mk index fb14ccea41..fecf4f6a28 100644 --- a/core/executable_internal.mk +++ b/core/executable_internal.mk @@ -45,7 +45,7 @@ ifeq ($(LOCAL_NO_CRT),true) my_target_crtbegin_dynamic_o := my_target_crtbegin_static_o := my_target_crtend_o := -else ifdef LOCAL_USE_VNDK +else ifeq (true,$(call module-in-vendor-or-product)) my_target_crtbegin_dynamic_o := $(SOONG_$(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_OBJECT_crtbegin_dynamic.vendor) my_target_crtbegin_static_o := $(SOONG_$(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_OBJECT_crtbegin_static.vendor) my_target_crtend_o := $(SOONG_$(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_OBJECT_crtend_android.vendor) diff --git a/core/generate_enforce_rro.mk b/core/generate_enforce_rro.mk index 9079981856..b002469ead 100644 --- a/core/generate_enforce_rro.mk +++ b/core/generate_enforce_rro.mk @@ -1,6 +1,6 @@ include $(CLEAR_VARS) -enforce_rro_module := $(enforce_rro_source_module)__auto_generated_rro_$(enforce_rro_partition) +enforce_rro_module := $(enforce_rro_source_module)__$(PRODUCT_NAME)__auto_generated_rro_$(enforce_rro_partition) LOCAL_PACKAGE_NAME := $(enforce_rro_module) intermediates := $(call intermediates-dir-for,APPS,$(LOCAL_PACKAGE_NAME),,COMMON) @@ -38,7 +38,7 @@ endif LOCAL_FULL_MANIFEST_FILE := $(rro_android_manifest_file) -LOCAL_AAPT_FLAGS += --auto-add-overlay +LOCAL_AAPT_FLAGS += --auto-add-overlay --keep-raw-values LOCAL_RESOURCE_DIR := $(enforce_rro_source_overlays) ifeq (product,$(enforce_rro_partition)) diff --git a/core/goma.mk b/core/goma.mk deleted file mode 100644 index 2b51d8be44..0000000000 --- a/core/goma.mk +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (C) 2015 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Notice: this works only with Google's Goma build infrastructure. -ifneq ($(filter-out false,$(USE_GOMA)),) - ifdef GOMA_DIR - goma_dir := $(GOMA_DIR) - else - goma_dir := $(HOME)/goma - endif - GOMA_CC := $(goma_dir)/gomacc - - # Append gomacc to existing *_WRAPPER variables so it's possible to - # use both ccache and gomacc. - CC_WRAPPER := $(strip $(CC_WRAPPER) $(GOMA_CC)) - CXX_WRAPPER := $(strip $(CXX_WRAPPER) $(GOMA_CC)) - # b/143658984: goma can't handle the --system argument to javac - #JAVAC_WRAPPER := $(strip $(JAVAC_WRAPPER) $(GOMA_CC)) - - goma_dir := -endif diff --git a/core/host_java_library.mk b/core/host_java_library.mk index 89aa53ce0c..d45da48c84 100644 --- a/core/host_java_library.mk +++ b/core/host_java_library.mk @@ -98,9 +98,7 @@ $(full_classes_combined_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_D $(full_classes_combined_jar): $(full_classes_compiled_jar) \ $(jar_manifest_file) \ $(full_static_java_libs) | $(MERGE_ZIPS) - $(if $(PRIVATE_JAR_MANIFEST), $(hide) sed -e "s/%BUILD_NUMBER%/$(BUILD_NUMBER_FROM_FILE)/" \ - $(PRIVATE_JAR_MANIFEST) > $(dir $@)/manifest.mf) - $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(dir $@)/manifest.mf) \ + $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(PRIVATE_JAR_MANIFEST)) \ $(if $(PRIVATE_DONT_DELETE_JAR_META_INF),,-stripDir META-INF -zipToNotStrip $<) \ $@ $< $(PRIVATE_STATIC_JAVA_LIBRARIES) diff --git a/core/host_java_library_common.mk b/core/host_java_library_common.mk index 0e62f60d24..006e6ecfae 100644 --- a/core/host_java_library_common.mk +++ b/core/host_java_library_common.mk @@ -46,5 +46,4 @@ else endif endif -LOCAL_INTERMEDIATE_SOURCE_DIR := $(intermediates.COMMON)/src LOCAL_JAVA_LIBRARIES := $(sort $(LOCAL_JAVA_LIBRARIES)) diff --git a/core/instrumentation_test_config_template.xml b/core/instrumentation_test_config_template.xml index 6ca964e414..379126c6de 100644 --- a/core/instrumentation_test_config_template.xml +++ b/core/instrumentation_test_config_template.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2017 The Android Open Source Project +<!-- Copyright (C) 2023 The Android Open Source Project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ </target_preparer> <test class="com.android.tradefed.testtype.{TEST_TYPE}" > - <option name="package" value="{PACKAGE}" /> + {EXTRA_TEST_RUNNER_CONFIGS}<option name="package" value="{PACKAGE}" /> <option name="runner" value="{RUNNER}" /> </test> </configuration> diff --git a/core/java.mk b/core/java.mk index b13ef4d403..5fbc916859 100644 --- a/core/java.mk +++ b/core/java.mk @@ -83,8 +83,6 @@ LOCAL_INTERMEDIATE_TARGETS += \ $(full_classes_stubs_jar) \ $(java_source_list_file) -LOCAL_INTERMEDIATE_SOURCE_DIR := $(intermediates.COMMON)/src - ########################################################### ## AIDL: Compile .aidl files to .java ########################################################### @@ -296,9 +294,7 @@ $(full_classes_combined_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_D $(full_classes_combined_jar): $(full_classes_compiled_jar) \ $(jar_manifest_file) \ $(full_static_java_libs) | $(MERGE_ZIPS) - $(if $(PRIVATE_JAR_MANIFEST), $(hide) sed -e "s/%BUILD_NUMBER%/$(BUILD_NUMBER_FROM_FILE)/" \ - $(PRIVATE_JAR_MANIFEST) > $(dir $@)/manifest.mf) - $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(dir $@)/manifest.mf) \ + $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(PRIVATE_JAR_MANIFEST)) \ $(if $(PRIVATE_DONT_DELETE_JAR_META_INF),,-stripDir META-INF -zipToNotStrip $<) \ $@ $< $(PRIVATE_STATIC_JAVA_LIBRARIES) diff --git a/core/java_common.mk b/core/java_common.mk index 5981b60929..65feb1532e 100644 --- a/core/java_common.mk +++ b/core/java_common.mk @@ -10,9 +10,7 @@ endif ## Java version ########################################################### # Use the LOCAL_JAVA_LANGUAGE_VERSION if it is set, otherwise -# use one based on the LOCAL_SDK_VERSION. If it is < 24 -# pass "1.7" to the tools, if it is unset, >= 24 or "current" -# pass "1.8". +# use one based on the LOCAL_SDK_VERSION. # # The LOCAL_SDK_VERSION behavior is to ensure that, by default, # code that is expected to run on older releases of Android @@ -25,15 +23,17 @@ ifeq (,$(LOCAL_JAVA_LANGUAGE_VERSION)) # Host modules always default to 1.9 LOCAL_JAVA_LANGUAGE_VERSION := 1.9 else - ifneq (,$(filter $(LOCAL_SDK_VERSION), $(TARGET_SDK_VERSIONS_WITHOUT_JAVA_18_SUPPORT))) - LOCAL_JAVA_LANGUAGE_VERSION := 1.7 - else ifneq (,$(filter $(LOCAL_SDK_VERSION), $(TARGET_SDK_VERSIONS_WITHOUT_JAVA_19_SUPPORT))) + ifneq (,$(filter $(LOCAL_SDK_VERSION), $(TARGET_SDK_VERSIONS_WITHOUT_JAVA_1_9_SUPPORT))) LOCAL_JAVA_LANGUAGE_VERSION := 1.8 + else ifneq (,$(filter $(LOCAL_SDK_VERSION), $(TARGET_SDK_VERSIONS_WITHOUT_JAVA_11_SUPPORT))) + LOCAL_JAVA_LANGUAGE_VERSION := 1.9 + else ifneq (,$(filter $(LOCAL_SDK_VERSION), $(TARGET_SDK_VERSIONS_WITHOUT_JAVA_17_SUPPORT))) + LOCAL_JAVA_LANGUAGE_VERSION := 11 else ifneq (,$(LOCAL_SDK_VERSION)$(TARGET_BUILD_USE_PREBUILT_SDKS)) # TODO(ccross): allow 1.9 for current and unbundled once we have SDK system modules LOCAL_JAVA_LANGUAGE_VERSION := 1.8 else - LOCAL_JAVA_LANGUAGE_VERSION := 1.9 + LOCAL_JAVA_LANGUAGE_VERSION := 17 endif endif endif @@ -190,7 +190,7 @@ endif ###################################### ## PRIVATE java vars # LOCAL_SOURCE_FILES_ALL_GENERATED is set only if the module does not have static source files, -# but generated source files in its LOCAL_INTERMEDIATE_SOURCE_DIR. +# but generated source files. # You have to set up the dependency in some other way. need_compile_java := $(strip $(all_java_sources)$(LOCAL_SRCJARS)$(all_res_assets)$(java_resource_sources))$(LOCAL_STATIC_JAVA_LIBRARIES)$(filter true,$(LOCAL_SOURCE_FILES_ALL_GENERATED)) ifdef need_compile_java @@ -234,8 +234,6 @@ $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_HAS_RS_SOURCES := $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_JAVA_SOURCES := $(all_java_sources) $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file) -$(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_RMTYPEDEFS := $(LOCAL_RMTYPEDEFS) - # Quickly check class path vars. disallowed_deps := $(foreach sdk,$(TARGET_AVAILABLE_SDK_VERSIONS),$(call resolve-prebuilt-sdk-module,$(sdk))) disallowed_deps += $(foreach sdk,$(TARGET_AVAILABLE_SDK_VERSIONS),\ @@ -296,16 +294,16 @@ ifndef LOCAL_IS_HOST_MODULE # Note: the lib naming scheme must be kept in sync with build/soong/java/sdk_library.go. sdk_lib_suffix = $(call pretty-error,sdk_lib_suffix was not set correctly) ifeq (current,$(LOCAL_SDK_VERSION)) - sdk_module := android_stubs_current + sdk_module := $(ANDROID_PUBLIC_STUBS) sdk_lib_suffix := .stubs else ifeq (system_current,$(LOCAL_SDK_VERSION)) - sdk_module := android_system_stubs_current + sdk_module := $(ANDROID_SYSTEM_STUBS) sdk_lib_suffix := .stubs.system else ifeq (test_current,$(LOCAL_SDK_VERSION)) - sdk_module := android_test_stubs_current + sdk_module := $(ANDROID_TEST_STUBS) sdk_lib_suffix := .stubs.test else ifeq (core_current,$(LOCAL_SDK_VERSION)) - sdk_module := core.current.stubs + sdk_module := $(ANDROID_CORE_STUBS) sdk_lib_suffix = $(call pretty-error,LOCAL_SDK_LIBRARIES not supported for LOCAL_SDK_VERSION = core_current) endif sdk_libs := $(foreach lib_name,$(LOCAL_SDK_LIBRARIES),$(lib_name)$(sdk_lib_suffix)) @@ -384,7 +382,7 @@ else # LOCAL_IS_HOST_MODULE endif # !LOCAL_IS_HOST_MODULE # (b/204397180) Record ALL_DEPS by default. -ALL_DEPS.$(LOCAL_MODULE).ALL_DEPS := $(ALL_DEPS.$(LOCAL_MODULE).ALL_DEPS) $(full_java_bootclasspath_libs) +ALL_MODULES.$(my_register_name).ALL_DEPS := $(ALL_MODULES.$(my_register_name).ALL_DEPS) $(full_java_bootclasspath_libs) # Export the SDK libs. The sdk library names listed in LOCAL_SDK_LIBRARIES are first exported. # Then sdk library names exported from dependencies are all re-exported. @@ -410,7 +408,7 @@ endif full_java_system_modules_deps := my_system_modules_dir := $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_USE_SYSTEM_MODULES := -ifeq ($(LOCAL_JAVA_LANGUAGE_VERSION),1.9) +ifeq (,$(filter $(LOCAL_JAVA_LANGUAGE_VERSION),$(JAVA_LANGUAGE_VERSIONS_WITHOUT_SYSTEM_MODULES))) $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_USE_SYSTEM_MODULES := true ifdef my_system_modules ifneq ($(my_system_modules),none) @@ -488,20 +486,6 @@ $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_ALL_JAVA_LIBRARIES := $(full_java_libs) $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(full_java_header_libs) $(LOCAL_INTERMEDIATE_TARGETS): PRIVATE_SHARED_JAVA_HEADER_LIBRARIES := $(full_shared_java_header_libs) -ALL_MODULES.$(my_register_name).INTERMEDIATE_SOURCE_DIR := \ - $(ALL_MODULES.$(my_register_name).INTERMEDIATE_SOURCE_DIR) $(LOCAL_INTERMEDIATE_SOURCE_DIR) - - -########################################################## -# Copy NOTICE files of transitive static dependencies -# Don't do this in mm, since many of the targets won't exist. -installed_static_library_notice_file_targets := \ - $(foreach lib,$(LOCAL_STATIC_JAVA_LIBRARIES), \ - NOTICE-$(if $(LOCAL_IS_HOST_MODULE),HOST$(if $(my_host_cross),_CROSS,),TARGET)-JAVA_LIBRARIES-$(lib)) - -$(notice_target): | $(installed_static_library_notice_file_targets) -$(LOCAL_INSTALLED_MODULE): | $(notice_target) - ########################################################### # Verify that all libraries are safe to use ########################################################### diff --git a/core/java_host_test_config_template.xml b/core/java_host_test_config_template.xml index 26c1cafa32..e123dc7ed1 100644 --- a/core/java_host_test_config_template.xml +++ b/core/java_host_test_config_template.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2018 The Android Open Source Project +<!-- Copyright (C) 2023 The Android Open Source Project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,6 @@ {EXTRA_CONFIGS} <test class="com.android.tradefed.testtype.HostTest" > - <option name="jar" value="{MODULE}.jar" /> + {EXTRA_TEST_RUNNER_CONFIGS}<option name="jar" value="{MODULE}.jar" /> </test> </configuration> diff --git a/core/java_prebuilt_internal.mk b/core/java_prebuilt_internal.mk index be733ff6c3..46393acb12 100644 --- a/core/java_prebuilt_internal.mk +++ b/core/java_prebuilt_internal.mk @@ -115,11 +115,6 @@ my_src_aar := $(filter %.aar, $(my_prebuilt_src_file)) ifneq ($(my_src_aar),) # This is .aar file, archive of classes.jar and Android resources. -# run Jetifier if needed -LOCAL_JETIFIER_INPUT_FILE := $(my_src_aar) -include $(BUILD_SYSTEM)/jetifier.mk -my_src_aar := $(LOCAL_JETIFIER_OUTPUT_FILE) - my_src_jar := $(intermediates.COMMON)/aar/classes.jar my_src_proguard_options := $(intermediates.COMMON)/aar/proguard.txt my_src_android_manifest := $(intermediates.COMMON)/aar/AndroidManifest.xml @@ -137,13 +132,6 @@ my_prebuilt_android_manifest := $(intermediates.COMMON)/manifest/AndroidManifest $(eval $(call copy-one-file,$(my_src_android_manifest),$(my_prebuilt_android_manifest))) $(call add-dependency,$(LOCAL_BUILT_MODULE),$(my_prebuilt_android_manifest)) -else - -# run Jetifier if needed -LOCAL_JETIFIER_INPUT_FILE := $(my_src_jar) -include $(BUILD_SYSTEM)/jetifier.mk -my_src_jar := $(LOCAL_JETIFIER_OUTPUT_FILE) - endif $(common_classes_jar) : $(my_src_jar) diff --git a/core/jetifier.mk b/core/jetifier.mk deleted file mode 100644 index fff4230860..0000000000 --- a/core/jetifier.mk +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file sets up the running of Jetifier - -# now add the rule to run jetifier -ifeq ($(strip $(LOCAL_JETIFIER_ENABLED)),true) - my_jetifier_input_path := $(LOCAL_JETIFIER_INPUT_FILE) - my_files := $(intermediates.COMMON)/jetifier - my_jetifier_output_path := $(my_files)/jetified-$(notdir $(my_jetifier_input_path)) - -$(my_jetifier_output_path) : $(my_jetifier_input_path) $(JETIFIER) - rm -rf $@ - $(JETIFIER) -l error -o $@ -i $< - - LOCAL_JETIFIER_OUTPUT_FILE := $(my_jetifier_output_path) - LOCAL_INTERMEDIATE_TARGETS += $(LOCAL_JETIFIER_OUTPUT_FILE) -else - LOCAL_JETIFIER_OUTPUT_FILE := $(LOCAL_JETIFIER_INPUT_FILE) -endif - diff --git a/core/layoutlib_data.mk b/core/layoutlib_data.mk new file mode 100644 index 0000000000..e45f7efe16 --- /dev/null +++ b/core/layoutlib_data.mk @@ -0,0 +1,153 @@ +# Data files for layoutlib + +FONT_TEMP := $(call intermediates-dir-for,PACKAGING,fonts,HOST,COMMON) + +# The font configuration files - system_fonts.xml, fallback_fonts.xml etc. +font_config := $(sort $(wildcard frameworks/base/data/fonts/*.xml)) +font_config := $(addprefix $(FONT_TEMP)/, $(notdir $(font_config))) + +$(font_config): $(FONT_TEMP)/%.xml: \ + frameworks/base/data/fonts/%.xml + $(hide) mkdir -p $(dir $@) + $(hide) cp -vf $< $@ + +# List of fonts on the device that we want to ship. This is all .ttf, .ttc and .otf fonts. +fonts_device := $(filter $(TARGET_OUT)/fonts/%.ttf $(TARGET_OUT)/fonts/%.ttc $(TARGET_OUT)/fonts/%.otf, $(INTERNAL_SYSTEMIMAGE_FILES)) +fonts_device := $(addprefix $(FONT_TEMP)/, $(notdir $(fonts_device))) + +# TODO: If the font file is a symlink, reuse the font renamed from the symlink +# target. +$(fonts_device): $(FONT_TEMP)/%: $(TARGET_OUT)/fonts/% + $(hide) mkdir -p $(dir $@) + $(hide) cp -vf $< $@ + +KEYBOARD_TEMP := $(call intermediates-dir-for,PACKAGING,keyboards,HOST,COMMON) + +# The key character map files needed for supporting KeyEvent +keyboards := $(sort $(wildcard frameworks/base/data/keyboards/*.kcm)) +keyboards := $(addprefix $(KEYBOARD_TEMP)/, $(notdir $(keyboards))) + +$(keyboards): $(KEYBOARD_TEMP)/%.kcm: frameworks/base/data/keyboards/%.kcm + $(hide) mkdir -p $(dir $@) + $(hide) cp -vf $< $@ + +# List of all data files - font files, font configuration files, key character map files +LAYOUTLIB_FILES := $(fonts_device) $(font_config) $(keyboards) + +.PHONY: layoutlib layoutlib-tests +layoutlib layoutlib-tests: $(LAYOUTLIB_FILES) + +$(call dist-for-goals, layoutlib, $(foreach m,$(fonts_device), $(m):layoutlib_native/fonts/$(notdir $(m)))) +$(call dist-for-goals, layoutlib, $(foreach m,$(font_config), $(m):layoutlib_native/fonts/$(notdir $(m)))) +$(call dist-for-goals, layoutlib, $(foreach m,$(keyboards), $(m):layoutlib_native/keyboards/$(notdir $(m)))) + +FONT_TEMP := +font_config := +fonts_device := +FONT_FILES := + +# The following build process of build.prop, layoutlib-res.zip is moved here from release_layoutlib.sh +# so the SBOM of all platform neutral artifacts and Linux/Windows artifacts of layoutlib can be built in Make/Soong. +# See go/layoutlib-sbom. + +# build.prop shipped with layoutlib +LAYOUTLIB_BUILD_PROP := $(call intermediates-dir-for,PACKAGING,layoutlib-build-prop,HOST,COMMON) +$(LAYOUTLIB_BUILD_PROP)/layoutlib-build.prop: $(INSTALLED_SDK_BUILD_PROP_TARGET) + rm -rf $@ + cp $< $@ + # Remove all the uncommon build properties + sed -i '/^ro\.\(build\|product\|config\|system\)/!d' $@ + # Mark the build as layoutlib. This can be read at runtime by apps + sed -i 's|ro.product.brand=generic|ro.product.brand=studio|' $@ + sed -i 's|ro.product.device=generic|ro.product.device=layoutlib|' $@ + +$(call dist-for-goals,layoutlib,$(LAYOUTLIB_BUILD_PROP)/layoutlib-build.prop:layoutlib_native/build.prop) + +# Resource files from frameworks/base/core/res/res +LAYOUTLIB_RES := $(call intermediates-dir-for,PACKAGING,layoutlib-res,HOST,COMMON) +LAYOUTLIB_RES_FILES := $(shell find frameworks/base/core/res/res -type f -not -path 'frameworks/base/core/res/res/values-m[nc]c*' | sort) +$(LAYOUTLIB_RES)/layoutlib-res.zip: $(SOONG_ZIP) $(HOST_OUT_EXECUTABLES)/aapt2 $(LAYOUTLIB_RES_FILES) + rm -rf $@ + echo $(LAYOUTLIB_RES_FILES) > $(LAYOUTLIB_RES)/filelist.txt + $(SOONG_ZIP) -C frameworks/base/core/res -l $(LAYOUTLIB_RES)/filelist.txt -o $(LAYOUTLIB_RES)/temp.zip + rm -rf $(LAYOUTLIB_RES)/data && unzip -q -d $(LAYOUTLIB_RES)/data $(LAYOUTLIB_RES)/temp.zip + rm -rf $(LAYOUTLIB_RES)/compiled && mkdir $(LAYOUTLIB_RES)/compiled && $(HOST_OUT_EXECUTABLES)/aapt2 compile $(LAYOUTLIB_RES)/data/res/**/*.9.png -o $(LAYOUTLIB_RES)/compiled + printf '<?xml version="1.0" encoding="utf-8"?>\n<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.google.android.layoutlib" />' > $(LAYOUTLIB_RES)/AndroidManifest.xml + $(HOST_OUT_EXECUTABLES)/aapt2 link -R $(LAYOUTLIB_RES)/compiled/* -o $(LAYOUTLIB_RES)/compiled.apk --manifest $(LAYOUTLIB_RES)/AndroidManifest.xml + rm -rf $(LAYOUTLIB_RES)/compiled_apk && unzip -q -d $(LAYOUTLIB_RES)/compiled_apk $(LAYOUTLIB_RES)/compiled.apk + for f in $(LAYOUTLIB_RES)/compiled_apk/res/*; do mv "$$f" "$${f/-v4/}";done + for f in $(LAYOUTLIB_RES)/compiled_apk/res/**/*.9.png; do mv "$$f" "$${f/.9.png/.compiled.9.png}";done + cp -r $(LAYOUTLIB_RES)/compiled_apk/res $(LAYOUTLIB_RES)/data + $(SOONG_ZIP) -C $(LAYOUTLIB_RES)/data -D $(LAYOUTLIB_RES)/data/res -o $@ + +$(call dist-for-goals,layoutlib,$(LAYOUTLIB_RES)/layoutlib-res.zip:layoutlib_native/res.zip) + +# SBOM of layoutlib artifacts +LAYOUTLIB_SBOM := $(call intermediates-dir-for,PACKAGING,layoutlib-sbom,HOST) +_layoutlib_font_config_files := $(sort $(wildcard frameworks/base/data/fonts/*.xml)) +_layoutlib_fonts_files := $(filter $(TARGET_OUT)/fonts/%.ttf $(TARGET_OUT)/fonts/%.ttc $(TARGET_OUT)/fonts/%.otf, $(INTERNAL_SYSTEMIMAGE_FILES)) +_layoutlib_keyboard_files := $(sort $(wildcard frameworks/base/data/keyboards/*.kcm)) + +# Find out files disted with layoutlib in Soong. +### Filter out static libraries for Windows and files already handled in make. +_layoutlib_filter_out_disted := $(addprefix layoutlib_native/,fonts/% keyboards/% build.prop res.zip windows/%.a) +_layoutlib_files_disted_by_soong := \ + $(strip \ + $(foreach p,$(_all_dist_src_dst_pairs), \ + $(if $(filter-out $(_layoutlib_filter_out_disted),$(filter layoutlib_native/% layoutlib.jar,$(call word-colon,2,$p))),$p))) + +$(LAYOUTLIB_SBOM)/sbom-metadata.csv: + rm -rf $@ + echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $@ + echo build.prop,,,,,,Y,$(LAYOUTLIB_BUILD_PROP)/layoutlib-build.prop,,, >> $@ + + $(foreach f,$(_layoutlib_font_config_files),\ + echo data/fonts/$(notdir $f),frameworks/base/data/fonts,prebuilt_etc,,,,,$f,,, >> $@; \ + ) + + $(foreach f,$(_layoutlib_fonts_files), \ + $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \ + $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \ + $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \ + echo data/fonts/$(notdir $f),$(_module_path),$(_soong_module_type),,,,,$f,,, >> $@; \ + ) + + $(foreach f,$(_layoutlib_keyboard_files), \ + echo data/keyboards/$(notdir $f),frameworks/base/data/keyboards,prebuilt_etc,,,,,$f,,, >> $@; \ + ) + + $(foreach f,$(_layoutlib_files_disted_by_soong), \ + $(eval _prebuilt_module_file := $(call word-colon,1,$f)) \ + $(eval _dist_file := $(call word-colon,2,$f)) \ + $(eval _dist_file := $(patsubst data/windows/%,data/win/lib64/%,$(patsubst layoutlib_native/%,data/%,$(_dist_file)))) \ + $(eval _dist_file := $(subst layoutlib.jar,data/layoutlib.jar,$(_dist_file))) \ + $(eval _module_name := $(strip $(foreach m,$(ALL_MODULES),$(if $(filter $(_prebuilt_module_file),$(ALL_MODULES.$m.CHECKED)),$m)))) \ + $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \ + $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \ + echo $(patsubst layoutlib_native/%,%,$(_dist_file)),$(_module_path),$(_soong_module_type),,,,,$(_prebuilt_module_file),,, >> $@; \ + ) + + $(foreach f,$(LAYOUTLIB_RES_FILES), \ + $(eval _path := $(subst frameworks/base/core/res,data,$f)) \ + echo $(_path),,,,,,Y,$f,,, >> $@; \ + ) + +.PHONY: layoutlib-sbom +layoutlib-sbom: $(LAYOUTLIB_SBOM)/layoutlib.spdx.json +$(LAYOUTLIB_SBOM)/layoutlib.spdx.json: $(PRODUCT_OUT)/always_dirty_file.txt $(GEN_SBOM) $(LAYOUTLIB_SBOM)/sbom-metadata.csv $(_layoutlib_font_config_files) $(_layoutlib_fonts_files) $(LAYOUTLIB_BUILD_PROP)/layoutlib-build.prop $(_layoutlib_keyboard_files) $(LAYOUTLIB_RES_FILES) + rm -rf $@ + $(GEN_SBOM) --output_file $@ --metadata $(LAYOUTLIB_SBOM)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --module_name "layoutlib" --json + +$(call dist-for-goals,layoutlib,$(LAYOUTLIB_SBOM)/layoutlib.spdx.json:layoutlib_native/sbom/layoutlib.spdx.json) + +# Generate SBOM of framework_res.jar that is created in release_layoutlib.sh. +# The generated SBOM contains placeholders for release_layotlib.sh to substitute, and the placeholders include: +# document name, document namespace, document creation info, organization and SHA1 value of framework_res.jar. +GEN_SBOM_FRAMEWORK_RES := $(HOST_OUT_EXECUTABLES)/generate-sbom-framework_res +.PHONY: layoutlib-framework_res-sbom +layoutlib-framework_res-sbom: $(LAYOUTLIB_SBOM)/framework_res.jar.spdx.json +$(LAYOUTLIB_SBOM)/framework_res.jar.spdx.json: $(LAYOUTLIB_SBOM)/layoutlib.spdx.json $(GEN_SBOM_FRAMEWORK_RES) + rm -rf $@ + $(GEN_SBOM_FRAMEWORK_RES) --output_file $(LAYOUTLIB_SBOM)/framework_res.jar.spdx.json --layoutlib_sbom $(LAYOUTLIB_SBOM)/layoutlib.spdx.json + +$(call dist-for-goals,layoutlib,$(LAYOUTLIB_SBOM)/framework_res.jar.spdx.json:layoutlib_native/sbom/framework_res.jar.spdx.json)
\ No newline at end of file diff --git a/core/layoutlib_fonts.mk b/core/layoutlib_fonts.mk deleted file mode 100644 index d2a814f85d..0000000000 --- a/core/layoutlib_fonts.mk +++ /dev/null @@ -1,35 +0,0 @@ -# Fonts for layoutlib - -FONT_TEMP := $(call intermediates-dir-for,PACKAGING,fonts,HOST,COMMON) - -# The font configuration files - system_fonts.xml, fallback_fonts.xml etc. -font_config := $(sort $(wildcard frameworks/base/data/fonts/*.xml)) -font_config := $(addprefix $(FONT_TEMP)/, $(notdir $(font_config))) - -$(font_config): $(FONT_TEMP)/%.xml: \ - frameworks/base/data/fonts/%.xml - $(hide) mkdir -p $(dir $@) - $(hide) cp -vf $< $@ - -# List of fonts on the device that we want to ship. This is all .ttf, .ttc and .otf fonts. -fonts_device := $(filter $(TARGET_OUT)/fonts/%.ttf $(TARGET_OUT)/fonts/%.ttc $(TARGET_OUT)/fonts/%.otf, $(INTERNAL_SYSTEMIMAGE_FILES)) -fonts_device := $(addprefix $(FONT_TEMP)/, $(notdir $(fonts_device))) - -# TODO: If the font file is a symlink, reuse the font renamed from the symlink -# target. -$(fonts_device): $(FONT_TEMP)/%: $(TARGET_OUT)/fonts/% - $(hide) mkdir -p $(dir $@) - $(hide) cp -vf $< $@ - -# List of all dependencies - all fonts and configuration files. -FONT_FILES := $(fonts_device) $(font_config) - -.PHONY: layoutlib layoutlib-tests -layoutlib layoutlib-tests: $(FONT_FILES) - -$(call dist-for-goals, layoutlib, $(foreach m,$(FONT_FILES), $(m):layoutlib_native/fonts/$(notdir $(m)))) - -FONT_TEMP := -font_config := -fonts_device := -FONT_FILES := diff --git a/core/local_current_sdk.mk b/core/local_current_sdk.mk index ea7da8a766..ccdbf77305 100644 --- a/core/local_current_sdk.mk +++ b/core/local_current_sdk.mk @@ -14,13 +14,24 @@ # limitations under the License. # ifdef BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES - ifneq (current,$(BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES)) + _override_to := $(BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES) + + # b/314011075: apks and jars in the vendor or odm partitions cannot use + # system SDK 35 and beyond. In order not to suddenly break those vendor + # modules using current or system_current as their LOCAL_SDK_VERSION, + # override it to 34, which is the maximum API level allowed for them. + ifneq (,$(filter JAVA_LIBRARIES APPS,$(LOCAL_MODULE_CLASS))) + _override_to := 34 + endif + + ifneq (current,$(_override_to)) ifneq (,$(filter true,$(LOCAL_VENDOR_MODULE) $(LOCAL_ODM_MODULE) $(LOCAL_PROPRIETARY_MODULE))) ifeq (current,$(LOCAL_SDK_VERSION)) - LOCAL_SDK_VERSION := $(BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES) + LOCAL_SDK_VERSION := $(_override_to) else ifeq (system_current,$(LOCAL_SDK_VERSION)) - LOCAL_SDK_VERSION := system_$(BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES) + LOCAL_SDK_VERSION := system_$(_override_to) endif endif endif + _override_to := endif diff --git a/core/local_systemsdk.mk b/core/local_systemsdk.mk index 460073daf7..3307e72b7c 100644 --- a/core/local_systemsdk.mk +++ b/core/local_systemsdk.mk @@ -33,6 +33,9 @@ ifdef BOARD_SYSTEMSDK_VERSIONS # Runtime resource overlays are exempted from building against System SDK. # TODO(b/155027019): remove this, after no product/vendor apps rely on this behavior. LOCAL_SDK_VERSION := system_current + # We have run below again since LOCAL_SDK_VERSION is newly set and the "_current" + # may have to be updated + include $(BUILD_SYSTEM)/local_current_sdk.mk endif endif endif @@ -54,10 +57,35 @@ ifneq (,$(call has-system-sdk-version,$(LOCAL_SDK_VERSION))) # If not, vendor apks are treated equally to system apps _supported_systemsdk_versions := $(PLATFORM_SYSTEMSDK_VERSIONS) endif + + # b/314011075: apks and jars in the vendor or odm partitions cannot use system SDK 35 and beyond. + # This is to discourage the use of Java APIs in the partitions, which hasn't been supported since + # the beginning of the project Treble back in Android 10. Ultimately, we'd like to completely + # disallow any Java API in the partitions, but it shall be done progressively. + ifneq (,$(filter true,$(LOCAL_VENDOR_MODULE) $(LOCAL_ODM_MODULE) $(LOCAL_PROPRIETARY_MODULE))) + # 28 is the API level when BOARD_SYSTEMSDK_VERSIONS was introduced. So, it's the oldset API + # we allow. + _supported_systemsdk_versions := $(call int_range_list, 28, 34) + endif + + # Extract version number from LOCAL_SDK_VERSION (ex: system_34 -> 34) _system_sdk_version := $(call get-numeric-sdk-version,$(LOCAL_SDK_VERSION)) + # However, the extraction may fail if it doesn't have any number (i.e. current, core_current, + # system_current, or similar) Then use the latest platform SDK version number or the actual + # codename. + ifeq (,$(_system_sdk_version) + ifeq (REL,$(PLATFORM_VERSION_CODENAME)) + _system_sdk_version := $(PLATFORM_SDK_VERSION) + else + _system_sdk_version := $(PLATFORM_VERSION_CODENAME) + endif + endif + ifneq ($(_system_sdk_version),$(filter $(_system_sdk_version),$(_supported_systemsdk_versions))) - $(call pretty-error,Incompatible LOCAL_SDK_VERSION '$(LOCAL_SDK_VERSION)'. \ - System SDK version '$(_system_sdk_version)' is not supported. Supported versions are: $(_supported_systemsdk_versions)) + ifneq (true,$(BUILD_BROKEN_DONT_CHECK_SYSTEMSDK) + $(call pretty-error,Incompatible LOCAL_SDK_VERSION '$(LOCAL_SDK_VERSION)'. \ + System SDK version '$(_system_sdk_version)' is not supported. Supported versions are: $(_supported_systemsdk_versions)) + endif endif _system_sdk_version := _supported_systemsdk_versions := diff --git a/core/local_vendor_product.mk b/core/local_vendor_product.mk new file mode 100644 index 0000000000..75982cd238 --- /dev/null +++ b/core/local_vendor_product.mk @@ -0,0 +1,22 @@ +# LOCAL_USE_VNDK is not the variable which set by module directly, but there are some modules do so. +# Set those as LOCAL_IN_VENDOR to make those modules work as expected. +ifeq (true,$(LOCAL_USE_VNDK)) + $(warning LOCAL_USE_VNDK must not be used. Please use LOCAL_VENDOR_MODULE or LOCAL_PRODUCT_MODULE instead.) + LOCAL_IN_VENDOR:=true +endif + +# Set LOCAL_IN_VENDOR for modules going into vendor or odm partition and LOCAL_IN_PRODUCT for product +# except for host modules. If LOCAL_SDK_VERSION is set, thats a more restrictive set, so they don't need +# LOCAL_IN_VENDOR or LOCAL_IN_PRODUCT +ifndef LOCAL_IS_HOST_MODULE +ifndef LOCAL_SDK_VERSION + ifneq (,$(filter true,$(LOCAL_VENDOR_MODULE) $(LOCAL_ODM_MODULE) $(LOCAL_OEM_MODULE) $(LOCAL_PROPRIETARY_MODULE))) + LOCAL_IN_VENDOR:=true + # Note: no need to check LOCAL_MODULE_PATH* since LOCAL_[VENDOR|ODM|OEM]_MODULE is already + # set correctly before this is included. + endif + ifeq (true,$(LOCAL_PRODUCT_MODULE)) + LOCAL_IN_PRODUCT:=true + endif +endif +endif diff --git a/core/local_vndk.mk b/core/local_vndk.mk deleted file mode 100644 index befbc59114..0000000000 --- a/core/local_vndk.mk +++ /dev/null @@ -1,48 +0,0 @@ - -#Set LOCAL_USE_VNDK for modules going into product, vendor or odm partition, except for host modules -#If LOCAL_SDK_VERSION is set, thats a more restrictive set, so they dont need LOCAL_USE_VNDK -ifndef LOCAL_IS_HOST_MODULE -ifndef LOCAL_SDK_VERSION - ifneq (,$(filter true,$(LOCAL_VENDOR_MODULE) $(LOCAL_ODM_MODULE) $(LOCAL_OEM_MODULE) $(LOCAL_PROPRIETARY_MODULE))) - LOCAL_USE_VNDK:=true - LOCAL_USE_VNDK_VENDOR:=true - # Note: no need to check LOCAL_MODULE_PATH* since LOCAL_[VENDOR|ODM|OEM]_MODULE is already - # set correctly before this is included. - endif - ifdef PRODUCT_PRODUCT_VNDK_VERSION - # Product modules also use VNDK when PRODUCT_PRODUCT_VNDK_VERSION is defined. - ifeq (true,$(LOCAL_PRODUCT_MODULE)) - LOCAL_USE_VNDK:=true - LOCAL_USE_VNDK_PRODUCT:=true - endif - endif -endif -endif - -# Verify LOCAL_USE_VNDK usage, and set LOCAL_SDK_VERSION if necessary - -ifdef LOCAL_IS_HOST_MODULE - ifdef LOCAL_USE_VNDK - $(shell echo $(LOCAL_MODULE_MAKEFILE): $(LOCAL_MODULE): Do not use LOCAL_USE_VNDK with host modules >&2) - $(error done) - endif -endif -ifdef LOCAL_USE_VNDK - ifneq ($(LOCAL_USE_VNDK),true) - $(shell echo '$(LOCAL_MODULE_MAKEFILE): $(LOCAL_MODULE): LOCAL_USE_VNDK must be "true" or empty, not "$(LOCAL_USE_VNDK)"' >&2) - $(error done) - endif - - ifdef LOCAL_SDK_VERSION - $(shell echo $(LOCAL_MODULE_MAKEFILE): $(LOCAL_MODULE): LOCAL_USE_VNDK must not be used with LOCAL_SDK_VERSION >&2) - $(error done) - endif - - # If we're not using the VNDK, drop all restrictions - ifndef BOARD_VNDK_VERSION - LOCAL_USE_VNDK:= - LOCAL_USE_VNDK_VENDOR:= - LOCAL_USE_VNDK_PRODUCT:= - endif -endif - diff --git a/core/main.mk b/core/main.mk index 3866037b4f..8182740321 100644 --- a/core/main.mk +++ b/core/main.mk @@ -4,7 +4,7 @@ $(warning Either use 'envsetup.sh; m' or 'build/soong/soong_ui.bash --make-mode' $(error done) endif -$(info [1/1] initializing build system ...) +$(info [1/1] initializing legacy Make module parser ...) # Absolute path of the present working direcotry. # This overrides the shell variable $PWD, which does not necessarily points to @@ -40,31 +40,23 @@ include $(BUILD_SYSTEM)/clang/config.mk # Write the build number to a file so it can be read back in # without changing the command line every time. Avoids rebuilds # when using ninja. -$(shell mkdir -p $(SOONG_OUT_DIR) && \ - echo -n $(BUILD_NUMBER) > $(SOONG_OUT_DIR)/build_number.tmp; \ - if ! cmp -s $(SOONG_OUT_DIR)/build_number.tmp $(SOONG_OUT_DIR)/build_number.txt; then \ - mv $(SOONG_OUT_DIR)/build_number.tmp $(SOONG_OUT_DIR)/build_number.txt; \ - else \ - rm $(SOONG_OUT_DIR)/build_number.tmp; \ - fi) BUILD_NUMBER_FILE := $(SOONG_OUT_DIR)/build_number.txt -.KATI_READONLY := BUILD_NUMBER_FILE $(KATI_obsolete_var BUILD_NUMBER,See https://android.googlesource.com/platform/build/+/master/Changes.md#BUILD_NUMBER) +BUILD_HOSTNAME_FILE := $(SOONG_OUT_DIR)/build_hostname.txt +$(KATI_obsolete_var BUILD_HOSTNAME,Use BUILD_HOSTNAME_FROM_FILE instead) +$(KATI_obsolete_var FILE_NAME_TAG,https://android.googlesource.com/platform/build/+/master/Changes.md#FILE_NAME_TAG) + $(BUILD_NUMBER_FILE): - touch $@ + # empty rule to prevent dangling rule error for a file that is written by soong_ui +$(BUILD_HOSTNAME_FILE): + # empty rule to prevent dangling rule error for a file that is written by soong_ui + +.KATI_RESTAT: $(BUILD_NUMBER_FILE) +.KATI_RESTAT: $(BUILD_HOSTNAME_FILE) DATE_FROM_FILE := date -d @$(BUILD_DATETIME_FROM_FILE) .KATI_READONLY := DATE_FROM_FILE -# Pick a reasonable string to use to identify files. -ifeq ($(strip $(HAS_BUILD_NUMBER)),false) - # BUILD_NUMBER has a timestamp in it, which means that - # it will change every time. Pick a stable value. - FILE_NAME_TAG := eng.$(BUILD_USERNAME) -else - FILE_NAME_TAG := $(file <$(BUILD_NUMBER_FILE)) -endif -.KATI_READONLY := FILE_NAME_TAG # Make an empty directory, which can be used to make empty jars EMPTY_DIRECTORY := $(OUT_DIR)/empty @@ -72,8 +64,6 @@ $(shell mkdir -p $(EMPTY_DIRECTORY) && rm -rf $(EMPTY_DIRECTORY)/*) # CTS-specific config. -include cts/build/config.mk -# VTS-specific config. --include test/vts/tools/vts-tradefed/build/config.mk # device-tests-specific-config. -include tools/tradefederation/build/suites/device-tests/config.mk # general-tests-specific-config. @@ -190,9 +180,13 @@ TARGET_BUILD_JAVA_SUPPORT_LEVEL :=$= platform ADDITIONAL_SYSTEM_PROPERTIES += ro.treble.enabled=${PRODUCT_FULL_TREBLE} $(KATI_obsolete_var PRODUCT_FULL_TREBLE,\ - Code should be written to work regardless of a device being Treble or \ - variables like PRODUCT_SEPOLICY_SPLIT should be used until that is \ - possible.) + Code should be written to work regardless of a device being Treble) + +# Set ro.llndk.api_level to show the maximum vendor API level that the LLNDK in +# the system partition supports. +ifdef RELEASE_BOARD_API_LEVEL +ADDITIONAL_SYSTEM_PROPERTIES += ro.llndk.api_level=$(RELEASE_BOARD_API_LEVEL) +endif # Sets ro.actionable_compatible_property.enabled to know on runtime whether the # allowed list of actionable compatible properties is enabled or not. @@ -203,6 +197,13 @@ ifneq (,$(PRODUCT_SYSTEM_SERVER_COMPILER_FILTER)) ADDITIONAL_PRODUCT_PROPERTIES += dalvik.vm.systemservercompilerfilter=$(PRODUCT_SYSTEM_SERVER_COMPILER_FILTER) endif +# Add the 16K developer option if it is defined for the product. +ifeq ($(PRODUCT_16K_DEVELOPER_OPTION),true) +ADDITIONAL_PRODUCT_PROPERTIES += ro.product.build.16k_page.enabled=true +else +ADDITIONAL_PRODUCT_PROPERTIES += ro.product.build.16k_page.enabled=false +endif + # Enable core platform API violation warnings on userdebug and eng builds. ifneq ($(TARGET_BUILD_VARIANT),user) ADDITIONAL_SYSTEM_PROPERTIES += persist.debug.dalvik.vm.core_platform_api_policy=just-warn @@ -220,18 +221,6 @@ ADDITIONAL_SYSTEM_PROPERTIES += $(foreach s,$(SANITIZE_TARGET),ro.sanitize.$(s)= # mount system_other partition. ADDITIONAL_SYSTEM_PROPERTIES += ro.postinstall.fstab.prefix=/system -# ----------------------------------------------------------------- -# ADDITIONAL_VENDOR_PROPERTIES will be installed in vendor/build.prop if -# property_overrides_split_enabled is true. Otherwise it will be installed in -# /system/build.prop -ifdef BOARD_VNDK_VERSION - ifeq ($(BOARD_VNDK_VERSION),current) - ADDITIONAL_VENDOR_PROPERTIES := ro.vndk.version=$(PLATFORM_VNDK_VERSION) - else - ADDITIONAL_VENDOR_PROPERTIES := ro.vndk.version=$(BOARD_VNDK_VERSION) - endif -endif - # Add cpu properties for bionic and ART. ADDITIONAL_VENDOR_PROPERTIES += ro.bionic.arch=$(TARGET_ARCH) ADDITIONAL_VENDOR_PROPERTIES += ro.bionic.cpu_variant=$(TARGET_CPU_VARIANT_RUNTIME) @@ -281,6 +270,11 @@ ADDITIONAL_VENDOR_PROPERTIES += \ ro.product.first_api_level=$(PRODUCT_SHIPPING_API_LEVEL) endif +ifdef PRODUCT_SHIPPING_VENDOR_API_LEVEL +ADDITIONAL_VENDOR_PROPERTIES += \ + ro.vendor.api_level=$(PRODUCT_SHIPPING_VENDOR_API_LEVEL) +endif + ifneq ($(TARGET_BUILD_VARIANT),user) ifdef PRODUCT_SET_DEBUGFS_RESTRICTIONS ADDITIONAL_VENDOR_PROPERTIES += \ @@ -290,16 +284,22 @@ endif # Vendors with GRF must define BOARD_SHIPPING_API_LEVEL for the vendor API level. # This must not be defined for the non-GRF devices. +# The values of the GRF properties will be verified by post_process_props.py ifdef BOARD_SHIPPING_API_LEVEL ADDITIONAL_VENDOR_PROPERTIES += \ ro.board.first_api_level=$(BOARD_SHIPPING_API_LEVEL) +endif -# To manually set the vendor API level of the vendor modules, BOARD_API_LEVEL can be used. -# The values of the GRF properties will be verified by post_process_props.py +# Build system set BOARD_API_LEVEL to show the api level of the vendor API surface. +# This must not be altered outside of build system. ifdef BOARD_API_LEVEL ADDITIONAL_VENDOR_PROPERTIES += \ ro.board.api_level=$(BOARD_API_LEVEL) endif +# RELEASE_BOARD_API_LEVEL_FROZEN is true when the vendor API surface is frozen. +ifdef RELEASE_BOARD_API_LEVEL_FROZEN +ADDITIONAL_VENDOR_PROPERTIES += \ + ro.board.api_frozen=$(RELEASE_BOARD_API_LEVEL_FROZEN) endif # Set build prop. This prop is read by ota_from_target_files when generating OTA, @@ -332,21 +332,19 @@ ADDITIONAL_VENDOR_PROPERTIES += \ ro.build.ab_update=$(AB_OTA_UPDATER) endif -# Set ro.product.vndk.version to know the VNDK version required by product -# modules. It uses the version in PRODUCT_PRODUCT_VNDK_VERSION. If the value -# is "current", use PLATFORM_VNDK_VERSION. -ifdef PRODUCT_PRODUCT_VNDK_VERSION -ifeq ($(PRODUCT_PRODUCT_VNDK_VERSION),current) -ADDITIONAL_PRODUCT_PROPERTIES += ro.product.vndk.version=$(PLATFORM_VNDK_VERSION) -else -ADDITIONAL_PRODUCT_PROPERTIES += ro.product.vndk.version=$(PRODUCT_PRODUCT_VNDK_VERSION) -endif -endif - ADDITIONAL_PRODUCT_PROPERTIES += ro.build.characteristics=$(TARGET_AAPT_CHARACTERISTICS) ifeq ($(AB_OTA_UPDATER),true) ADDITIONAL_PRODUCT_PROPERTIES += ro.product.ab_ota_partitions=$(subst $(space),$(comma),$(sort $(AB_OTA_PARTITIONS))) +ADDITIONAL_VENDOR_PROPERTIES += ro.vendor.build.ab_ota_partitions=$(subst $(space),$(comma),$(sort $(AB_OTA_PARTITIONS))) +endif + +# Set this property for VTS to skip large page size tests on unsupported devices. +ADDITIONAL_PRODUCT_PROPERTIES += \ + ro.product.cpu.pagesize.max=$(TARGET_MAX_PAGE_SIZE_SUPPORTED) + +ifeq ($(PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO),true) +ADDITIONAL_PRODUCT_PROPERTIES += ro.product.build.no_bionic_page_size_macro=true endif # ----------------------------------------------------------------- @@ -419,6 +417,8 @@ ifndef is_sdk_build # To speedup startup of non-preopted builds, don't verify or compile the boot image. ADDITIONAL_SYSTEM_PROPERTIES += dalvik.vm.image-dex2oat-filter=extract endif +# b/323566535 +ADDITIONAL_SYSTEM_PROPERTIES += init.svc_debug.no_fatal.zygote=true endif ## asan ## @@ -553,6 +553,7 @@ $(foreach mk,$(subdir_makefiles),$(info [$(call inc_and_print,subdir_makefiles_i # sources or dependencies for these tools may be missing from the tree. ifeq (,$(TARGET_BUILD_UNBUNDLED_IMAGE)) droid_targets : blueprint_tools +checkbuild: blueprint_tests endif endif # dont_bother @@ -561,7 +562,7 @@ ifndef subdir_makefiles_total subdir_makefiles_total := $(words init post finish) endif -$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] finishing build rules ...) +$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] finishing legacy Make module parsing ...) # ------------------------------------------------------------------- # All module makefiles have been included at this point. @@ -764,6 +765,9 @@ ifneq (,$(_nonexistent_required)) $(info $(word 1,$(r)) module $(word 2,$(r)) requires non-existent $(word 3,$(r)) module: $(word 4,$(r))) \ ) $(warning Set BUILD_BROKEN_MISSING_REQUIRED_MODULES := true to bypass this check if this is intentional) + ifneq (,$(PRODUCT_SOURCE_ROOT_DIRS)) + $(warning PRODUCT_SOURCE_ROOT_DIRS is non-empty. Some necessary modules may have been skipped by Soong) + endif $(error Build failed) endif # _nonexistent_required != empty endif # check_missing_required_modules == true @@ -814,12 +818,14 @@ $(call add-all-host-cross-to-host-cross-required-modules-deps) # Sets up dependencies such that whenever a target module is installed, # any other target modules listed in $(ALL_MODULES.$(m).REQUIRED_FROM_TARGET) will also be installed +# This doesn't apply to ORDERONLY_INSTALLED items. define add-all-target-to-target-required-modules-deps $(foreach m,$(ALL_MODULES), \ $(eval r := $(ALL_MODULES.$(m).REQUIRED_FROM_TARGET)) \ $(if $(r), \ $(eval r := $(call module-installed-files,$(r))) \ $(eval t_m := $(filter $(TARGET_OUT_ROOT)/%, $(ALL_MODULES.$(m).INSTALLED))) \ + $(eval t_m := $(filter-out $(ALL_MODULES.$(m).ORDERONLY_INSTALLED), $(ALL_MODULES.$(m).INSTALLED))) \ $(eval t_r := $(filter $(TARGET_OUT_ROOT)/%, $(r))) \ $(eval t_r := $(filter-out $(t_m), $(t_r))) \ $(if $(t_m), $(eval $(call add-required-deps, $(t_m),$(t_r)))) \ @@ -1220,7 +1226,7 @@ endef # Returns modules included automatically as a result of certain BoardConfig # variables being set. define auto-included-modules - $(if $(BOARD_VNDK_VERSION),vndk_package) \ + llndk_in_system \ $(if $(DEVICE_MANIFEST_FILE),vendor_manifest.xml) \ $(if $(DEVICE_MANIFEST_SKUS),$(foreach sku, $(DEVICE_MANIFEST_SKUS),vendor_manifest_$(sku).xml)) \ $(if $(ODM_MANIFEST_FILES),odm_manifest.xml) \ @@ -1228,9 +1234,7 @@ define auto-included-modules endef -# Lists most of the files a particular product installs, including: -# - PRODUCT_PACKAGES, and their LOCAL_REQUIRED_MODULES -# - PRODUCT_COPY_FILES +# Lists the modules particular product installs. # The base list of modules to build for this product is specified # by the appropriate product definition file, which was included # by product_config.mk. @@ -1242,8 +1246,7 @@ endef # Name resolution for LOCAL_REQUIRED_MODULES: # See the select-bitness-of-required-modules definition. # $(1): product makefile - -define product-installed-files +define product-installed-modules $(eval _pif_modules := \ $(call get-product-var,$(1),PRODUCT_PACKAGES) \ $(if $(filter eng,$(tags_to_install)),$(call get-product-var,$(1),PRODUCT_PACKAGES_ENG)) \ @@ -1251,6 +1254,12 @@ define product-installed-files $(if $(filter tests,$(tags_to_install)),$(call get-product-var,$(1),PRODUCT_PACKAGES_TESTS)) \ $(if $(filter asan,$(tags_to_install)),$(call get-product-var,$(1),PRODUCT_PACKAGES_DEBUG_ASAN)) \ $(if $(filter java_coverage,$(tags_to_install)),$(call get-product-var,$(1),PRODUCT_PACKAGES_DEBUG_JAVA_COVERAGE)) \ + $(if $(filter arm64,$(TARGET_ARCH) $(TARGET_2ND_ARCH)),$(call get-product-var,$(1),PRODUCT_PACKAGES_ARM64)) \ + $(if $(PRODUCT_SHIPPING_API_LEVEL), \ + $(if $(call math_gt_or_eq,29,$(PRODUCT_SHIPPING_API_LEVEL)),$(call get-product-var,$(1),PRODUCT_PACKAGES_SHIPPING_API_LEVEL_29)) \ + $(if $(call math_gt_or_eq,33,$(PRODUCT_SHIPPING_API_LEVEL)),$(call get-product-var,$(1),PRODUCT_PACKAGES_SHIPPING_API_LEVEL_33)) \ + $(if $(call math_gt_or_eq,34,$(PRODUCT_SHIPPING_API_LEVEL)),$(call get-product-var,$(1),PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34)) \ + ) \ $(call auto-included-modules) \ ) \ $(eval ### Filter out the overridden packages and executables before doing expansion) \ @@ -1259,7 +1268,14 @@ define product-installed-files $(eval ### Resolve the :32 :64 module name) \ $(eval _pif_modules := $(sort $(call resolve-bitness-for-modules,TARGET,$(_pif_modules)))) \ $(call expand-required-modules,_pif_modules,$(_pif_modules),$(_pif_overrides)) \ - $(filter-out $(HOST_OUT_ROOT)/%,$(call module-installed-files, $(_pif_modules))) \ + $(_pif_modules) +endef + +# Lists most of the files a particular product installs. +# It gives all the installed files for all modules returned by product-installed-modules, +# and also includes PRODUCT_COPY_FILES. +define product-installed-files + $(filter-out $(HOST_OUT_ROOT)/%,$(call module-installed-files, $(call product-installed-modules,$(1)))) \ $(call resolve-product-relative-paths,\ $(foreach cf,$(call get-product-var,$(1),PRODUCT_COPY_FILES),$(call word-colon,2,$(cf)))) endef @@ -1345,6 +1361,13 @@ else ifdef FULL_BUILD $(if $(ALL_MODULES.$(m).INSTALLED),\ $(if $(filter-out $(HOST_OUT_ROOT)/%,$(ALL_MODULES.$(m).INSTALLED)),,\ $(m)))) + ifeq ($(TARGET_ARCH),riscv64) + # HACK: riscv64 can't build the device version of bcc and ld.mc due to a + # dependency on an old version of LLVM, but they are listed in + # base_system.mk which can't add them conditionally based on the target + # architecture. + _host_modules := $(filter-out bcc ld.mc,$(_host_modules)) + endif $(call maybe-print-list-and-error,$(sort $(_host_modules)),\ Host modules should be in PRODUCT_HOST_PACKAGES$(comma) not PRODUCT_PACKAGES) endif @@ -1352,6 +1375,7 @@ else ifdef FULL_BUILD product_host_FILES := $(call host-installed-files,$(INTERNAL_PRODUCT)) product_target_FILES := $(call product-installed-files, $(INTERNAL_PRODUCT)) # WARNING: The product_MODULES variable is depended on by external files. + # It contains the list of register names that will be installed on the device product_MODULES := $(_pif_modules) # Verify the artifact path requirements made by included products. @@ -1377,56 +1401,12 @@ modules_to_install := $(sort \ $(CUSTOM_MODULES) \ ) -ifdef FULL_BUILD -# -# Used by the cleanup logic in soong_ui to remove files that should no longer -# be installed. -# - -# Include all tests, so that we remove them from the test suites / testcase -# folders when they are removed. -test_files := $(foreach ts,$(ALL_COMPATIBILITY_SUITES),$(COMPATIBILITY.$(ts).FILES)) - -$(shell mkdir -p $(PRODUCT_OUT) $(HOST_OUT)) - -$(file >$(PRODUCT_OUT)/.installable_files$(if $(filter address,$(SANITIZE_TARGET)),_asan), \ - $(sort $(patsubst $(PRODUCT_OUT)/%,%,$(filter $(PRODUCT_OUT)/%, \ - $(modules_to_install) $(test_files))))) - -$(file >$(HOST_OUT)/.installable_test_files,$(sort \ - $(patsubst $(HOST_OUT)/%,%,$(filter $(HOST_OUT)/%, \ - $(test_files))))) - -test_files := -endif - -# Dedpulicate compatibility suite dist files across modules and packages before +# Deduplicate compatibility suite dist files across modules and packages before # copying them to their requested locations. Assign the eval result to an unused # var to prevent Make from trying to make a sense of it. _unused := $(call copy-many-files, $(sort $(ALL_COMPATIBILITY_DIST_FILES))) -# Don't include any GNU General Public License shared objects or static -# libraries in SDK images. GPL executables (not static/dynamic libraries) -# are okay if they don't link against any closed source libraries (directly -# or indirectly) - -# It's ok (and necessary) to build the host tools, but nothing that's -# going to be installed on the target (including static libraries). - ifdef is_sdk_build - target_gnu_MODULES := \ - $(filter \ - $(TARGET_OUT_INTERMEDIATES)/% \ - $(TARGET_OUT)/% \ - $(TARGET_OUT_DATA)/%, \ - $(sort $(call get-tagged-modules,gnu))) - target_gnu_MODULES := $(filter-out $(TARGET_OUT_EXECUTABLES)/%,$(target_gnu_MODULES)) - target_gnu_MODULES := $(filter-out %/libopenjdkjvmti.so,$(target_gnu_MODULES)) - target_gnu_MODULES := $(filter-out %/libopenjdkjvmtid.so,$(target_gnu_MODULES)) - $(info Removing from sdk:)$(foreach d,$(target_gnu_MODULES),$(info : $(d))) - modules_to_install := \ - $(filter-out $(target_gnu_MODULES),$(modules_to_install)) - # Ensure every module listed in PRODUCT_PACKAGES* gets something installed # TODO: Should we do this for all builds and not just the sdk? dangling_modules := @@ -1447,6 +1427,16 @@ ifdef is_sdk_build $(warning $(ALL_MODULES.$(m).MAKEFILE): Module '$(m)' in PRODUCT_PACKAGES_TESTS has nothing to install!))) endif +ifneq ($(TARGET_BUILD_APPS),) + # If this build is just for apps, only build apps and not the full system by default. + ifneq ($(filter all,$(TARGET_BUILD_APPS)),) + # If they used the magic goal "all" then build all apps in the source tree. + unbundled_build_modules := $(foreach m,$(sort $(ALL_MODULES)),$(if $(filter APPS,$(ALL_MODULES.$(m).CLASS)),$(m))) + else + unbundled_build_modules := $(sort $(TARGET_BUILD_APPS)) + endif +endif + # build/make/core/Makefile contains extra stuff that we don't want to pollute this # top-level makefile with. It expects that ALL_DEFAULT_INSTALLED_MODULES # contains everything that's built during the current make, but it also further @@ -1458,6 +1448,28 @@ endif modules_to_install := $(sort $(ALL_DEFAULT_INSTALLED_MODULES)) ALL_DEFAULT_INSTALLED_MODULES := +ifdef FULL_BUILD +# +# Used by the cleanup logic in soong_ui to remove files that should no longer +# be installed. +# + +# Include all tests, so that we remove them from the test suites / testcase +# folders when they are removed. +test_files := $(foreach ts,$(ALL_COMPATIBILITY_SUITES),$(COMPATIBILITY.$(ts).FILES)) + +$(shell mkdir -p $(PRODUCT_OUT) $(HOST_OUT)) + +$(file >$(PRODUCT_OUT)/.installable_files$(if $(filter address,$(SANITIZE_TARGET)),_asan), \ + $(sort $(patsubst $(PRODUCT_OUT)/%,%,$(filter $(PRODUCT_OUT)/%, \ + $(modules_to_install) $(test_files))))) + +$(file >$(HOST_OUT)/.installable_test_files,$(sort \ + $(patsubst $(HOST_OUT)/%,%,$(filter $(HOST_OUT)/%, \ + $(test_files))))) + +test_files := +endif # Some notice deps refer to module names without prefix or arch suffix where # only the variants with them get built. @@ -1642,6 +1654,7 @@ droidcore-unbundled: $(filter $(HOST_OUT_ROOT)/%,$(modules_to_install)) \ $(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) \ $(INSTALLED_SUPERIMAGE_EMPTY_TARGET) \ $(INSTALLED_PRODUCTIMAGE_TARGET) \ + $(INSTALLED_SYSTEM_EXTIMAGE_TARGET) \ $(INSTALLED_SYSTEMOTHERIMAGE_TARGET) \ $(INSTALLED_TEST_HARNESS_RAMDISK_TARGET) \ $(INSTALLED_TEST_HARNESS_BOOTIMAGE_TARGET) \ @@ -1687,10 +1700,8 @@ droidcore: droidcore-unbundled # dist_files only for putting your library into the dist directory with a full build. .PHONY: dist_files -ifeq ($(SOONG_COLLECT_JAVA_DEPS), true) - $(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json) - $(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json) -endif +$(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json) +$(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json) .PHONY: apps_only ifeq ($(HOST_OS),darwin) @@ -1700,16 +1711,10 @@ ifeq ($(HOST_OS),darwin) else ifneq ($(TARGET_BUILD_APPS),) # If this build is just for apps, only build apps and not the full system by default. - unbundled_build_modules := - ifneq ($(filter all,$(TARGET_BUILD_APPS)),) - # If they used the magic goal "all" then build all apps in the source tree. - unbundled_build_modules := $(foreach m,$(sort $(ALL_MODULES)),$(if $(filter APPS,$(ALL_MODULES.$(m).CLASS)),$(m))) - else - unbundled_build_modules := $(TARGET_BUILD_APPS) - endif - - # Dist the installed files if they exist. - apps_only_installed_files := $(foreach m,$(unbundled_build_modules),$(ALL_MODULES.$(m).INSTALLED)) + # Dist the installed files if they exist, except the installed symlinks. dist-for-goals emits + # `cp src dest` commands, which will fail to copy dangling symlinks. + apps_only_installed_files := $(foreach m,$(unbundled_build_modules),\ + $(filter-out $(ALL_MODULES.$(m).INSTALLED_SYMLINKS),$(ALL_MODULES.$(m).INSTALLED))) $(call dist-for-goals,apps_only, $(apps_only_installed_files)) # Dist the bundle files if they exist. @@ -1739,15 +1744,15 @@ else ifneq ($(TARGET_BUILD_APPS),) endif $(PROGUARD_DICT_ZIP) : $(apps_only_installed_files) - $(call dist-for-goals,apps_only, $(PROGUARD_DICT_ZIP) $(PROGUARD_DICT_MAPPING)) + $(call dist-for-goals-with-filenametag,apps_only, $(PROGUARD_DICT_ZIP) $(PROGUARD_DICT_ZIP) $(PROGUARD_DICT_MAPPING)) $(call declare-container-license-deps,$(PROGUARD_DICT_ZIP),$(apps_only_installed_files),$(PRODUCT_OUT)/:/) $(PROGUARD_USAGE_ZIP) : $(apps_only_installed_files) - $(call dist-for-goals,apps_only, $(PROGUARD_USAGE_ZIP)) + $(call dist-for-goals-with-filenametag,apps_only, $(PROGUARD_USAGE_ZIP)) $(call declare-container-license-deps,$(PROGUARD_USAGE_ZIP),$(apps_only_installed_files),$(PRODUCT_OUT)/:/) $(SYMBOLS_ZIP) : $(apps_only_installed_files) - $(call dist-for-goals,apps_only, $(SYMBOLS_ZIP) $(SYMBOLS_MAPPING)) + $(call dist-for-goals-with-filenametag,apps_only, $(SYMBOLS_ZIP) $(SYMBOLS_MAPPING)) $(call declare-container-license-deps,$(SYMBOLS_ZIP),$(apps_only_installed_files),$(PRODUCT_OUT)/:/) $(COVERAGE_ZIP) : $(apps_only_installed_files) @@ -1793,17 +1798,23 @@ else ifeq ($(TARGET_BUILD_UNBUNDLED),$(TARGET_BUILD_UNBUNDLED_IMAGE)) # avoid disting targets that would cause building framework java sources, # which we want to avoid in an unbundled build. - $(call dist-for-goals, droidcore-unbundled, \ + $(call dist-for-goals-with-filenametag, droidcore-unbundled, \ $(INTERNAL_UPDATE_PACKAGE_TARGET) \ $(INTERNAL_OTA_PACKAGE_TARGET) \ - $(INTERNAL_OTA_METADATA) \ $(INTERNAL_OTA_PARTIAL_PACKAGE_TARGET) \ + $(BUILT_RAMDISK_16K_TARGET) \ + $(BUILT_KERNEL_16K_TARGET) \ $(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET) \ $(SYMBOLS_ZIP) \ $(SYMBOLS_MAPPING) \ $(PROGUARD_DICT_ZIP) \ $(PROGUARD_DICT_MAPPING) \ $(PROGUARD_USAGE_ZIP) \ + $(BUILT_TARGET_FILES_PACKAGE) \ + ) + + $(call dist-for-goals, droidcore-unbundled, \ + $(INTERNAL_OTA_METADATA) \ $(COVERAGE_ZIP) \ $(INSTALLED_FILES_FILE) \ $(INSTALLED_FILES_JSON) \ @@ -1825,13 +1836,12 @@ else ifeq ($(TARGET_BUILD_UNBUNDLED),$(TARGET_BUILD_UNBUNDLED_IMAGE)) $(INSTALLED_FILES_JSON_SYSTEMOTHER) \ $(INSTALLED_FILES_FILE_RECOVERY) \ $(INSTALLED_FILES_JSON_RECOVERY) \ - $(INSTALLED_BUILD_PROP_TARGET):build.prop \ - $(INSTALLED_VENDOR_BUILD_PROP_TARGET):build.prop-vendor \ - $(INSTALLED_PRODUCT_BUILD_PROP_TARGET):build.prop-product \ - $(INSTALLED_ODM_BUILD_PROP_TARGET):build.prop-odm \ - $(INSTALLED_SYSTEM_EXT_BUILD_PROP_TARGET):build.prop-system_ext \ - $(INSTALLED_RAMDISK_BUILD_PROP_TARGET):build.prop-ramdisk \ - $(BUILT_TARGET_FILES_PACKAGE) \ + $(if $(BUILDING_SYSTEM_IMAGE), $(INSTALLED_BUILD_PROP_TARGET):build.prop) \ + $(if $(BUILDING_VENDOR_IMAGE), $(INSTALLED_VENDOR_BUILD_PROP_TARGET):build.prop-vendor) \ + $(if $(BUILDING_PRODUCT_IMAGE), $(INSTALLED_PRODUCT_BUILD_PROP_TARGET):build.prop-product) \ + $(if $(BUILDING_ODM_IMAGE), $(INSTALLED_ODM_BUILD_PROP_TARGET):build.prop-odm) \ + $(if $(BUILDING_SYSTEM_EXT_IMAGE), $(INSTALLED_SYSTEM_EXT_BUILD_PROP_TARGET):build.prop-system_ext) \ + $(if $(BUILDING_RAMDISK_IMAGE), $(INSTALLED_RAMDISK_BUILD_PROP_TARGET):build.prop-ramdisk) \ $(INSTALLED_ANDROID_INFO_TXT_TARGET) \ $(INSTALLED_MISC_INFO_TARGET) \ $(INSTALLED_RAMDISK_TARGET) \ @@ -1843,7 +1853,7 @@ else ifeq ($(TARGET_BUILD_UNBUNDLED),$(TARGET_BUILD_UNBUNDLED_IMAGE)) $(call dist-for-goals, droidcore-unbundled, $(f))) ifneq ($(ANDROID_BUILD_EMBEDDED),true) - $(call dist-for-goals, droidcore, \ + $(call dist-for-goals-with-filenametag, droidcore, \ $(APPS_ZIP) \ $(INTERNAL_EMULATOR_PACKAGE_TARGET) \ ) @@ -1892,17 +1902,17 @@ else ifeq ($(TARGET_BUILD_UNBUNDLED),$(TARGET_BUILD_UNBUNDLED_IMAGE)) endif # Put XML formatted API files in the dist dir. - $(TARGET_OUT_COMMON_INTERMEDIATES)/api.xml: $(call java-lib-files,android_stubs_current) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/system-api.xml: $(call java-lib-files,android_system_stubs_current) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/module-lib-api.xml: $(call java-lib-files,android_module_lib_stubs_current) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/system-server-api.xml: $(call java-lib-files,android_system_server_stubs_current) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/test-api.xml: $(call java-lib-files,android_test_stubs_current) $(APICHECK) + $(TARGET_OUT_COMMON_INTERMEDIATES)/api.xml: $(call java-lib-files,$(ANDROID_PUBLIC_STUBS)) $(APICHECK) + $(TARGET_OUT_COMMON_INTERMEDIATES)/system-api.xml: $(call java-lib-files,$(ANDROID_SYSTEM_STUBS)) $(APICHECK) + $(TARGET_OUT_COMMON_INTERMEDIATES)/module-lib-api.xml: $(call java-lib-files,$(ANDROID_MODULE_LIB_STUBS)) $(APICHECK) + $(TARGET_OUT_COMMON_INTERMEDIATES)/system-server-api.xml: $(call java-lib-files,$(ANDROID_SYSTEM_SERVER_STUBS)) $(APICHECK) + $(TARGET_OUT_COMMON_INTERMEDIATES)/test-api.xml: $(call java-lib-files,$(ANDROID_TEST_STUBS)) $(APICHECK) api_xmls := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/,api.xml system-api.xml module-lib-api.xml system-server-api.xml test-api.xml) $(api_xmls): $(hide) echo "Converting API file to XML: $@" $(hide) mkdir -p $(dir $@) - $(hide) $(APICHECK_COMMAND) --input-api-jar $< --api-xml $@ + $(hide) $(APICHECK_COMMAND) jar-to-jdiff $< $@ $(foreach xml,$(sort $(api_xmls)),$(call declare-1p-target,$(xml),)) @@ -1943,10 +1953,8 @@ docs: $(ALL_DOCS) ifeq ($(HOST_OS),linux) ALL_SDK_TARGETS := $(INTERNAL_SDK_TARGET) sdk: $(ALL_SDK_TARGETS) -$(call dist-for-goals,sdk, \ - $(ALL_SDK_TARGETS) \ - $(INSTALLED_BUILD_PROP_TARGET) \ -) +$(call dist-for-goals-with-filenametag,sdk,$(ALL_SDK_TARGETS)) +$(call dist-for-goals,sdk,$(INSTALLED_BUILD_PROP_TARGET)) endif # umbrella targets to assit engineers in verifying builds @@ -1971,27 +1979,9 @@ tests : host-tests target-tests .PHONY: findbugs findbugs: $(INTERNAL_FINDBUGS_HTML_TARGET) $(INTERNAL_FINDBUGS_XML_TARGET) -LSDUMP_PATHS_FILE := $(PRODUCT_OUT)/lsdump_paths.txt - -.PHONY: findlsdumps -# LSDUMP_PATHS is a list of tag:path. -findlsdumps: $(LSDUMP_PATHS_FILE) $(foreach p,$(LSDUMP_PATHS),$(call word-colon,2,$(p))) - -$(LSDUMP_PATHS_FILE): PRIVATE_LSDUMP_PATHS := $(LSDUMP_PATHS) -$(LSDUMP_PATHS_FILE): - @echo "Generate $@" - @rm -rf $@ && echo -e "$(subst :,:$(space),$(subst $(space),\n,$(PRIVATE_LSDUMP_PATHS)))" > $@ - .PHONY: check-elf-files check-elf-files: -#xxx scrape this from ALL_MODULE_NAME_TAGS -.PHONY: modules -modules: - @echo "Available sub-modules:" - @echo "$(call module-names-for-tag-list,$(ALL_MODULE_TAGS))" | \ - tr -s ' ' '\n' | sort -u - .PHONY: dump-files dump-files: @echo "Target files for $(TARGET_PRODUCT)-$(TARGET_BUILD_VARIANT) ($(INTERNAL_PRODUCT)):" @@ -2019,6 +2009,244 @@ endif # missing dependency errors. $(call build-license-metadata) +# Generate SBOM in SPDX format +product_copy_files_without_owner := $(foreach pcf,$(PRODUCT_COPY_FILES),$(call word-colon,1,$(pcf)):$(call word-colon,2,$(pcf))) +ifeq ($(TARGET_BUILD_APPS),) +dest_files_without_source := $(sort $(foreach pcf,$(product_copy_files_without_owner),$(if $(wildcard $(call word-colon,1,$(pcf))),,$(call word-colon,2,$(pcf))))) +dest_files_without_source := $(addprefix $(PRODUCT_OUT)/,$(dest_files_without_source)) +filter_out_files := \ + $(PRODUCT_OUT)/apex/% \ + $(PRODUCT_OUT)/fake_packages/% \ + $(PRODUCT_OUT)/testcases/% \ + $(dest_files_without_source) +# Check if each partition image is built, if not filter out all its installed files +# Also check if a partition uses prebuilt image file, save the info if prebuilt image is used. +PREBUILT_PARTITION_COPY_FILES := +# product.img +ifndef BUILDING_PRODUCT_IMAGE +filter_out_files += $(PRODUCT_OUT)/product/% +ifdef BOARD_PREBUILT_PRODUCTIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_PRODUCTIMAGE):$(INSTALLED_PRODUCTIMAGE_TARGET) +endif +endif + +# system.img +ifndef BUILDING_SYSTEM_IMAGE +filter_out_files += $(PRODUCT_OUT)/system/% +endif +# system_dlkm.img +ifndef BUILDING_SYSTEM_DLKM_IMAGE +filter_out_files += $(PRODUCT_OUT)/system_dlkm/% +ifdef BOARD_PREBUILT_SYSTEM_DLKMIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_SYSTEM_DLKMIMAGE):$(INSTALLED_SYSTEM_DLKMIMAGE_TARGET) +endif +endif +# system_ext.img +ifndef BUILDING_SYSTEM_EXT_IMAGE +filter_out_files += $(PRODUCT_OUT)/system_ext/% +ifdef BOARD_PREBUILT_SYSTEM_EXTIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_SYSTEM_EXTIMAGE):$(INSTALLED_SYSTEM_EXTIMAGE_TARGET) +endif +endif +# system_other.img +ifndef BUILDING_SYSTEM_OTHER_IMAGE +filter_out_files += $(PRODUCT_OUT)/system_other/% +endif + +# odm.img +ifndef BUILDING_ODM_IMAGE +filter_out_files += $(PRODUCT_OUT)/odm/% +ifdef BOARD_PREBUILT_ODMIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_ODMIMAGE):$(INSTALLED_ODMIMAGE_TARGET) +endif +endif +# odm_dlkm.img +ifndef BUILDING_ODM_DLKM_IMAGE +filter_out_files += $(PRODUCT_OUT)/odm_dlkm/% +ifdef BOARD_PREBUILT_ODM_DLKMIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_ODM_DLKMIMAGE):$(INSTALLED_ODM_DLKMIMAGE_TARGET) +endif +endif + +# vendor.img +ifndef BUILDING_VENDOR_IMAGE +filter_out_files += $(PRODUCT_OUT)/vendor/% +ifdef BOARD_PREBUILT_VENDORIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_VENDORIMAGE):$(INSTALLED_VENDORIMAGE_TARGET) +endif +endif +# vendor_dlkm.img +ifndef BUILDING_VENDOR_DLKM_IMAGE +filter_out_files += $(PRODUCT_OUT)/vendor_dlkm/% +ifdef BOARD_PREBUILT_VENDOR_DLKMIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_VENDOR_DLKMIMAGE):$(INSTALLED_VENDOR_DLKMIMAGE_TARGET) +endif +endif + +# cache.img +ifndef BUILDING_CACHE_IMAGE +filter_out_files += $(PRODUCT_OUT)/cache/% +endif + +# boot.img +ifndef BUILDING_BOOT_IMAGE +ifdef BOARD_PREBUILT_BOOTIMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_BOOTIMAGE):$(INSTALLED_BOOTIMAGE_TARGET) +endif +endif +# init_boot.img +ifndef BUILDING_INIT_BOOT_IMAGE +ifdef BOARD_PREBUILT_INIT_BOOT_IMAGE +PREBUILT_PARTITION_COPY_FILES += $(BOARD_PREBUILT_INIT_BOOT_IMAGE):$(INSTALLED_INIT_BOOT_IMAGE_TARGET) +endif +endif + +# ramdisk.img +ifndef BUILDING_RAMDISK_IMAGE +filter_out_files += $(PRODUCT_OUT)/ramdisk/% +endif + +# recovery.img +ifndef INSTALLED_RECOVERYIMAGE_TARGET +filter_out_files += $(PRODUCT_OUT)/recovery/% +endif + +installed_files := $(sort $(filter-out $(filter_out_files),$(filter $(PRODUCT_OUT)/%,$(modules_to_install)))) +else +installed_files := $(apps_only_installed_files) +endif # TARGET_BUILD_APPS + +# sbom-metadata.csv contains all raw data collected in Make for generating SBOM in generate-sbom.py. +# There are multiple columns and each identifies the source of an installed file for a specific case. +# The columns and their uses are described as below: +# installed_file: the file path on device, e.g. /product/app/Browser2/Browser2.apk +# module_path: the path of the module that generates the installed file, e.g. packages/apps/Browser2 +# soong_module_type: Soong module type, e.g. android_app, cc_binary +# is_prebuilt_make_module: Y, if the installed file is from a prebuilt Make module, see prebuilt_internal.mk +# product_copy_files: the installed file is from variable PRODUCT_COPY_FILES, e.g. device/google/cuttlefish/shared/config/init.product.rc:product/etc/init/init.rc +# kernel_module_copy_files: the installed file is from variable KERNEL_MODULE_COPY_FILES, similar to product_copy_files +# is_platform_generated: this is an aggregated value including some small cases instead of adding more columns. It is set to Y if any case is Y +# is_build_prop: build.prop in each partition, see sysprop.mk. +# is_notice_file: NOTICE.xml.gz in each partition, see Makefile. +# is_dexpreopt_image_profile: see the usage of DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED in Soong and Make +# is_product_system_other_avbkey: see INSTALLED_PRODUCT_SYSTEM_OTHER_AVBKEY_TARGET +# is_system_other_odex_marker: see INSTALLED_SYSTEM_OTHER_ODEX_MARKER +# is_event_log_tags_file: see variable event_log_tags_file in Makefile +# is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile. +# is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile. +# is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile. +# build_output_path: the path of the built file, used to calculate checksum +# static_libraries/whole_static_libraries: list of module name of the static libraries the file links against, e.g. libclang_rt.builtins or libclang_rt.builtins_32 +# Info of all static libraries of all installed files are collected in variable _all_static_libs that is used to list all the static library files in sbom-metadata.csv. +# See the second foreach loop in the rule of sbom-metadata.csv for the detailed info of static libraries collected in _all_static_libs. +# is_static_lib: whether the file is a static library + +metadata_list := $(OUT_DIR)/.module_paths/METADATA.list +metadata_files := $(subst $(newline),$(space),$(file <$(metadata_list))) +$(PRODUCT_OUT)/sbom-metadata.csv: + rm -f $@ + echo 'installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib' >> $@ + $(eval _all_static_libs :=) + $(foreach f,$(installed_files),\ + $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \ + $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \ + $(eval _build_output_path := $(PRODUCT_OUT)/$(_path_on_device)) \ + $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \ + $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \ + $(eval _is_prebuilt_make_module := $(ALL_MODULES.$(_module_name).IS_PREBUILT_MAKE_MODULE)) \ + $(eval _product_copy_files := $(sort $(filter %:$(_path_on_device),$(product_copy_files_without_owner)))) \ + $(eval _kernel_module_copy_files := $(sort $(filter %$(_path_on_device),$(KERNEL_MODULE_COPY_FILES)))) \ + $(eval _is_build_prop := $(call is-build-prop,$f)) \ + $(eval _is_notice_file := $(call is-notice-file,$f)) \ + $(eval _is_dexpreopt_image_profile := $(if $(filter %:/$(_path_on_device),$(DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED)),Y)) \ + $(eval _is_product_system_other_avbkey := $(if $(findstring $f,$(INSTALLED_PRODUCT_SYSTEM_OTHER_AVBKEY_TARGET)),Y)) \ + $(eval _is_event_log_tags_file := $(if $(findstring $f,$(event_log_tags_file)),Y)) \ + $(eval _is_system_other_odex_marker := $(if $(findstring $f,$(INSTALLED_SYSTEM_OTHER_ODEX_MARKER)),Y)) \ + $(eval _is_kernel_modules_blocklist := $(if $(findstring $f,$(ALL_KERNEL_MODULES_BLOCKLIST)),Y)) \ + $(eval _is_fsverity_build_manifest_apk := $(if $(findstring $f,$(ALL_FSVERITY_BUILD_MANIFEST_APK)),Y)) \ + $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \ + $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \ + $(eval _is_flags_file := $(if $(findstring $f, $(ALL_FLAGS_FILES)),Y)) \ + $(eval _is_rootdir_symlink := $(if $(findstring $f, $(ALL_ROOTDIR_SYMLINKS)),Y)) \ + $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)$(_is_flags_file)$(_is_rootdir_symlink)) \ + $(eval _static_libs := $(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES)) \ + $(eval _whole_static_libs := $(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES)) \ + $(foreach l,$(_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \ + $(foreach l,$(_whole_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \ + echo '/$(_path_on_device),$(_module_path),$(_soong_module_type),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_build_output_path),$(_static_libs),$(_whole_static_libs),' >> $@; \ + ) + $(foreach l,$(sort $(_all_static_libs)), \ + $(eval _lib_stem := $(call word-colon,1,$l)) \ + $(eval _module_path := $(call word-colon,2,$l)) \ + $(eval _soong_module_type := $(call word-colon,3,$l)) \ + $(eval _built_file := $(call word-colon,4,$l)) \ + $(eval _static_libs := $(ALL_STATIC_LIBRARIES.$l.STATIC_LIBRARIES)) \ + $(eval _whole_static_libs := $(ALL_STATIC_LIBRARIES.$l.WHOLE_STATIC_LIBRARIES)) \ + $(eval _is_static_lib := Y) \ + echo '$(_lib_stem).a,$(_module_path),$(_soong_module_type),,,,,$(_built_file),$(_static_libs),$(_whole_static_libs),$(_is_static_lib)' >> $@; \ + ) + +# (TODO: b/272358583 find another way of always rebuilding sbom.spdx) +# Remove the always_dirty_file.txt whenever the makefile is evaluated +$(shell rm -f $(PRODUCT_OUT)/always_dirty_file.txt) +$(PRODUCT_OUT)/always_dirty_file.txt: + touch $@ + +.PHONY: sbom +ifeq ($(TARGET_BUILD_APPS),) +sbom: $(PRODUCT_OUT)/sbom.spdx.json +$(PRODUCT_OUT)/sbom.spdx.json: $(PRODUCT_OUT)/sbom.spdx +$(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) $(installed_files) $(metadata_list) $(metadata_files) $(PRODUCT_OUT)/always_dirty_file.txt + rm -rf $@ + $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json + +$(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json) +else +# Create build rules for generating SBOMs of unbundled APKs and APEXs +# $1: sbom file +# $2: sbom fragment file +# $3: installed file +# $4: sbom-metadata.csv file +define generate-app-sbom +$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$(3))) +$(eval _module_name := $(ALL_INSTALLED_FILES.$(3))) +$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) +$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) +$(eval _dep_modules := $(filter %.$(_module_name),$(ALL_MODULES)) $(filter %.$(_module_name)$(TARGET_2ND_ARCH_MODULE_SUFFIX),$(ALL_MODULES))) +$(eval _is_apex := $(filter %.apex,$(3))) + +$(4): + rm -rf $$@ + echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $$@ + echo /$(_path_on_device),$(_module_path),$(_soong_module_type),,,,,$(3),,, >> $$@ + $(if $(filter %.apex,$(3)),\ + $(foreach m,$(_dep_modules),\ + echo $(patsubst $(PRODUCT_OUT)/apex/$(_module_name)/%,%,$(ALL_MODULES.$m.INSTALLED)),$(sort $(ALL_MODULES.$m.PATH)),$(sort $(ALL_MODULES.$m.SOONG_MODULE_TYPE)),,,,,$(strip $(ALL_MODULES.$m.BUILT)),,, >> $$@;)) + +$(2): $(1) +$(1): $(4) $(3) $(GEN_SBOM) $(installed_files) $(metadata_list) $(metadata_files) + rm -rf $$@ + $(GEN_SBOM) --output_file $$@ --metadata $(4) --build_version $$(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(if $(filter %.apk,$(3)),--unbundled_apk,--unbundled_apex) +endef + +apps_only_sbom_files := +apps_only_fragment_files := +$(foreach f,$(filter %.apk %.apex,$(installed_files)), \ + $(eval _metadata_csv_file := $(patsubst %,%-sbom-metadata.csv,$f)) \ + $(eval _sbom_file := $(patsubst %,%.spdx.json,$f)) \ + $(eval _fragment_file := $(patsubst %,%-fragment.spdx,$f)) \ + $(eval apps_only_sbom_files += $(_sbom_file)) \ + $(eval apps_only_fragment_files += $(_fragment_file)) \ + $(eval $(call generate-app-sbom,$(_sbom_file),$(_fragment_file),$f,$(_metadata_csv_file))) \ +) + +sbom: $(apps_only_sbom_files) + +$(foreach f,$(apps_only_fragment_files),$(eval apps_only_fragment_dist_files += :sbom/$(notdir $f))) +$(foreach f,$(apps_only_sbom_files),$(eval apps_only_sbom_dist_files += :sbom/$(notdir $f))) +$(call dist-for-goals,apps_only,$(join $(apps_only_sbom_files),$(apps_only_sbom_dist_files)) $(join $(apps_only_fragment_files),$(apps_only_fragment_dist_files))) +endif + $(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk) -$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] writing build rules ...) +$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] writing legacy Make module rules ...) diff --git a/core/misc_prebuilt_internal.mk b/core/misc_prebuilt_internal.mk index 921ea52781..a56220772c 100644 --- a/core/misc_prebuilt_internal.mk +++ b/core/misc_prebuilt_internal.mk @@ -27,9 +27,6 @@ include $(BUILD_SYSTEM)/base_rules.mk ifneq ($(filter init%rc,$(notdir $(LOCAL_INSTALLED_MODULE)))$(filter %/etc/init,$(dir $(LOCAL_INSTALLED_MODULE))),) $(eval $(call copy-init-script-file-checked,$(my_prebuilt_src_file),$(LOCAL_BUILT_MODULE))) -else ifneq ($(LOCAL_PREBUILT_STRIP_COMMENTS),) -$(LOCAL_BUILT_MODULE) : $(my_prebuilt_src_file) - $(transform-prebuilt-to-target-strip-comments) else $(LOCAL_BUILT_MODULE) : $(my_prebuilt_src_file) $(transform-prebuilt-to-target) diff --git a/core/native_test_config_template.xml b/core/native_test_config_template.xml index ea982cf2f9..788157c611 100644 --- a/core/native_test_config_template.xml +++ b/core/native_test_config_template.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2017 The Android Open Source Project +<!-- Copyright (C) 2023 The Android Open Source Project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ </target_preparer> <test class="com.android.tradefed.testtype.GTest" > - <option name="native-test-device-path" value="{TEST_INSTALL_BASE}" /> + {EXTRA_TEST_RUNNER_CONFIGS}<option name="native-test-device-path" value="{TEST_INSTALL_BASE}" /> <option name="module-name" value="{MODULE}" /> </test> </configuration> diff --git a/core/ninja_config.mk b/core/ninja_config.mk index 2b5ceee796..d4b7c6df11 100644 --- a/core/ninja_config.mk +++ b/core/ninja_config.mk @@ -14,7 +14,6 @@ PARSE_TIME_MAKE_GOALS := \ $(PARSE_TIME_MAKE_GOALS) \ $(dont_bother_goals) \ all \ - ECLIPSE-% \ brillo_tests \ btnod \ build-art% \ diff --git a/core/notice_files.mk b/core/notice_files.mk index a5852cc6ad..6935115b0a 100644 --- a/core/notice_files.mk +++ b/core/notice_files.mk @@ -1,167 +1,144 @@ ########################################################### ## Track NOTICE files ########################################################### -$(call record-module-type,NOTICE_FILE) -ifneq ($(LOCAL_NOTICE_FILE),) -notice_file:=$(strip $(LOCAL_NOTICE_FILE)) -else -notice_file:=$(strip $(wildcard $(LOCAL_PATH)/LICENSE $(LOCAL_PATH)/LICENCE $(LOCAL_PATH)/NOTICE)) -endif +module_license_metadata := $(call local-meta-intermediates-dir)/$(my_register_name).meta_lic -ifneq (,$(strip $(LOCAL_LICENSE_PACKAGE_NAME))) -license_package_name:=$(strip $(LOCAL_LICENSE_PACKAGE_NAME)) -else -license_package_name:= -endif +$(foreach target,$(ALL_MODULES.$(my_register_name).BUILT) $(ALL_MODULES.$(my_register_name).INSTALLED) $(foreach bi,$(LOCAL_SOONG_BUILT_INSTALLED),$(call word-colon,1,$(bi))),\ + $(eval ALL_TARGETS.$(target).META_LIC := $(module_license_metadata))) -ifneq (,$(strip $(LOCAL_LICENSE_INSTALL_MAP))) -install_map:=$(strip $(LOCAL_LICENSE_INSTALL_MAP)) -else -install_map:= -endif +$(foreach f,$(my_test_data) $(my_test_config),\ + $(if $(strip $(ALL_TARGETS.$(call word-colon,1,$(f)).META_LIC)), \ + $(call declare-copy-target-license-metadata,$(call word-colon,2,$(f)),$(call word-colon,1,$(f))), \ + $(eval ALL_TARGETS.$(call word-colon,2,$(f)).META_LIC := $(module_license_metadata)))) -ifneq (,$(strip $(LOCAL_LICENSE_KINDS))) -license_kinds:=$(strip $(LOCAL_LICENSE_KINDS)) -else -license_kinds:=legacy_by_exception_only -endif +ALL_MODULES.$(my_register_name).META_LIC := $(strip $(ALL_MODULES.$(my_register_name).META_LIC) $(module_license_metadata)) -ifneq (,$(strip $(LOCAL_LICENSE_CONDITIONS))) -license_conditions:=$(strip $(LOCAL_LICENSE_CONDITIONS)) +ifdef LOCAL_SOONG_LICENSE_METADATA + # Soong modules have already produced a license metadata file, copy it to where Make expects it. + $(eval $(call copy-one-license-metadata-file, $(LOCAL_SOONG_LICENSE_METADATA), $(module_license_metadata),$(ALL_MODULES.$(my_register_name).BUILT),$(ALL_MODUES.$(my_register_name).INSTALLED))) else -license_conditions:=by_exception_only -endif - -ifeq ($(LOCAL_MODULE_CLASS),GYP) - # We ignore NOTICE files for modules of type GYP. - notice_file := -endif + # Make modules don't have enough information to produce a license metadata rule until after fix-notice-deps + # has been called, store the necessary information until later. -ifeq ($(LOCAL_MODULE_CLASS),FAKE) - # We ignore NOTICE files for modules of type FAKE. - notice_file := -endif - -# Soong generates stub libraries that don't need NOTICE files -ifdef LOCAL_NO_NOTICE_FILE - ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) - $(call pretty-error,LOCAL_NO_NOTICE_FILE should not be used by Android.mk files) + ifneq ($(LOCAL_NOTICE_FILE),) + notice_file:=$(strip $(LOCAL_NOTICE_FILE)) + else + notice_file:=$(strip $(wildcard $(LOCAL_PATH)/LICENSE $(LOCAL_PATH)/LICENCE $(LOCAL_PATH)/NOTICE)) endif - notice_file := -endif -ifeq ($(LOCAL_MODULE_CLASS),NOTICE_FILES) -# If this is a NOTICE-only module, we don't include base_rule.mk, -# so my_prefix is not set at this point. -ifeq ($(LOCAL_IS_HOST_MODULE),true) - my_prefix := HOST_ - LOCAL_HOST_PREFIX := -else - my_prefix := TARGET_ -endif -endif - -installed_notice_file := - -is_container:=$(strip $(LOCAL_MODULE_IS_CONTAINER)) -ifeq (,$(is_container)) -ifneq (,$(strip $(filter %.zip %.tar %.tgz %.tar.gz %.apk %.img %.srcszip %.apex, $(LOCAL_BUILT_MODULE)))) -is_container:=true -else -is_container:=false -endif -else ifneq (,$(strip $(filter-out true false,$(is_container)))) -$(error Unrecognized value '$(is_container)' for LOCAL_MODULE_IS_CONTAINER) -endif - -ifeq (true,$(is_container)) -# Include shared libraries' notices for "container" types, but not for binaries etc. -notice_deps := \ - $(strip \ - $(foreach d, \ - $(LOCAL_REQUIRED_MODULES) \ - $(LOCAL_STATIC_LIBRARIES) \ - $(LOCAL_WHOLE_STATIC_LIBRARIES) \ - $(LOCAL_SHARED_LIBRARIES) \ - $(LOCAL_DYLIB_LIBRARIES) \ - $(LOCAL_RLIB_LIBRARIES) \ - $(LOCAL_PROC_MACRO_LIBRARIES) \ - $(LOCAL_HEADER_LIBRARIES) \ - $(LOCAL_STATIC_JAVA_LIBRARIES) \ - $(LOCAL_JAVA_LIBRARIES) \ - $(LOCAL_JNI_SHARED_LIBRARIES) \ - ,$(subst :,_,$(d)):static \ - ) \ - ) -else -notice_deps := \ - $(strip \ - $(foreach d, \ - $(LOCAL_REQUIRED_MODULES) \ - $(LOCAL_STATIC_LIBRARIES) \ - $(LOCAL_WHOLE_STATIC_LIBRARIES) \ - $(LOCAL_RLIB_LIBRARIES) \ - $(LOCAL_PROC_MACRO_LIBRARIES) \ - $(LOCAL_HEADER_LIBRARIES) \ - $(LOCAL_STATIC_JAVA_LIBRARIES) \ - ,$(subst :,_,$(d)):static \ - )$(foreach d, \ - $(LOCAL_SHARED_LIBRARIES) \ - $(LOCAL_DYLIB_LIBRARIES) \ - $(LOCAL_JAVA_LIBRARIES) \ - $(LOCAL_JNI_SHARED_LIBRARIES) \ - ,$(subst :,_,$(d)):dynamic \ - ) \ - ) -endif -ifeq ($(LOCAL_IS_HOST_MODULE),true) -notice_deps := $(strip $(notice_deps) $(foreach d,$(LOCAL_HOST_REQUIRED_MODULES),$(subst :,_,$(d)):static)) -else -notice_deps := $(strip $(notice_deps) $(foreach d,$(LOCAL_TARGET_REQUIRED_MODULES),$(subst :,_,$(d)):static)) -endif + ifeq ($(LOCAL_MODULE_CLASS),GYP) + # We ignore NOTICE files for modules of type GYP. + notice_file := + endif -local_path := $(LOCAL_PATH) + ifeq ($(LOCAL_MODULE_CLASS),FAKE) + # We ignore NOTICE files for modules of type FAKE. + notice_file := + endif + # Soong generates stub libraries that don't need NOTICE files + ifdef LOCAL_NO_NOTICE_FILE + ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) + $(call pretty-error,LOCAL_NO_NOTICE_FILE should not be used by Android.mk files) + endif + notice_file := + endif -module_license_metadata := + ifneq (,$(strip $(LOCAL_LICENSE_PACKAGE_NAME))) + license_package_name:=$(strip $(LOCAL_LICENSE_PACKAGE_NAME)) + else + license_package_name:= + endif -ifdef my_register_name - module_license_metadata := $(call local-meta-intermediates-dir)/$(my_register_name).meta_lic + ifneq (,$(strip $(LOCAL_LICENSE_INSTALL_MAP))) + install_map:=$(strip $(LOCAL_LICENSE_INSTALL_MAP)) + else + install_map:= + endif - $(foreach target,$(ALL_MODULES.$(my_register_name).BUILT) $(ALL_MODULES.$(my_register_name).INSTALLED) $(foreach bi,$(LOCAL_SOONG_BUILT_INSTALLED),$(call word-colon,1,$(bi))),\ - $(eval ALL_TARGETS.$(target).META_LIC := $(module_license_metadata))) + ifneq (,$(strip $(LOCAL_LICENSE_KINDS))) + license_kinds:=$(strip $(LOCAL_LICENSE_KINDS)) + else + license_kinds:=legacy_by_exception_only + endif - $(foreach f,$(my_test_data) $(my_test_config),\ - $(if $(strip $(ALL_TARGETS.$(call word-colon,1,$(f)).META_LIC)), \ - $(call declare-copy-target-license-metadata,$(call word-colon,2,$(f)),$(call word-colon,1,$(f))), \ - $(eval ALL_TARGETS.$(call word-colon,2,$(f)).META_LIC := $(module_license_metadata)))) + ifneq (,$(strip $(LOCAL_LICENSE_CONDITIONS))) + license_conditions:=$(strip $(LOCAL_LICENSE_CONDITIONS)) + else + license_conditions:=by_exception_only + endif - ALL_MODULES.$(my_register_name).META_LIC := $(strip $(ALL_MODULES.$(my_register_name).META_LIC) $(module_license_metadata)) + is_container:=$(strip $(LOCAL_MODULE_IS_CONTAINER)) + ifeq (,$(is_container)) + ifneq (,$(strip $(filter %.zip %.tar %.tgz %.tar.gz %.apk %.img %.srcszip %.apex, $(LOCAL_BUILT_MODULE)))) + is_container:=true + else + is_container:=false + endif + else ifneq (,$(strip $(filter-out true false,$(is_container)))) + $(error Unrecognized value '$(is_container)' for LOCAL_MODULE_IS_CONTAINER) + endif - ifdef LOCAL_SOONG_LICENSE_METADATA - # Soong modules have already produced a license metadata file, copy it to where Make expects it. - $(eval $(call copy-one-license-metadata-file, $(LOCAL_SOONG_LICENSE_METADATA), $(module_license_metadata),$(ALL_MODULES.$(my_register_name).BUILT),$(ALL_MODUES.$(my_register_name).INSTALLED))) + ifeq (true,$(is_container)) + # Include shared libraries' notices for "container" types, but not for binaries etc. + notice_deps := \ + $(strip \ + $(foreach d, \ + $(LOCAL_REQUIRED_MODULES) \ + $(LOCAL_STATIC_LIBRARIES) \ + $(LOCAL_WHOLE_STATIC_LIBRARIES) \ + $(LOCAL_SHARED_LIBRARIES) \ + $(LOCAL_DYLIB_LIBRARIES) \ + $(LOCAL_RLIB_LIBRARIES) \ + $(LOCAL_PROC_MACRO_LIBRARIES) \ + $(LOCAL_HEADER_LIBRARIES) \ + $(LOCAL_STATIC_JAVA_LIBRARIES) \ + $(LOCAL_JAVA_LIBRARIES) \ + $(LOCAL_JNI_SHARED_LIBRARIES) \ + ,$(subst :,_,$(d)):static \ + ) \ + ) else - # Make modules don't have enough information to produce a license metadata rule until after fix-notice-deps - # has been called, store the necessary information until later. - ALL_MODULES.$(my_register_name).DELAYED_META_LIC := $(strip $(ALL_MODULES.$(my_register_name).DELAYED_META_LIC) $(module_license_metadata)) - ALL_MODULES.$(my_register_name).LICENSE_PACKAGE_NAME := $(strip $(license_package_name)) - ALL_MODULES.$(my_register_name).MODULE_TYPE := $(strip $(ALL_MODULES.$(my_register_name).MODULE_TYPE) $(LOCAL_MODULE_TYPE)) - ALL_MODULES.$(my_register_name).MODULE_CLASS := $(strip $(ALL_MODULES.$(my_register_name).MODULE_CLASS) $(LOCAL_MODULE_CLASS)) - ALL_MODULES.$(my_register_name).LICENSE_KINDS := $(ALL_MODULES.$(my_register_name).LICENSE_KINDS) $(license_kinds) - ALL_MODULES.$(my_register_name).LICENSE_CONDITIONS := $(ALL_MODULES.$(my_register_name).LICENSE_CONDITIONS) $(license_conditions) - ALL_MODULES.$(my_register_name).LICENSE_INSTALL_MAP := $(ALL_MODULES.$(my_register_name).LICENSE_INSTALL_MAP) $(install_map) - ALL_MODULES.$(my_register_name).NOTICE_DEPS := $(ALL_MODULES.$(my_register_name).NOTICE_DEPS) $(notice_deps) - ALL_MODULES.$(my_register_name).IS_CONTAINER := $(strip $(filter-out false,$(ALL_MODULES.$(my_register_name).IS_CONTAINER) $(is_container))) - ALL_MODULES.$(my_register_name).PATH := $(strip $(ALL_MODULES.$(my_register_name).PATH) $(local_path)) + notice_deps := \ + $(strip \ + $(foreach d, \ + $(LOCAL_REQUIRED_MODULES) \ + $(LOCAL_STATIC_LIBRARIES) \ + $(LOCAL_WHOLE_STATIC_LIBRARIES) \ + $(LOCAL_RLIB_LIBRARIES) \ + $(LOCAL_PROC_MACRO_LIBRARIES) \ + $(LOCAL_HEADER_LIBRARIES) \ + $(LOCAL_STATIC_JAVA_LIBRARIES) \ + ,$(subst :,_,$(d)):static \ + )$(foreach d, \ + $(LOCAL_SHARED_LIBRARIES) \ + $(LOCAL_DYLIB_LIBRARIES) \ + $(LOCAL_JAVA_LIBRARIES) \ + $(LOCAL_JNI_SHARED_LIBRARIES) \ + ,$(subst :,_,$(d)):dynamic \ + ) \ + ) + endif + ifeq ($(LOCAL_IS_HOST_MODULE),true) + notice_deps := $(strip $(notice_deps) $(foreach d,$(LOCAL_HOST_REQUIRED_MODULES),$(subst :,_,$(d)):static)) + else + notice_deps := $(strip $(notice_deps) $(foreach d,$(LOCAL_TARGET_REQUIRED_MODULES),$(subst :,_,$(d)):static)) endif -endif -ifdef notice_file + ALL_MODULES.$(my_register_name).DELAYED_META_LIC := $(strip $(ALL_MODULES.$(my_register_name).DELAYED_META_LIC) $(module_license_metadata)) + ALL_MODULES.$(my_register_name).LICENSE_PACKAGE_NAME := $(strip $(license_package_name)) + ALL_MODULES.$(my_register_name).MODULE_TYPE := $(strip $(ALL_MODULES.$(my_register_name).MODULE_TYPE) $(LOCAL_MODULE_TYPE)) + ALL_MODULES.$(my_register_name).MODULE_CLASS := $(strip $(ALL_MODULES.$(my_register_name).MODULE_CLASS) $(LOCAL_MODULE_CLASS)) + ALL_MODULES.$(my_register_name).LICENSE_KINDS := $(ALL_MODULES.$(my_register_name).LICENSE_KINDS) $(license_kinds) + ALL_MODULES.$(my_register_name).LICENSE_CONDITIONS := $(ALL_MODULES.$(my_register_name).LICENSE_CONDITIONS) $(license_conditions) + ALL_MODULES.$(my_register_name).LICENSE_INSTALL_MAP := $(ALL_MODULES.$(my_register_name).LICENSE_INSTALL_MAP) $(install_map) + ALL_MODULES.$(my_register_name).NOTICE_DEPS := $(ALL_MODULES.$(my_register_name).NOTICE_DEPS) $(notice_deps) + ALL_MODULES.$(my_register_name).IS_CONTAINER := $(strip $(filter-out false,$(ALL_MODULES.$(my_register_name).IS_CONTAINER) $(is_container))) + ALL_MODULES.$(my_register_name).PATH := $(strip $(ALL_MODULES.$(my_register_name).PATH) $(local_path)) -ifdef my_register_name -ALL_MODULES.$(my_register_name).NOTICES := $(ALL_MODULES.$(my_register_name).NOTICES) $(notice_file) + ifdef notice_file + ALL_MODULES.$(my_register_name).NOTICES := $(ALL_MODULES.$(my_register_name).NOTICES) $(notice_file) + endif # notice_file endif -endif # notice_file diff --git a/core/package_internal.mk b/core/package_internal.mk index c7a173b71f..a03a62b7fa 100644 --- a/core/package_internal.mk +++ b/core/package_internal.mk @@ -111,24 +111,26 @@ include $(BUILD_SYSTEM)/support_libraries.mk # Determine whether auto-RRO is enabled for this package. enforce_rro_enabled := -ifneq (,$(filter *, $(PRODUCT_ENFORCE_RRO_TARGETS))) - # * means all system and system_ext APKs, so enable conditionally based on module path. - - # Note that base_rules.mk has not yet been included, so it's likely that only - # one of LOCAL_MODULE_PATH and the LOCAL_X_MODULE flags has been set. - ifeq (,$(LOCAL_MODULE_PATH)) - non_rro_target_module := $(filter true,\ - $(LOCAL_ODM_MODULE) \ - $(LOCAL_OEM_MODULE) \ - $(LOCAL_PRODUCT_MODULE) \ - $(LOCAL_PROPRIETARY_MODULE) \ - $(LOCAL_VENDOR_MODULE)) - enforce_rro_enabled := $(if $(non_rro_target_module),,true) - else ifneq ($(filter $(TARGET_OUT)/%,$(LOCAL_MODULE_PATH)),) +ifeq (,$(filter tests,$(LOCAL_MODULE_TAGS))) + ifneq (,$(filter *, $(PRODUCT_ENFORCE_RRO_TARGETS))) + # * means all system and system_ext APKs, so enable conditionally based on module path. + + # Note that base_rules.mk has not yet been included, so it's likely that only + # one of LOCAL_MODULE_PATH and the LOCAL_X_MODULE flags has been set. + ifeq (,$(LOCAL_MODULE_PATH)) + non_rro_target_module := $(filter true,\ + $(LOCAL_ODM_MODULE) \ + $(LOCAL_OEM_MODULE) \ + $(LOCAL_PRODUCT_MODULE) \ + $(LOCAL_PROPRIETARY_MODULE) \ + $(LOCAL_VENDOR_MODULE)) + enforce_rro_enabled := $(if $(non_rro_target_module),,true) + else ifneq ($(filter $(TARGET_OUT)/%,$(LOCAL_MODULE_PATH)),) + enforce_rro_enabled := true + endif + else ifneq (,$(filter $(LOCAL_PACKAGE_NAME), $(PRODUCT_ENFORCE_RRO_TARGETS))) enforce_rro_enabled := true endif -else ifneq (,$(filter $(LOCAL_PACKAGE_NAME), $(PRODUCT_ENFORCE_RRO_TARGETS))) - enforce_rro_enabled := true endif product_package_overlays := $(strip \ @@ -201,10 +203,10 @@ my_res_resources := $(if $(my_res_dir),$(strip \ all_resources := $(strip $(my_res_resources) $(my_overlay_resources)) # The linked resource package. -my_res_package := $(intermediates)/package-res.apk +my_res_package := $(intermediates.COMMON)/package-res.apk LOCAL_INTERMEDIATE_TARGETS += $(my_res_package) -my_bundle_module := $(intermediates)/base.zip +my_bundle_module := $(intermediates.COMMON)/base.zip LOCAL_INTERMEDIATE_TARGETS += $(my_bundle_module) # Always run aapt2, because we need to at least compile the AndroidManifest.xml. @@ -529,7 +531,7 @@ $(LOCAL_BUILT_MODULE) : $(JAR_ARGS) $(SOONG_ZIP) $(MERGE_ZIPS) $(ZIP2ZIP) $(LOCAL_BUILT_MODULE): PRIVATE_RES_PACKAGE := $(my_res_package) $(LOCAL_BUILT_MODULE) : $(my_res_package) $(AAPT2) ifdef LOCAL_COMPRESSED_MODULE -$(LOCAL_BUILT_MODULE) : $(MINIGZIP) +$(LOCAL_BUILT_MODULE) : $(GZIP) endif ifeq (true, $(LOCAL_UNCOMPRESS_DEX)) $(LOCAL_BUILT_MODULE) : $(ZIP2ZIP) @@ -570,7 +572,7 @@ ifdef LOCAL_COMPRESSED_MODULE $(compress-package) endif # LOCAL_COMPRESSED_MODULE -my_package_res_pb := $(intermediates)/package-res.pb.apk +my_package_res_pb := $(intermediates.COMMON)/package-res.pb.apk $(my_package_res_pb): $(my_res_package) $(AAPT2) $(AAPT2) convert --output-format proto $< -o $@ diff --git a/core/packaging/flags.mk b/core/packaging/flags.mk new file mode 100644 index 0000000000..e715fd1292 --- /dev/null +++ b/core/packaging/flags.mk @@ -0,0 +1,205 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file is included by build/make/core/Makefile, and contains the logic for +# the combined flags files. +# + +# TODO: Should we do all of the images in $(IMAGES_TO_BUILD)? +_FLAG_PARTITIONS := product system system_ext vendor + + +# ----------------------------------------------------------------- +# Release Config Flags + +# Create a summary file of build flags for each partition +# $(1): built build flags json file +# $(2): installed build flags json file +# $(3): flag names +define generate-partition-build-flag-file +$(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1))) +$(eval $(strip $(1)): PRIVATE_FLAG_NAMES := $(strip $(3))) +$(strip $(1)): + mkdir -p $$(dir $$(PRIVATE_OUT)) + echo '{' > $$(PRIVATE_OUT) + echo '"flags": [' >> $$(PRIVATE_OUT) + $$(foreach flag, $$(PRIVATE_FLAG_NAMES), \ + ( \ + printf ' { "name": "%s", "value": "%s", ' \ + '$$(flag)' \ + '$$(_ALL_RELEASE_FLAGS.$$(flag).VALUE)' \ + ; \ + printf '"set": "%s", "default": "%s", "declared": "%s" }' \ + '$$(_ALL_RELEASE_FLAGS.$$(flag).SET_IN)' \ + '$$(_ALL_RELEASE_FLAGS.$$(flag).DEFAULT)' \ + '$$(_ALL_RELEASE_FLAGS.$$(flag).DECLARED_IN)' \ + ; \ + printf '$$(if $$(filter $$(lastword $$(PRIVATE_FLAG_NAMES)),$$(flag)),,$$(comma))\n' ; \ + ) >> $$(PRIVATE_OUT) ; \ + ) + echo "]" >> $$(PRIVATE_OUT) + echo "}" >> $$(PRIVATE_OUT) +$(call copy-one-file, $(1), $(2)) +endef + +$(foreach partition, $(_FLAG_PARTITIONS), \ + $(eval build_flag_summaries.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/build_flags.json) \ + $(eval $(call generate-partition-build-flag-file, \ + $(TARGET_OUT_FLAGS)/$(partition)/build_flags.json, \ + $(build_flag_summaries.$(partition)), \ + $(_ALL_RELEASE_FLAGS.PARTITIONS.$(partition)) \ + ) \ + ) \ +) + + +# ----------------------------------------------------------------- +# Aconfig Flags + +# Create a summary file of build flags for each partition +# $(1): built aconfig flags file (out) +# $(2): installed aconfig flags file (out) +# $(3): input aconfig files for the partition (in) +define generate-partition-aconfig-flag-file +$(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1))) +$(eval $(strip $(1)): PRIVATE_IN := $(strip $(3))) +$(strip $(1)): $(ACONFIG) $(strip $(3)) + mkdir -p $$(dir $$(PRIVATE_OUT)) + $$(if $$(PRIVATE_IN), \ + $$(ACONFIG) dump --dedup --format protobuf --out $$(PRIVATE_OUT) \ + $$(addprefix --cache ,$$(PRIVATE_IN)), \ + echo -n > $$(PRIVATE_OUT) \ + ) +$(call copy-one-file, $(1), $(2)) +endef + + +$(foreach partition, $(_FLAG_PARTITIONS), \ + $(eval aconfig_flag_summaries_protobuf.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig_flags.pb) \ + $(eval $(call generate-partition-aconfig-flag-file, \ + $(TARGET_OUT_FLAGS)/$(partition)/aconfig_flags.pb, \ + $(aconfig_flag_summaries_protobuf.$(partition)), \ + $(sort $(foreach m,$(call register-names-for-partition, $(partition)), \ + $(ALL_MODULES.$(m).ACONFIG_FILES) \ + )), \ + )) \ +) + +# Collect the on-device flags into a single file, similar to all_aconfig_declarations. +required_aconfig_flags_files := \ + $(sort $(foreach partition, $(filter $(IMAGES_TO_BUILD), $(_FLAG_PARTITIONS)), \ + $(aconfig_flag_summaries_protobuf.$(partition)) \ + )) + +.PHONY: device_aconfig_declarations +device_aconfig_declarations: $(PRODUCT_OUT)/device_aconfig_declarations.pb +$(eval $(call generate-partition-aconfig-flag-file, \ + $(TARGET_OUT_FLAGS)/device_aconfig_declarations.pb, \ + $(PRODUCT_OUT)/device_aconfig_declarations.pb, \ + $(sort $(required_aconfig_flags_files)) \ +)) \ + +# Create a set of storage file for each partition +# $(1): built aconfig flags storage package map file (out) +# $(2): built aconfig flags storage flag map file (out) +# $(3): built aconfig flags storage flag val file (out) +# $(4): installed aconfig flags storage package map file (out) +# $(5): installed aconfig flags storage flag map file (out) +# $(6): installed aconfig flags storage flag value file (out) +# $(7): input aconfig files for the partition (in) +# $(8): partition name +define generate-partition-aconfig-storage-file +$(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1))) +$(eval $(strip $(1)): PRIVATE_IN := $(strip $(7))) +$(strip $(1)): $(ACONFIG) $(strip $(7)) + mkdir -p $$(dir $$(PRIVATE_OUT)) + $$(if $$(PRIVATE_IN), \ + $$(ACONFIG) create-storage --container $(8) --file package_map --out $$(PRIVATE_OUT) \ + $$(addprefix --cache ,$$(PRIVATE_IN)), \ + ) + touch $$(PRIVATE_OUT) +$(eval $(strip $(2)): PRIVATE_OUT := $(strip $(2))) +$(eval $(strip $(2)): PRIVATE_IN := $(strip $(7))) +$(strip $(2)): $(ACONFIG) $(strip $(7)) + mkdir -p $$(dir $$(PRIVATE_OUT)) + $$(if $$(PRIVATE_IN), \ + $$(ACONFIG) create-storage --container $(8) --file flag_map --out $$(PRIVATE_OUT) \ + $$(addprefix --cache ,$$(PRIVATE_IN)), \ + ) + touch $$(PRIVATE_OUT) +$(eval $(strip $(3)): PRIVATE_OUT := $(strip $(3))) +$(eval $(strip $(3)): PRIVATE_IN := $(strip $(7))) +$(strip $(3)): $(ACONFIG) $(strip $(7)) + mkdir -p $$(dir $$(PRIVATE_OUT)) + $$(if $$(PRIVATE_IN), \ + $$(ACONFIG) create-storage --container $(8) --file flag_val --out $$(PRIVATE_OUT) \ + $$(addprefix --cache ,$$(PRIVATE_IN)), \ + ) + touch $$(PRIVATE_OUT) +$(call copy-one-file, $(strip $(1)), $(4)) +$(call copy-one-file, $(strip $(2)), $(5)) +$(call copy-one-file, $(strip $(3)), $(6)) +endef + +ifeq ($(RELEASE_CREATE_ACONFIG_STORAGE_FILE),true) +$(foreach partition, $(_FLAG_PARTITIONS), \ + $(eval aconfig_storage_package_map.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/package.map) \ + $(eval aconfig_storage_flag_map.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/flag.map) \ + $(eval aconfig_storage_flag_val.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/flag.val) \ + $(eval $(call generate-partition-aconfig-storage-file, \ + $(TARGET_OUT_FLAGS)/$(partition)/package.map, \ + $(TARGET_OUT_FLAGS)/$(partition)/flag.map, \ + $(TARGET_OUT_FLAGS)/$(partition)/flag.val, \ + $(aconfig_storage_package_map.$(partition)), \ + $(aconfig_storage_flag_map.$(partition)), \ + $(aconfig_storage_flag_val.$(partition)), \ + $(sort $(foreach m,$(call register-names-for-partition, $(partition)), \ + $(ALL_MODULES.$(m).ACONFIG_FILES) \ + )), \ + $(partition), \ + )) \ +) +endif + +# ----------------------------------------------------------------- +# Install the ones we need for the configured product +required_flags_files := \ + $(sort $(foreach partition, $(filter $(IMAGES_TO_BUILD), $(_FLAG_PARTITIONS)), \ + $(build_flag_summaries.$(partition)) \ + $(aconfig_flag_summaries_protobuf.$(partition)) \ + $(aconfig_storage_package_map.$(partition)) \ + $(aconfig_storage_flag_map.$(partition)) \ + $(aconfig_storage_flag_val.$(partition)) \ + )) + +ALL_DEFAULT_INSTALLED_MODULES += $(required_flags_files) +ALL_FLAGS_FILES := $(required_flags_files) + +# TODO: Remove +.PHONY: flag-files +flag-files: $(required_flags_files) + + +# Clean up +required_flags_files:= +required_aconfig_flags_files:= +$(foreach partition, $(_FLAG_PARTITIONS), \ + $(eval build_flag_summaries.$(partition):=) \ + $(eval aconfig_flag_summaries_protobuf.$(partition):=) \ + $(eval aconfig_storage_package_map.$(partition):=) \ + $(eval aconfig_storage_flag_map.$(partition):=) \ + $(eval aconfig_storage_flag_val.$(partition):=) \ +) + diff --git a/core/prebuilt_internal.mk b/core/prebuilt_internal.mk index ef1471d503..94626402c9 100644 --- a/core/prebuilt_internal.mk +++ b/core/prebuilt_internal.mk @@ -57,7 +57,9 @@ else $(error $(LOCAL_MODULE) : unexpected LOCAL_MODULE_CLASS for prebuilts: $(LOCAL_MODULE_CLASS)) endif +$(if $(filter-out $(SOONG_ANDROID_MK),$(LOCAL_MODULE_MAKEFILE)), \ + $(eval ALL_MODULES.$(my_register_name).IS_PREBUILT_MAKE_MODULE := Y)) + $(built_module) : $(LOCAL_ADDITIONAL_DEPENDENCIES) my_prebuilt_src_file := -my_preopt_for_extracted_apk := diff --git a/core/product.mk b/core/product.mk index f4d5a4fe8c..0a761fb44e 100644 --- a/core/product.mk +++ b/core/product.mk @@ -27,7 +27,16 @@ _product_single_value_vars += PRODUCT_MODEL _product_single_value_vars += PRODUCT_NAME_FOR_ATTESTATION _product_single_value_vars += PRODUCT_MODEL_FOR_ATTESTATION -# The resoure configuration options to use for this product. +# Defines the ELF segment alignment for binaries (executables and shared libraries). +# The ELF segment alignment has to be a PAGE_SIZE multiple. For example, if +# PRODUCT_MAX_PAGE_SIZE_SUPPORTED=65536, the possible values for PAGE_SIZE could be +# 4096, 16384 and 65536. +_product_single_value_vars += PRODUCT_MAX_PAGE_SIZE_SUPPORTED + +# Boolean variable determining if AOSP relies on bionic's PAGE_SIZE macro. +_product_single_value_vars += PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO + +# The resource configuration options to use for this product. _product_list_vars += PRODUCT_LOCALES _product_list_vars += PRODUCT_AAPT_CONFIG _product_single_value_vars += PRODUCT_AAPT_PREF_CONFIG @@ -36,6 +45,14 @@ _product_list_vars += PRODUCT_HOST_PACKAGES _product_list_vars += PRODUCT_PACKAGES _product_list_vars += PRODUCT_PACKAGES_DEBUG _product_list_vars += PRODUCT_PACKAGES_DEBUG_ASAN +_product_list_vars += PRODUCT_PACKAGES_ARM64 + +# packages that are added to PRODUCT_PACKAGES based on the PRODUCT_SHIPPING_API_LEVEL +# These are only added if the shipping API level is that level or lower +_product_list_vars += PRODUCT_PACKAGES_SHIPPING_API_LEVEL_29 +_product_list_vars += PRODUCT_PACKAGES_SHIPPING_API_LEVEL_33 +_product_list_vars += PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34 + # Packages included only for eng/userdebug builds, when building with EMMA_INSTRUMENT=true _product_list_vars += PRODUCT_PACKAGES_DEBUG_JAVA_COVERAGE _product_list_vars += PRODUCT_PACKAGES_ENG @@ -139,6 +156,9 @@ _product_list_vars += PRODUCT_BOOT_JARS # PRODUCT_BOOT_JARS, so that device-specific jars go after common jars. _product_list_vars += PRODUCT_BOOT_JARS_EXTRA +# List of jars to be included in the ART boot image for testing. +_product_list_vars += PRODUCT_TEST_ONLY_ART_BOOT_IMAGE_JARS + _product_single_value_vars += PRODUCT_SUPPORTS_VBOOT _product_list_vars += PRODUCT_SYSTEM_SERVER_APPS # List of system_server classpath jars on the platform. @@ -162,8 +182,6 @@ _product_list_vars += PRODUCT_SYSTEM_SERVER_JARS_EXTRA # Set to true to disable <uses-library> checks for a product. _product_list_vars += PRODUCT_BROKEN_VERIFY_USES_LIBRARIES -# All of the apps that we force preopt, this overrides WITH_DEXPREOPT. -_product_list_vars += PRODUCT_ALWAYS_PREOPT_EXTRACTED_APK _product_list_vars += PRODUCT_DEXPREOPT_SPEED_APPS _product_list_vars += PRODUCT_LOADED_BY_PRIVILEGED_MODULES _product_single_value_vars += PRODUCT_VBOOT_SIGNING_KEY @@ -211,6 +229,9 @@ _product_single_value_vars += PRODUCT_SYSTEM_DLKM_BASE_FS_PATH # The first API level this product shipped with _product_single_value_vars += PRODUCT_SHIPPING_API_LEVEL +# The first vendor API level this product shipped with +_product_single_value_vars += PRODUCT_SHIPPING_VENDOR_API_LEVEL + _product_list_vars += VENDOR_PRODUCT_RESTRICT_VENDOR_FILES _product_list_vars += VENDOR_EXCEPTION_MODULES _product_list_vars += VENDOR_EXCEPTION_PATHS @@ -237,6 +258,22 @@ _product_list_vars += PRODUCT_CFI_INCLUDE_PATHS # Whether any paths are excluded from sanitization when SANITIZE_TARGET=cfi _product_list_vars += PRODUCT_CFI_EXCLUDE_PATHS +# Whether any paths should have HWASan enabled for components +_product_list_vars += PRODUCT_HWASAN_INCLUDE_PATHS + +# Whether any paths are excluded from sanitization when SANITIZE_TARGET=hwaddress +_product_list_vars += PRODUCT_HWASAN_EXCLUDE_PATHS + +# Whether any paths should have Memtag_heap enabled for components +_product_list_vars += PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS +_product_list_vars += PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS +_product_list_vars += PRODUCT_MEMTAG_HEAP_SYNC_INCLUDE_PATHS +_product_list_vars += PRODUCT_MEMTAG_HEAP_SYNC_DEFAULT_INCLUDE_PATHS +_product_list_vars += PRODUCT_MEMTAG_HEAP_EXCLUDE_PATHS + +# Whether this product wants to start with an empty list of default memtag_heap include paths +_product_single_value_vars += PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS + # Whether the Scudo hardened allocator is disabled platform-wide _product_single_value_vars += PRODUCT_DISABLE_SCUDO @@ -269,6 +306,9 @@ _product_single_value_vars += PRODUCT_RETROFIT_DYNAMIC_PARTITIONS # List of tags that will be used to gate blueprint modules from the build graph _product_list_vars += PRODUCT_INCLUDE_TAGS +# List of directories that will be used to gate blueprint modules from the build graph +_product_list_vars += PRODUCT_SOURCE_ROOT_DIRS + # When this is true, various build time as well as runtime debugfs restrictions are enabled. _product_single_value_vars += PRODUCT_SET_DEBUGFS_RESTRICTIONS @@ -285,6 +325,13 @@ _product_single_value_vars += \ # set this variable to prevent OTA failures. _product_list_vars += PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS +# If set to true, this product forces HIDL to be enabled by declaring android.hidl.manager +# and android.hidl.token in the framework manifest. The product will also need to add the +# 'hwservicemanager' service to PRODUCT_PACKAGES if its SHIPPING_API_LEVEL is greater than 34. +# This should only be used during bringup for devices that are targeting FCM 202404 and still +# have partner-owned HIDL interfaces that are being converted to AIDL. +_product_single_value_vars += PRODUCT_HIDL_ENABLED + # If set to true, this product builds a generic OTA package, which installs generic system images # onto matching devices. The product may only build a subset of system images (e.g. only # system.img), so devices need to install the package in a system-only OTA manner. @@ -294,6 +341,10 @@ _product_list_vars += PRODUCT_MANIFEST_PACKAGE_NAME_OVERRIDES _product_list_vars += PRODUCT_PACKAGE_NAME_OVERRIDES _product_list_vars += PRODUCT_CERTIFICATE_OVERRIDES +# Overrides the (apex, jar) pairs above when determining the on-device location. The format is: +# <old_apex>:<old_jar>:<new_apex>:<new_jar> +_product_list_vars += PRODUCT_CONFIGURED_JAR_LOCATION_OVERRIDES + # Controls for whether different partitions are built for the current product. _product_single_value_vars += PRODUCT_BUILD_SYSTEM_IMAGE _product_single_value_vars += PRODUCT_BUILD_SYSTEM_OTHER_IMAGE @@ -355,8 +406,6 @@ _product_single_value_vars += PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY # a java_sdk_library module. _product_list_vars += PRODUCT_INTER_PARTITION_JAVA_LIBRARY_ALLOWLIST -_product_single_value_vars += PRODUCT_INSTALL_EXTRA_FLATTENED_APEXES - # Install a copy of the debug policy to the system_ext partition, and allow # init-second-stage to load debug policy from system_ext. # This option is only meant to be set by compliance GSI targets. @@ -369,13 +418,23 @@ _product_single_value_vars += PRODUCT_INSTALL_DEBUG_POLICY_TO_SYSTEM_EXT # /system/etc/security/fsverity/BuildManifest.apk _product_single_value_vars += PRODUCT_FSVERITY_GENERATE_METADATA -# If true, sets the default for MODULE_BUILD_FROM_SOURCE. This overrides -# BRANCH_DEFAULT_MODULE_BUILD_FROM_SOURCE but not an explicitly set value. +# If true, this builds the mainline modules from source. This overrides any +# prebuilts selected via RELEASE_APEX_CONTRIBUTIONS_* build flags for the +# current release config. _product_single_value_vars += PRODUCT_MODULE_BUILD_FROM_SOURCE # If true, installs a full version of com.android.virt APEX. _product_single_value_vars += PRODUCT_AVF_ENABLED +# If true, kernel with modules will be used for Microdroid VMs. +_product_single_value_vars += PRODUCT_AVF_KERNEL_MODULES_ENABLED + +# If true, the memory controller will be force-enabled in the cgroup v2 hierarchy +_product_single_value_vars += PRODUCT_MEMCG_V2_FORCE_ENABLED + +# If true, the cgroup v2 hierarchy will be split into apps/system subtrees +_product_single_value_vars += PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED + # List of .json files to be merged/compiled into vendor/etc/linker.config.pb _product_list_vars += PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS @@ -391,6 +450,39 @@ _product_list_vars += PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS # supports it _product_single_value_vars += PRODUCT_ENABLE_UFFD_GC +# Specifies COW version to be used by update_engine and libsnapshot. If this value is not +# specified we default to COW version 2 in update_engine for backwards compatibility +_product_single_value_vars += PRODUCT_VIRTUAL_AB_COW_VERSION + +# Specifies maximum bytes to be compressed at once during ota. Options: 4096, 8192, 16384, 32768, 65536, 131072, 262144. +_product_single_value_vars += PRODUCT_VIRTUAL_AB_COMPRESSION_FACTOR + +# If set, determines whether the build system checks vendor seapp contexts violations. +_product_single_value_vars += PRODUCT_CHECK_VENDOR_SEAPP_VIOLATIONS + +# If set, determines whether the build system checks dev type violations. +_product_single_value_vars += PRODUCT_CHECK_DEV_TYPE_VIOLATIONS + +_product_list_vars += PRODUCT_AFDO_PROFILES + +_product_single_value_vars += PRODUCT_SCUDO_ALLOCATION_RING_BUFFER_SIZE + +_product_list_vars += PRODUCT_RELEASE_CONFIG_MAPS + +_product_list_vars += PRODUCT_VALIDATION_CHECKS + +_product_single_value_vars += PRODUCT_BUILD_FROM_SOURCE_STUB + +_product_single_value_vars += PRODUCT_BUILD_IGNORE_APEX_CONTRIBUTION_CONTENTS + +_product_single_value_vars += PRODUCT_HIDDEN_API_EXPORTABLE_STUBS + +_product_single_value_vars += PRODUCT_EXPORT_RUNTIME_APIS + +# If set, determines which version of the GKI is used as guest kernel for Microdroid VMs. +# TODO(b/325991735): link to documentation once it is done. +_product_single_value_vars += PRODUCT_AVF_MICRODROID_GUEST_GKI_VERSION + .KATI_READONLY := _product_single_value_vars _product_list_vars _product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars) @@ -488,7 +580,7 @@ endef # be cleaned up to not be product variables. _readonly_late_variables := \ DEVICE_PACKAGE_OVERLAYS \ - WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY \ + WITH_DEXPREOPT_ART_BOOT_IMG_ONLY \ # Modified internally in the build system _readonly_late_variables += \ diff --git a/core/product_config.mk b/core/product_config.mk index 7055a1e958..f21c1c4188 100644 --- a/core/product_config.mk +++ b/core/product_config.mk @@ -74,7 +74,7 @@ endif ########################################################### define find-copy-subdir-files -$(sort $(shell find $(2) -name "$(1)" -type f | $(SED_EXTENDED) "s:($(2)/?(.*)):\\1\\:$(3)/\\2:" | sed "s://:/:g")) +$(shell find $(2) -name "$(1)" -type f | $(SED_EXTENDED) "s:($(2)/?(.*)):\\1\\:$(3)/\\2:" | sed "s://:/:g" | sort) endef # @@ -144,7 +144,6 @@ endif # include $(BUILD_SYSTEM)/node_fns.mk include $(BUILD_SYSTEM)/product.mk -include $(BUILD_SYSTEM)/device.mk # Read all product definitions. # @@ -224,7 +223,7 @@ _product_config_saved_KATI_ALLOW_RULES := $(.KATI_ALLOW_RULES) endif ifeq (,$(current_product_makefile)) - $(error Can not locate config makefile for product "$(TARGET_PRODUCT)") + $(error Cannot locate config makefile for product "$(TARGET_PRODUCT)") endif ifneq (,$(filter $(TARGET_PRODUCT),$(products_using_starlark_config))) @@ -237,14 +236,22 @@ else $(shell mkdir -p $(OUT_DIR)/rbc) $(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_product_config.mk) - $(shell build/soong/scripts/update_out \ - $(OUT_DIR)/rbc/rbc_product_config_results.mk \ - build/soong/scripts/rbc-run \ - $(current_product_makefile) \ - $(OUT_DIR)/rbc/make_vars_pre_product_config.mk) + $(shell $(OUT_DIR)/mk2rbc \ + --mode=write -r --outdir $(OUT_DIR)/rbc \ + --launcher=$(OUT_DIR)/rbc/launcher.rbc \ + --input_variables=$(OUT_DIR)/rbc/make_vars_pre_product_config.mk \ + --makefile_list=$(OUT_DIR)/.module_paths/configuration.list \ + $(current_product_makefile)) ifneq ($(.SHELLSTATUS),0) $(error product configuration converter failed: $(.SHELLSTATUS)) endif + + $(shell build/soong/scripts/update_out $(OUT_DIR)/rbc/rbc_product_config_results.mk \ + $(OUT_DIR)/rbcrun --mode=rbc $(OUT_DIR)/rbc/launcher.rbc) + ifneq ($(.SHELLSTATUS),0) + $(error product configuration runner failed: $(.SHELLSTATUS)) + endif + include $(OUT_DIR)/rbc/rbc_product_config_results.mk endif @@ -275,11 +282,50 @@ current_product_makefile := ############################################################################# # Check product include tag allowlist -BLUEPRINT_INCLUDE_TAGS_ALLOWLIST := com.android.mainline_go com.android.mainline +BLUEPRINT_INCLUDE_TAGS_ALLOWLIST := \ + com.android.mainline_go \ + com.android.mainline \ + mainline_module_prebuilt_nightly \ + mainline_module_prebuilt_monthly_release .KATI_READONLY := BLUEPRINT_INCLUDE_TAGS_ALLOWLIST $(foreach include_tag,$(PRODUCT_INCLUDE_TAGS), \ $(if $(filter $(include_tag),$(BLUEPRINT_INCLUDE_TAGS_ALLOWLIST)),,\ $(call pretty-error, $(include_tag) is not in BLUEPRINT_INCLUDE_TAGS_ALLOWLIST: $(BLUEPRINT_INCLUDE_TAGS_ALLOWLIST)))) +# Create default PRODUCT_INCLUDE_TAGS +ifeq (, $(PRODUCT_INCLUDE_TAGS)) +# Soong analysis is global: even though a module might not be relevant to a specific product (e.g. build_tools for aosp_arm), +# we still analyse it. +# This means that in setups where we two have two prebuilts of module_sdk, we need a "default" to use in analysis +# This should be a no-op in aosp and internal since no Android.bp file contains blueprint_package_includes +# Use the big android one and main-based prebuilts by default +PRODUCT_INCLUDE_TAGS += com.android.mainline mainline_module_prebuilt_nightly +endif + +# AOSP and Google products currently share the same `apex_contributions` in next. +# This causes issues when building <aosp_product>-next-userdebug in main. +# Create a temporary allowlist to ignore the google apexes listed in `contents` of apex_contributions of `next` +# *for aosp products*. +# TODO(b/308187268): Remove this denylist mechanism +# Use PRODUCT_PACKAGES to determine if this is an aosp product. aosp products do not use google signed apexes. +ignore_apex_contributions := +ifeq (,$(findstring com.google.android.conscrypt,$(PRODUCT_PACKAGES))$(findstring com.google.android.go.conscrypt,$(PRODUCT_PACKAGES))) + ignore_apex_contributions := true +endif +ifeq (true,$(PRODUCT_MODULE_BUILD_FROM_SOURCE)) + ignore_apex_contributions := true +endif +ifneq ($(EMMA_INSTRUMENT)$(EMMA_INSTRUMENT_STATIC)$(EMMA_INSTRUMENT_FRAMEWORK)$(CLANG_COVERAGE)$(NATIVE_COVERAGE_PATHS),) +# Coverage builds for TARGET_RELEASE=foo should always build from source, +# even if TARGET_RELEASE=foo uses prebuilt mainline modules. +# This is necessary because the checked-in prebuilts were generated with +# instrumentation turned off. + ignore_apex_contributions := true +endif + +ifeq (true, $(ignore_apex_contributions)) +PRODUCT_BUILD_IGNORE_APEX_CONTRIBUTION_CONTENTS := true +endif + ############################################################################# # Quick check and assign default values @@ -399,6 +445,8 @@ PRODUCT_OTA_PUBLIC_KEYS := $(sort $(PRODUCT_OTA_PUBLIC_KEYS)) PRODUCT_EXTRA_OTA_KEYS := $(sort $(PRODUCT_EXTRA_OTA_KEYS)) PRODUCT_EXTRA_RECOVERY_KEYS := $(sort $(PRODUCT_EXTRA_RECOVERY_KEYS)) +PRODUCT_VALIDATION_CHECKS := $(sort $(PRODUCT_VALIDATION_CHECKS)) + # Resolve and setup per-module dex-preopt configs. DEXPREOPT_DISABLED_MODULES := # If a module has multiple setups, the first takes precedence. @@ -477,15 +525,6 @@ ifeq ($(PRODUCT_SET_DEBUGFS_RESTRICTIONS),) endif endif -ifdef PRODUCT_SHIPPING_API_LEVEL - ifneq (,$(call math_gt_or_eq,29,$(PRODUCT_SHIPPING_API_LEVEL))) - PRODUCT_PACKAGES += $(PRODUCT_PACKAGES_SHIPPING_API_LEVEL_29) - endif - ifneq (,$(call math_gt_or_eq,33,$(PRODUCT_SHIPPING_API_LEVEL))) - PRODUCT_PACKAGES += $(PRODUCT_PACKAGES_SHIPPING_API_LEVEL_33) - endif -endif - # If build command defines OVERRIDE_PRODUCT_EXTRA_VNDK_VERSIONS, # override PRODUCT_EXTRA_VNDK_VERSIONS with it. ifdef OVERRIDE_PRODUCT_EXTRA_VNDK_VERSIONS @@ -518,7 +557,8 @@ ifdef OVERRIDE_PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE := $(OVERRIDE_PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE) endif else ifeq ($(PRODUCT_SHIPPING_API_LEVEL),) - # No shipping level defined + # No shipping level defined. Enforce the product interface by default. + PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE := true else ifeq ($(call math_gt,$(PRODUCT_SHIPPING_API_LEVEL),29),true) # Enforce product interface if PRODUCT_SHIPPING_API_LEVEL is greater than 29. PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE := true @@ -526,33 +566,58 @@ endif $(KATI_obsolete_var OVERRIDE_PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE,Use PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE instead) -# If build command defines PRODUCT_USE_PRODUCT_VNDK_OVERRIDE as `false`, -# PRODUCT_PRODUCT_VNDK_VERSION will not be defined automatically. -# PRODUCT_USE_PRODUCT_VNDK_OVERRIDE can be used for testing only. -PRODUCT_USE_PRODUCT_VNDK := false -ifneq ($(PRODUCT_USE_PRODUCT_VNDK_OVERRIDE),) - PRODUCT_USE_PRODUCT_VNDK := $(PRODUCT_USE_PRODUCT_VNDK_OVERRIDE) -else ifeq ($(PRODUCT_SHIPPING_API_LEVEL),) - # No shipping level defined -else ifeq ($(call math_gt,$(PRODUCT_SHIPPING_API_LEVEL),29),true) - # Enforce product interface for VNDK if PRODUCT_SHIPPING_API_LEVEL is greater - # than 29. - PRODUCT_USE_PRODUCT_VNDK := true +# From Android V, Define PRODUCT_PRODUCT_VNDK_VERSION as current by default. +# This is required to make all devices have product variants. +ifndef PRODUCT_PRODUCT_VNDK_VERSION + PRODUCT_PRODUCT_VNDK_VERSION := current endif -ifeq ($(PRODUCT_USE_PRODUCT_VNDK),true) - ifndef PRODUCT_PRODUCT_VNDK_VERSION - PRODUCT_PRODUCT_VNDK_VERSION := current +ifdef PRODUCT_ENFORCE_RRO_EXEMPTED_TARGETS + $(error PRODUCT_ENFORCE_RRO_EXEMPTED_TARGETS is deprecated, consider using RRO for \ + $(PRODUCT_ENFORCE_RRO_EXEMPTED_TARGETS)) +endif + +# This table maps sdk version 35 to vendor api level 202404 and assumes yearly +# release for the same month. +define sdk-to-vendor-api-level + $(if $(call math_lt_or_eq,$(1),34),$(1),20$(call int_subtract,$(1),11)04) +endef + +ifdef PRODUCT_SHIPPING_VENDOR_API_LEVEL +# Follow the version that is set manually. + VSR_VENDOR_API_LEVEL := $(PRODUCT_SHIPPING_VENDOR_API_LEVEL) +else + # VSR API level is the vendor api level of the product shipping API level. + VSR_VENDOR_API_LEVEL := $(call sdk-to-vendor-api-level,$(PLATFORM_SDK_VERSION)) + ifdef PRODUCT_SHIPPING_API_LEVEL + VSR_VENDOR_API_LEVEL := $(call sdk-to-vendor-api-level,$(PRODUCT_SHIPPING_API_LEVEL)) + endif + ifdef BOARD_SHIPPING_API_LEVEL + # Vendors with GRF must define BOARD_SHIPPING_API_LEVEL for the vendor API level. + # In this case, the VSR API level is the minimum of the PRODUCT_SHIPPING_API_LEVEL + # and RELEASE_BOARD_API_LEVEL + VSR_VENDOR_API_LEVEL := $(call math_min,$(VSR_VENDOR_API_LEVEL),$(RELEASE_BOARD_API_LEVEL)) endif endif +.KATI_READONLY := VSR_VENDOR_API_LEVEL -$(KATI_obsolete_var PRODUCT_USE_PRODUCT_VNDK,Use PRODUCT_PRODUCT_VNDK_VERSION instead) -$(KATI_obsolete_var PRODUCT_USE_PRODUCT_VNDK_OVERRIDE,Use PRODUCT_PRODUCT_VNDK_VERSION instead) +# Boolean variable determining if vendor seapp contexts is enforced +CHECK_VENDOR_SEAPP_VIOLATIONS := false +ifneq ($(call math_gt,$(VSR_VENDOR_API_LEVEL),34),) + CHECK_VENDOR_SEAPP_VIOLATIONS := true +else ifneq ($(PRODUCT_CHECK_VENDOR_SEAPP_VIOLATIONS),) + CHECK_VENDOR_SEAPP_VIOLATIONS := $(PRODUCT_CHECK_VENDOR_SEAPP_VIOLATIONS) +endif +.KATI_READONLY := CHECK_VENDOR_SEAPP_VIOLATIONS -ifdef PRODUCT_ENFORCE_RRO_EXEMPTED_TARGETS - $(error PRODUCT_ENFORCE_RRO_EXEMPTED_TARGETS is deprecated, consider using RRO for \ - $(PRODUCT_ENFORCE_RRO_EXEMPTED_TARGETS)) +# Boolean variable determining if selinux labels of /dev are enforced +CHECK_DEV_TYPE_VIOLATIONS := false +ifneq ($(call math_gt,$(VSR_VENDOR_API_LEVEL),202404),) + CHECK_DEV_TYPE_VIOLATIONS := true +else ifneq ($(PRODUCT_CHECK_DEV_TYPE_VIOLATIONS),) + CHECK_DEV_TYPE_VIOLATIONS := $(PRODUCT_CHECK_DEV_TYPE_VIOLATIONS) endif +.KATI_READONLY := CHECK_DEV_TYPE_VIOLATIONS define product-overrides-config $$(foreach rule,$$(PRODUCT_$(1)_OVERRIDES),\ @@ -573,6 +638,15 @@ ifneq ($$(filter-out true false,$$(PRODUCT_BUILD_$(1)_IMAGE)),) endif endef +ifndef PRODUCT_VIRTUAL_AB_COW_VERSION + PRODUCT_VIRTUAL_AB_COW_VERSION := 2 + ifdef PRODUCT_SHIPPING_API_LEVEL + ifeq (true,$(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),34)) + PRODUCT_VIRTUAL_AB_COW_VERSION := 3 + endif + endif +endif + # Copy and check the value of each PRODUCT_BUILD_*_IMAGE variable $(foreach image, \ PVMFW \ diff --git a/core/product_config.rbc b/core/product_config.rbc index 97c1d00471..59e2c95903 100644 --- a/core/product_config.rbc +++ b/core/product_config.rbc @@ -54,25 +54,16 @@ def __print_attr(attr, value): if value == None: return if type(value) == "list": - if _options.rearrange: - value = __printvars_rearrange_list(value) - if _options.format == "pretty": - print(attr, "=", repr(value)) - elif _options.format == "make": - value = list(value) - for i, x in enumerate(value): - if type(x) == "tuple" and len(x) == 1: - value[i] = "@inherit:" + x[0] + ".mk" - elif type(x) != "string": - fail("Wasn't a list of strings:", attr, " value:", value) - print(attr, ":=", " ".join(value)) - elif _options.format == "pretty": - print(attr, "=", repr(value)) - elif _options.format == "make": + value = list(value) + for i, x in enumerate(value): + if type(x) == "tuple" and len(x) == 1: + value[i] = "@inherit:" + x[0] + ".mk" + elif type(x) != "string": + fail("Wasn't a list of strings:", attr, " value:", value) + print(attr, ":=", " ".join(value)) + else: # Trim all spacing to a single space print(attr, ":=", _mkstrip(value)) - else: - fail("bad output format", _options.format) def _printvars(state): """Prints configuration and global variables.""" @@ -83,8 +74,7 @@ def _printvars(state): for nsname, nsvars in sorted(val.items()): # Define SOONG_CONFIG_<ns> for Make, othewise # it cannot be added to .KATI_READONLY list - if _options.format == "make": - print("SOONG_CONFIG_" + nsname, ":=", " ".join(nsvars.keys())) + print("SOONG_CONFIG_" + nsname, ":=", " ".join(nsvars.keys())) for var, val in sorted(nsvars.items()): if val: __print_attr("SOONG_CONFIG_%s_%s" % (nsname, var), val) @@ -105,11 +95,6 @@ def _printvars(state): elif attr not in globals_base or globals_base[attr] != val: __print_attr(attr, val) -def __printvars_rearrange_list(value_list): - """Rearrange value list: return only distinct elements, maybe sorted.""" - seen = {item: 0 for item in value_list} - return sorted(seen.keys()) if _options.rearrange == "sort" else seen.keys() - def __sort_pcm_names(pcm_names): # We have to add an extension back onto the pcm names when sorting, # or else the sort order could be wrong when one is a prefix of another. @@ -366,6 +351,7 @@ def _percolate_inherited(configs, cfg_name, cfg, children_names): if cfg.get(attr, "") == "": cfg[attr] = value percolated_attrs[attr] = True + child_cfg.pop(attr) for attr in _options.trace_variables: if attr in percolated_attrs: @@ -375,7 +361,7 @@ def __move_items(to_list, from_cfg, attr): value = from_cfg.get(attr, []) if value: to_list.extend(value) - from_cfg[attr] = [] + from_cfg.pop(attr) def _indirect(pcm_name): """Returns configuration item for the inherited module.""" @@ -394,7 +380,7 @@ def _soong_config_namespace(g, nsname): def _soong_config_set(g, nsname, var, value): """Assigns the value to the variable in the namespace.""" _soong_config_namespace(g, nsname) - g[_soong_config_namespaces_key][nsname][var]=value + g[_soong_config_namespaces_key][nsname][var]=_mkstrip(value) def _soong_config_append(g, nsname, var, value): """Appends to the value of the variable in the namespace.""" @@ -402,9 +388,9 @@ def _soong_config_append(g, nsname, var, value): ns = g[_soong_config_namespaces_key][nsname] oldv = ns.get(var) if oldv == None: - ns[var] = value + ns[var] = _mkstrip(value) else: - ns[var] += " " + value + ns[var] += " " + _mkstrip(value) def _soong_config_get(g, nsname, var): @@ -691,16 +677,8 @@ def _mkwarning(file, message = ""): rblf_log(file, "warning", message, sep = ':') def _mk2rbc_error(loc, message): - """Prints a message about conversion error and stops. - - If RBC_MK2RBC_CONTINUE environment variable is set, - the execution will continue after the message is printed. - """ - if _options.mk2rbc_continue: - rblf_log(loc, message, sep = ':') - else: - _mkerror(loc, message) - + """Prints a message about conversion error and stops.""" + _mkerror(loc, message) def _mkinfo(file, message = ""): """Prints info.""" @@ -873,39 +851,12 @@ def _clear_var_list(g, h, var_list): # Cause the variable to appear set like the make version does g[v] = "" - -def __get_options(): - """Returns struct containing runtime global settings.""" - settings = dict( - format = "pretty", - rearrange = "", - trace_modules = False, - trace_variables = [], - mk2rbc_continue = False, - ) - for x in getattr(rblf_cli, "RBC_OUT", "").split(","): - if x == "sort" or x == "unique": - if settings["rearrange"]: - fail("RBC_OUT: either sort or unique is allowed (and sort implies unique)") - settings["rearrange"] = x - elif x == "pretty" or x == "make": - settings["format"] = x - elif x == "global": - # TODO: Remove this, kept for backwards compatibility - pass - elif x != "": - fail("RBC_OUT: got %s, should be one of: [pretty|make] [sort|unique]" % x) - for x in getattr(rblf_cli, "RBC_DEBUG", "").split(","): - if x == "!trace": - settings["trace_modules"] = True - elif x != "": - settings["trace_variables"].append(x) - if getattr(rblf_cli, "RBC_MK2RBC_CONTINUE", ""): - settings["mk2rbc_continue"] = True - return struct(**settings) - # Settings used during debugging. -_options = __get_options() +_options = struct( + trace_modules = False, + trace_variables = [], +) + rblf = struct( soong_config_namespace = _soong_config_namespace, soong_config_append = _soong_config_append, diff --git a/core/product_validation_checks.mk b/core/product_validation_checks.mk new file mode 100644 index 0000000000..e0d976f156 --- /dev/null +++ b/core/product_validation_checks.mk @@ -0,0 +1,72 @@ +# PRODUCT_VALIDATION_CHECKS allows you to enforce that your product config variables follow some +# rules. To use it, add the paths to starlark configuration language (scl) files in +# PRODUCT_VALIDATION_CHECKS. A validate_product_variables function in those files will be called +# with a single "context" object. +# +# The context object currently 2 attributes: +# - product_variables: This has all the product variables. All the variables are either of type +# string or list, more accurate typing (like bool) isn't known. +# - board_variables: This only has a small subset of the board variables, because there isn't a +# known list of board variables. Feel free to expand the subset if you need a +# new variable. +# +# You can then inspect (but not modify) these variables and fail() if they don't meet your +# requirements. Example: +# +# In a product config file: PRODUCT_VALIDATION_CHECKS += //path/to/my_validations.scl +# In my_validations.scl: +# def validate_product_variables(ctx): +# for dir in ctx.board_variables.BOARD_SEPOLICY_DIRS: +# if not dir.startswith('system/sepolicy/'): +# fail('Only sepolicies in system/seplicy are allowed, found: ' + dir) + +ifdef PRODUCT_VALIDATION_CHECKS + +$(if $(filter-out //%.scl,$(PRODUCT_VALIDATION_CHECKS)), \ + $(error All PRODUCT_VALIDATION_CHECKS files must start with // and end with .scl, exceptions: $(filter-out //%.scl,$(PRODUCT_VALIDATION_CHECKS)))) + +known_board_variables := \ + BOARD_VENDOR_SEPOLICY_DIRS BOARD_SEPOLICY_DIRS \ + SYSTEM_EXT_PRIVATE_SEPOLICY_DIRS \ + SYSTEM_EXT_PUBLIC_SEPOLICY_DIRS \ + PRODUCT_PUBLIC_SEPOLICY_DIRS \ + PRODUCT_PRIVATE_SEPOLICY_DIRS + +known_board_list_variables := \ + BOARD_VENDOR_SEPOLICY_DIRS BOARD_SEPOLICY_DIRS \ + SYSTEM_EXT_PRIVATE_SEPOLICY_DIRS \ + SYSTEM_EXT_PUBLIC_SEPOLICY_DIRS \ + PRODUCT_PUBLIC_SEPOLICY_DIRS \ + PRODUCT_PRIVATE_SEPOLICY_DIRS + +escape_starlark_string=$(subst ",\",$(subst \,\\,$(1))) +product_variable_starlark_value=$(if $(filter $(1),$(_product_list_vars) $(known_board_list_variables)),[$(foreach w,$($(1)),"$(call escape_starlark_string,$(w))", )],"$(call escape_starlark_string,$(1))") +filename_to_starlark=$(subst -,_,$(subst /,_,$(subst .,_,$(1)))) +_c:=$(foreach f,$(PRODUCT_VALIDATION_CHECKS),$(newline)load("$(f)", validate_product_variables_$(call filename_to_starlark,$(f)) = "validate_product_variables")) +# TODO: we should freeze the context because it contains mutable lists, so that validation checks can't affect each other +_c+=$(newline)_ctx = struct( +_c+=$(newline)product_variables = struct( +_c+=$(foreach v,$(_product_var_list),$(newline) $(v) = $(call product_variable_starlark_value,$(v)),) +_c+=$(newline)), +_c+=$(newline)board_variables = struct( +_c+=$(foreach v,$(known_board_variables),$(newline) $(v) = $(call product_variable_starlark_value,$(v)),) +_c+=$(newline)) +_c+=$(newline)) +_c+=$(foreach f,$(PRODUCT_VALIDATION_CHECKS),$(newline)validate_product_variables_$(call filename_to_starlark,$(f))(_ctx)) +_c+=$(newline)variables_to_export_to_make = {} +$(KATI_file_no_rerun >$(OUT_DIR)/product_validation_checks_entrypoint.scl,$(_c)) +filename_to_starlark:= +escape_starlark_string:= +product_variable_starlark_value:= +known_board_variables := +known_board_list_variables := + +# Exclude the entrypoint file as a dependency (by passing it as the 2nd argument) so that we don't +# rerun kati every build. Even though we're using KATI_file_no_rerun, product config is run every +# build, so the file will still be rewritten. +# +# We also need to pass --allow_external_entrypoint to rbcrun in case the OUT_DIR is set to something +# outside of the source tree. +$(call run-starlark,$(OUT_DIR)/product_validation_checks_entrypoint.scl,$(OUT_DIR)/product_validation_checks_entrypoint.scl,--allow_external_entrypoint) + +endif # ifdef PRODUCT_VALIDATION_CHECKS diff --git a/core/proguard.flags b/core/proguard.flags index d790061524..aa406b983e 100644 --- a/core/proguard.flags +++ b/core/proguard.flags @@ -15,6 +15,13 @@ @com.android.internal.annotations.VisibleForTesting *; } +# Keep classes and members with platform @TestApi annotations, similar to +# @VisibleForTesting. +-keep @android.annotation.TestApi class * +-keepclassmembers class * { + @android.annotation.TestApi *; +} + # Keep classes and members with non-platform @VisibleForTesting annotations, but # only within platform-defined packages. This avoids keeping external, library-specific # test code that isn't actually needed for platform testing. @@ -51,4 +58,14 @@ @**android**.annotation*.Keep <init>(...); } +# Keep Dalvik optimization annotations. These annotations are special in that +# 1) we want them preserved for visibility with ART, but 2) they don't have +# RUNTIME retention. These minimal keep rules ensure they're not stripped by R8. +# TODO(b/215417388): Export this rule from the owning library, core-libart, +# via export_proguard_flags_files. +-keepclassmembers,allowshrinking,allowoptimization,allowobfuscation,allowaccessmodification class * { + @dalvik.annotation.optimization.** *; +} + -include proguard_basic_keeps.flags +-include proguard/kotlin.flags diff --git a/core/proguard/kotlin.flags b/core/proguard/kotlin.flags new file mode 100644 index 0000000000..ef6bf0e9e3 --- /dev/null +++ b/core/proguard/kotlin.flags @@ -0,0 +1,21 @@ +# Ignore missing Kotlin meta-annotations so that Java-only projects can depend +# on projects that happen to be written in Kotlin but do not have a run-time +# dependency on the Kotlin standard library. Note these annotations are RUNTIME +# retention, but we won't need them available in Java-only projects. +-dontwarn kotlin.Metadata +-dontwarn kotlin.annotation.AnnotationRetention +-dontwarn kotlin.annotation.AnnotationTarget +-dontwarn kotlin.annotation.Retention +-dontwarn kotlin.annotation.Target + +# Kotlin DebugMetadata has no value in release builds, these two rules, will +# allow AppReduce to strip out DebutMetadata. +# TODO(b/302383328): Restore the below checkdiscard after resolving transitive +# inclusion of kotlin-stdlib from androidx.annotation library deps. +# -checkdiscard interface kotlin.coroutines.jvm.internal.DebugMetadata +-assumenosideeffects class kotlin.coroutines.jvm.internal.DebugMetadataKt { + *** getDebugMetadataAnnotation(...); +} +-assumevalues class kotlin.coroutines.jvm.internal.DebugMetadataKt { + *** getDebugMetadataAnnotation(...) return null; +} diff --git a/core/proguard_basic_keeps.flags b/core/proguard_basic_keeps.flags index 7e7b2701ca..f6b34b8217 100644 --- a/core/proguard_basic_keeps.flags +++ b/core/proguard_basic_keeps.flags @@ -10,6 +10,12 @@ # and RuntimeVisibleTypeAnnotations, as well as associated defaults. -keepattributes RuntimeVisible*Annotation*,AnnotationDefault +# With R8 full mode, certain attributes are only kept when matched with an +# explicit keep rule for that target, even with a global -keepattributes rule. +# As such, we can add the global keep rule here with minimal cost while +# simplifying incremental development. +-keepattributes Exceptions + # For enumeration classes, see http://proguard.sourceforge.net/manual/examples.html#enumerations -keepclassmembers enum * { public static **[] values(); @@ -41,6 +47,11 @@ java.lang.Object readResolve(); } +# Keep all Javascript API methods +-keepclassmembers class * { + @android.webkit.JavascriptInterface <methods>; +} + # Keep Throwable's constructor that takes a String argument. -keepclassmembers class * extends java.lang.Throwable { <init>(java.lang.String); diff --git a/core/python_binary_host_mobly_test_config_template.xml b/core/python_binary_host_mobly_test_config_template.xml new file mode 100644 index 0000000000..a986df2db1 --- /dev/null +++ b/core/python_binary_host_mobly_test_config_template.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Config for {MODULE} mobly test"> + {EXTRA_CONFIGS} + + <device name="AndroidRealDevice"></device> + <device name="AndroidRealDevice"></device> + + <option name="mobly_pkg" key="file" value="{MODULE}" /> + <test class="MoblyAospPackageTest" /> +</configuration> diff --git a/core/ravenwood_test_config_template.xml b/core/ravenwood_test_config_template.xml new file mode 100644 index 0000000000..16a22c0628 --- /dev/null +++ b/core/ravenwood_test_config_template.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- This test config file is auto-generated. --> +<configuration description="Runs {MODULE}"> + <option name="test-suite-tag" value="ravenwood" /> + <option name="test-suite-tag" value="ravenwood-tests" /> + + <option name="java-folder" value="prebuilts/jdk/jdk17/linux-x86/" /> + <option name="use-ravenwood-resources" value="true" /> + <option name="exclude-paths" value="java" /> + <option name="socket-timeout" value="10000" /> + <option name="null-device" value="true" /> + + {EXTRA_CONFIGS} + + <test class="com.android.tradefed.testtype.IsolatedHostTest" > + <option name="jar" value="{MODULE}.jar" /> + <option name="java-flags" value="--add-modules=jdk.compiler"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED"/> + <option name="java-flags" value="--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"/> + + <!-- Needed for supporting ParcelFileDescriptor internals --> + <option name="java-flags" value="--add-exports=java.base/jdk.internal.access=ALL-UNNAMED"/> + </test> +</configuration> diff --git a/core/rbe.mk b/core/rbe.mk index 6754b0a171..0f90ddd6f9 100644 --- a/core/rbe.mk +++ b/core/rbe.mk @@ -64,7 +64,7 @@ ifneq ($(filter-out false,$(USE_RBE)),) d8_exec_strategy := remote_local_fallback endif - platform := container-image=docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62 + platform := container-image=docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:1eb7f64b9e17102b970bd7a1af7daaebdb01c3fb777715899ef462d6c6d01a45 cxx_platform := $(platform),Pool=$(cxx_pool) java_r8_d8_platform := $(platform),Pool=$(java_pool) diff --git a/core/release_config.mk b/core/release_config.mk new file mode 100644 index 0000000000..bb519804d1 --- /dev/null +++ b/core/release_config.mk @@ -0,0 +1,344 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----------------------------------------------------------------- +# Choose the flag files +# ----------------------------------------------------------------- +# Release configs are defined in reflease_config_map files, which map +# the short name (e.g. -next) used in lunch to the starlark files +# defining the build flag values. +# +# (If you're thinking about aconfig flags, there is one build flag, +# RELEASE_ACONFIG_VALUE_SETS, that sets which aconfig_value_set +# module to use to set the aconfig flag values.) +# +# The short release config names *can* appear multiple times, to allow +# for AOSP and vendor specific flags under the same name, but the +# individual flag values must appear in exactly one config. Vendor +# does not override AOSP, or anything like that. This is because +# vendor code usually includes prebuilts, and having vendor compile +# with different flags from AOSP increases the likelihood of flag +# mismatch. + +# Do this first, because we're going to unset TARGET_RELEASE before +# including anyone, so they don't start making conditionals based on it. +# This logic is in make because starlark doesn't understand optional +# vendor files. + +# If this is a google source tree, restrict it to only the one file +# which has OWNERS control. If it isn't let others define their own. +# TODO: Remove wildcard for build/release one when all branch manifests +# have updated. +_must_protobuf := +config_map_files := $(wildcard build/release/release_config_map.mk) \ + $(wildcard vendor/google_shared/build/release/release_config_map.mk) \ + $(if $(wildcard vendor/google/release/release_config_map.mk), \ + vendor/google/release/release_config_map.mk, \ + $(sort \ + $(wildcard device/*/release/release_config_map.mk) \ + $(wildcard device/*/*/release/release_config_map.mk) \ + $(wildcard vendor/*/release/release_config_map.mk) \ + $(wildcard vendor/*/*/release/release_config_map.mk) \ + ) \ + ) + +protobuf_map_files := $(wildcard build/release/release_config_map.textproto) \ + $(wildcard vendor/google_shared/build/release/release_config_map.textproto) \ + $(if $(wildcard vendor/google/release/release_config_map.textproto), \ + vendor/google/release/release_config_map.textproto, \ + $(sort \ + $(wildcard device/*/release/release_config_map.textproto) \ + $(wildcard device/*/*/release/release_config_map.textproto) \ + $(wildcard vendor/*/release/release_config_map.textproto) \ + $(wildcard vendor/*/*/release/release_config_map.textproto) \ + ) \ + ) + +# PRODUCT_RELEASE_CONFIG_MAPS is set by Soong using an initial run of product +# config to capture only the list of config maps needed by the build. +# Keep them in the order provided, but remove duplicates. +# Treat .mk and .textproto as equal for duplicate elimination, but force +# protobuf if any PRODUCT_RELEASE_CONFIG_MAPS specify .textproto. +$(foreach map,$(PRODUCT_RELEASE_CONFIG_MAPS), \ + $(if $(filter $(basename $(map)),$(basename $(config_map_files))),, \ + $(eval config_map_files += $(map))) \ + $(if $(filter $(basename $(map)).textproto,$(map)),$(eval _must_protobuf := true)) \ +) + + +# If we are missing the textproto version of any of $(config_map_files), we cannot use protobuf. +_can_protobuf := true +$(foreach map,$(config_map_files), \ + $(if $(wildcard $(basename $(map)).textproto),,$(eval _can_protobuf :=)) \ +) +# If we are missing the mk version of any of $(protobuf_map_files), we must use protobuf. +$(foreach map,$(protobuf_map_files), \ + $(if $(wildcard $(basename $(map)).mk),,$(eval _must_protobuf := true)) \ +) + +ifneq (,$(_must_protobuf)) + ifeq (,$(_can_protobuf)) + # We must use protobuf, but we cannot use protobuf. + $(error release config is a mixture of .scl and .textproto) + endif +endif + +_use_protobuf := +ifneq (,$(_must_protobuf)) + _use_protobuf := true +else + ifneq ($(_can_protobuf),) + # Determine the default + $(foreach map,$(config_map_files), \ + $(if $(wildcard $(dir $(map))/build_config/DEFAULT=proto),$(eval _use_protobuf := true)) \ + $(if $(wildcard $(dir $(map))/build_config/DEFAULT=make),$(eval _use_protobuf := )) \ + ) + # Update for this specific release config only (no inheritance). + $(foreach map,$(config_map_files), \ + $(if $(wildcard $(dir $(map))/build_config/$(TARGET_RELEASE)=proto),$(eval _use_protobuf := true)) \ + $(if $(wildcard $(dir $(map))/build_config/$(TARGET_RELEASE)=make),$(eval _use_protobuf := )) \ + ) + endif +endif + +ifneq (,$(_use_protobuf)) + # The .textproto files are the canonical source of truth. + _args := $(foreach map,$(config_map_files), --map $(map) ) + ifneq (,$(_must_protobuf)) + # Disable the build flag in release-config. + _args += --guard=false + endif + $(KATI_shell_no_rerun $(OUT_DIR)/release-config $(_args) >$(OUT_DIR)/release-config.out && touch -t 200001010000 $(OUT_DIR)/release-config.out) + $(if $(filter-out 0,$(.SHELLSTATUS)),$(error release-config failed to run)) + # This will also set _all_release_configs for us. + $(eval include $(OUT_DIR)/soong/release-config/release_config-$(TARGET_PRODUCT)-$(TARGET_RELEASE).mk) + $(KATI_extra_file_deps $(OUT_DIR)/release-config $(config_map_files)) + ifeq (,$(_must_protobuf)$(RELEASE_BUILD_FLAGS_IN_PROTOBUF)) + _use_protobuf := + endif +endif +ifeq (,$(_use_protobuf)) + # The .mk files are the canonical source of truth. + + +# Declare an alias release-config +# +# This should be used to declare a release as an alias of another, meaning no +# release config files should be present. +# +# $1 config name +# $2 release config for which it is an alias +define alias-release-config + $(call _declare-release-config,$(1),,$(2),true) +endef + +# Declare or extend a release-config. +# +# The order of processing is: +# 1. Recursively apply any overridden release configs. Only apply each config +# the first time we reach it. +# 2. Apply any files for this release config, in the order they were added to +# the declaration. +# +# Example: +# With these declarations: +# $(declare-release-config foo, foo.scl) +# $(declare-release-config bar, bar.scl, foo) +# $(declare-release-config baz, baz.scl, bar) +# $(declare-release-config bif, bif.scl, foo baz) +# $(declare-release-config bop, bop.scl, bar baz) +# +# TARGET_RELEASE: +# - bar will use: foo.scl bar.scl +# - baz will use: foo.scl bar.scl baz.scl +# - bif will use: foo.scl bar.scl baz.scl bif.scl +# - bop will use: foo.scl bar.scl baz.scl bop.scl +# +# $1 config name +# $2 release config files +# $3 overridden release config +define declare-release-config + $(call _declare-release-config,$(1),$(2),$(3),) +endef + +define _declare-release-config + $(if $(strip $(2)$(3)),, \ + $(error declare-release-config: config $(strip $(1)) must have release config files, override another release config, or both) \ + ) + $(if $(strip $(4)),$(eval _all_release_configs.$(strip $(1)).ALIAS := true)) + $(eval _all_release_configs := $(sort $(_all_release_configs) $(strip $(1)))) + $(if $(strip $(3)), \ + $(if $(filter $(_all_release_configs), $(strip $(3))), + $(if $(filter $(_all_release_configs.$(strip $(1)).OVERRIDES),$(strip $(3))),, + $(eval _all_release_configs.$(strip $(1)).OVERRIDES := $(_all_release_configs.$(strip $(1)).OVERRIDES) $(strip $(3)))), \ + $(error No release config $(strip $(3))) \ + ) \ + ) + $(eval _all_release_configs.$(strip $(1)).DECLARED_IN := $(_included) $(_all_release_configs.$(strip $(1)).DECLARED_IN)) + $(eval _all_release_configs.$(strip $(1)).FILES := $(_all_release_configs.$(strip $(1)).FILES) $(strip $(2))) +endef + +# Include the config map files and populate _flag_declaration_files. +# If the file is found more than once, only include it the first time. +_flag_declaration_files := +_included_config_map_files := +$(foreach f, $(config_map_files), \ + $(eval FLAG_DECLARATION_FILES:= ) \ + $(if $(filter $(_included_config_map_files),$(f)),,\ + $(eval _included := $(f)) \ + $(eval include $(f)) \ + $(eval _flag_declaration_files += $(FLAG_DECLARATION_FILES)) \ + $(eval _included_config_map_files += $(f)) \ + ) \ +) +FLAG_DECLARATION_FILES := + +# Verify that all inherited/overridden release configs are declared. +$(foreach config,$(_all_release_configs),\ + $(foreach r,$(all_release_configs.$(r).OVERRIDES),\ + $(if $(strip $(_all_release_configs.$(r).FILES)$(_all_release_configs.$(r).OVERRIDES)),,\ + $(error Release config $(config) [declared in: $(_all_release_configs.$(r).DECLARED_IN)] inherits from non-existent $(r).)\ +))) +# Verify that alias configs do not have config files. +$(foreach r,$(_all_release_configs),\ + $(if $(_all_release_configs.$(r).ALIAS),$(if $(_all_release_configs.$(r).FILES),\ + $(error Alias release config "$(r)" may not specify release config files $(_all_release_configs.$(r).FILES))\ +))) + +# Use makefiles +endif + +ifeq ($(TARGET_RELEASE),) + # We allow some internal paths to explicitly set TARGET_RELEASE to the + # empty string. For the most part, 'make' treats unset and empty string as + # the same. But the following line differentiates, and will only assign + # if the variable was completely unset. + TARGET_RELEASE ?= was_unset + ifeq ($(TARGET_RELEASE),was_unset) + $(error No release config set for target; please set TARGET_RELEASE, or if building on the command line use 'lunch <target>-<release>-<build_type>', where release is one of: $(_all_release_configs)) + endif + # Instead of leaving this string empty, we want to default to a valid + # setting. Full builds coming through this path is a bug, but in case + # of such a bug, we want to at least get consistent, valid results. + TARGET_RELEASE = trunk_staging +endif + +# During pass 1 of product config, using a non-existent release config is not an error. +# We can safely assume that we are doing pass 1 if DUMP_MANY_VARS=="PRODUCT_RELEASE_CONFIG_MAPS". +ifneq (PRODUCT_RELEASE_CONFIG_MAPS,$(DUMP_MANY_VARS)) + ifeq ($(filter $(_all_release_configs), $(TARGET_RELEASE)),) + $(error No release config found for TARGET_RELEASE: $(TARGET_RELEASE). Available releases are: $(_all_release_configs)) + endif +endif + +ifeq (,$(_use_protobuf)) +# Choose flag files +# Don't sort this, use it in the order they gave us. +# Do allow duplicate entries, retaining only the first usage. +flag_value_files := + +# Apply overrides recursively +# +# $1 release config that we override +applied_releases := +define _apply-release-config-overrides +$(foreach r,$(1), \ + $(if $(filter $(r),$(applied_releases)),, \ + $(foreach o,$(_all_release_configs.$(r).OVERRIDES),$(call _apply-release-config-overrides,$(o)))\ + $(eval applied_releases += $(r))\ + $(foreach f,$(_all_release_configs.$(r).FILES), \ + $(if $(filter $(f),$(flag_value_files)),,$(eval flag_value_files += $(f)))\ + )\ + )\ +) +endef +$(call _apply-release-config-overrides,$(TARGET_RELEASE)) +# Unset variables so they can't use them +define declare-release-config +$(error declare-release-config can only be called from inside release_config_map.mk files) +endef +define _apply-release-config-overrides +$(error invalid use of apply-release-config-overrides) +endef + +# use makefiles +endif + +# TODO: Remove this check after enough people have sourced lunch that we don't +# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023 +ifneq ($(CALLED_FROM_SETUP),true) +define TARGET_RELEASE +$(error TARGET_RELEASE may not be accessed directly. Use individual flags.) +endef +else +TARGET_RELEASE:= +endif +.KATI_READONLY := TARGET_RELEASE + +ifeq (,$(_use_protobuf)) +$(foreach config, $(_all_release_configs), \ + $(eval _all_release_configs.$(config).DECLARED_IN:= ) \ + $(eval _all_release_configs.$(config).FILES:= ) \ +) +applied_releases:= +# use makefiles +endif +_all_release_configs:= +config_map_files:= +protobuf_map_files:= + + +ifeq (,$(_use_protobuf)) +# ----------------------------------------------------------------- +# Flag declarations and values +# ----------------------------------------------------------------- +# This part is in starlark. We generate a root starlark file that loads +# all of the flags declaration files that we found, and the flag_value_files +# that we chose from the config map above. Then we run that, and load the +# results of that into the make environment. + +# _flag_declaration_files is the combined list of FLAG_DECLARATION_FILES set by +# release_config_map.mk files above. + +# Because starlark can't find files with $(wildcard), write an entrypoint starlark script that +# contains the result of the above wildcards for the starlark code to use. +filename_to_starlark=$(subst /,_,$(subst .,_,$(1))) +_c:=load("//build/make/core/release_config.scl", "release_config") +_c+=$(newline)def add(d, k, v): +_c+=$(newline)$(space)d = dict(d) +_c+=$(newline)$(space)d[k] = v +_c+=$(newline)$(space)return d +_c+=$(foreach f,$(_flag_declaration_files),$(newline)load("$(f)", flags_$(call filename_to_starlark,$(f)) = "flags")) +_c+=$(newline)all_flags = [] $(foreach f,$(_flag_declaration_files),+ [add(x, "declared_in", "$(f)") for x in flags_$(call filename_to_starlark,$(f))]) +_c+=$(foreach f,$(flag_value_files),$(newline)load("//$(f)", values_$(call filename_to_starlark,$(f)) = "values")) +_c+=$(newline)all_values = [] $(foreach f,$(flag_value_files),+ [add(x, "set_in", "$(f)") for x in values_$(call filename_to_starlark,$(f))]) +_c+=$(newline)variables_to_export_to_make = release_config(all_flags, all_values) +$(file >$(OUT_DIR)/release_config_entrypoint.scl,$(_c)) +_c:= +filename_to_starlark:= + +# Exclude the entrypoint file as a dependency (by passing it as the 2nd argument) so that we don't +# rerun kati every build. Kati will replay the $(file) command that generates it every build, +# updating its timestamp. +# +# We also need to pass --allow_external_entrypoint to rbcrun in case the OUT_DIR is set to something +# outside of the source tree. +$(call run-starlark,$(OUT_DIR)/release_config_entrypoint.scl,$(OUT_DIR)/release_config_entrypoint.scl,--allow_external_entrypoint) + +# use makefiles +endif +_can_protobuf := +_must_protobuf := +_use_protobuf := diff --git a/core/release_config.scl b/core/release_config.scl new file mode 100644 index 0000000000..c5815dfe30 --- /dev/null +++ b/core/release_config.scl @@ -0,0 +1,243 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Export build flags (with values) to make. +""" + +load("//build/bazel/utils:schema_validation.scl", "validate") + +# Partitions that get build system flag summaries +_flag_partitions = [ + "product", + "system", + "system_ext", + "vendor", +] + +ALL = ["all"] +PRODUCT = ["product"] +SYSTEM = ["system"] +SYSTEM_EXT = ["system_ext"] +VENDOR = ["vendor"] + +_valid_types = ["NoneType", "bool", "list", "string", "int"] + +_all_flags_schema = { + "type": "list", + "of": { + "type": "dict", + "required_keys": { + "name": {"type": "string"}, + "partitions": { + "type": "list", + "of": { + "type": "string", + "choices": _flag_partitions + ["all"], + }, + "unique": True, + }, + "default": { + "or": [ + {"type": t} + for t in _valid_types + ], + }, + "origin": {"type": "string"}, + "declared_in": {"type": "string"}, + }, + "optional_keys": { + "appends": { + "type": "bool", + }, + }, + }, +} + +_all_values_schema = { + "type": "list", + "of": { + "type": "dict", + "required_keys": { + "name": {"type": "string"}, + "value": { + "or": [ + {"type": t} + for t in _valid_types + ], + }, + "set_in": {"type": "string"}, + }, + }, +} + +def flag(name, partitions, default, *, origin = "Unknown", appends = False): + """Declare a flag. + + Args: + name: name of the flag + partitions: the partitions where this should be recorded. + default: the default value of the flag. + origin: The origin of this flag. + appends: Whether new values should be append (not replace) the old. + + Returns: + A dictionary containing the flag declaration. + """ + if not partitions: + fail("At least 1 partition is required") + if not name.startswith("RELEASE_"): + fail("Release flag names must start with RELEASE_") + if " " in name or "\t" in name or "\n" in name: + fail("Flag names must not contain whitespace: \"" + name + "\"") + for partition in partitions: + if partition == "all": + if len(partitions) > 1: + fail("\"all\" can't be combined with other partitions: " + str(partitions)) + elif partition not in _flag_partitions: + fail("Invalid partition: " + partition + ", allowed partitions: " + + str(_flag_partitions)) + if type(default) not in _valid_types: + fail("Invalid type of default for flag \"" + name + "\" (" + type(default) + ")") + return { + "name": name, + "partitions": partitions, + "default": default, + "appends": appends, + "origin": origin, + } + +def value(name, value): + """Define the flag value for a particular configuration. + + Args: + name: The name of the flag. + value: The value for the flag. + + Returns: + A dictionary containing the name and value to be used. + """ + return { + "name": name, + "value": value, + } + +def _format_value(val): + """Format the starlark type correctly for make. + + Args: + val: The value to format + + Returns: + The value, formatted correctly for make. + """ + if type(val) == "NoneType": + return "" + elif type(val) == "bool": + return "true" if val else "" + else: + return val + +def equal_flag_declaration(flag, other): + """Return true if the flag declarations are equal. + + Args: + flag: This flag declaration. + other: Another flag declaration. + + Returns: + Whether the declarations are the same. + """ + for key in "name", "partitions", "default", "appends": + if flag[key] != other[key]: + return False + # For now, allow Unknown to match any other origin. + if flag["origin"] == "Unknown" or other["origin"] == "Unknown": + return True + return flag["origin"] == other["origin"] + +def release_config(all_flags, all_values): + """Return the make variables that should be set for this release config. + + Args: + all_flags: A list of flag objects (from flag() calls). + all_values: A list of value objects (from value() calls). + + Returns: + A dictionary of {name: value} variables for make. + """ + validate(all_flags, _all_flags_schema) + validate(all_values, _all_values_schema) + + # Final values. + values = {} + # Validate flags + flag_names = [] + flags_dict = {} + for flag in all_flags: + name = flag["name"] + if name in flag_names: + if equal_flag_declaration(flag, flags_dict[name]): + continue + else: + fail(flag["declared_in"] + ": Duplicate declaration of flag " + name + + " (declared first in " + flags_dict[name]["declared_in"] + ")") + flag_names.append(name) + flags_dict[name] = flag + # Set the flag value to the default value. + values[name] = {"name": name, "value": _format_value(flag["default"]), "set_in": flag["declared_in"]} + + # Record which flags go on which partition + partitions = {} + for flag in all_flags: + for partition in flag["partitions"]: + if partition == "all": + if len(flag["partitions"]) > 1: + fail("\"all\" can't be combined with other partitions: " + str(flag["partitions"])) + for partition in _flag_partitions: + partitions.setdefault(partition, []).append(flag["name"]) + else: + partitions.setdefault(partition, []).append(flag["name"]) + + # Generate final values. + # Only declared flags may have a value. + for value in all_values: + name = value["name"] + if name not in flag_names: + fail(value["set_in"] + ": Value set for undeclared build flag: " + name) + if flags_dict[name]["appends"]: + if name in values: + values[name]["value"] += " " + value["value"] + values[name]["set_in"] += " " + value["set_in"] + else: + values[name] = value + else: + values[name] = value + + # Collect values + result = { + "_ALL_RELEASE_FLAGS": sorted(flag_names), + } + for partition, names in partitions.items(): + result["_ALL_RELEASE_FLAGS.PARTITIONS." + partition] = names + for flag in all_flags: + val = _format_value(values[flag["name"]]["value"]) + result[flag["name"]] = val + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".PARTITIONS"] = flag["partitions"] + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".DEFAULT"] = _format_value(flag["default"]) + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".VALUE"] = val + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".DECLARED_IN"] = flag["declared_in"] + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".SET_IN"] = values[flag["name"]]["set_in"] + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".ORIGIN"] = flag["origin"] + + return result diff --git a/core/rust_device_benchmark_config_template.xml b/core/rust_device_benchmark_config_template.xml index 2055df2096..541630cc68 100644 --- a/core/rust_device_benchmark_config_template.xml +++ b/core/rust_device_benchmark_config_template.xml @@ -16,7 +16,7 @@ <!-- This test config file is auto-generated. --> <configuration description="Config to run {MODULE} rust benchmark tests."> <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer"> - <option name="cleanup" value="false" /> + <option name="cleanup" value="true" /> <option name="push" value="{MODULE}->/data/local/tmp/{MODULE}" /> </target_preparer> diff --git a/core/sbom.mk b/core/sbom.mk new file mode 100644 index 0000000000..39c251ae0c --- /dev/null +++ b/core/sbom.mk @@ -0,0 +1,22 @@ +# For SBOM generation +# This is included by base_rules.mk and is not necessary to be included in other .mk files +# unless a .mk file changes its installed file after including base_rules.mk. + +ifdef my_register_name + # ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle libclang_rt.builtins, for primary arch + # ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle_32 libclang_rt.builtins_32, for 2nd arch. + ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED))) + $(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\ + $(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\ + $(eval ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\ + $(eval ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\ + ) + endif + ifeq (STATIC_LIBRARIES,$(LOCAL_MODULE_CLASS)) + ALL_STATIC_LIBRARIES.$(my_register_name).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX)) + ALL_STATIC_LIBRARIES.$(my_register_name).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX)) + ifdef LOCAL_SOONG_MODULE_TYPE + ALL_STATIC_LIBRARIES.$(my_register_name).BUILT_FILE := $(LOCAL_PREBUILT_MODULE_FILE) + endif + endif +endif
\ No newline at end of file diff --git a/core/shared_library_internal.mk b/core/shared_library_internal.mk index 139de1077d..2f510d9b30 100644 --- a/core/shared_library_internal.mk +++ b/core/shared_library_internal.mk @@ -42,7 +42,7 @@ endif ifeq ($(LOCAL_NO_CRT),true) my_target_crtbegin_so_o := my_target_crtend_so_o := -else ifdef LOCAL_USE_VNDK +else ifeq ($(call module-in-vendor-or-product),true) my_target_crtbegin_so_o := $(SOONG_$(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_OBJECT_crtbegin_so.vendor) my_target_crtend_so_o := $(SOONG_$(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_OBJECT_crtend_so.vendor) else diff --git a/core/soong_app_prebuilt.mk b/core/soong_app_prebuilt.mk index 786a755b35..3aa244c77f 100644 --- a/core/soong_app_prebuilt.mk +++ b/core/soong_app_prebuilt.mk @@ -73,12 +73,14 @@ endif # We skip it for unbundled app builds where we cannot build veridex. module_run_appcompat := ifeq (true,$(non_system_module)) -ifeq (,$(TARGET_BUILD_APPS)) # ! unbundled app build +ifeq (,$(TARGET_BUILD_APPS)) # not unbundled app build +ifeq (,$(filter sdk,$(MAKECMDGOALS))) # not sdk build (which is another form of unbundled build) ifneq ($(UNSAFE_DISABLE_HIDDENAPI_FLAGS),true) module_run_appcompat := true endif endif endif +endif ifeq ($(module_run_appcompat),true) $(LOCAL_BUILT_MODULE): $(appcompat-files) @@ -100,31 +102,24 @@ ifdef LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR endif ifdef LOCAL_SOONG_PROGUARD_DICT - my_proguard_dictionary_directory := $(local-proguard-dictionary-directory) - my_proguard_dictionary_mapping_directory := $(local-proguard-dictionary-mapping-directory) - $(eval $(call copy-one-file,$(LOCAL_SOONG_PROGUARD_DICT),\ - $(intermediates.COMMON)/proguard_dictionary)) $(eval $(call copy-r8-dictionary-file-with-mapping,\ $(LOCAL_SOONG_PROGUARD_DICT),\ - $(my_proguard_dictionary_directory)/proguard_dictionary,\ - $(my_proguard_dictionary_mapping_directory)/proguard_dictionary.textproto)) - $(eval $(call copy-one-file,$(LOCAL_SOONG_CLASSES_JAR),\ - $(my_proguard_dictionary_directory)/classes.jar)) - $(call add-dependency,$(LOCAL_BUILT_MODULE),\ - $(intermediates.COMMON)/proguard_dictionary) - $(call add-dependency,$(LOCAL_BUILT_MODULE),\ - $(my_proguard_dictionary_directory)/proguard_dictionary) - $(call add-dependency,$(LOCAL_BUILT_MODULE),\ - $(my_proguard_dictionary_mapping_directory)/proguard_dictionary.textproto) - $(call add-dependency,$(LOCAL_BUILT_MODULE),\ - $(my_proguard_dictionary_directory)/classes.jar) + $(intermediates.COMMON)/proguard_dictionary,\ + $(intermediates.COMMON)/proguard_dictionary.textproto)) + + ALL_MODULES.$(my_register_name).PROGUARD_DICTIONARY_FILES := \ + $(intermediates.COMMON)/proguard_dictionary \ + $(LOCAL_SOONG_CLASSES_JAR) + ALL_MODULES.$(my_register_name).PROGUARD_DICTIONARY_SOONG_ZIP_ARGUMENTS := \ + -e out/target/common/obj/$(LOCAL_MODULE_CLASS)/$(LOCAL_MODULE)_intermediates/proguard_dictionary \ + -f $(intermediates.COMMON)/proguard_dictionary \ + -e out/target/common/obj/$(LOCAL_MODULE_CLASS)/$(LOCAL_MODULE)_intermediates/classes.jar \ + -f $(LOCAL_SOONG_CLASSES_JAR) + ALL_MODULES.$(my_register_name).PROGUARD_DICTIONARY_MAPPING := $(intermediates.COMMON)/proguard_dictionary.textproto endif ifdef LOCAL_SOONG_PROGUARD_USAGE_ZIP - $(eval $(call copy-one-file,$(LOCAL_SOONG_PROGUARD_USAGE_ZIP),\ - $(call local-packaging-dir,proguard_usage)/proguard_usage.zip)) - $(call add-dependency,$(LOCAL_BUILT_MODULE),\ - $(call local-packaging-dir,proguard_usage)/proguard_usage.zip) + ALL_MODULES.$(my_register_name).PROGUARD_USAGE_ZIP := $(LOCAL_SOONG_PROGUARD_USAGE_ZIP) endif ifdef LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE @@ -162,19 +157,21 @@ $(LOCAL_BUILT_MODULE): | $(call copy-many-files, $(my_jni_lib_symbols_copy_files # embedded JNI will already have been handled by soong my_embed_jni := my_prebuilt_jni_libs := -ifdef LOCAL_SOONG_JNI_LIBS_$(TARGET_ARCH) - my_2nd_arch_prefix := - LOCAL_JNI_SHARED_LIBRARIES := $(LOCAL_SOONG_JNI_LIBS_$(TARGET_ARCH)) - partition_lib_pairs := $(LOCAL_SOONG_JNI_LIBS_PARTITION_$(TARGET_ARCH)) - include $(BUILD_SYSTEM)/install_jni_libs_internal.mk -endif -ifdef TARGET_2ND_ARCH - ifdef LOCAL_SOONG_JNI_LIBS_$(TARGET_2ND_ARCH) - my_2nd_arch_prefix := $(TARGET_2ND_ARCH_VAR_PREFIX) - LOCAL_JNI_SHARED_LIBRARIES := $(LOCAL_SOONG_JNI_LIBS_$(TARGET_2ND_ARCH)) - partition_lib_pairs := $(LOCAL_SOONG_JNI_LIBS_PARTITION_$(TARGET_2ND_ARCH)) +ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) + ifdef LOCAL_SOONG_JNI_LIBS_$(TARGET_ARCH) + my_2nd_arch_prefix := + LOCAL_JNI_SHARED_LIBRARIES := $(LOCAL_SOONG_JNI_LIBS_$(TARGET_ARCH)) + partition_lib_pairs := $(LOCAL_SOONG_JNI_LIBS_PARTITION_$(TARGET_ARCH)) include $(BUILD_SYSTEM)/install_jni_libs_internal.mk endif + ifdef TARGET_2ND_ARCH + ifdef LOCAL_SOONG_JNI_LIBS_$(TARGET_2ND_ARCH) + my_2nd_arch_prefix := $(TARGET_2ND_ARCH_VAR_PREFIX) + LOCAL_JNI_SHARED_LIBRARIES := $(LOCAL_SOONG_JNI_LIBS_$(TARGET_2ND_ARCH)) + partition_lib_pairs := $(LOCAL_SOONG_JNI_LIBS_PARTITION_$(TARGET_2ND_ARCH)) + include $(BUILD_SYSTEM)/install_jni_libs_internal.mk + endif + endif endif LOCAL_SHARED_JNI_LIBRARIES := my_embed_jni := @@ -237,26 +234,28 @@ my_common := COMMON include $(BUILD_SYSTEM)/link_type.mk endif # !LOCAL_IS_HOST_MODULE -ifdef LOCAL_SOONG_DEVICE_RRO_DIRS - $(call append_enforce_rro_sources, \ - $(my_register_name), \ - false, \ - $(LOCAL_FULL_MANIFEST_FILE), \ - $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \ - $(LOCAL_SOONG_DEVICE_RRO_DIRS), \ - vendor \ - ) -endif +ifeq (,$(filter tests,$(LOCAL_MODULE_TAGS))) + ifdef LOCAL_SOONG_DEVICE_RRO_DIRS + $(call append_enforce_rro_sources, \ + $(my_register_name), \ + false, \ + $(LOCAL_FULL_MANIFEST_FILE), \ + $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \ + $(LOCAL_SOONG_DEVICE_RRO_DIRS), \ + vendor \ + ) + endif -ifdef LOCAL_SOONG_PRODUCT_RRO_DIRS - $(call append_enforce_rro_sources, \ - $(my_register_name), \ - false, \ - $(LOCAL_FULL_MANIFEST_FILE), \ - $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \ - $(LOCAL_SOONG_PRODUCT_RRO_DIRS), \ - product \ - ) + ifdef LOCAL_SOONG_PRODUCT_RRO_DIRS + $(call append_enforce_rro_sources, \ + $(my_register_name), \ + false, \ + $(LOCAL_FULL_MANIFEST_FILE), \ + $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \ + $(LOCAL_SOONG_PRODUCT_RRO_DIRS), \ + product \ + ) + endif endif ifdef LOCAL_PREBUILT_COVERAGE_ARCHIVE @@ -267,3 +266,8 @@ ifdef LOCAL_PREBUILT_COVERAGE_ARCHIVE endif SOONG_ALREADY_CONV += $(LOCAL_MODULE) + +########################################################### +## SBOM generation +########################################################### +include $(BUILD_SBOM_GEN) diff --git a/core/soong_cc_rust_prebuilt.mk b/core/soong_cc_rust_prebuilt.mk index 05b4b6b596..a1c64786ee 100644 --- a/core/soong_cc_rust_prebuilt.mk +++ b/core/soong_cc_rust_prebuilt.mk @@ -38,14 +38,6 @@ ifndef LOCAL_UNINSTALLABLE_MODULE endif endif -# Don't install modules of current VNDK when it is told so -ifeq ($(TARGET_SKIP_CURRENT_VNDK),true) - ifeq ($(LOCAL_SOONG_VNDK_VERSION),$(PLATFORM_VNDK_VERSION)) - LOCAL_UNINSTALLABLE_MODULE := true - endif -endif - - # Use the Soong output as the checkbuild target instead of LOCAL_BUILT_MODULE # to avoid checkbuilds making an extra copy of every module. LOCAL_CHECKED_MODULE := $(LOCAL_PREBUILT_MODULE_FILE) @@ -99,7 +91,7 @@ ifneq ($(filter STATIC_LIBRARIES SHARED_LIBRARIES RLIB_LIBRARIES DYLIB_LIBRARIES include $(BUILD_SYSTEM)/link_type.mk endif -ifdef LOCAL_USE_VNDK +ifeq ($(call module-in-vendor-or-product),true) ifneq ($(LOCAL_VNDK_DEPEND_ON_CORE_VARIANT),true) name_without_suffix := $(patsubst %.vendor,%,$(LOCAL_MODULE)) ifneq ($(name_without_suffix),$(LOCAL_MODULE)) @@ -128,9 +120,14 @@ endif ifdef LOCAL_INSTALLED_MODULE ifdef LOCAL_SHARED_LIBRARIES my_shared_libraries := $(LOCAL_SHARED_LIBRARIES) - ifdef LOCAL_USE_VNDK - my_shared_libraries := $(foreach l,$(my_shared_libraries),\ - $(if $(SPLIT_VENDOR.SHARED_LIBRARIES.$(l)),$(l).vendor,$(l))) + ifeq ($(call module-in-vendor-or-product),true) + ifdef LOCAL_IN_PRODUCT + my_shared_libraries := $(foreach l,$(my_shared_libraries),\ + $(if $(SPLIT_PRODUCT.SHARED_LIBRARIES.$(l)),$(l).product,$(l))) + else + my_shared_libraries := $(foreach l,$(my_shared_libraries),\ + $(if $(SPLIT_VENDOR.SHARED_LIBRARIES.$(l)),$(l).vendor,$(l))) + endif endif $(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)DEPENDENCIES_ON_SHARED_LIBRARIES += \ $(my_register_name):$(LOCAL_INSTALLED_MODULE):$(subst $(space),$(comma),$(my_shared_libraries)) @@ -138,9 +135,14 @@ ifdef LOCAL_INSTALLED_MODULE ifdef LOCAL_DYLIB_LIBRARIES my_dylibs := $(LOCAL_DYLIB_LIBRARIES) # Treat these as shared library dependencies for installation purposes. - ifdef LOCAL_USE_VNDK - my_dylibs := $(foreach l,$(my_dylibs),\ - $(if $(SPLIT_VENDOR.SHARED_LIBRARIES.$(l)),$(l).vendor,$(l))) + ifeq ($(call module-in-vendor-or-product),true) + ifdef LOCAL_IN_PRODUCT + my_dylibs := $(foreach l,$(my_dylibs),\ + $(if $(SPLIT_PRODUCT.SHARED_LIBRARIES.$(l)),$(l).product,$(l))) + else + my_dylibs := $(foreach l,$(my_dylibs),\ + $(if $(SPLIT_VENDOR.SHARED_LIBRARIES.$(l)),$(l).vendor,$(l))) + endif endif $(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)DEPENDENCIES_ON_SHARED_LIBRARIES += \ $(my_register_name):$(LOCAL_INSTALLED_MODULE):$(subst $(space),$(comma),$(my_dylibs)) @@ -251,30 +253,6 @@ endif $(LOCAL_BUILT_MODULE): $(LOCAL_ADDITIONAL_DEPENDENCIES) -# We don't care about installed rlib/static libraries, since the libraries have -# already been linked into the module at that point. We do, however, care -# about the NOTICE files for any rlib/static libraries that we use. -# (see notice_files.mk) -# -# Filter out some NDK libraries that are not being exported. -my_static_libraries := \ - $(filter-out ndk_libc++_static ndk_libc++abi ndk_libandroid_support ndk_libunwind \ - ndk_libc++_static.native_bridge ndk_libc++abi.native_bridge \ - ndk_libandroid_support.native_bridge ndk_libunwind.native_bridge, \ - $(LOCAL_STATIC_LIBRARIES)) -installed_static_library_notice_file_targets := \ - $(foreach lib,$(my_static_libraries) $(LOCAL_WHOLE_STATIC_LIBRARIES), \ - NOTICE-$(if $(LOCAL_IS_HOST_MODULE),HOST$(if $(my_host_cross),_CROSS,),TARGET)-STATIC_LIBRARIES-$(lib)) -installed_static_library_notice_file_targets += \ - $(foreach lib,$(LOCAL_RLIB_LIBRARIES), \ - NOTICE-$(if $(LOCAL_IS_HOST_MODULE),HOST$(if $(my_host_cross),_CROSS,),TARGET)-RLIB_LIBRARIES-$(lib)) -installed_static_library_notice_file_targets += \ - $(foreach lib,$(LOCAL_PROC_MACRO_LIBRARIES), \ - NOTICE-$(if $(LOCAL_IS_HOST_MODULE),HOST$(if $(my_host_cross),_CROSS,),TARGET)-PROC_MACRO_LIBRARIES-$(lib)) - -$(notice_target): | $(installed_static_library_notice_file_targets) -$(LOCAL_INSTALLED_MODULE): | $(notice_target) - # Reinstall shared library dependencies of fuzz targets to /data/fuzz/ (for # target) or /data/ (for host). ifdef LOCAL_IS_FUZZ_TARGET diff --git a/core/soong_config.mk b/core/soong_config.mk index 091fa343d5..acd213c776 100644 --- a/core/soong_config.mk +++ b/core/soong_config.mk @@ -1,17 +1,25 @@ SOONG_MAKEVARS_MK := $(SOONG_OUT_DIR)/make_vars-$(TARGET_PRODUCT).mk -SOONG_VARIABLES := $(SOONG_OUT_DIR)/soong.variables +SOONG_VARIABLES := $(SOONG_OUT_DIR)/soong.$(TARGET_PRODUCT).variables SOONG_ANDROID_MK := $(SOONG_OUT_DIR)/Android-$(TARGET_PRODUCT).mk -BINDER32BIT := -ifneq ($(TARGET_USES_64_BIT_BINDER),true) -ifneq ($(TARGET_IS_64_BIT),true) -BINDER32BIT := true -endif -endif - include $(BUILD_SYSTEM)/art_config.mk include $(BUILD_SYSTEM)/dex_preopt_config.mk +ifndef AFDO_PROFILES +# Set AFDO_PROFILES +-include vendor/google_data/pgo_profile/sampling/afdo_profiles.mk +include toolchain/pgo-profiles/sampling/afdo_profiles.mk +else +$(error AFDO_PROFILES can only be set from soong_config.mk. For product-specific fdo_profiles, please use PRODUCT_AFDO_PROFILES) +endif + +# PRODUCT_AFDO_PROFILES takes precedence over product-agnostic profiles in AFDO_PROFILES +ALL_AFDO_PROFILES := $(PRODUCT_AFDO_PROFILES) $(AFDO_PROFILES) + +ifneq (,$(filter-out environment undefined,$(origin GENRULE_SANDBOXING))) + $(error GENRULE_SANDBOXING can only be provided via an environment variable, use BUILD_BROKEN_GENRULE_SANDBOXING to disable genrule sandboxing in board config) +endif + ifeq ($(WRITE_SOONG_VARIABLES),true) # Create soong.variables with copies of makefile settings. Runs every build, @@ -23,7 +31,11 @@ $(call add_json_str, Make_suffix, -$(TARGET_PRODUCT)) $(call add_json_str, BuildId, $(BUILD_ID)) $(call add_json_str, BuildNumberFile, build_number.txt) +$(call add_json_str, BuildHostnameFile, build_hostname.txt) +$(call add_json_str, BuildThumbprintFile, build_thumbprint.txt) +$(call add_json_bool, DisplayBuildNumber, $(filter true,$(DISPLAY_BUILD_NUMBER))) +$(call add_json_str, Platform_display_version_name, $(PLATFORM_DISPLAY_VERSION)) $(call add_json_str, Platform_version_name, $(PLATFORM_VERSION)) $(call add_json_val, Platform_sdk_version, $(PLATFORM_SDK_VERSION)) $(call add_json_str, Platform_sdk_codename, $(PLATFORM_VERSION_CODENAME)) @@ -31,13 +43,14 @@ $(call add_json_bool, Platform_sdk_final, $(filter REL,$(PLATFORM $(call add_json_val, Platform_sdk_extension_version, $(PLATFORM_SDK_EXTENSION_VERSION)) $(call add_json_val, Platform_base_sdk_extension_version, $(PLATFORM_BASE_SDK_EXTENSION_VERSION)) $(call add_json_csv, Platform_version_active_codenames, $(PLATFORM_VERSION_ALL_CODENAMES)) +$(call add_json_csv, Platform_version_all_preview_codenames, $(PLATFORM_VERSION_ALL_PREVIEW_CODENAMES)) $(call add_json_str, Platform_security_patch, $(PLATFORM_SECURITY_PATCH)) $(call add_json_str, Platform_preview_sdk_version, $(PLATFORM_PREVIEW_SDK_VERSION)) $(call add_json_str, Platform_base_os, $(PLATFORM_BASE_OS)) $(call add_json_str, Platform_version_last_stable, $(PLATFORM_VERSION_LAST_STABLE)) $(call add_json_str, Platform_version_known_codenames, $(PLATFORM_VERSION_KNOWN_CODENAMES)) -$(call add_json_str, Platform_min_supported_target_sdk_version, $(PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION)) +$(call add_json_bool, Release_aidl_use_unfrozen, $(RELEASE_AIDL_USE_UNFROZEN)) $(call add_json_bool, Allow_missing_dependencies, $(filter true,$(ALLOW_MISSING_DEPENDENCIES))) $(call add_json_bool, Unbundled_build, $(TARGET_BUILD_UNBUNDLED)) @@ -47,6 +60,7 @@ $(call add_json_bool, Always_use_prebuilt_sdks, $(TARGET_BUILD_USE_PREB $(call add_json_bool, Debuggable, $(filter userdebug eng,$(TARGET_BUILD_VARIANT))) $(call add_json_bool, Eng, $(filter eng,$(TARGET_BUILD_VARIANT))) +$(call add_json_str, BuildType, $(TARGET_BUILD_TYPE)) $(call add_json_str, DeviceName, $(TARGET_DEVICE)) $(call add_json_str, DeviceProduct, $(TARGET_PRODUCT)) @@ -109,10 +123,12 @@ $(call add_json_bool, EnableCFI, $(call invert_bool,$(fi $(call add_json_list, CFIExcludePaths, $(CFI_EXCLUDE_PATHS) $(PRODUCT_CFI_EXCLUDE_PATHS)) $(call add_json_list, CFIIncludePaths, $(CFI_INCLUDE_PATHS) $(PRODUCT_CFI_INCLUDE_PATHS)) $(call add_json_list, IntegerOverflowExcludePaths, $(INTEGER_OVERFLOW_EXCLUDE_PATHS) $(PRODUCT_INTEGER_OVERFLOW_EXCLUDE_PATHS)) +$(call add_json_list, HWASanIncludePaths, $(HWASAN_INCLUDE_PATHS) $(PRODUCT_HWASAN_INCLUDE_PATHS)) +$(call add_json_list, HWASanExcludePaths, $(HWASAN_EXCLUDE_PATHS) $(PRODUCT_HWASAN_EXCLUDE_PATHS)) $(call add_json_list, MemtagHeapExcludePaths, $(MEMTAG_HEAP_EXCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_EXCLUDE_PATHS)) -$(call add_json_list, MemtagHeapAsyncIncludePaths, $(MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS)) -$(call add_json_list, MemtagHeapSyncIncludePaths, $(MEMTAG_HEAP_SYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_SYNC_INCLUDE_PATHS)) +$(call add_json_list, MemtagHeapAsyncIncludePaths, $(MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) $(if $(filter true,$(PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS)),,$(PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS))) +$(call add_json_list, MemtagHeapSyncIncludePaths, $(MEMTAG_HEAP_SYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_SYNC_INCLUDE_PATHS) $(if $(filter true,$(PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS)),,$(PRODUCT_MEMTAG_HEAP_SYNC_DEFAULT_INCLUDE_PATHS))) $(call add_json_bool, DisableScudo, $(filter true,$(PRODUCT_DISABLE_SCUDO))) @@ -131,30 +147,34 @@ $(call add_json_list, NativeCoverageExcludePaths, $(NATIVE_COVERAGE_EXCLU $(call add_json_bool, SamplingPGO, $(filter true,$(SAMPLING_PGO))) $(call add_json_bool, ArtUseReadBarrier, $(call invert_bool,$(filter false,$(PRODUCT_ART_USE_READ_BARRIER)))) -$(call add_json_bool, Binder32bit, $(BINDER32BIT)) $(call add_json_str, BtConfigIncludeDir, $(BOARD_BLUETOOTH_BDROID_BUILDCFG_INCLUDE_DIR)) $(call add_json_list, DeviceKernelHeaders, $(TARGET_DEVICE_KERNEL_HEADERS) $(TARGET_BOARD_KERNEL_HEADERS) $(TARGET_PRODUCT_KERNEL_HEADERS)) -$(call add_json_str, DeviceVndkVersion, $(BOARD_VNDK_VERSION)) -$(call add_json_str, Platform_vndk_version, $(PLATFORM_VNDK_VERSION)) -$(call add_json_str, ProductVndkVersion, $(PRODUCT_PRODUCT_VNDK_VERSION)) +$(call add_json_str, VendorApiLevel, $(BOARD_API_LEVEL)) $(call add_json_list, ExtraVndkVersions, $(PRODUCT_EXTRA_VNDK_VERSIONS)) $(call add_json_list, DeviceSystemSdkVersions, $(BOARD_SYSTEMSDK_VERSIONS)) $(call add_json_str, RecoverySnapshotVersion, $(RECOVERY_SNAPSHOT_VERSION)) $(call add_json_list, Platform_systemsdk_versions, $(PLATFORM_SYSTEMSDK_VERSIONS)) -$(call add_json_bool, Malloc_not_svelte, $(call invert_bool,$(filter true,$(MALLOC_SVELTE)))) +$(call add_json_bool, Malloc_low_memory, $(findstring true,$(MALLOC_SVELTE) $(MALLOC_LOW_MEMORY))) $(call add_json_bool, Malloc_zero_contents, $(call invert_bool,$(filter false,$(MALLOC_ZERO_CONTENTS)))) $(call add_json_bool, Malloc_pattern_fill_contents, $(MALLOC_PATTERN_FILL_CONTENTS)) $(call add_json_str, Override_rs_driver, $(OVERRIDE_RS_DRIVER)) +$(call add_json_str, DeviceMaxPageSizeSupported, $(TARGET_MAX_PAGE_SIZE_SUPPORTED)) +$(call add_json_bool, DeviceNoBionicPageSizeMacro, $(filter true,$(TARGET_NO_BIONIC_PAGE_SIZE_MACRO))) $(call add_json_bool, UncompressPrivAppDex, $(call invert_bool,$(filter true,$(DONT_UNCOMPRESS_PRIV_APPS_DEXS)))) $(call add_json_list, ModulesLoadedByPrivilegedModules, $(PRODUCT_LOADED_BY_PRIVILEGED_MODULES)) $(call add_json_list, BootJars, $(PRODUCT_BOOT_JARS)) -$(call add_json_list, ApexBootJars, $(PRODUCT_APEX_BOOT_JARS)) +$(call add_json_list, ApexBootJars, $(filter-out $(APEX_BOOT_JARS_EXCLUDED), $(PRODUCT_APEX_BOOT_JARS))) $(call add_json_bool, VndkUseCoreVariant, $(TARGET_VNDK_USE_CORE_VARIANT)) $(call add_json_bool, VndkSnapshotBuildArtifacts, $(VNDK_SNAPSHOT_BUILD_ARTIFACTS)) +$(call add_json_map, BuildFlags) +$(foreach flag,$(_ALL_RELEASE_FLAGS),\ + $(call add_json_str,$(flag),$(_ALL_RELEASE_FLAGS.$(flag).VALUE))) +$(call end_json_map) + $(call add_json_bool, DirectedVendorSnapshot, $(DIRECTED_VENDOR_SNAPSHOT)) $(call add_json_map, VendorSnapshotModules) $(foreach module,$(VENDOR_SNAPSHOT_MODULES),\ @@ -199,17 +219,8 @@ $(call add_json_list, NamespacesToExport, $(PRODUCT_SOONG_NAMESPA $(call add_json_list, PgoAdditionalProfileDirs, $(PGO_ADDITIONAL_PROFILE_DIRS)) -$(call add_json_list, BoardPlatVendorPolicy, $(BOARD_PLAT_VENDOR_POLICY)) -$(call add_json_list, BoardReqdMaskPolicy, $(BOARD_REQD_MASK_POLICY)) -$(call add_json_list, BoardSystemExtPublicPrebuiltDirs, $(BOARD_SYSTEM_EXT_PUBLIC_PREBUILT_DIRS)) -$(call add_json_list, BoardSystemExtPrivatePrebuiltDirs, $(BOARD_SYSTEM_EXT_PRIVATE_PREBUILT_DIRS)) -$(call add_json_list, BoardProductPublicPrebuiltDirs, $(BOARD_PRODUCT_PUBLIC_PREBUILT_DIRS)) -$(call add_json_list, BoardProductPrivatePrebuiltDirs, $(BOARD_PRODUCT_PRIVATE_PREBUILT_DIRS)) $(call add_json_list, BoardVendorSepolicyDirs, $(BOARD_VENDOR_SEPOLICY_DIRS) $(BOARD_SEPOLICY_DIRS)) $(call add_json_list, BoardOdmSepolicyDirs, $(BOARD_ODM_SEPOLICY_DIRS)) -$(call add_json_list, BoardVendorDlkmSepolicyDirs, $(BOARD_VENDOR_DLKM_SEPOLICY_DIRS)) -$(call add_json_list, BoardOdmDlkmSepolicyDirs, $(BOARD_ODM_DLKM_SEPOLICY_DIRS)) -$(call add_json_list, BoardSystemDlkmSepolicyDirs, $(BOARD_SYSTEM_DLKM_SEPOLICY_DIRS)) $(call add_json_list, SystemExtPublicSepolicyDirs, $(SYSTEM_EXT_PUBLIC_SEPOLICY_DIRS)) $(call add_json_list, SystemExtPrivateSepolicyDirs, $(SYSTEM_EXT_PRIVATE_SEPOLICY_DIRS)) $(call add_json_list, BoardSepolicyM4Defs, $(BOARD_SEPOLICY_M4DEFS)) @@ -218,10 +229,8 @@ $(call add_json_str, SystemExtSepolicyPrebuiltApiDir, $(BOARD_SYSTEM_EXT_PREB $(call add_json_str, ProductSepolicyPrebuiltApiDir, $(BOARD_PRODUCT_PREBUILT_DIR)) $(call add_json_str, PlatformSepolicyVersion, $(PLATFORM_SEPOLICY_VERSION)) -$(call add_json_str, TotSepolicyVersion, $(TOT_SEPOLICY_VERSION)) $(call add_json_list, PlatformSepolicyCompatVersions, $(PLATFORM_SEPOLICY_COMPAT_VERSIONS)) -$(call add_json_bool, Flatten_apex, $(filter true,$(TARGET_FLATTEN_APEX))) $(call add_json_bool, ForceApexSymlinkOptimization, $(filter true,$(TARGET_FORCE_APEX_SYMLINK_OPTIMIZATION))) $(call add_json_str, DexpreoptGlobalConfig, $(DEX_PREOPT_CONFIG)) @@ -231,6 +240,7 @@ $(call add_json_bool, WithDexpreopt, $(filter true,$(WITH_DE $(call add_json_list, ManifestPackageNameOverrides, $(PRODUCT_MANIFEST_PACKAGE_NAME_OVERRIDES)) $(call add_json_list, PackageNameOverrides, $(PRODUCT_PACKAGE_NAME_OVERRIDES)) $(call add_json_list, CertificateOverrides, $(PRODUCT_CERTIFICATE_OVERRIDES)) +$(call add_json_list, ConfiguredJarLocationOverrides, $(PRODUCT_CONFIGURED_JAR_LOCATION_OVERRIDES)) $(call add_json_str, ApexGlobalMinSdkVersionOverride, $(APEX_GLOBAL_MIN_SDK_VERSION_OVERRIDE)) @@ -246,8 +256,6 @@ $(call add_json_list, ProductPrivateSepolicyDirs, $(PRODUCT_PRIVATE_SEPOL $(call add_json_list, TargetFSConfigGen, $(TARGET_FS_CONFIG_GEN)) -$(call add_json_list, MissingUsesLibraries, $(INTERNAL_PLATFORM_MISSING_USES_LIBRARIES)) - $(call add_json_map, VendorVars) $(foreach namespace,$(sort $(SOONG_CONFIG_NAMESPACES)),\ $(call add_json_map, $(namespace))\ @@ -262,8 +270,6 @@ $(call add_json_str, DeviceCurrentApiLevelForVendorModules, $(BOARD_CURRENT_AP $(call add_json_bool, EnforceInterPartitionJavaSdkLibrary, $(filter true,$(PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY))) $(call add_json_list, InterPartitionJavaLibraryAllowList, $(PRODUCT_INTER_PARTITION_JAVA_LIBRARY_ALLOWLIST)) -$(call add_json_bool, InstallExtraFlattenedApexes, $(PRODUCT_INSTALL_EXTRA_FLATTENED_APEXES)) - $(call add_json_bool, CompressedApex, $(filter true,$(PRODUCT_COMPRESSED_APEX))) ifndef APEX_BUILD_FOR_PRE_S_DEVICES @@ -280,14 +286,21 @@ $(call add_json_str, PrebuiltHiddenApiDir, $(BOARD_PREBUILT_HIDDENAPI_DIR)) $(call add_json_str, ShippingApiLevel, $(PRODUCT_SHIPPING_API_LEVEL)) -$(call add_json_bool, BuildBrokenClangProperty, $(filter true,$(BUILD_BROKEN_CLANG_PROPERTY))) -$(call add_json_bool, BuildBrokenClangAsFlags, $(filter true,$(BUILD_BROKEN_CLANG_ASFLAGS))) -$(call add_json_bool, BuildBrokenClangCFlags, $(filter true,$(BUILD_BROKEN_CLANG_CFLAGS))) -$(call add_json_bool, BuildBrokenDepfile, $(filter true,$(BUILD_BROKEN_DEPFILE))) -$(call add_json_bool, BuildBrokenEnforceSyspropOwner, $(filter true,$(BUILD_BROKEN_ENFORCE_SYSPROP_OWNER))) -$(call add_json_bool, BuildBrokenTrebleSyspropNeverallow, $(filter true,$(BUILD_BROKEN_TREBLE_SYSPROP_NEVERALLOW))) -$(call add_json_bool, BuildBrokenVendorPropertyNamespace, $(filter true,$(BUILD_BROKEN_VENDOR_PROPERTY_NAMESPACE))) -$(call add_json_list, BuildBrokenInputDirModules, $(BUILD_BROKEN_INPUT_DIR_MODULES)) +$(call add_json_list, BuildBrokenPluginValidation, $(BUILD_BROKEN_PLUGIN_VALIDATION)) +$(call add_json_bool, BuildBrokenClangProperty, $(filter true,$(BUILD_BROKEN_CLANG_PROPERTY))) +$(call add_json_bool, BuildBrokenClangAsFlags, $(filter true,$(BUILD_BROKEN_CLANG_ASFLAGS))) +$(call add_json_bool, BuildBrokenClangCFlags, $(filter true,$(BUILD_BROKEN_CLANG_CFLAGS))) +# Use the value of GENRULE_SANDBOXING if set, otherwise use the inverse of BUILD_BROKEN_GENRULE_SANDBOXING +$(call add_json_bool, GenruleSandboxing, $(if $(GENRULE_SANDBOXING),$(filter true,$(GENRULE_SANDBOXING)),$(if $(filter true,$(BUILD_BROKEN_GENRULE_SANDBOXING)),,true))) +$(call add_json_bool, BuildBrokenEnforceSyspropOwner, $(filter true,$(BUILD_BROKEN_ENFORCE_SYSPROP_OWNER))) +$(call add_json_bool, BuildBrokenTrebleSyspropNeverallow, $(filter true,$(BUILD_BROKEN_TREBLE_SYSPROP_NEVERALLOW))) +$(call add_json_bool, BuildBrokenUsesSoongPython2Modules, $(filter true,$(BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES))) +$(call add_json_bool, BuildBrokenVendorPropertyNamespace, $(filter true,$(BUILD_BROKEN_VENDOR_PROPERTY_NAMESPACE))) +$(call add_json_bool, BuildBrokenIncorrectPartitionImages, $(filter true,$(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES))) +$(call add_json_list, BuildBrokenInputDirModules, $(BUILD_BROKEN_INPUT_DIR_MODULES)) +$(call add_json_bool, BuildBrokenDontCheckSystemSdk, $(filter true,$(BUILD_BROKEN_DONT_CHECK_SYSTEMSDK))) + +$(call add_json_list, BuildWarningBadOptionalUsesLibsAllowlist, $(BUILD_WARNING_BAD_OPTIONAL_USES_LIBS_ALLOWLIST)) $(call add_json_bool, BuildDebugfsRestrictionsEnabled, $(filter true,$(PRODUCT_SET_DEBUGFS_RESTRICTIONS))) @@ -295,8 +308,6 @@ $(call add_json_bool, RequiresInsecureExecmemForSwiftshader, $(filter true,$(PRO $(call add_json_bool, SelinuxIgnoreNeverallows, $(filter true,$(SELINUX_IGNORE_NEVERALLOWS))) -$(call add_json_bool, SepolicySplit, $(filter true,$(PRODUCT_SEPOLICY_SPLIT))) - $(call add_json_list, SepolicyFreezeTestExtraDirs, $(SEPOLICY_FREEZE_TEST_EXTRA_DIRS)) $(call add_json_list, SepolicyFreezeTestExtraPrebuiltDirs, $(SEPOLICY_FREEZE_TEST_EXTRA_PREBUILT_DIRS)) @@ -305,6 +316,101 @@ $(call add_json_bool, GenerateAidlNdkPlatformBackend, $(filter true,$(NEED_AIDL_ $(call add_json_bool, IgnorePrefer32OnDevice, $(filter true,$(IGNORE_PREFER32_ON_DEVICE))) $(call add_json_list, IncludeTags, $(PRODUCT_INCLUDE_TAGS)) +$(call add_json_list, SourceRootDirs, $(PRODUCT_SOURCE_ROOT_DIRS)) + +$(call add_json_list, AfdoProfiles, $(ALL_AFDO_PROFILES)) + +$(call add_json_str, ProductManufacturer, $(PRODUCT_MANUFACTURER)) +$(call add_json_str, ProductBrand, $(PRODUCT_BRAND)) + +$(call add_json_str, ReleaseVersion, $(_RELEASE_VERSION)) +$(call add_json_list, ReleaseAconfigValueSets, $(RELEASE_ACONFIG_VALUE_SETS)) +$(call add_json_str, ReleaseAconfigFlagDefaultPermission, $(RELEASE_ACONFIG_FLAG_DEFAULT_PERMISSION)) + +$(call add_json_bool, ReleaseDefaultModuleBuildFromSource, $(RELEASE_DEFAULT_MODULE_BUILD_FROM_SOURCE)) + +$(call add_json_bool, KeepVndk, $(filter true,$(KEEP_VNDK))) + +$(call add_json_bool, CheckVendorSeappViolations, $(filter true,$(CHECK_VENDOR_SEAPP_VIOLATIONS))) + +$(call add_json_bool, BuildIgnoreApexContributionContents, $(PRODUCT_BUILD_IGNORE_APEX_CONTRIBUTION_CONTENTS)) + +$(call add_json_map, PartitionVarsForBazelMigrationOnlyDoNotUse) + $(call add_json_str, ProductDirectory, $(dir $(INTERNAL_PRODUCT))) + + $(call add_json_map,PartitionQualifiedVariables) + $(foreach image_type,SYSTEM VENDOR CACHE USERDATA PRODUCT SYSTEM_EXT OEM ODM VENDOR_DLKM ODM_DLKM SYSTEM_DLKM, \ + $(call add_json_map,$(call to-lower,$(image_type))) \ + $(call add_json_bool, BuildingImage, $(filter true,$(BUILDING_$(image_type)_IMAGE))) \ + $(call add_json_str, BoardErofsCompressor, $(BOARD_$(image_type)IMAGE_EROFS_COMPRESSOR)) \ + $(call add_json_str, BoardErofsCompressHints, $(BOARD_$(image_type)IMAGE_EROFS_COMPRESS_HINTS)) \ + $(call add_json_str, BoardErofsPclusterSize, $(BOARD_$(image_type)IMAGE_EROFS_PCLUSTER_SIZE)) \ + $(call add_json_str, BoardExtfsInodeCount, $(BOARD_$(image_type)IMAGE_EXTFS_INODE_COUNT)) \ + $(call add_json_str, BoardExtfsRsvPct, $(BOARD_$(image_type)IMAGE_EXTFS_RSV_PCT)) \ + $(call add_json_str, BoardF2fsSloadCompressFlags, $(BOARD_$(image_type)IMAGE_F2FS_SLOAD_COMPRESS_FLAGS)) \ + $(call add_json_str, BoardFileSystemCompress, $(BOARD_$(image_type)IMAGE_FILE_SYSTEM_COMPRESS)) \ + $(call add_json_str, BoardFileSystemType, $(BOARD_$(image_type)IMAGE_FILE_SYSTEM_TYPE)) \ + $(call add_json_str, BoardJournalSize, $(BOARD_$(image_type)IMAGE_JOURNAL_SIZE)) \ + $(call add_json_str, BoardPartitionReservedSize, $(BOARD_$(image_type)IMAGE_PARTITION_RESERVED_SIZE)) \ + $(call add_json_str, BoardPartitionSize, $(BOARD_$(image_type)IMAGE_PARTITION_SIZE)) \ + $(call add_json_str, BoardSquashfsBlockSize, $(BOARD_$(image_type)IMAGE_SQUASHFS_BLOCK_SIZE)) \ + $(call add_json_str, BoardSquashfsCompressor, $(BOARD_$(image_type)IMAGE_SQUASHFS_COMPRESSOR)) \ + $(call add_json_str, BoardSquashfsCompressorOpt, $(BOARD_$(image_type)IMAGE_SQUASHFS_COMPRESSOR_OPT)) \ + $(call add_json_str, BoardSquashfsDisable4kAlign, $(BOARD_$(image_type)IMAGE_SQUASHFS_DISABLE_4K_ALIGN)) \ + $(call add_json_str, ProductBaseFsPath, $(PRODUCT_$(image_type)_BASE_FS_PATH)) \ + $(call add_json_str, ProductHeadroom, $(PRODUCT_$(image_type)_HEADROOM)) \ + $(call add_json_str, ProductVerityPartition, $(PRODUCT_$(image_type)_VERITY_PARTITION)) \ + $(call add_json_str, BoardAvbAddHashtreeFooterArgs, $(BOARD_AVB_$(image_type)_ADD_HASHTREE_FOOTER_ARGS)) \ + $(call add_json_str, BoardAvbKeyPath, $(BOARD_AVB_$(image_type)_KEY_PATH)) \ + $(call add_json_str, BoardAvbAlgorithm, $(BOARD_AVB_$(image_type)_ALGORITHM)) \ + $(call add_json_str, BoardAvbRollbackIndex, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX)) \ + $(call add_json_str, BoardAvbRollbackIndexLocation, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX_LOCATION)) \ + $(call end_json_map) \ + ) + $(call end_json_map) + + $(call add_json_bool, TargetUserimagesUseExt2, $(filter true,$(TARGET_USERIMAGES_USE_EXT2))) + $(call add_json_bool, TargetUserimagesUseExt3, $(filter true,$(TARGET_USERIMAGES_USE_EXT3))) + $(call add_json_bool, TargetUserimagesUseExt4, $(filter true,$(TARGET_USERIMAGES_USE_EXT4))) + + $(call add_json_bool, TargetUserimagesSparseExtDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_EXT_DISABLED))) + $(call add_json_bool, TargetUserimagesSparseErofsDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_EROFS_DISABLED))) + $(call add_json_bool, TargetUserimagesSparseSquashfsDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_SQUASHFS_DISABLED))) + $(call add_json_bool, TargetUserimagesSparseF2fsDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_F2FS_DISABLED))) + + $(call add_json_str, BoardErofsCompressor, $(BOARD_EROFS_COMPRESSOR)) + $(call add_json_str, BoardErofsCompressorHints, $(BOARD_EROFS_COMPRESS_HINTS)) + $(call add_json_str, BoardErofsPclusterSize, $(BOARD_EROFS_PCLUSTER_SIZE)) + $(call add_json_str, BoardErofsShareDupBlocks, $(BOARD_EROFS_SHARE_DUP_BLOCKS)) + $(call add_json_str, BoardErofsUseLegacyCompression, $(BOARD_EROFS_USE_LEGACY_COMPRESSION)) + $(call add_json_str, BoardExt4ShareDupBlocks, $(BOARD_EXT4_SHARE_DUP_BLOCKS)) + $(call add_json_str, BoardFlashLogicalBlockSize, $(BOARD_FLASH_LOGICAL_BLOCK_SIZE)) + $(call add_json_str, BoardFlashEraseBlockSize, $(BOARD_FLASH_ERASE_BLOCK_SIZE)) + + $(call add_json_bool, BoardUsesRecoveryAsBoot, $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT))) + $(call add_json_bool, ProductUseDynamicPartitionSize, $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITION_SIZE))) + $(call add_json_bool, CopyImagesForTargetFilesZip, $(filter true,$(COPY_IMAGES_FOR_TARGET_FILES_ZIP))) + + $(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE))) + + $(call add_json_list, ProductPackages, $(sort $(PRODUCT_PACKAGES))) +$(call end_json_map) + +$(call add_json_bool, BuildFromSourceStub, $(findstring true,$(PRODUCT_BUILD_FROM_SOURCE_STUB) $(BUILD_FROM_SOURCE_STUB))) + +$(call add_json_bool, HiddenapiExportableStubs, $(filter true,$(PRODUCT_HIDDEN_API_EXPORTABLE_STUBS))) + +$(call add_json_bool, ExportRuntimeApis, $(filter true,$(PRODUCT_EXPORT_RUNTIME_APIS))) + +$(call add_json_str, AconfigContainerValidation, $(ACONFIG_CONTAINER_VALIDATION)) + +$(call add_json_list, ProductLocales, $(subst _,-,$(PRODUCT_LOCALES))) + +$(call add_json_list, ProductDefaultWifiChannels, $(PRODUCT_DEFAULT_WIFI_CHANNELS)) + +$(call add_json_bool, BoardUseVbmetaDigestInFingerprint, $(filter true,$(BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT))) + +$(call add_json_list, OemProperties, $(PRODUCT_OEM_PROPERTIES)) $(call json_end) diff --git a/core/soong_java_prebuilt.mk b/core/soong_java_prebuilt.mk index a8f475fd9a..7f85231543 100644 --- a/core/soong_java_prebuilt.mk +++ b/core/soong_java_prebuilt.mk @@ -62,31 +62,24 @@ ifdef LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR endif ifdef LOCAL_SOONG_PROGUARD_DICT - my_proguard_dictionary_directory := $(local-proguard-dictionary-directory) - my_proguard_dictionary_mapping_directory := $(local-proguard-dictionary-mapping-directory) - $(eval $(call copy-one-file,$(LOCAL_SOONG_PROGUARD_DICT),\ - $(intermediates.COMMON)/proguard_dictionary)) $(eval $(call copy-r8-dictionary-file-with-mapping,\ $(LOCAL_SOONG_PROGUARD_DICT),\ - $(my_proguard_dictionary_directory)/proguard_dictionary,\ - $(my_proguard_dictionary_mapping_directory)/proguard_dictionary.textproto)) - $(eval $(call copy-one-file,$(LOCAL_SOONG_CLASSES_JAR),\ - $(my_proguard_dictionary_directory)/classes.jar)) - $(call add-dependency,$(common_javalib.jar),\ - $(intermediates.COMMON)/proguard_dictionary) - $(call add-dependency,$(common_javalib.jar),\ - $(my_proguard_dictionary_directory)/proguard_dictionary) - $(call add-dependency,$(common_javalib.jar),\ - $(my_proguard_dictionary_mapping_directory)/proguard_dictionary.textproto) - $(call add-dependency,$(common_javalib.jar),\ - $(my_proguard_dictionary_directory)/classes.jar) + $(intermediates.COMMON)/proguard_dictionary,\ + $(intermediates.COMMON)/proguard_dictionary.textproto)) + + ALL_MODULES.$(my_register_name).PROGUARD_DICTIONARY_FILES := \ + $(intermediates.COMMON)/proguard_dictionary \ + $(LOCAL_SOONG_CLASSES_JAR) + ALL_MODULES.$(my_register_name).PROGUARD_DICTIONARY_SOONG_ZIP_ARGUMENTS := \ + -e out/target/common/obj/$(LOCAL_MODULE_CLASS)/$(LOCAL_MODULE)_intermediates/proguard_dictionary \ + -f $(intermediates.COMMON)/proguard_dictionary \ + -e out/target/common/obj/$(LOCAL_MODULE_CLASS)/$(LOCAL_MODULE)_intermediates/classes.jar \ + -f $(LOCAL_SOONG_CLASSES_JAR) + ALL_MODULES.$(my_register_name).PROGUARD_DICTIONARY_MAPPING := $(intermediates.COMMON)/proguard_dictionary.textproto endif ifdef LOCAL_SOONG_PROGUARD_USAGE_ZIP - $(eval $(call copy-one-file,$(LOCAL_SOONG_PROGUARD_USAGE_ZIP),\ - $(call local-packaging-dir,proguard_usage)/proguard_usage.zip)) - $(call add-dependency,$(common_javalib.jar),\ - $(call local-packaging-dir,proguard_usage)/proguard_usage.zip) + ALL_MODULES.$(my_register_name).PROGUARD_USAGE_ZIP := $(LOCAL_SOONG_PROGUARD_USAGE_ZIP) endif @@ -99,16 +92,12 @@ ifdef LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE $(call add-dependency,$(LOCAL_BUILT_MODULE),$(my_res_package)) - my_proguard_flags := $(intermediates.COMMON)/export_proguard_flags - $(my_proguard_flags): $(LOCAL_SOONG_EXPORT_PROGUARD_FLAGS) - @echo "Export proguard flags: $@" - rm -f $@ - touch $@ - for f in $+; do \ - echo -e "\n# including $$f" >>$@; \ - cat $$f >>$@; \ - done + my_transitive_res_packages := $(intermediates.COMMON)/transitive-res-packages + $(eval $(call copy-one-file,$(LOCAL_SOONG_TRANSITIVE_RES_PACKAGES),$(my_transitive_res_packages))) + $(call add-dependency,$(my_res_package),$(my_transitive_res_packages)) + my_proguard_flags := $(intermediates.COMMON)/export_proguard_flags + $(eval $(call copy-one-file,$(LOCAL_SOONG_EXPORT_PROGUARD_FLAGS),$(my_proguard_flags))) $(call add-dependency,$(LOCAL_BUILT_MODULE),$(my_proguard_flags)) my_static_library_extra_packages := $(intermediates.COMMON)/extra_packages diff --git a/core/static_java_library.mk b/core/static_java_library.mk index 4053985a3a..4a72a1fc31 100644 --- a/core/static_java_library.mk +++ b/core/static_java_library.mk @@ -127,6 +127,12 @@ framework_res_package_export := \ endif endif +# transitive-res-packages is only populated for Soong modules for now, but needs +# to exist so that other Make modules can depend on it. Create an empty file. +my_transitive_res_packages := $(intermediates.COMMON)/transitive-res-packages +$(my_transitive_res_packages): + touch $@ + import_proguard_flag_files := $(strip $(foreach l,$(LOCAL_STATIC_ANDROID_LIBRARIES) $(LOCAL_STATIC_JAVA_AAR_LIBRARIES),\ $(call intermediates-dir-for,JAVA_LIBRARIES,$(l),,COMMON)/export_proguard_flags)) $(intermediates.COMMON)/export_proguard_flags: $(import_proguard_flag_files) $(addprefix $(LOCAL_PATH)/,$(LOCAL_EXPORT_PROGUARD_FLAG_FILES)) diff --git a/core/sysprop.mk b/core/sysprop.mk index 6e2caed318..47d8a41a38 100644 --- a/core/sysprop.mk +++ b/core/sysprop.mk @@ -23,7 +23,6 @@ ifeq ($(BOARD_PROPERTY_OVERRIDES_SPLIT_ENABLED), true) property_overrides_split_enabled := true endif -BUILDINFO_SH := build/make/tools/buildinfo.sh POST_PROCESS_PROPS := $(HOST_OUT_EXECUTABLES)/post_process_props$(HOST_EXECUTABLE_SUFFIX) # Emits a set of sysprops common to all partitions to a file. @@ -46,10 +45,21 @@ define generate-common-build-props echo "ro.product.$(1).manufacturer=$(PRODUCT_MANUFACTURER)" >> $(2);\ echo "ro.product.$(1).model=$(PRODUCT_MODEL)" >> $(2);\ echo "ro.product.$(1).name=$(TARGET_PRODUCT)" >> $(2);\ - # Attestation specific properties for AOSP/GSI build running on device. - echo "ro.product.model_for_attestation=$(PRODUCT_MODEL_FOR_ATTESTATION)" >> $(2);\ - echo "ro.product.brand_for_attestation=$(PRODUCT_BRAND_FOR_ATTESTATION)" >> $(2);\ - echo "ro.product.name_for_attestation=$(PRODUCT_NAME_FOR_ATTESTATION)" >> $(2);\ + if [ -n "$(strip $(PRODUCT_MODEL_FOR_ATTESTATION))" ]; then \ + echo "ro.product.model_for_attestation=$(PRODUCT_MODEL_FOR_ATTESTATION)" >> $(2);\ + fi; \ + if [ -n "$(strip $(PRODUCT_BRAND_FOR_ATTESTATION))" ]; then \ + echo "ro.product.brand_for_attestation=$(PRODUCT_BRAND_FOR_ATTESTATION)" >> $(2);\ + fi; \ + if [ -n "$(strip $(PRODUCT_NAME_FOR_ATTESTATION))" ]; then \ + echo "ro.product.name_for_attestation=$(PRODUCT_NAME_FOR_ATTESTATION)" >> $(2);\ + fi; \ + if [ -n "$(strip $(PRODUCT_DEVICE_FOR_ATTESTATION))" ]; then \ + echo "ro.product.device_for_attestation=$(PRODUCT_DEVICE_FOR_ATTESTATION)" >> $(2);\ + fi; \ + if [ -n "$(strip $(PRODUCT_MANUFACTURER_FOR_ATTESTATION))" ]; then \ + echo "ro.product.manufacturer_for_attestation=$(PRODUCT_MANUFACTURER_FOR_ATTESTATION)" >> $(2);\ + fi; \ )\ $(if $(filter true,$(ZYGOTE_FORCE_64)),\ $(if $(filter vendor,$(1)),\ @@ -66,9 +76,12 @@ define generate-common-build-props )\ echo "ro.$(1).build.date=`$(DATE_FROM_FILE)`" >> $(2);\ echo "ro.$(1).build.date.utc=`$(DATE_FROM_FILE) +%s`" >> $(2);\ - echo "ro.$(1).build.fingerprint=$(BUILD_FINGERPRINT_FROM_FILE)" >> $(2);\ - echo "ro.$(1).build.id=$(BUILD_ID)" >> $(2);\ - echo "ro.$(1).build.tags=$(BUILD_VERSION_TAGS)" >> $(2);\ + # Allow optional assignments for ARC forward-declarations (b/249168657) + # TODO: Remove any tag-related inconsistencies once the goals from + # go/arc-android-sigprop-changes have been achieved. + echo "ro.$(1).build.fingerprint?=$(BUILD_FINGERPRINT_FROM_FILE)" >> $(2);\ + echo "ro.$(1).build.id?=$(BUILD_ID)" >> $(2);\ + echo "ro.$(1).build.tags?=$(BUILD_VERSION_TAGS)" >> $(2);\ echo "ro.$(1).build.type=$(TARGET_BUILD_VARIANT)" >> $(2);\ echo "ro.$(1).build.version.incremental=$(BUILD_NUMBER_FROM_FILE)" >> $(2);\ echo "ro.$(1).build.version.release=$(PLATFORM_VERSION_LAST_STABLE)" >> $(2);\ @@ -110,7 +123,7 @@ $(if $(filter true,$(BUILD_BROKEN_DUP_SYSPROP)),\ $(eval _option := --allow-dup)\ ) -$(2): $(POST_PROCESS_PROPS) $(INTERNAL_BUILD_ID_MAKEFILE) $(3) $(6) +$(2): $(POST_PROCESS_PROPS) $(INTERNAL_BUILD_ID_MAKEFILE) $(3) $(6) $(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC) $(hide) echo Building $$@ $(hide) mkdir -p $$(dir $$@) $(hide) rm -f $$@ && touch $$@ @@ -134,7 +147,10 @@ endif echo "$$(line)" >> $$@;\ )\ ) - $(hide) $(POST_PROCESS_PROPS) $$(_option) --sdk-version $(PLATFORM_SDK_VERSION) $$@ $(5) + $(hide) $(POST_PROCESS_PROPS) $$(_option) \ + --sdk-version $(PLATFORM_SDK_VERSION) \ + --kernel-version-file-for-uffd-gc "$(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC)" \ + $$@ $(5) $(hide) $(foreach file,$(strip $(6)),\ if [ -f "$(file)" ]; then\ cat $(file) >> $$@;\ @@ -171,15 +187,8 @@ BUILD_VERSION_TAGS := $(subst $(space),$(comma),$(sort $(BUILD_VERSION_TAGS))) # BUILD_FINGERPRINT is used used to uniquely identify the combined build and # product; used by the OTA server. ifeq (,$(strip $(BUILD_FINGERPRINT))) - ifeq ($(strip $(HAS_BUILD_NUMBER)),false) - BF_BUILD_NUMBER := $(BUILD_USERNAME)$$($(DATE_FROM_FILE) +%m%d%H%M) - else - BF_BUILD_NUMBER := $(file <$(BUILD_NUMBER_FILE)) - endif - BUILD_FINGERPRINT := $(PRODUCT_BRAND)/$(TARGET_PRODUCT)/$(TARGET_DEVICE):$(PLATFORM_VERSION)/$(BUILD_ID)/$(BF_BUILD_NUMBER):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS) + BUILD_FINGERPRINT := $(PRODUCT_BRAND)/$(TARGET_PRODUCT)/$(TARGET_DEVICE):$(PLATFORM_VERSION)/$(BUILD_ID)/$(BUILD_NUMBER_FROM_FILE):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS) endif -# unset it for safety. -BF_BUILD_NUMBER := BUILD_FINGERPRINT_FILE := $(PRODUCT_OUT)/build_fingerprint.txt ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_FINGERPRINT) >$(BUILD_FINGERPRINT_FILE) && grep " " $(BUILD_FINGERPRINT_FILE))) @@ -196,47 +205,16 @@ ifeq (,$(strip $(BUILD_THUMBPRINT))) endif BUILD_THUMBPRINT_FILE := $(PRODUCT_OUT)/build_thumbprint.txt +ifeq ($(strip $(HAS_BUILD_NUMBER)),true) +$(BUILD_THUMBPRINT_FILE): $(BUILD_NUMBER_FILE) +endif ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_THUMBPRINT) >$(BUILD_THUMBPRINT_FILE) && grep " " $(BUILD_THUMBPRINT_FILE))) $(error BUILD_THUMBPRINT cannot contain spaces: "$(file <$(BUILD_THUMBPRINT_FILE))") endif -BUILD_THUMBPRINT_FROM_FILE := $$(cat $(BUILD_THUMBPRINT_FILE)) # unset it for safety. +BUILD_THUMBPRINT_FILE := BUILD_THUMBPRINT := -# ----------------------------------------------------------------- -# Define human readable strings that describe this build -# - -# BUILD_ID: detail info; has the same info as the build fingerprint -BUILD_DESC := $(TARGET_PRODUCT)-$(TARGET_BUILD_VARIANT) $(PLATFORM_VERSION) $(BUILD_ID) $(BUILD_NUMBER_FROM_FILE) $(BUILD_VERSION_TAGS) - -# BUILD_DISPLAY_ID is shown under Settings -> About Phone -ifeq ($(TARGET_BUILD_VARIANT),user) - # User builds should show: - # release build number or branch.buld_number non-release builds - - # Dev. branches should have DISPLAY_BUILD_NUMBER set - ifeq (true,$(DISPLAY_BUILD_NUMBER)) - BUILD_DISPLAY_ID := $(BUILD_ID).$(BUILD_NUMBER_FROM_FILE) $(BUILD_KEYS) - else - BUILD_DISPLAY_ID := $(BUILD_ID) $(BUILD_KEYS) - endif -else - # Non-user builds should show detailed build information - BUILD_DISPLAY_ID := $(BUILD_DESC) -endif - -# TARGET_BUILD_FLAVOR and ro.build.flavor are used only by the test -# harness to distinguish builds. Only add _asan for a sanitized build -# if it isn't already a part of the flavor (via a dedicated lunch -# config for example). -TARGET_BUILD_FLAVOR := $(TARGET_PRODUCT)-$(TARGET_BUILD_VARIANT) -ifneq (, $(filter address, $(SANITIZE_TARGET))) -ifeq (,$(findstring _asan,$(TARGET_BUILD_FLAVOR))) -TARGET_BUILD_FLAVOR := $(TARGET_BUILD_FLAVOR)_asan -endif -endif - KNOWN_OEM_THUMBPRINT_PROPERTIES := \ ro.product.brand \ ro.product.name \ @@ -251,50 +229,7 @@ KNOWN_OEM_THUMBPRINT_PROPERTIES:= # Note: parts of this file that can't be generated by the build-properties # macro are manually created as separate files and then fed into the macro -# Accepts a whitespace separated list of product locales such as -# (en_US en_AU en_GB...) and returns the first locale in the list with -# underscores replaced with hyphens. In the example above, this will -# return "en-US". -define get-default-product-locale -$(strip $(subst _,-, $(firstword $(1)))) -endef - -gen_from_buildinfo_sh := $(call intermediates-dir-for,PACKAGING,system_build_prop)/buildinfo.prop -$(gen_from_buildinfo_sh): $(INTERNAL_BUILD_ID_MAKEFILE) $(API_FINGERPRINT) | $(BUILD_DATETIME_FILE) $(BUILD_NUMBER_FILE) - $(hide) TARGET_BUILD_TYPE="$(TARGET_BUILD_VARIANT)" \ - TARGET_BUILD_FLAVOR="$(TARGET_BUILD_FLAVOR)" \ - TARGET_DEVICE="$(TARGET_DEVICE)" \ - PRODUCT_DEFAULT_LOCALE="$(call get-default-product-locale,$(PRODUCT_LOCALES))" \ - PRODUCT_DEFAULT_WIFI_CHANNELS="$(PRODUCT_DEFAULT_WIFI_CHANNELS)" \ - PRIVATE_BUILD_DESC="$(BUILD_DESC)" \ - BUILD_ID="$(BUILD_ID)" \ - BUILD_DISPLAY_ID="$(BUILD_DISPLAY_ID)" \ - DATE="$(DATE_FROM_FILE)" \ - BUILD_USERNAME="$(BUILD_USERNAME)" \ - BUILD_HOSTNAME="$(BUILD_HOSTNAME)" \ - BUILD_NUMBER="$(BUILD_NUMBER_FROM_FILE)" \ - BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT="$(BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT)" \ - PLATFORM_VERSION="$(PLATFORM_VERSION)" \ - PLATFORM_DISPLAY_VERSION="$(PLATFORM_DISPLAY_VERSION)" \ - PLATFORM_VERSION_LAST_STABLE="$(PLATFORM_VERSION_LAST_STABLE)" \ - PLATFORM_SECURITY_PATCH="$(PLATFORM_SECURITY_PATCH)" \ - PLATFORM_BASE_OS="$(PLATFORM_BASE_OS)" \ - PLATFORM_SDK_VERSION="$(PLATFORM_SDK_VERSION)" \ - PLATFORM_PREVIEW_SDK_VERSION="$(PLATFORM_PREVIEW_SDK_VERSION)" \ - PLATFORM_PREVIEW_SDK_FINGERPRINT="$$(cat $(API_FINGERPRINT))" \ - PLATFORM_VERSION_CODENAME="$(PLATFORM_VERSION_CODENAME)" \ - PLATFORM_VERSION_ALL_CODENAMES="$(PLATFORM_VERSION_ALL_CODENAMES)" \ - PLATFORM_VERSION_KNOWN_CODENAMES="$(PLATFORM_VERSION_KNOWN_CODENAMES)" \ - PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION="$(PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION)" \ - BUILD_VERSION_TAGS="$(BUILD_VERSION_TAGS)" \ - $(if $(OEM_THUMBPRINT_PROPERTIES),BUILD_THUMBPRINT="$(BUILD_THUMBPRINT_FROM_FILE)") \ - TARGET_CPU_ABI_LIST="$(TARGET_CPU_ABI_LIST)" \ - TARGET_CPU_ABI_LIST_32_BIT="$(TARGET_CPU_ABI_LIST_32_BIT)" \ - TARGET_CPU_ABI_LIST_64_BIT="$(TARGET_CPU_ABI_LIST_64_BIT)" \ - TARGET_CPU_ABI="$(TARGET_CPU_ABI)" \ - TARGET_CPU_ABI2="$(TARGET_CPU_ABI2)" \ - ZYGOTE_FORCE_64_BIT="$(ZYGOTE_FORCE_64_BIT)" \ - bash $(BUILDINFO_SH) > $@ +buildinfo_prop := $(call intermediates-dir-for,ETC,buildinfo.prop)/buildinfo.prop ifdef TARGET_SYSTEM_PROP system_prop_file := $(TARGET_SYSTEM_PROP) @@ -303,7 +238,7 @@ system_prop_file := $(wildcard $(TARGET_DEVICE_DIR)/system.prop) endif _prop_files_ := \ - $(gen_from_buildinfo_sh) \ + $(buildinfo_prop) \ $(system_prop_file) # Order matters here. When there are duplicates, the last one wins. @@ -543,3 +478,19 @@ $(eval $(call build-properties,\ $(empty))) $(eval $(call declare-1p-target,$(INSTALLED_RAMDISK_BUILD_PROP_TARGET))) + +ALL_INSTALLED_BUILD_PROP_FILES := \ + $(INSTALLED_BUILD_PROP_TARGET) \ + $(INSTALLED_VENDOR_BUILD_PROP_TARGET) \ + $(INSTALLED_PRODUCT_BUILD_PROP_TARGET) \ + $(INSTALLED_ODM_BUILD_PROP_TARGET) \ + $(INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET) \ + $(INSTALLED_ODM_DLKM_BUILD_PROP_TARGET) \ + $(INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET) \ + $(INSTALLED_SYSTEM_EXT_BUILD_PROP_TARGET) \ + $(INSTALLED_RAMDISK_BUILD_PROP_TARGET) + +# $1 installed file path, e.g. out/target/product/vsoc_x86_64/system/build.prop +define is-build-prop +$(if $(findstring $1,$(ALL_INSTALLED_BUILD_PROP_FILES)),Y) +endef diff --git a/core/tasks/art-host-tests.mk b/core/tasks/art-host-tests.mk index 2af1deda7c..c95f6e7878 100644 --- a/core/tasks/art-host-tests.mk +++ b/core/tasks/art-host-tests.mk @@ -24,25 +24,56 @@ my_host_shared_lib_for_art_host_tests := $(foreach f,$(COMPATIBILITY.art-host-te $(eval _cmf_src := $(word 1,$(_cmf_tuple))) \ $(_cmf_src))) -$(art_host_tests_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_art_host_tests) +# Create an artifact to include a list of test config files in art-host-tests. +art_host_tests_list_zip := $(PRODUCT_OUT)/art-host-tests_list.zip +# Create an artifact to include all test config files in art-host-tests. +art_host_tests_configs_zip := $(PRODUCT_OUT)/art-host-tests_configs.zip +# Create an artifact to include all shared library files in art-host-tests. +art_host_tests_host_shared_libs_zip := $(PRODUCT_OUT)/art-host-tests_host-shared-libs.zip +$(art_host_tests_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_art_host_tests) +$(art_host_tests_zip) : PRIVATE_art_host_tests_list_zip := $(art_host_tests_list_zip) +$(art_host_tests_zip) : PRIVATE_art_host_tests_configs_zip := $(art_host_tests_configs_zip) +$(art_host_tests_zip) : PRIVATE_art_host_tests_host_shared_libs_zip := $(art_host_tests_host_shared_libs_zip) +$(art_host_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(art_host_tests_list_zip) $(art_host_tests_configs_zip) $(art_host_tests_host_shared_libs_zip) +$(art_host_tests_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir) $(art_host_tests_zip) : $(COMPATIBILITY.art-host-tests.FILES) $(my_host_shared_lib_for_art_host_tests) $(SOONG_ZIP) - echo $(sort $(COMPATIBILITY.art-host-tests.FILES)) | tr " " "\n" > $@.list - grep $(HOST_OUT_TESTCASES) $@.list > $@-host.list || true - $(hide) touch $@-host-libs.list + rm -rf $(PRIVATE_INTERMEDIATES_DIR) + rm -f $@ $(PRIVATE_art_host_tests_list_zip) + mkdir -p $(PRIVATE_INTERMEDIATES_DIR) + echo $(sort $(COMPATIBILITY.art-host-tests.FILES)) | tr " " "\n" > $(PRIVATE_INTERMEDIATES_DIR)/list + grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/host.list || true + $(hide) touch $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ - echo $$shared_lib >> $@-host-libs.list; \ + echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \ done - grep $(TARGET_OUT_TESTCASES) $@.list > $@-target.list || true - $(hide) $(SOONG_ZIP) -d -o $@ -P host -C $(HOST_OUT) -l $@-host.list \ - -P target -C $(PRODUCT_OUT) -l $@-target.list \ - -P host/testcases -C $(HOST_OUT) -l $@-host-libs.list - rm -f $@.list $@-host.list $@-target.list $@-host-libs.list + grep $(TARGET_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/target.list || true + $(hide) $(SOONG_ZIP) -d -o $@ -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host.list \ + -P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target.list \ + -P host/testcases -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list \ + -sha256 + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list > $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list || true + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/target.list > $(PRIVATE_INTERMEDIATES_DIR)/target-test-configs.list || true + $(hide) $(SOONG_ZIP) -d -o $(PRIVATE_art_host_tests_configs_zip) \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list \ + -P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target-test-configs.list + grep $(HOST_OUT) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true + $(hide) $(SOONG_ZIP) -d -o $(PRIVATE_art_host_tests_host_shared_libs_zip) \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list | sed s%$(HOST_OUT)%host%g > $(PRIVATE_INTERMEDIATES_DIR)/art-host-tests_list + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/target.list | sed s%$(PRODUCT_OUT)%target%g >> $(PRIVATE_INTERMEDIATES_DIR)/art-host-tests_list + $(hide) $(SOONG_ZIP) -d -o $(PRIVATE_art_host_tests_list_zip) -C $(PRIVATE_INTERMEDIATES_DIR) -f $(PRIVATE_INTERMEDIATES_DIR)/art-host-tests_list art-host-tests: $(art_host_tests_zip) -$(call dist-for-goals, art-host-tests, $(art_host_tests_zip)) +$(call dist-for-goals, art-host-tests, $(art_host_tests_zip) $(art_host_tests_list_zip) $(art_host_tests_configs_zip) $(art_host_tests_host_shared_libs_zip)) $(call declare-1p-container,$(art_host_tests_zip),) $(call declare-container-license-deps,$(art_host_tests_zip),$(COMPATIBILITY.art-host-tests.FILES) $(my_host_shared_lib_for_art_host_tests),$(PRODUCT_OUT)/:/) tests: art-host-tests + +intermediates_dir := +art_host_tests_zip := +art_host_tests_list_zip := +art_host_tests_configs_zip := +art_host_tests_host_shared_libs_zip := diff --git a/core/tasks/automotive-general-tests.mk b/core/tasks/automotive-general-tests.mk new file mode 100644 index 0000000000..44b62bef78 --- /dev/null +++ b/core/tasks/automotive-general-tests.mk @@ -0,0 +1,89 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: automotive-general-tests + +automotive_general_tests_tools := \ + $(HOST_OUT_JAVA_LIBRARIES)/cts-tradefed.jar \ + $(HOST_OUT_JAVA_LIBRARIES)/compatibility-host-util.jar \ + $(HOST_OUT_JAVA_LIBRARIES)/vts-tradefed.jar \ + +intermediates_dir := $(call intermediates-dir-for,PACKAGING,automotive-general-tests) +automotive_general_tests_zip := $(PRODUCT_OUT)/automotive-general-tests.zip +# Create an artifact to include a list of test config files in automotive-general-tests. +automotive_general_tests_list_zip := $(PRODUCT_OUT)/automotive-general-tests_list.zip + +# Filter shared entries between automotive-general-tests and automotive-tests's HOST_SHARED_LIBRARY.FILES, +# to avoid warning about overriding commands. +my_host_shared_lib_for_automotive_general_tests := \ + $(foreach m,$(filter $(COMPATIBILITY.automotive-tests.HOST_SHARED_LIBRARY.FILES),\ + $(COMPATIBILITY.automotive-general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m))) +my_automotive_general_tests_shared_lib_files := \ + $(filter-out $(COMPATIBILITY.automotive-tests.HOST_SHARED_LIBRARY.FILES),\ + $(COMPATIBILITY.automotive-general-tests.HOST_SHARED_LIBRARY.FILES)) + +my_host_shared_lib_for_automotive_general_tests += $(call copy-many-files,$(my_automotive_general_tests_shared_lib_files)) + +# Create an artifact to include all test config files in automotive-general-tests. +automotive_general_tests_configs_zip := $(PRODUCT_OUT)/automotive-general-tests_configs.zip +# Create an artifact to include all shared librariy files in automotive-general-tests. +automotive_general_tests_host_shared_libs_zip := $(PRODUCT_OUT)/automotive-general-tests_host-shared-libs.zip + +$(automotive_general_tests_zip) : PRIVATE_automotive_general_tests_list_zip := $(automotive_general_tests_list_zip) +$(automotive_general_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(automotive_general_tests_list_zip) $(automotive_general_tests_configs_zip) $(automotive_general_tests_host_shared_libs_zip) +$(automotive_general_tests_zip) : PRIVATE_TOOLS := $(automotive_general_tests_tools) +$(automotive_general_tests_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir) +$(automotive_general_tests_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_automotive_general_tests) +$(automotive_general_tests_zip) : PRIVATE_automotive_general_tests_configs_zip := $(automotive_general_tests_configs_zip) +$(automotive_general_tests_zip) : PRIVATE_general_host_shared_libs_zip := $(automotive_general_tests_host_shared_libs_zip) +$(automotive_general_tests_zip) : $(COMPATIBILITY.automotive-general-tests.FILES) $(automotive_general_tests_tools) $(my_host_shared_lib_for_automotive_general_tests) $(SOONG_ZIP) + rm -rf $(PRIVATE_INTERMEDIATES_DIR) + rm -f $@ $(PRIVATE_automotive_general_tests_list_zip) + mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools + echo $(sort $(COMPATIBILITY.automotive-general-tests.FILES)) | tr " " "\n" > $(PRIVATE_INTERMEDIATES_DIR)/list + grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/host.list || true + grep $(TARGET_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/target.list || true + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list > $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list || true + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/target.list > $(PRIVATE_INTERMEDIATES_DIR)/target-test-configs.list || true + $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ + echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/host.list; \ + echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \ + done + grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true + cp -fp $(PRIVATE_TOOLS) $(PRIVATE_INTERMEDIATES_DIR)/tools/ + $(SOONG_ZIP) -d -o $@ \ + -P host -C $(PRIVATE_INTERMEDIATES_DIR) -D $(PRIVATE_INTERMEDIATES_DIR)/tools \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host.list \ + -P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target.list + $(SOONG_ZIP) -d -o $(PRIVATE_automotive_general_tests_configs_zip) \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list \ + -P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target-test-configs.list + $(SOONG_ZIP) -d -o $(PRIVATE_general_host_shared_libs_zip) \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list | sed s%$(HOST_OUT)%host%g > $(PRIVATE_INTERMEDIATES_DIR)/automotive-general-tests_list + grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/target.list | sed s%$(PRODUCT_OUT)%target%g >> $(PRIVATE_INTERMEDIATES_DIR)/automotive-general-tests_list + $(SOONG_ZIP) -d -o $(PRIVATE_automotive_general_tests_list_zip) -C $(PRIVATE_INTERMEDIATES_DIR) -f $(PRIVATE_INTERMEDIATES_DIR)/automotive-general-tests_list + +automotive-general-tests: $(automotive_general_tests_zip) +$(call dist-for-goals, automotive-general-tests, $(automotive_general_tests_zip) $(automotive_general_tests_list_zip) $(automotive_general_tests_configs_zip) $(automotive_general_tests_host_shared_libs_zip)) + +$(call declare-1p-container,$(automotive_general_tests_zip),) +$(call declare-container-license-deps,$(automotive_general_tests_zip),$(COMPATIBILITY.automotive-general-tests.FILES) $(automotive_general_tests_tools) $(my_host_shared_lib_for_automotive_general_tests),$(PRODUCT_OUT)/:/) + +intermediates_dir := +automotive_general_tests_tools := +automotive_general_tests_zip := +automotive_general_tests_list_zip := +automotive_general_tests_configs_zip := +automotive_general_tests_host_shared_libs_zip := diff --git a/core/tasks/automotive-sdv-tests.mk b/core/tasks/automotive-sdv-tests.mk new file mode 100644 index 0000000000..12706ce33d --- /dev/null +++ b/core/tasks/automotive-sdv-tests.mk @@ -0,0 +1,61 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +.PHONY: automotive-sdv-tests + +automotive-sdv-tests-zip := $(PRODUCT_OUT)/automotive-sdv-tests.zip +# Create an artifact to include a list of test config files in automotive-sdv-tests. +automotive-sdv-tests-list-zip := $(PRODUCT_OUT)/automotive-sdv-tests_list.zip +# Create an artifact to include all test config files in automotive-sdv-tests. +automotive-sdv-tests-configs-zip := $(PRODUCT_OUT)/automotive-sdv-tests_configs.zip +my_host_shared_lib_for_automotive_sdv_tests := $(call copy-many-files,$(COMPATIBILITY.automotive-sdv-tests.HOST_SHARED_LIBRARY.FILES)) +automotive_sdv_tests_host_shared_libs_zip := $(PRODUCT_OUT)/automotive-sdv-tests_host-shared-libs.zip + +$(automotive-sdv-tests-zip) : .KATI_IMPLICIT_OUTPUTS := $(automotive-sdv-tests-list-zip) $(automotive-sdv-tests-configs-zip) $(automotive_sdv_tests_host_shared_libs_zip) +$(automotive-sdv-tests-zip) : PRIVATE_automotive_sdv_tests_list := $(PRODUCT_OUT)/automotive-sdv-tests_list +$(automotive-sdv-tests-zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_automotive_sdv_tests) +$(automotive-sdv-tests-zip) : PRIVATE_automotive_host_shared_libs_zip := $(automotive_sdv_tests_host_shared_libs_zip) +$(automotive-sdv-tests-zip) : $(COMPATIBILITY.automotive-sdv-tests.FILES) $(my_host_shared_lib_for_automotive_sdv_tests) $(SOONG_ZIP) + rm -f $@-shared-libs.list + echo $(sort $(COMPATIBILITY.automotive-sdv-tests.FILES)) | tr " " "\n" > $@.list + grep $(HOST_OUT_TESTCASES) $@.list > $@-host.list || true + grep -e .*\\.config$$ $@-host.list > $@-host-test-configs.list || true + $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ + echo $$shared_lib >> $@-host.list; \ + echo $$shared_lib >> $@-shared-libs.list; \ + done + grep $(HOST_OUT_TESTCASES) $@-shared-libs.list > $@-host-shared-libs.list || true + grep $(TARGET_OUT_TESTCASES) $@.list > $@-target.list || true + grep -e .*\\.config$$ $@-target.list > $@-target-test-configs.list || true + $(hide) $(SOONG_ZIP) -d -o $@ -P host -C $(HOST_OUT) -l $@-host.list -P target -C $(PRODUCT_OUT) -l $@-target.list + $(hide) $(SOONG_ZIP) -d -o $(automotive-sdv-tests-configs-zip) \ + -P host -C $(HOST_OUT) -l $@-host-test-configs.list \ + -P target -C $(PRODUCT_OUT) -l $@-target-test-configs.list + $(SOONG_ZIP) -d -o $(PRIVATE_automotive_host_shared_libs_zip) \ + -P host -C $(HOST_OUT) -l $@-host-shared-libs.list + rm -f $(PRIVATE_automotive_sdv_tests_list) + $(hide) grep -e .*\\.config$$ $@-host.list | sed s%$(HOST_OUT)%host%g > $(PRIVATE_automotive_sdv_tests_list) + $(hide) grep -e .*\\.config$$ $@-target.list | sed s%$(PRODUCT_OUT)%target%g >> $(PRIVATE_automotive_sdv_tests_list) + $(hide) $(SOONG_ZIP) -d -o $(automotive-sdv-tests-list-zip) -C $(dir $@) -f $(PRIVATE_automotive_sdv_tests_list) + rm -f $@.list $@-host.list $@-target.list $@-host-test-configs.list $@-target-test-configs.list \ + $@-shared-libs.list $@-host-shared-libs.list $(PRIVATE_automotive_sdv_tests_list) + +automotive-sdv-tests: $(automotive-sdv-tests-zip) +$(call dist-for-goals, automotive-sdv-tests, $(automotive-sdv-tests-zip) $(automotive-sdv-tests-list-zip) $(automotive-sdv-tests-configs-zip) $(automotive_sdv_tests_host_shared_libs_zip)) + +$(call declare-1p-container,$(automotive-sdv-tests-zip),) +$(call declare-container-license-deps,$(automotive-sdv-tests-zip),$(COMPATIBILITY.automotive-sdv-tests.FILES) $(my_host_shared_lib_for_automotive_sdv_tests),$(PRODUCT_OUT)/:/) + +tests: automotive-sdv-tests diff --git a/core/tasks/automotive-tests.mk b/core/tasks/automotive-tests.mk new file mode 100644 index 0000000000..da6af6bb3c --- /dev/null +++ b/core/tasks/automotive-tests.mk @@ -0,0 +1,61 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +.PHONY: automotive-tests + +automotive-tests-zip := $(PRODUCT_OUT)/automotive-tests.zip +# Create an artifact to include a list of test config files in automotive-tests. +automotive-tests-list-zip := $(PRODUCT_OUT)/automotive-tests_list.zip +# Create an artifact to include all test config files in automotive-tests. +automotive-tests-configs-zip := $(PRODUCT_OUT)/automotive-tests_configs.zip +my_host_shared_lib_for_automotive_tests := $(call copy-many-files,$(COMPATIBILITY.automotive-tests.HOST_SHARED_LIBRARY.FILES)) +automotive_tests_host_shared_libs_zip := $(PRODUCT_OUT)/automotive-tests_host-shared-libs.zip + +$(automotive-tests-zip) : .KATI_IMPLICIT_OUTPUTS := $(automotive-tests-list-zip) $(automotive-tests-configs-zip) $(automotive_tests_host_shared_libs_zip) +$(automotive-tests-zip) : PRIVATE_automotive_tests_list := $(PRODUCT_OUT)/automotive-tests_list +$(automotive-tests-zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_automotive_tests) +$(automotive-tests-zip) : PRIVATE_automotive_host_shared_libs_zip := $(automotive_tests_host_shared_libs_zip) +$(automotive-tests-zip) : $(COMPATIBILITY.automotive-tests.FILES) $(my_host_shared_lib_for_automotive_tests) $(SOONG_ZIP) + rm -f $@-shared-libs.list + echo $(sort $(COMPATIBILITY.automotive-tests.FILES)) | tr " " "\n" > $@.list + grep $(HOST_OUT_TESTCASES) $@.list > $@-host.list || true + grep -e .*\\.config$$ $@-host.list > $@-host-test-configs.list || true + $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ + echo $$shared_lib >> $@-host.list; \ + echo $$shared_lib >> $@-shared-libs.list; \ + done + grep $(HOST_OUT_TESTCASES) $@-shared-libs.list > $@-host-shared-libs.list || true + grep $(TARGET_OUT_TESTCASES) $@.list > $@-target.list || true + grep -e .*\\.config$$ $@-target.list > $@-target-test-configs.list || true + $(hide) $(SOONG_ZIP) -d -o $@ -P host -C $(HOST_OUT) -l $@-host.list -P target -C $(PRODUCT_OUT) -l $@-target.list + $(hide) $(SOONG_ZIP) -d -o $(automotive-tests-configs-zip) \ + -P host -C $(HOST_OUT) -l $@-host-test-configs.list \ + -P target -C $(PRODUCT_OUT) -l $@-target-test-configs.list + $(SOONG_ZIP) -d -o $(PRIVATE_automotive_host_shared_libs_zip) \ + -P host -C $(HOST_OUT) -l $@-host-shared-libs.list + rm -f $(PRIVATE_automotive_tests_list) + $(hide) grep -e .*\\.config$$ $@-host.list | sed s%$(HOST_OUT)%host%g > $(PRIVATE_automotive_tests_list) + $(hide) grep -e .*\\.config$$ $@-target.list | sed s%$(PRODUCT_OUT)%target%g >> $(PRIVATE_automotive_tests_list) + $(hide) $(SOONG_ZIP) -d -o $(automotive-tests-list-zip) -C $(dir $@) -f $(PRIVATE_automotive_tests_list) + rm -f $@.list $@-host.list $@-target.list $@-host-test-configs.list $@-target-test-configs.list \ + $@-shared-libs.list $@-host-shared-libs.list $(PRIVATE_automotive_tests_list) + +automotive-tests: $(automotive-tests-zip) +$(call dist-for-goals, automotive-tests, $(automotive-tests-zip) $(automotive-tests-list-zip) $(automotive-tests-configs-zip) $(automotive_tests_host_shared_libs_zip)) + +$(call declare-1p-container,$(automotive-tests-zip),) +$(call declare-container-license-deps,$(automotive-tests-zip),$(COMPATIBILITY.automotive-tests.FILES) $(my_host_shared_lib_for_automotive_tests),$(PRODUCT_OUT)/:/) + +tests: automotive-tests diff --git a/core/tasks/berberis_test.mk b/core/tasks/berberis_test.mk new file mode 100644 index 0000000000..860470980d --- /dev/null +++ b/core/tasks/berberis_test.mk @@ -0,0 +1,25 @@ +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BERBERIS_DIR := frameworks/libs/binary_translation + +# Berberis includes some components which may conflict with other packages. +# Only build it when requested explicitly. +ifeq ($(BUILD_BERBERIS),true) + +include $(BERBERIS_DIR)/tests/run_host_tests.mk + +endif # BUILD_BERBERIS diff --git a/core/tasks/collect_gpl_sources.mk b/core/tasks/collect_gpl_sources.mk deleted file mode 100644 index 9e9ab8ecc2..0000000000 --- a/core/tasks/collect_gpl_sources.mk +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2011 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The rule below doesn't have dependenices on the files that it copies, -# so manually generate into a PACKAGING intermediate dir, which is wiped -# in installclean between incremental builds on build servers. -gpl_source_tgz := $(call intermediates-dir-for,PACKAGING,gpl_source)/gpl_source.tgz - -ALL_GPL_MODULE_LICENSE_FILES := $(sort $(ALL_GPL_MODULE_LICENSE_FILES)) - -# FORCE since we can't know whether any of the sources changed -$(gpl_source_tgz): PRIVATE_PATHS := $(sort $(patsubst %/, %, $(dir $(ALL_GPL_MODULE_LICENSE_FILES)))) -$(gpl_source_tgz) : $(ALL_GPL_MODULE_LICENSE_FILES) - @echo Package GPL sources: $@ - $(hide) tar cfz $@ --exclude ".git*" $(PRIVATE_PATHS) - -# Dist the tgz only if we are doing a full build -$(call dist-for-goals,droidcore-unbundled,$(gpl_source_tgz)) diff --git a/core/tasks/cts.mk b/core/tasks/cts.mk index c8b1183e28..b9f0988740 100644 --- a/core/tasks/cts.mk +++ b/core/tasks/cts.mk @@ -37,16 +37,18 @@ ifneq (,$(wildcard cts/)) cts_platform_release_path := cts/tests/tests/os/assets/platform_releases.txt cts_platform_release_string := $(shell cat $(cts_platform_release_path)) - ifeq (,$(findstring $(PLATFORM_VERSION),$(cts_platform_version_string))) - define error_msg - ============================================================ - Could not find version "$(PLATFORM_VERSION)" in CTS platform version file: - $(cts_platform_version_path) - Most likely PLATFORM_VERSION in build/core/version_defaults.mk - has changed and a new version must be added to this CTS file. - ============================================================ - endef - $(error $(error_msg)) + ifneq (REL,$(PLATFORM_VERSION_CODENAME)) + ifeq (,$(findstring $(PLATFORM_VERSION),$(cts_platform_version_string))) + define error_msg + ============================================================ + Could not find version "$(PLATFORM_VERSION)" in CTS platform version file: + $(cts_platform_version_path) + Most likely PLATFORM_VERSION in build/core/version_defaults.mk + has changed and a new version must be added to this CTS file. + ============================================================ + endef + $(error $(error_msg)) + endif endif ifeq (,$(findstring $(PLATFORM_VERSION_LAST_STABLE),$(cts_platform_release_string))) define error_msg @@ -142,30 +144,30 @@ $(cts-system-api-xml-coverage-report) : $(android_cts_zip) $(cts_system_api_cove $(call generate-coverage-report-cts,"CTS System API Coverage Report - XML",\ $(PRIVATE_TEST_CASES),xml) -$(cts-verifier-coverage-report): PRIVATE_TEST_CASES := $(cts_verifier_apk) +$(cts-verifier-coverage-report): PRIVATE_TEST_CASES := $(foreach c, $(cts_verifier_apk) $(verifier-dir), $(c)) $(cts-verifier-coverage-report): PRIVATE_CTS_API_COVERAGE_EXE := $(cts_api_coverage_exe) $(cts-verifier-coverage-report): PRIVATE_DEXDEPS_EXE := $(dexdeps_exe) $(cts-verifier-coverage-report): PRIVATE_API_XML_DESC := $(api_xml_description) $(cts-verifier-coverage-report): PRIVATE_NAPI_XML_DESC := $(napi_xml_description) -$(cts-verifier-coverage-report) : $(cts_verifier_apk) $(cts_api_coverage_dependencies) | $(ACP) +$(cts-verifier-coverage-report) : $(cts_verifier_apk) $(verifier-zip) $(cts_api_coverage_dependencies) | $(ACP) $(call generate-coverage-report-cts,"CTS Verifier API Coverage Report",\ $(PRIVATE_TEST_CASES),html) -$(cts-combined-coverage-report): PRIVATE_TEST_CASES := $(foreach c, $(cts_verifier_apk) $(COMPATIBILITY_TESTCASES_OUT_cts), $(c)) +$(cts-combined-coverage-report): PRIVATE_TEST_CASES := $(foreach c, $(cts_verifier_apk) $(COMPATIBILITY_TESTCASES_OUT_cts) $(verifier-dir), $(c)) $(cts-combined-coverage-report): PRIVATE_CTS_API_COVERAGE_EXE := $(cts_api_coverage_exe) $(cts-combined-coverage-report): PRIVATE_DEXDEPS_EXE := $(dexdeps_exe) $(cts-combined-coverage-report): PRIVATE_API_XML_DESC := $(api_xml_description) $(cts-combined-coverage-report): PRIVATE_NAPI_XML_DESC := $(napi_xml_description) -$(cts-combined-coverage-report) : $(android_cts_zip) $(cts_verifier_apk) $(cts_api_coverage_dependencies) | $(ACP) +$(cts-combined-coverage-report) : $(android_cts_zip) $(cts_verifier_apk) $(verifier-zip) $(cts_api_coverage_dependencies) | $(ACP) $(call generate-coverage-report-cts,"CTS Combined API Coverage Report",\ $(PRIVATE_TEST_CASES),html) -$(cts-combined-xml-coverage-report): PRIVATE_TEST_CASES := $(foreach c, $(cts_verifier_apk) $(COMPATIBILITY_TESTCASES_OUT_cts), $(c)) +$(cts-combined-xml-coverage-report): PRIVATE_TEST_CASES := $(foreach c, $(cts_verifier_apk) $(COMPATIBILITY_TESTCASES_OUT_cts) $(verifier-dir), $(c)) $(cts-combined-xml-coverage-report): PRIVATE_CTS_API_COVERAGE_EXE := $(cts_api_coverage_exe) $(cts-combined-xml-coverage-report): PRIVATE_DEXDEPS_EXE := $(dexdeps_exe) $(cts-combined-xml-coverage-report): PRIVATE_API_XML_DESC := $(api_xml_description) $(cts-combined-xml-coverage-report): PRIVATE_NAPI_XML_DESC := $(napi_xml_description) -$(cts-combined-xml-coverage-report) : $(android_cts_zip) $(cts_verifier_apk) $(cts_api_coverage_dependencies) | $(ACP) +$(cts-combined-xml-coverage-report) : $(android_cts_zip) $(cts_verifier_apk) $(verifier-zip) $(cts_api_coverage_dependencies) | $(ACP) $(call generate-coverage-report-cts,"CTS Combined API Coverage Report - XML",\ $(PRIVATE_TEST_CASES),xml) @@ -234,3 +236,8 @@ dexdeps_exe := cts_api_coverage_exe := cts_verifier_apk := android_cts_zip := +cts-dir := +verifier-dir-name := +verifier-dir := +verifier-zip-name := +verifier-zip := diff --git a/core/tasks/fontchain_lint.mk b/core/tasks/fontchain_lint.mk new file mode 100644 index 0000000000..a4c396daa0 --- /dev/null +++ b/core/tasks/fontchain_lint.mk @@ -0,0 +1,43 @@ +# Copyright (C) 2011 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run sanity tests on fonts on checkbuild +checkbuild: fontchain_lint + +FONTCHAIN_LINTER := $(HOST_OUT_EXECUTABLES)/fontchain_linter +ifeq ($(MINIMAL_FONT_FOOTPRINT),true) +CHECK_EMOJI := false +else +CHECK_EMOJI := true +endif + +fontchain_lint_timestamp := $(call intermediates-dir-for,PACKAGING,fontchain_lint)/stamp + +.PHONY: fontchain_lint +fontchain_lint: $(fontchain_lint_timestamp) + +fontchain_lint_deps := \ + external/unicode/DerivedAge.txt \ + external/unicode/emoji-data.txt \ + external/unicode/emoji-sequences.txt \ + external/unicode/emoji-variation-sequences.txt \ + external/unicode/emoji-zwj-sequences.txt \ + external/unicode/additions/emoji-data.txt \ + external/unicode/additions/emoji-sequences.txt \ + external/unicode/additions/emoji-zwj-sequences.txt \ + +$(fontchain_lint_timestamp): $(FONTCHAIN_LINTER) $(TARGET_OUT)/etc/fonts.xml $(PRODUCT_OUT)/system.img $(fontchain_lint_deps) + @echo Running fontchain lint + $(FONTCHAIN_LINTER) $(TARGET_OUT) $(CHECK_EMOJI) external/unicode + touch $@ diff --git a/core/tasks/general-tests-shared-libs.mk b/core/tasks/general-tests-shared-libs.mk new file mode 100644 index 0000000000..240514073e --- /dev/null +++ b/core/tasks/general-tests-shared-libs.mk @@ -0,0 +1,52 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: general-tests-shared-libs + +intermediates_dir := $(call intermediates-dir-for,PACKAGING,general-tests-shared-libs) + +general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip + +# Filter shared entries between general-tests and device-tests's HOST_SHARED_LIBRARY.FILES, +# to avoid warning about overriding commands. +my_host_shared_lib_for_general_tests := \ + $(foreach m,$(filter $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ + $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m))) +my_general_tests_shared_lib_files := \ + $(filter-out $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ + $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)) + +my_host_shared_lib_for_general_tests += $(call copy-many-files,$(my_general_tests_shared_lib_files)) + +$(general_tests_shared_libs_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir) +$(general_tests_shared_libs_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_general_tests) +$(general_tests_shared_libs_zip) : PRIVATE_general_host_shared_libs_zip := $(general_tests_shared_libs_zip) +$(general_tests_shared_libs_zip) : $(my_host_shared_lib_for_general_tests) $(SOONG_ZIP) + rm -rf $(PRIVATE_INTERMEDIATES_DIR) + mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools + $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ + echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \ + done + grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true + $(SOONG_ZIP) -d -o $(PRIVATE_general_host_shared_libs_zip) \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list + +general-tests-shared-libs: $(general_tests_shared_libs_zip) +$(call dist-for-goals, general-tests-shared-libs, $(general_tests_shared_libs_zip)) + +$(call declare-1p-container,$(general_tests_shared_libs_zip),) +$(call declare-container-license-deps,$(general_tests_shared_libs_zip),$(my_host_shared_lib_for_general_tests),$(PRODUCT_OUT)/:/) + +intermediates_dir := +general_tests_shared_libs_zip := diff --git a/core/tasks/general-tests.mk b/core/tasks/general-tests.mk index 8dbc76f131..cae71e4046 100644 --- a/core/tasks/general-tests.mk +++ b/core/tasks/general-tests.mk @@ -24,65 +24,39 @@ general_tests_zip := $(PRODUCT_OUT)/general-tests.zip # Create an artifact to include a list of test config files in general-tests. general_tests_list_zip := $(PRODUCT_OUT)/general-tests_list.zip -# Filter shared entries between general-tests and device-tests's HOST_SHARED_LIBRARY.FILES, -# to avoid warning about overriding commands. -my_host_shared_lib_for_general_tests := \ - $(foreach m,$(filter $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ - $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m))) -my_general_tests_shared_lib_files := \ - $(filter-out $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ - $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)) - -my_host_shared_lib_for_general_tests += $(call copy-many-files,$(my_general_tests_shared_lib_files)) - # Create an artifact to include all test config files in general-tests. general_tests_configs_zip := $(PRODUCT_OUT)/general-tests_configs.zip -# Create an artifact to include all shared librariy files in general-tests. -general_tests_host_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip # Copy kernel test modules to testcases directories include $(BUILD_SYSTEM)/tasks/tools/vts-kernel-tests.mk ltp_copy_pairs := \ $(call target-native-copy-pairs,$(kernel_ltp_modules),$(kernel_ltp_host_out)) -kselftest_copy_pairs := \ - $(call target-native-copy-pairs,$(kernel_kselftest_modules),$(kernel_kselftest_host_out)) copy_ltp_tests := $(call copy-many-files,$(ltp_copy_pairs)) -copy_kselftest_tests := $(call copy-many-files,$(kselftest_copy_pairs)) -# PHONY target to be used to build and test `vts_ltp_tests` and `vts_kselftest_tests` without building full vts +# PHONY target to be used to build and test `vts_ltp_tests` without building full vts .PHONY: vts_kernel_ltp_tests vts_kernel_ltp_tests: $(copy_ltp_tests) -.PHONY: vts_kernel_kselftest_tests -vts_kernel_kselftest_tests: $(copy_kselftest_tests) +general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip +$(general_tests_zip) : $(general_tests_shared_libs_zip) $(general_tests_zip) : $(copy_ltp_tests) -$(general_tests_zip) : $(copy_kselftest_tests) $(general_tests_zip) : PRIVATE_KERNEL_LTP_HOST_OUT := $(kernel_ltp_host_out) -$(general_tests_zip) : PRIVATE_KERNEL_KSELFTEST_HOST_OUT := $(kernel_kselftest_host_out) $(general_tests_zip) : PRIVATE_general_tests_list_zip := $(general_tests_list_zip) -$(general_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(general_tests_list_zip) $(general_tests_configs_zip) $(general_tests_host_shared_libs_zip) +$(general_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(general_tests_list_zip) $(general_tests_configs_zip) $(general_tests_zip) : PRIVATE_TOOLS := $(general_tests_tools) $(general_tests_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir) -$(general_tests_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_general_tests) $(general_tests_zip) : PRIVATE_general_tests_configs_zip := $(general_tests_configs_zip) -$(general_tests_zip) : PRIVATE_general_host_shared_libs_zip := $(general_tests_host_shared_libs_zip) -$(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(general_tests_tools) $(my_host_shared_lib_for_general_tests) $(SOONG_ZIP) +$(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(general_tests_tools) $(SOONG_ZIP) rm -rf $(PRIVATE_INTERMEDIATES_DIR) rm -f $@ $(PRIVATE_general_tests_list_zip) mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools echo $(sort $(COMPATIBILITY.general-tests.FILES)) | tr " " "\n" > $(PRIVATE_INTERMEDIATES_DIR)/list find $(PRIVATE_KERNEL_LTP_HOST_OUT) >> $(PRIVATE_INTERMEDIATES_DIR)/list - find $(PRIVATE_KERNEL_KSELFTEST_HOST_OUT) >> $(PRIVATE_INTERMEDIATES_DIR)/list grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/host.list || true grep $(TARGET_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/target.list || true grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list > $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list || true grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/target.list > $(PRIVATE_INTERMEDIATES_DIR)/target-test-configs.list || true - $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ - echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/host.list; \ - echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \ - done - grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true cp -fp $(PRIVATE_TOOLS) $(PRIVATE_INTERMEDIATES_DIR)/tools/ $(SOONG_ZIP) -d -o $@ \ -P host -C $(PRIVATE_INTERMEDIATES_DIR) -D $(PRIVATE_INTERMEDIATES_DIR)/tools \ @@ -92,21 +66,19 @@ $(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(general_tests_tool $(SOONG_ZIP) -d -o $(PRIVATE_general_tests_configs_zip) \ -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list \ -P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target-test-configs.list - $(SOONG_ZIP) -d -o $(PRIVATE_general_host_shared_libs_zip) \ - -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list | sed s%$(HOST_OUT)%host%g > $(PRIVATE_INTERMEDIATES_DIR)/general-tests_list grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/target.list | sed s%$(PRODUCT_OUT)%target%g >> $(PRIVATE_INTERMEDIATES_DIR)/general-tests_list $(SOONG_ZIP) -d -o $(PRIVATE_general_tests_list_zip) -C $(PRIVATE_INTERMEDIATES_DIR) -f $(PRIVATE_INTERMEDIATES_DIR)/general-tests_list general-tests: $(general_tests_zip) -$(call dist-for-goals, general-tests, $(general_tests_zip) $(general_tests_list_zip) $(general_tests_configs_zip) $(general_tests_host_shared_libs_zip)) +$(call dist-for-goals, general-tests, $(general_tests_zip) $(general_tests_list_zip) $(general_tests_configs_zip) $(general_tests_shared_libs_zip)) $(call declare-1p-container,$(general_tests_zip),) -$(call declare-container-license-deps,$(general_tests_zip),$(COMPATIBILITY.general-tests.FILES) $(general_tests_tools) $(my_host_shared_lib_for_general_tests),$(PRODUCT_OUT)/:/) +$(call declare-container-license-deps,$(general_tests_zip),$(COMPATIBILITY.general-tests.FILES) $(general_tests_tools),$(PRODUCT_OUT)/:/) intermediates_dir := general_tests_tools := general_tests_zip := general_tests_list_zip := general_tests_configs_zip := -general_tests_host_shared_libs_zip := +general_tests_shared_libs_zip := diff --git a/core/tasks/ide.mk b/core/tasks/ide.mk deleted file mode 100644 index a3aa0cdc03..0000000000 --- a/core/tasks/ide.mk +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (C) 2010 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -define filter-ide-goals -$(strip $(filter $(1)-%,$(MAKECMDGOALS))) -endef - -define filter-ide-modules -$(strip $(subst -,$(space),$(patsubst $(1)-%,%,$(2)))) -endef - -# eclipse -eclipse_project_goals := $(call filter-ide-goals,ECLIPSE) -ifdef eclipse_project_goals - ifneq ($(words $(eclipse_project_goals)),1) - $(error Only one ECLIPSE- goal may be specified: $(eclipse_project_goals)) - endif - eclipse_project_modules := $(call filter-ide-modules,ECLIPSE,$(eclipse_project_goals)) - - ifneq ($(filter lunch,$(eclipse_project_modules)),) - eclipse_project_modules := $(filter-out lunch,$(eclipse_project_modules)) - installed_modules := $(foreach m,$(ALL_DEFAULT_INSTALLED_MODULES),\ - $(INSTALLABLE_FILES.$(m).MODULE)) - java_modules := $(foreach m,$(installed_modules),\ - $(if $(filter JAVA_LIBRARIES APPS,$(ALL_MODULES.$(m).CLASS)),$(m),)) - eclipse_project_modules := $(sort $(eclipse_project_modules) $(java_modules)) - endif - - source_paths := $(foreach m,$(eclipse_project_modules),$(ALL_MODULES.$(m).PATH)) \ - $(foreach m,$(eclipse_project_modules),$(ALL_MODULES.$(m).INTERMEDIATE_SOURCE_DIR)) - source_paths := $(sort $(source_paths)) - -.classpath: PRIVATE_MODULES := $(eclipse_project_modules) -.classpath: PRIVATE_DIRS := $(source_paths) - -# the mess below with ./src tries to guess whether the src -$(eclipse_project_goals): .classpath -.classpath: FORCE - $(hide) echo Generating .classpath for eclipse - $(hide) echo '<classpath>' > $@ - $(hide) for p in $(PRIVATE_DIRS) ; do \ - echo -n ' <classpathentry kind="src" path="' >> $@ ; \ - ( if [ -d $$p/src ] ; then echo -n $$p/src ; else echo -n $$p ; fi ) >> $@ ; \ - echo '"/>' >> $@ ; \ - done - $(hide) echo '</classpath>' >> $@ -endif - diff --git a/core/tasks/meta-lic.mk b/core/tasks/meta-lic.mk new file mode 100644 index 0000000000..2126bd02ba --- /dev/null +++ b/core/tasks/meta-lic.mk @@ -0,0 +1,142 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Declare license metadata for non-module files released with products. + +# Moved here from device/generic/car/Android.mk +$(eval $(call declare-1p-copy-files,device/generic/car,)) + +# Moved here from device/generic/trusty/Android.mk +$(eval $(call declare-1p-copy-files,device/generic/trusty,)) + +# Moved here from device/generic/uml/Android.mk +$(eval $(call declare-1p-copy-files,device/generic/uml,)) + +# Moved here from device/google_car/common/Android.mk +$(eval $(call declare-1p-copy-files,device/google_car/common,)) + +# Moved here from device/google/atv/Android.mk +$(eval $(call declare-1p-copy-files,device/google/atv,atv-component-overrides.xml)) +$(eval $(call declare-1p-copy-files,device/google/atv,tv_core_hardware.xml)) + +# Moved here from device/google/barbet/Android.mk +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,default-permissions.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,libnfc-nci.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,fstab.postinstall,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,ueventd.rc,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,hals.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,media_profiles_V1_0.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,media_codecs_performance.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,device_state_configuration.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,task_profiles.json,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,p2p_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/barbet,wpa_supplicant_overlay.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) + +$(eval $(call declare-1p-copy-files,device/google/barbet,audio_policy_configuration.xml)) + +# Moved here from device/google/coral/Android.mk +$(eval $(call declare-copy-files-license-metadata,device/google/coral,default-permissions.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,libnfc-nci.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,fstab.postinstall,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,ueventd.rc,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,hals.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,media_profiles_V1_0.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,media_codecs_performance.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,device_state_configuration.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,task_profiles.json,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,p2p_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,wpa_supplicant_overlay.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/coral,display_19261132550654593.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) + +$(eval $(call declare-1p-copy-files,device/google/coral,audio_policy_configuration.xml)) +$(eval $(call declare-1p-copy-files,device/google/coral,display_19260504575090817.xml)) + +# Moved here from device/google/gs101/Android.mk +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,default-permissions.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,libnfc-nci.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,fstab.postinstall,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,ueventd.rc,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,hals.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,media_profiles_V1_0.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,media_codecs_performance.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,device_state_configuration.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,task_profiles.json,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,p2p_supplicant_overlay.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/gs101,wpa_supplicant_overlay.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) + +$(eval $(call declare-1p-copy-files,device/google/gs101,audio_policy_configuration.xml)) + +# Move here from device/google/raviole/Android.mk +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,default-permissions.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,libnfc-nci-raven.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,libnfc-nci.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,fstab.postinstall,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,ueventd.rc,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,hals.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,media_profiles_V1_0.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,media_codecs_performance.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,device_state_configuration.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,task_profiles.json,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,p2p_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/raviole,wpa_supplicant_overlay.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) + +$(eval $(call declare-1p-copy-files,device/google/raviole,audio_policy_configuration.xml)) + +# Moved here from device/google/redfin/Android.mk +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,default-permissions.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,libnfc-nci.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,fstab.postinstall,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,ueventd.rc,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,hals.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,media_profiles_V1_0.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,media_codecs_performance.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,device_state_configuration.xml,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,task_profiles.json,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,p2p_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,wpa_supplicant.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) +$(eval $(call declare-copy-files-license-metadata,device/google/redfin,wpa_supplicant_overlay.conf,SPDX-license-identifier-Apache-2.0,notice,build/soong/licenses/LICENSE,)) + +$(eval $(call declare-1p-copy-files,device/google/redfin,audio_policy_configuration.xml)) + +# Moved here from device/sample/Android.mk +$(eval $(call declare-1p-copy-files,device/sample,)) + +# Moved here from device/google/trout/Android.mk +$(eval $(call declare-1p-copy-files,device/google/trout,)) + +# Moved here from frameworks/av/media/Android.mk +$(eval $(call declare-1p-copy-files,frameworks/av/media/libeffects,audio_effects.conf)) +$(eval $(call declare-1p-copy-files,frameworks/av/media/libeffects,audio_effects.xml)) +$(eval $(call declare-1p-copy-files,frameworks/av/media/libstagefright,)) + +# Moved here from frameworks/av/services/Android.mk +$(eval $(call declare-1p-copy-files,frameworks/av/services/audiopolicy,)) + +# Moved here from frameworks/base/Android.mk +$(eval $(call declare-1p-copy-files,frameworks/base,.ogg)) +$(eval $(call declare-1p-copy-files,frameworks/base,.kl)) +$(eval $(call declare-1p-copy-files,frameworks/base,.kcm)) +$(eval $(call declare-1p-copy-files,frameworks/base,.idc)) +$(eval $(call declare-1p-copy-files,frameworks/base,dirty-image-objects)) +$(eval $(call declare-1p-copy-files,frameworks/base/config,)) +$(eval $(call declare-1p-copy-files,frameworks/native/data,)) diff --git a/core/tasks/module-info.mk b/core/tasks/module-info.mk index e83d408577..aa695eb31c 100644 --- a/core/tasks/module-info.mk +++ b/core/tasks/module-info.mk @@ -2,39 +2,55 @@ # Currently runtime_dependencies only include the runtime libs information for cc binaries. MODULE_INFO_JSON := $(PRODUCT_OUT)/module-info.json +COMMA := , +_NEWLINE := '\n' -$(MODULE_INFO_JSON): +define write-optional-json-list +$(if $(strip $(2)),'$(COMMA)$(strip $(1)): [$(KATI_foreach_sep w,$(COMMA) ,$(2),"$(w)")]') +endef + +define write-optional-json-bool +$(if $(strip $(2)),'$(COMMA)$(strip $(1)): "$(strip $(2))"') +endef + +SOONG_MODULE_INFO := $(SOONG_OUT_DIR)/module-info-$(TARGET_PRODUCT).json + +$(MODULE_INFO_JSON): PRIVATE_SOONG_MODULE_INFO := $(SOONG_MODULE_INFO) +$(MODULE_INFO_JSON): PRIVATE_MERGE_JSON_OBJECTS := $(HOST_OUT_EXECUTABLES)/merge_module_info_json +$(MODULE_INFO_JSON): $(HOST_OUT_EXECUTABLES)/merge_module_info_json +$(MODULE_INFO_JSON): $(SOONG_MODULE_INFO) @echo Generating $@ - $(hide) echo -ne '{\n ' > $@ - $(hide) echo -ne $(foreach m, $(sort $(ALL_MODULES)), \ - ' "$(m)": {' \ - '"class": [$(foreach w,$(sort $(ALL_MODULES.$(m).CLASS)),"$(w)", )], ' \ - '"path": [$(foreach w,$(sort $(ALL_MODULES.$(m).PATH)),"$(w)", )], ' \ - '"tags": [$(foreach w,$(sort $(ALL_MODULES.$(m).TAGS)),"$(w)", )], ' \ - '"installed": [$(foreach w,$(sort $(ALL_MODULES.$(m).INSTALLED)),"$(w)", )], ' \ - '"compatibility_suites": [$(foreach w,$(sort $(ALL_MODULES.$(m).COMPATIBILITY_SUITES)),"$(w)", )], ' \ - '"auto_test_config": [$(ALL_MODULES.$(m).auto_test_config)], ' \ - '"module_name": "$(ALL_MODULES.$(m).MODULE_NAME)", ' \ - '"test_config": [$(foreach w,$(strip $(ALL_MODULES.$(m).TEST_CONFIG) $(ALL_MODULES.$(m).EXTRA_TEST_CONFIGS)),"$(w)", )], ' \ - '"dependencies": [$(foreach w,$(sort $(ALL_DEPS.$(m).ALL_DEPS)),"$(w)", )], ' \ - '"shared_libs": [$(foreach w,$(sort $(ALL_MODULES.$(m).SHARED_LIBS)),"$(w)", )], ' \ - '"system_shared_libs": [$(foreach w,$(sort $(ALL_MODULES.$(m).SYSTEM_SHARED_LIBS)),"$(w)", )], ' \ - '"srcs": [$(foreach w,$(sort $(ALL_MODULES.$(m).SRCS)),"$(w)", )], ' \ - '"srcjars": [$(foreach w,$(sort $(ALL_MODULES.$(m).SRCJARS)),"$(w)", )], ' \ - '"classes_jar": [$(foreach w,$(sort $(ALL_MODULES.$(m).CLASSES_JAR)),"$(w)", )], ' \ - '"test_mainline_modules": [$(foreach w,$(sort $(ALL_MODULES.$(m).TEST_MAINLINE_MODULES)),"$(w)", )], ' \ - '"is_unit_test": "$(ALL_MODULES.$(m).IS_UNIT_TEST)", ' \ - '"test_options_tags": [$(foreach w,$(sort $(ALL_MODULES.$(m).TEST_OPTIONS_TAGS)),"$(w)", )], ' \ - '"data": [$(foreach w,$(sort $(ALL_MODULES.$(m).TEST_DATA)),"$(w)", )], ' \ - '"runtime_dependencies": [$(foreach w,$(sort $(ALL_MODULES.$(m).LOCAL_RUNTIME_LIBRARIES)),"$(w)", )], ' \ - '"static_dependencies": [$(foreach w,$(sort $(ALL_MODULES.$(m).LOCAL_STATIC_LIBRARIES)),"$(w)", )], ' \ - '"data_dependencies": [$(foreach w,$(sort $(ALL_MODULES.$(m).TEST_DATA_BINS)),"$(w)", )], ' \ - '"supported_variants": [$(foreach w,$(sort $(ALL_MODULES.$(m).SUPPORTED_VARIANTS)),"$(w)", )], ' \ - '"host_dependencies": [$(foreach w,$(sort $(ALL_MODULES.$(m).HOST_REQUIRED_FROM_TARGET)),"$(w)", )], ' \ - '"target_dependencies": [$(foreach w,$(sort $(ALL_MODULES.$(m).TARGET_REQUIRED_FROM_HOST)),"$(w)", )], ' \ - '},\n' \ - ) | sed -e 's/, *\]/]/g' -e 's/, *\}/ }/g' -e '$$s/,$$//' >> $@ - $(hide) echo '}' >> $@ + $(hide) echo -ne '{\n ' > $@.tmp + $(hide) echo -ne $(KATI_foreach_sep m,$(COMMA)$(_NEWLINE), $(sort $(ALL_MAKE_MODULE_INFO_JSON_MODULES)),\ + '"$(m)": {' \ + '"module_name": "$(ALL_MODULES.$(m).MODULE_NAME)"' \ + $(call write-optional-json-list, "class", $(sort $(ALL_MODULES.$(m).CLASS))) \ + $(call write-optional-json-list, "path", $(sort $(ALL_MODULES.$(m).PATH))) \ + $(call write-optional-json-list, "tags", $(sort $(ALL_MODULES.$(m).TAGS))) \ + $(call write-optional-json-list, "installed", $(sort $(ALL_MODULES.$(m).INSTALLED))) \ + $(call write-optional-json-list, "compatibility_suites", $(sort $(ALL_MODULES.$(m).COMPATIBILITY_SUITES))) \ + $(call write-optional-json-list, "auto_test_config", $(sort $(ALL_MODULES.$(m).auto_test_config))) \ + $(call write-optional-json-list, "test_config", $(strip $(ALL_MODULES.$(m).TEST_CONFIG) $(ALL_MODULES.$(m).EXTRA_TEST_CONFIGS))) \ + $(call write-optional-json-list, "dependencies", $(sort $(ALL_MODULES.$(m).ALL_DEPS))) \ + $(call write-optional-json-list, "shared_libs", $(sort $(ALL_MODULES.$(m).SHARED_LIBS))) \ + $(call write-optional-json-list, "static_libs", $(sort $(ALL_MODULES.$(m).STATIC_LIBS))) \ + $(call write-optional-json-list, "system_shared_libs", $(sort $(ALL_MODULES.$(m).SYSTEM_SHARED_LIBS))) \ + $(call write-optional-json-list, "srcs", $(sort $(ALL_MODULES.$(m).SRCS))) \ + $(call write-optional-json-list, "srcjars", $(sort $(ALL_MODULES.$(m).SRCJARS))) \ + $(call write-optional-json-list, "classes_jar", $(sort $(ALL_MODULES.$(m).CLASSES_JAR))) \ + $(call write-optional-json-list, "test_mainline_modules", $(sort $(ALL_MODULES.$(m).TEST_MAINLINE_MODULES))) \ + $(call write-optional-json-bool, "is_unit_test", $(ALL_MODULES.$(m).IS_UNIT_TEST)) \ + $(call write-optional-json-list, "test_options_tags", $(sort $(ALL_MODULES.$(m).TEST_OPTIONS_TAGS))) \ + $(call write-optional-json-list, "data", $(sort $(ALL_MODULES.$(m).TEST_DATA))) \ + $(call write-optional-json-list, "runtime_dependencies", $(sort $(ALL_MODULES.$(m).LOCAL_RUNTIME_LIBRARIES))) \ + $(call write-optional-json-list, "static_dependencies", $(sort $(ALL_MODULES.$(m).LOCAL_STATIC_LIBRARIES))) \ + $(call write-optional-json-list, "data_dependencies", $(sort $(ALL_MODULES.$(m).TEST_DATA_BINS))) \ + $(call write-optional-json-list, "supported_variants", $(sort $(ALL_MODULES.$(m).SUPPORTED_VARIANTS))) \ + $(call write-optional-json-list, "host_dependencies", $(sort $(ALL_MODULES.$(m).HOST_REQUIRED_FROM_TARGET))) \ + $(call write-optional-json-list, "target_dependencies", $(sort $(ALL_MODULES.$(m).TARGET_REQUIRED_FROM_HOST))) \ + '}')'\n}\n' >> $@.tmp + $(PRIVATE_MERGE_JSON_OBJECTS) -o $@ $(PRIVATE_SOONG_MODULE_INFO) $@.tmp + rm $@.tmp droidcore-unbundled: $(MODULE_INFO_JSON) diff --git a/core/tasks/offline-sdk-docs.mk b/core/tasks/offline-sdk-docs.mk new file mode 100644 index 0000000000..d9e8006af2 --- /dev/null +++ b/core/tasks/offline-sdk-docs.mk @@ -0,0 +1,26 @@ +# +# Copyright (C) 2008 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# sdk.atree needs to copy the whole dir: $(OUT_DOCS)/offline-sdk to the final zip. +# So keep offline-sdk-timestamp target here, and unzip offline-sdk-docs.zip to +# $(OUT_DOCS)/offline-sdk. +$(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip + $(hide) rm -rf $(OUT_DOCS)/offline-sdk + $(hide) mkdir -p $(OUT_DOCS)/offline-sdk + ( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1 + +.PHONY: docs offline-sdk-docs +docs offline-sdk-docs: $(OUT_DOCS)/offline-sdk-timestamp diff --git a/core/tasks/sdk-addon.mk b/core/tasks/sdk-addon.mk index 5097f1266f..2fd4ce9ec7 100644 --- a/core/tasks/sdk-addon.mk +++ b/core/tasks/sdk-addon.mk @@ -19,12 +19,13 @@ addon_name := $(PRODUCT_SDK_ADDON_NAME) ifneq ($(addon_name),) -addon_dir_leaf := $(addon_name)-$(FILE_NAME_TAG)-$(INTERNAL_SDK_HOST_OS_NAME) -addon_dir_img := $(addon_dir_leaf)-img -intermediates := $(HOST_OUT_INTERMEDIATES)/SDK_ADDON/$(addon_name)_intermediates -full_target := $(HOST_OUT_SDK_ADDON)/$(addon_dir_leaf).zip -full_target_img := $(HOST_OUT_SDK_ADDON)/$(addon_dir_img).zip -staging := $(intermediates) +addon_dir_leaf := $(addon_name)-$(INTERNAL_SDK_HOST_OS_NAME) +addon_dir_img := $(addon_dir_leaf)-img +intermediates := $(HOST_OUT_INTERMEDIATES)/SDK_ADDON/$(addon_name)_intermediates +full_target := $(HOST_OUT_SDK_ADDON)/$(addon_dir_leaf).zip +full_target_dist_name := $(addon_name)-FILE_NAME_TAG_PLACEHOLDER-$(INTERNAL_SDK_HOST_OS_NAME) +full_target_img := $(HOST_OUT_SDK_ADDON)/$(addon_dir_img).zip +staging := $(intermediates) sdk_addon_deps := files_to_copy := @@ -125,7 +126,7 @@ $(full_target_img): PRIVATE_STAGING_DIR := $(call append-path,$(staging),$(addon $(full_target_img): $(full_target) $(addon_img_source_prop) | $(SOONG_ZIP) @echo Packaging SDK Addon System-Image: $@ $(hide) mkdir -p $(dir $@) - cp -R $(PRODUCT_OUT)/data $(PRIVATE_STAGING_DIR)/data + cp -R $(PRODUCT_OUT)/data $(PRIVATE_STAGING_DIR) $(hide) $(SOONG_ZIP) -o $@ -C $(dir $(PRIVATE_STAGING_DIR)) -D $(PRIVATE_STAGING_DIR) @@ -140,7 +141,7 @@ ADDON_SDK_IMG_ZIP := $(full_target_img) else # When not building an sdk_repo, just dist the addon zip file # as-is. -$(call dist-for-goals, sdk_addon, $(full_target)) +$(call dist-for-goals, sdk_addon, $(full_target):$(full_target_dist_name)) endif else # addon_name diff --git a/core/tasks/sts-lite.mk b/core/tasks/sts-lite.mk index dee25d4802..65c65c3dc6 100644 --- a/core/tasks/sts-lite.mk +++ b/core/tasks/sts-lite.mk @@ -29,7 +29,8 @@ $(sts_sdk_zip): $(MERGE_ZIPS) $(ZIP2ZIP) $(compatibility_zip) $(sts_sdk_samples) $(ZIP2ZIP) -i $(STS_LITE_ZIP) -o $(STS_LITE_ZIP)_filtered \ -x android-sts-lite/tools/sts-tradefed-tests.jar \ 'android-sts-lite/tools/*:sts-test/libs/' \ - 'android-sts-lite/testcases/*:sts-test/utils/' + 'android-sts-lite/testcases/*:sts-test/utils/' \ + 'android-sts-lite/jdk/**/*:sts-test/jdk/' $(MERGE_ZIPS) $@ $(STS_LITE_ZIP)_filtered $(STS_SDK_SAMPLES) rm -f $(STS_LITE_ZIP)_filtered diff --git a/core/tasks/test_mapping.mk b/core/tasks/test_mapping.mk index 0b0c93ca0a..eb2a585880 100644 --- a/core/tasks/test_mapping.mk +++ b/core/tasks/test_mapping.mk @@ -21,17 +21,17 @@ intermediates := $(call intermediates-dir-for,PACKAGING,test_mapping) test_mappings_zip := $(intermediates)/test_mappings.zip test_mapping_list := $(OUT_DIR)/.module_paths/TEST_MAPPING.list -test_mappings := $(file <$(test_mapping_list)) -$(test_mappings_zip) : PRIVATE_test_mappings := $(subst $(newline),\n,$(test_mappings)) $(test_mappings_zip) : PRIVATE_all_disabled_presubmit_tests := $(ALL_DISABLED_PRESUBMIT_TESTS) +$(test_mappings_zip) : PRIVATE_test_mapping_list := $(test_mapping_list) -$(test_mappings_zip) : $(test_mappings) $(SOONG_ZIP) +$(test_mappings_zip) : .KATI_DEPFILE := $(test_mappings_zip).d +$(test_mappings_zip) : $(test_mapping_list) $(SOONG_ZIP) @echo "Building artifact to include TEST_MAPPING files and tests to skip in presubmit check." rm -rf $@ $(dir $@)/disabled-presubmit-tests echo $(sort $(PRIVATE_all_disabled_presubmit_tests)) | tr " " "\n" > $(dir $@)/disabled-presubmit-tests - echo -e "$(PRIVATE_test_mappings)" > $@.list - $(SOONG_ZIP) -o $@ -C . -l $@.list -C $(dir $@) -f $(dir $@)/disabled-presubmit-tests - rm -f $@.list $(dir $@)/disabled-presubmit-tests + $(SOONG_ZIP) -o $@ -C . -l $(PRIVATE_test_mapping_list) -C $(dir $@) -f $(dir $@)/disabled-presubmit-tests + echo "$@ : " $$(cat $(PRIVATE_test_mapping_list)) > $@.d + rm -f $(dir $@)/disabled-presubmit-tests test_mapping : $(test_mappings_zip) diff --git a/core/tasks/tools/build_custom_image.mk b/core/tasks/tools/build_custom_image.mk index 2626120eb0..ba97e8a626 100644 --- a/core/tasks/tools/build_custom_image.mk +++ b/core/tasks/tools/build_custom_image.mk @@ -105,6 +105,9 @@ ifeq (true,$(filter true, $(CUSTOM_IMAGE_AVB_HASH_ENABLE) $(CUSTOM_IMAGE_AVB_HAS else ifneq (,$(filter true, $(CUSTOM_IMAGE_AVB_HASH_ENABLE) $(CUSTOM_IMAGE_AVB_HASHTREE_ENABLE))) $(error Cannot set both CUSTOM_IMAGE_AVB_HASH_ENABLE and CUSTOM_IMAGE_AVB_HASHTREE_ENABLE to true) endif +ifeq ($(strip $(HAS_BUILD_NUMBER)),true) +$(my_built_custom_image): $(BUILD_NUMBER_FILE) +endif $(my_built_custom_image): $(INTERNAL_USERIMAGES_DEPS) $(my_built_modules) $(my_image_copy_files) $(my_custom_image_modules_dep) \ $(CUSTOM_IMAGE_DICT_FILE) @echo "Build image $@" diff --git a/core/tasks/tools/compatibility.mk b/core/tasks/tools/compatibility.mk index 94008909cc..4e78d89eb3 100644 --- a/core/tasks/tools/compatibility.mk +++ b/core/tasks/tools/compatibility.mk @@ -30,14 +30,13 @@ test_suite_subdir := android-$(test_suite_name) out_dir := $(HOST_OUT)/$(test_suite_name)/$(test_suite_subdir) test_artifacts := $(COMPATIBILITY.$(test_suite_name).FILES) test_tools := $(HOST_OUT_JAVA_LIBRARIES)/tradefed.jar \ - $(HOST_OUT_JAVA_LIBRARIES)/tradefed-no-fwk.jar \ - $(HOST_OUT_JAVA_LIBRARIES)/tradefed-test-framework.jar \ $(HOST_OUT_JAVA_LIBRARIES)/loganalysis.jar \ $(HOST_OUT_JAVA_LIBRARIES)/compatibility-host-util.jar \ $(HOST_OUT_JAVA_LIBRARIES)/compatibility-tradefed.jar \ $(HOST_OUT_JAVA_LIBRARIES)/$(test_suite_tradefed).jar \ $(HOST_OUT_JAVA_LIBRARIES)/$(test_suite_tradefed)-tests.jar \ $(HOST_OUT_EXECUTABLES)/$(test_suite_tradefed) \ + $(HOST_OUT_EXECUTABLES)/test-utils-script \ $(test_suite_readme) $(foreach f,$(test_suite_readme),$(if $(strip $(ALL_TARGETS.$(f).META_LIC)),,$(eval ALL_TARGETS.$(f).META_LIC := $(module_license_metadata)))) @@ -46,10 +45,16 @@ test_tools += $(test_suite_tools) # The JDK to package into the test suite zip file. Always package the linux JDK. test_suite_jdk_dir := $(ANDROID_JAVA_HOME)/../linux-x86 +ifndef test_suite_jdk_files + # This file gets included many times, so make sure we only run the $(shell) once. + # Otherwise it will slow down every build due to all copies of it being rerun when kati + # checks the stamp file. + test_suite_jdk_files :=$= $(shell find $(test_suite_jdk_dir) -type f | sort) +endif test_suite_jdk := $(call intermediates-dir-for,PACKAGING,$(test_suite_name)_jdk,HOST)/jdk.zip $(test_suite_jdk): PRIVATE_JDK_DIR := $(test_suite_jdk_dir) $(test_suite_jdk): PRIVATE_SUBDIR := $(test_suite_subdir) -$(test_suite_jdk): $(shell find $(test_suite_jdk_dir) -type f | sort) +$(test_suite_jdk): $(test_suite_jdk_files) $(test_suite_jdk): $(SOONG_ZIP) $(SOONG_ZIP) -o $@ -P $(PRIVATE_SUBDIR)/jdk -C $(PRIVATE_JDK_DIR) -D $(PRIVATE_JDK_DIR) -sha256 @@ -114,6 +119,9 @@ $(compatibility_zip): PRIVATE_RESOURCES := $(compatibility_zip_resources) $(compatibility_zip): PRIVATE_JDK := $(test_suite_jdk) $(compatibility_zip): PRIVATE_tests_list := $(out_dir)-tests_list $(compatibility_zip): PRIVATE_tests_list_zip := $(compatibility_tests_list_zip) +ifeq ($(strip $(HAS_BUILD_NUMBER)),true) +$(compatibility_zip): $(BUILD_NUMBER_FILE) +endif $(compatibility_zip): $(compatibility_zip_deps) | $(ADB) $(ACP) # Make dir structure mkdir -p $(PRIVATE_OUT_DIR)/tools $(PRIVATE_OUT_DIR)/testcases diff --git a/core/tasks/tools/package-modules.mk b/core/tasks/tools/package-modules.mk index c41aec5fc4..4ec552047a 100644 --- a/core/tasks/tools/package-modules.mk +++ b/core/tasks/tools/package-modules.mk @@ -50,12 +50,12 @@ ifneq ($(filter-out true false,$(my_modules_strict)),) $(error done) endif -my_missing_files = $(shell $(call echo-warning,$(my_makefile),$(my_package_name): Unknown installed file for module '$(1)')) +my_missing_files = $(shell $(call echo-warning,$(my_makefile),$(my_package_name): Unknown installed file for module '$(1)'))$(shell$(call echo-warning,$(my_makefile),$(my_package_name): Some necessary modules may have been skipped by Soong. Check if PRODUCT_SOURCE_ROOT_DIRS is pruning necessary Android.bp files.)) ifeq ($(ALLOW_MISSING_DEPENDENCIES),true) # Ignore unknown installed files on partial builds my_missing_files = else ifneq ($(my_modules_strict),false) - my_missing_files = $(shell $(call echo-error,$(my_makefile),$(my_package_name): Unknown installed file for module '$(1)'))$(eval my_missing_error := true) + my_missing_files = $(shell $(call echo-error,$(my_makefile),$(my_package_name): Unknown installed file for module '$(1)'))$(shell$(call echo-warning,$(my_makefile),$(my_package_name): Some necessary modules may have been skipped by Soong. Check if PRODUCT_SOURCE_ROOT_DIRS is pruning necessary Android.bp files.))$(eval my_missing_error := true) endif # Iterate over modules' built files and installed files; @@ -106,6 +106,7 @@ $(my_package_zip) : $(my_built_modules) $(hide) $(foreach f, $(PRIVATE_PICKUP_FILES),\ cp -RfL $(f) $(PRIVATE_STAGING_DIR) && ) true $(hide) cd $(PRIVATE_STAGING_DIR) && zip -rqX ../$(notdir $@) * + rm -rf $(PRIVATE_STAGING_DIR) my_makefile := my_staging_dir := diff --git a/core/tasks/tools/vts-kernel-tests.mk b/core/tasks/tools/vts-kernel-tests.mk index bd115c9000..e727dc1f55 100644 --- a/core/tasks/tools/vts-kernel-tests.mk +++ b/core/tasks/tools/vts-kernel-tests.mk @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. --include external/linux-kselftest/android/kselftest_test_list.mk -include external/ltp/android/ltp_package_list.mk include $(BUILD_SYSTEM)/tasks/tools/vts_package_utils.mk @@ -23,7 +22,3 @@ kernel_ltp_vts_out := $(HOST_OUT)/$(test_suite_name)/android-$(test_suite_name)/ kernel_ltp_modules := \ ltp \ $(ltp_packages) - -kernel_kselftest_host_out := $(HOST_OUT_TESTCASES)/vts_kernel_kselftest_tests -kernel_kselftest_vts_out := $(HOST_OUT)/$(test_suite_name)/android-$(test_suite_name)/testcases/vts_kernel_kselftest_tests -kernel_kselftest_modules := $(kselftest_modules) diff --git a/core/tasks/tools/vts_package_utils.mk b/core/tasks/tools/vts_package_utils.mk index 06161f0b3c..1a819f2172 100644 --- a/core/tasks/tools/vts_package_utils.mk +++ b/core/tasks/tools/vts_package_utils.mk @@ -21,7 +21,7 @@ define target-native-copy-pairs $(foreach m,$(1),\ $(eval _built_files := $(strip $(ALL_MODULES.$(m).BUILT_INSTALLED)\ $(ALL_MODULES.$(m)$(TARGET_2ND_ARCH_MODULE_SUFFIX).BUILT_INSTALLED)))\ - $(foreach i, $(_built_files),\ + $(foreach i, $(sort $(_built_files)),\ $(eval bui_ins := $(subst :,$(space),$(i)))\ $(eval ins := $(word 2,$(bui_ins)))\ $(if $(filter $(TARGET_OUT_ROOT)/%,$(ins)),\ diff --git a/core/tasks/vndk.mk b/core/tasks/vndk.mk deleted file mode 100644 index ebe9bd4736..0000000000 --- a/core/tasks/vndk.mk +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -current_makefile := $(lastword $(MAKEFILE_LIST)) - -# BOARD_VNDK_VERSION must be set to 'current' in order to generate a VNDK snapshot. -ifeq ($(BOARD_VNDK_VERSION),current) - -# PLATFORM_VNDK_VERSION must be set. -ifneq (,$(PLATFORM_VNDK_VERSION)) - -.PHONY: vndk -vndk: $(SOONG_VNDK_SNAPSHOT_ZIP) - -$(call dist-for-goals, vndk, $(SOONG_VNDK_SNAPSHOT_ZIP)) - -else # PLATFORM_VNDK_VERSION is NOT set -error_msg := "CANNOT generate VNDK snapshot. PLATFORM_VNDK_VERSION must be set." -endif # PLATFORM_VNDK_VERSION - -else # BOARD_VNDK_VERSION is NOT set to 'current' -error_msg := "CANNOT generate VNDK snapshot. BOARD_VNDK_VERSION must be set to 'current'." -endif # BOARD_VNDK_VERSION - -ifneq (,$(error_msg)) - -.PHONY: vndk -vndk: PRIVATE_MAKEFILE := $(current_makefile) -vndk: - $(call echo-error,$(PRIVATE_MAKEFILE),$(error_msg)) - exit 1 - -endif diff --git a/core/tasks/vts-core-tests.mk b/core/tasks/vts-core-tests.mk index bd7652bf40..1eeb0789ec 100644 --- a/core/tasks/vts-core-tests.mk +++ b/core/tasks/vts-core-tests.mk @@ -20,13 +20,10 @@ include $(BUILD_SYSTEM)/tasks/tools/vts-kernel-tests.mk ltp_copy_pairs := \ $(call target-native-copy-pairs,$(kernel_ltp_modules),$(kernel_ltp_vts_out)) -kselftest_copy_pairs := \ - $(call target-native-copy-pairs,$(kernel_kselftest_modules),$(kernel_kselftest_vts_out)) copy_ltp_tests := $(call copy-many-files,$(ltp_copy_pairs)) -copy_kselftest_tests := $(call copy-many-files,$(kselftest_copy_pairs)) -test_suite_extra_deps := $(copy_ltp_tests) $(copy_kselftest_tests) +test_suite_extra_deps := $(copy_ltp_tests) include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk diff --git a/core/tasks/with-license.mk b/core/tasks/with-license.mk index d41e77a1ce..5ca974a1a0 100644 --- a/core/tasks/with-license.mk +++ b/core/tasks/with-license.mk @@ -20,7 +20,8 @@ ifeq ($(TARGET_BUILD_TYPE),debug) name := $(name)_debug endif -name := $(name)-flashable-$(FILE_NAME_TAG)-with-license +dist_name := $(name)-flashable-FILE_NAME_TAG_PLACEHOLDER-with-license +name := $(name)-flashable-with-license with_license_intermediates := \ $(call intermediates-dir-for,PACKAGING,with_license) @@ -42,6 +43,7 @@ $(call declare-1p-container,$(license_image_input_zip),build) $(call declare-container-deps,$(license_image_input_zip),$(BUILT_TARGET_FILES_PACKAGE)) with_license_zip := $(PRODUCT_OUT)/$(name).sh +dist_name := $(dist_name).sh $(with_license_zip): PRIVATE_NAME := $(name) $(with_license_zip): PRIVATE_INPUT_ZIP := $(license_image_input_zip) $(with_license_zip): PRIVATE_VENDOR_BLOBS_LICENSE := $(VENDOR_BLOBS_LICENSE) @@ -51,7 +53,7 @@ $(with_license_zip): $(HOST_OUT_EXECUTABLES)/generate-self-extracting-archive $(HOST_OUT_EXECUTABLES)/generate-self-extracting-archive $@ \ $(PRIVATE_INPUT_ZIP) $(PRIVATE_NAME) $(PRIVATE_VENDOR_BLOBS_LICENSE) with-license : $(with_license_zip) -$(call dist-for-goals, with-license, $(with_license_zip)) +$(call dist-for-goals, with-license, $(with_license_zip):$(dist_name)) $(call declare-1p-container,$(with_license_zip),) $(call declare-container-license-deps,$(with_license_zip),$(license_image_input_zip),$(with_license_zip):) diff --git a/core/version_defaults.mk b/core/version_defaults.mk deleted file mode 100644 index a664b9deb1..0000000000 --- a/core/version_defaults.mk +++ /dev/null @@ -1,109 +0,0 @@ -# -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Handle various build version information. -# -# Guarantees that the following are defined: -# PLATFORM_VERSION -# PLATFORM_DISPLAY_VERSION -# PLATFORM_SDK_VERSION -# PLATFORM_VERSION_CODENAME -# DEFAULT_APP_TARGET_SDK -# BUILD_ID -# BUILD_NUMBER -# PLATFORM_SECURITY_PATCH -# PLATFORM_VNDK_VERSION -# PLATFORM_SYSTEMSDK_VERSIONS -# - -# Look for an optional file containing overrides of the defaults, -# but don't cry if we don't find it. We could just use -include, but -# the build.prop target also wants INTERNAL_BUILD_ID_MAKEFILE to be set -# if the file exists. -# -INTERNAL_BUILD_ID_MAKEFILE := $(wildcard $(BUILD_SYSTEM)/build_id.mk) -ifdef INTERNAL_BUILD_ID_MAKEFILE - include $(INTERNAL_BUILD_ID_MAKEFILE) -endif - -DEFAULT_PLATFORM_VERSION := UP1A -.KATI_READONLY := DEFAULT_PLATFORM_VERSION -MIN_PLATFORM_VERSION := UP1A -MAX_PLATFORM_VERSION := UP1A - -# The last stable version name of the platform that was released. During -# development, this stays at that previous version, while the codename indicates -# further work based on the previous version. -PLATFORM_VERSION_LAST_STABLE := 13 -.KATI_READONLY := PLATFORM_VERSION_LAST_STABLE - -# These are the current development codenames, if the build is not a final -# release build. If this is a final release build, it is simply "REL". -PLATFORM_VERSION_CODENAME.UP1A := UpsideDownCake - -# This is the user-visible version. In a final release build it should -# be empty to use PLATFORM_VERSION as the user-visible version. For -# a preview release it can be set to a user-friendly value like `12 Preview 1` -PLATFORM_DISPLAY_VERSION := 13 - -ifndef PLATFORM_SDK_VERSION - # This is the canonical definition of the SDK version, which defines - # the set of APIs and functionality available in the platform. It - # is a single integer that increases monotonically as updates to - # the SDK are released. It should only be incremented when the APIs for - # the new release are frozen (so that developers don't write apps against - # intermediate builds). During development, this number remains at the - # SDK version the branch is based on and PLATFORM_VERSION_CODENAME holds - # the code-name of the new development work. - - # When you increment the PLATFORM_SDK_VERSION please ensure you also - # clear out the following text file of all older PLATFORM_VERSION's: - # cts/tests/tests/os/assets/platform_versions.txt - PLATFORM_SDK_VERSION := 33 -endif -.KATI_READONLY := PLATFORM_SDK_VERSION - -# This is the sdk extension version of this tree. -PLATFORM_SDK_EXTENSION_VERSION := 3 -.KATI_READONLY := PLATFORM_SDK_EXTENSION_VERSION - -# This is the sdk extension version that PLATFORM_SDK_VERSION ships with. -PLATFORM_BASE_SDK_EXTENSION_VERSION := $(PLATFORM_SDK_EXTENSION_VERSION) -.KATI_READONLY := PLATFORM_BASE_SDK_EXTENSION_VERSION - -# This are all known codenames. -PLATFORM_VERSION_KNOWN_CODENAMES := \ -Base Base11 Cupcake Donut Eclair Eclair01 EclairMr1 Froyo Gingerbread GingerbreadMr1 \ -Honeycomb HoneycombMr1 HoneycombMr2 IceCreamSandwich IceCreamSandwichMr1 \ -JellyBean JellyBeanMr1 JellyBeanMr2 Kitkat KitkatWatch Lollipop LollipopMr1 M N NMr1 O OMr1 P \ -Q R S Sv2 Tiramisu UpsideDownCake - -# Convert from space separated list to comma separated -PLATFORM_VERSION_KNOWN_CODENAMES := \ - $(call normalize-comma-list,$(PLATFORM_VERSION_KNOWN_CODENAMES)) -.KATI_READONLY := PLATFORM_VERSION_KNOWN_CODENAMES - -ifndef PLATFORM_SECURITY_PATCH - # Used to indicate the security patch that has been applied to the device. - # It must signify that the build includes all security patches issued up through the designated Android Public Security Bulletin. - # It must be of the form "YYYY-MM-DD" on production devices. - # It must match one of the Android Security Patch Level strings of the Public Security Bulletins. - # If there is no $PLATFORM_SECURITY_PATCH set, keep it empty. - PLATFORM_SECURITY_PATCH := 2023-02-05 -endif - -include $(BUILD_SYSTEM)/version_util.mk diff --git a/core/version_util.mk b/core/version_util.mk index cbfef96593..eb568becc4 100644 --- a/core/version_util.mk +++ b/core/version_util.mk @@ -14,88 +14,92 @@ # limitations under the License. # -# -ALLOWED_VERSIONS := $(call allowed-platform-versions,\ - $(MIN_PLATFORM_VERSION),\ - $(MAX_PLATFORM_VERSION),\ - $(DEFAULT_PLATFORM_VERSION)) +# +# Handle various build version information. +# +# Guarantees that the following are defined: +# PLATFORM_VERSION +# PLATFORM_DISPLAY_VERSION +# PLATFORM_SDK_VERSION +# PLATFORM_SDK_EXTENSION_VERSION +# PLATFORM_VERSION_CODENAME +# DEFAULT_APP_TARGET_SDK +# BUILD_ID +# BUILD_NUMBER +# PLATFORM_SECURITY_PATCH +# PLATFORM_SYSTEMSDK_VERSIONS +# PLATFORM_VERSION_LAST_STABLE +# PLATFORM_VERSION_KNOWN_CODENAMES +# -ifndef TARGET_PLATFORM_VERSION - TARGET_PLATFORM_VERSION := $(DEFAULT_PLATFORM_VERSION) +# Look for an optional file containing overrides of the defaults, +# but don't cry if we don't find it. We could just use -include, but +# the build.prop target also wants INTERNAL_BUILD_ID_MAKEFILE to be set +# if the file exists. +# +INTERNAL_BUILD_ID_MAKEFILE := $(wildcard $(BUILD_SYSTEM)/build_id.mk) +ifdef INTERNAL_BUILD_ID_MAKEFILE + include $(INTERNAL_BUILD_ID_MAKEFILE) endif -ifeq (,$(filter $(ALLOWED_VERSIONS), $(TARGET_PLATFORM_VERSION))) - $(warning Invalid TARGET_PLATFORM_VERSION '$(TARGET_PLATFORM_VERSION)', must be one of) - $(error $(ALLOWED_VERSIONS)) +ifdef TARGET_PLATFORM_VERSION + $(error Do not set TARGET_PLATFORM_VERSION directly. Use RELEASE_PLATFORM_VERSION. value: $(TARGET_PLATFORM_VERSION)) endif -ALLOWED_VERSIONS := -MIN_PLATFORM_VERSION := -MAX_PLATFORM_VERSION := - +TARGET_PLATFORM_VERSION := $(RELEASE_PLATFORM_VERSION) .KATI_READONLY := TARGET_PLATFORM_VERSION -# Default versions for each TARGET_PLATFORM_VERSION -# TODO: PLATFORM_VERSION, PLATFORM_SDK_VERSION, etc. should be conditional -# on this +ifdef PLATFORM_SECURITY_PATCH + $(error Do not set PLATFORM_SECURITY_PATCH directly. Use RELEASE_PLATFORM_SECURITY_PATCH. value: $(PLATFORM_SECURITY_PATCH)) +endif +PLATFORM_SECURITY_PATCH := $(RELEASE_PLATFORM_SECURITY_PATCH) +.KATI_READONLY := PLATFORM_SECURITY_PATCH -# This is the canonical definition of the platform version, -# which is the version that we reveal to the end user. -# Update this value when the platform version changes (rather -# than overriding it somewhere else). Can be an arbitrary string. +ifdef PLATFORM_SDK_VERSION + $(error Do not set PLATFORM_SDK_VERSION directly. Use RELEASE_PLATFORM_SDK_VERSION. value: $(PLATFORM_SDK_VERSION)) +endif +PLATFORM_SDK_VERSION := $(RELEASE_PLATFORM_SDK_VERSION) +.KATI_READONLY := PLATFORM_SDK_VERSION -# When you change PLATFORM_VERSION for a given PLATFORM_SDK_VERSION -# please add that PLATFORM_VERSION as well as clean up obsolete PLATFORM_VERSION's -# in the following text file: -# cts/tests/tests/os/assets/platform_versions.txt +ifdef PLATFORM_SDK_EXTENSION_VERSION + $(error Do not set PLATFORM_SDK_EXTENSION_VERSION directly. Use RELEASE_PLATFORM_SDK_EXTENSION_VERSION. value: $(PLATFORM_SDK_EXTENSION_VERSION)) +endif +PLATFORM_SDK_EXTENSION_VERSION := $(RELEASE_PLATFORM_SDK_EXTENSION_VERSION) +.KATI_READONLY := PLATFORM_SDK_EXTENSION_VERSION -# Note that there should be one PLATFORM_VERSION and PLATFORM_VERSION_CODENAME -# entry for each unreleased API level, regardless of -# MIN_PLATFORM_VERSION/MAX_PLATFORM_VERSION. PLATFORM_VERSION is used to -# generate the range of allowed SDK versions, so it must have an entry for every -# unreleased API level targetable by this branch, not just those that are valid -# lunch targets for this branch. +# This is the sdk extension version that PLATFORM_SDK_VERSION ships with. +PLATFORM_BASE_SDK_EXTENSION_VERSION := $(PLATFORM_SDK_EXTENSION_VERSION) +.KATI_READONLY := PLATFORM_BASE_SDK_EXTENSION_VERSION -PLATFORM_VERSION_CODENAME := $(PLATFORM_VERSION_CODENAME.$(TARGET_PLATFORM_VERSION)) -ifndef PLATFORM_VERSION_CODENAME - # PLATFORM_VERSION_CODENAME falls back to TARGET_PLATFORM_VERSION - PLATFORM_VERSION_CODENAME := $(TARGET_PLATFORM_VERSION) +ifdef PLATFORM_VERSION_CODENAME + $(error Do not set PLATFORM_VERSION_CODENAME directly. Use RELEASE_PLATFORM_VERSION. value: $(PLATFORM_VERSION_CODENAME)) endif +PLATFORM_VERSION_CODENAME := $(RELEASE_PLATFORM_VERSION_CODENAME) +.KATI_READONLY := PLATFORM_VERSION_CODENAME -# This is all of the *active* development codenames. -# This confusing name is needed because -# all_codenames has been baked into build.prop for ages. -# -# Should be either the same as PLATFORM_VERSION_CODENAME or a comma-separated -# list of additional codenames after PLATFORM_VERSION_CODENAME. -PLATFORM_VERSION_ALL_CODENAMES := - -# Build a list of all active code names. Avoid duplicates, and stop when we -# reach a codename that matches PLATFORM_VERSION_CODENAME (anything beyond -# that is not included in our build). -_versions_in_target := \ - $(call find_and_earlier,$(ALL_VERSIONS),$(TARGET_PLATFORM_VERSION)) -$(foreach version,$(_versions_in_target),\ - $(eval _codename := $(PLATFORM_VERSION_CODENAME.$(version)))\ - $(if $(filter $(_codename),$(PLATFORM_VERSION_ALL_CODENAMES)),,\ - $(eval PLATFORM_VERSION_ALL_CODENAMES += $(_codename)))) +ifdef PLATFORM_VERSION_ALL_CODENAMES + $(error Do not set PLATFORM_VERSION_ALL_CODENAMES directly. Use RELEASE_PLATFORM_VERSION_ALL_CODENAMES. value: $(PLATFORM_VERSION_ALL_CODENAMES)) +endif +PLATFORM_VERSION_ALL_CODENAMES := $(RELEASE_PLATFORM_VERSION_ALL_CODENAMES) +.KATI_READONLY := PLATFORM_VERSION_ALL_CODENAMES -# And convert from space separated to comma separated. -PLATFORM_VERSION_ALL_CODENAMES := \ - $(subst $(space),$(comma),$(strip $(PLATFORM_VERSION_ALL_CODENAMES))) +ifdef PLATFORM_VERSION_ALL_PREVIEW_CODENAMES + $(error Do not set PLATFORM_VERSION_ALL_PREVIEW_CODENAMES directly. Use RELEASE_PLATFORM_VERSION_ALL_PREVIEW_CODENAMES. value: $(PLATFORM_VERSION_ALL_PREVIEW_CODENAMES)) +endif +PLATFORM_VERSION_ALL_PREVIEW_CODENAMES := $(RELEASE_PLATFORM_VERSION_ALL_PREVIEW_CODENAMES) +.KATI_READONLY := PLATFORM_VERSION_ALL_PREVIEW_CODENAMES -.KATI_READONLY := \ - PLATFORM_VERSION_CODENAME \ - PLATFORM_VERSION_ALL_CODENAMES +ifdef PLATFORM_VERSION_LAST_STABLE + $(error Do not set PLATFORM_VERSION_LAST_STABLE directly. Use RELEASE_PLATFORM_VERSION_LAST_STABLE. value: $(PLATFORM_VERSION_CODENAME)) +endif +PLATFORM_VERSION_LAST_STABLE := $(RELEASE_PLATFORM_VERSION_LAST_STABLE) +.KATI_READONLY := PLATFORM_VERSION_LAST_STABLE -ifneq (REL,$(PLATFORM_VERSION_CODENAME)) - codenames := \ - $(subst $(comma),$(space),$(strip $(PLATFORM_VERSION_KNOWN_CODENAMES))) - ifeq ($(filter $(PLATFORM_VERSION_CODENAME),$(codenames)),) - $(error '$(PLATFORM_VERSION_CODENAME)' is not in '$(codenames)'. \ - Add PLATFORM_VERSION_CODENAME to PLATFORM_VERSION_KNOWN_CODENAMES) - endif +ifdef PLATFORM_VERSION_KNOWN_CODENAMES + $(error Do not set PLATFORM_VERSION_KNOWN_CODENAMES directly. Use RELEASE_PLATFORM_VERSION_KNOWN_CODENAMES. value: $(PLATFORM_VERSION_KNOWN_CODENAMES)) endif +PLATFORM_VERSION_KNOWN_CODENAMES := $(RELEASE_PLATFORM_VERSION_KNOWN_CODENAMES) +.KATI_READONLY := PLATFORM_VERSION_KNOWN_CODENAMES ifndef PLATFORM_VERSION ifeq (REL,$(PLATFORM_VERSION_CODENAME)) @@ -146,31 +150,12 @@ ifndef DEFAULT_APP_TARGET_SDK endif .KATI_READONLY := DEFAULT_APP_TARGET_SDK -ifndef PLATFORM_VNDK_VERSION - # This is the definition of the VNDK version for the current VNDK libraries. - # The version is only available when PLATFORM_VERSION_CODENAME == REL. - # Otherwise, it will be set to a CODENAME version. The ABI is allowed to be - # changed only before the Android version is released. Once - # PLATFORM_VNDK_VERSION is set to actual version, the ABI for this version - # will be frozon and emit build errors if any ABI for the VNDK libs are - # changed. - # After that the snapshot of the VNDK with this version will be generated. - # - # The VNDK version follows PLATFORM_SDK_VERSION. - ifeq (REL,$(PLATFORM_VERSION_CODENAME)) - PLATFORM_VNDK_VERSION := $(PLATFORM_SDK_VERSION) - else - PLATFORM_VNDK_VERSION := $(PLATFORM_VERSION_CODENAME) - endif -endif -.KATI_READONLY := PLATFORM_VNDK_VERSION - ifndef PLATFORM_SYSTEMSDK_MIN_VERSION # This is the oldest version of system SDK that the platform supports. Contrary # to the public SDK where platform essentially supports all previous SDK versions, # platform supports only a few number of recent system SDK versions as some of # old system APIs are gradually deprecated, removed and then deleted. - PLATFORM_SYSTEMSDK_MIN_VERSION := 28 + PLATFORM_SYSTEMSDK_MIN_VERSION := 29 endif .KATI_READONLY := PLATFORM_SYSTEMSDK_MIN_VERSION @@ -231,26 +216,13 @@ DATE := date -d @$(BUILD_DATETIME) # to soong_ui. $(KATI_obsolete_var BUILD_DATETIME,Use BUILD_DATETIME_FROM_FILE) -HAS_BUILD_NUMBER := true -ifndef BUILD_NUMBER - # BUILD_NUMBER should be set to the source control value that - # represents the current state of the source code. E.g., a - # perforce changelist number or a git hash. Can be an arbitrary string - # (to allow for source control that uses something other than numbers), - # but must be a single word and a valid file name. - # - # If no BUILD_NUMBER is set, create a useful "I am an engineering build - # from this date/time" value. Make it start with a non-digit so that - # anyone trying to parse it as an integer will probably get "0". - BUILD_NUMBER := eng.$(shell echo $${BUILD_USERNAME:0:6}).$(shell $(DATE) +%Y%m%d.%H%M%S) +ifndef HAS_BUILD_NUMBER HAS_BUILD_NUMBER := false endif -.KATI_READONLY := BUILD_NUMBER HAS_BUILD_NUMBER +.KATI_READONLY := HAS_BUILD_NUMBER -ifndef PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION - # Used to set minimum supported target sdk version. Apps targeting sdk - # version lower than the set value will result in a warning being shown - # when any activity from the app is started. - PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION := 23 +ifdef PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION + $(error Do not set PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION directly. Use RELEASE_PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION. value: $(PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION)) endif +PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION := $(RELEASE_PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION) .KATI_READONLY := PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION diff --git a/envsetup.sh b/envsetup.sh index 905635c890..50fec5146a 100644 --- a/envsetup.sh +++ b/envsetup.sh @@ -56,14 +56,14 @@ cat <<EOF Run "m help" for help with the build system itself. Invoke ". build/envsetup.sh" from your shell to add the following functions to your environment: -- lunch: lunch <product_name>-<build_variant> +- lunch: lunch <product_name>-<release_type>-<build_variant> Selects <product_name> as the product to build, and <build_variant> as the variant to build, and stores those selections in the environment to be read by subsequent invocations of 'm' etc. - tapas: tapas [<App1> <App2> ...] [arm|x86|arm64|x86_64] [eng|userdebug|user] Sets up the build environment for building unbundled apps (APKs). -- banchan: banchan <module1> [<module2> ...] [arm|x86|arm64|x86_64|arm64_only|x86_64only] \ - [eng|userdebug|user] +- banchan: banchan <module1> [<module2> ...] \\ + [arm|x86|arm64|riscv64|x86_64|arm64_only|x86_64only] [eng|userdebug|user] Sets up the build environment for building unbundled modules (APEXes). - croot: Changes directory to the top of the tree, or a subdirectory thereof. - m: Makes from the top of the tree. @@ -205,6 +205,7 @@ function check_product() return fi TARGET_PRODUCT=$1 \ + TARGET_RELEASE= \ TARGET_BUILD_VARIANT= \ TARGET_BUILD_TYPE= \ TARGET_BUILD_APPS= \ @@ -253,7 +254,7 @@ function set_lunch_paths() # Note: on windows/cygwin, ANDROID_LUNCH_BUILD_PATHS will contain spaces # due to "C:\Program Files" being in the path. - # Handle compat with the old ANDROID_BUILD_PATHS variable. + # Handle compat with the old ANDROID_BUILD_PATHS variable. # TODO: Remove this after we think everyone has lunched again. if [ -z "$ANDROID_LUNCH_BUILD_PATHS" -a -n "$ANDROID_BUILD_PATHS" ] ; then ANDROID_LUNCH_BUILD_PATHS="$ANDROID_BUILD_PATHS" @@ -312,7 +313,7 @@ function set_lunch_paths() # would prevent exporting type info from those packages. # # http://b/266688086 - export ANDROID_PYTHONPATH=$T/development/python-packages/adb:$T/development/python-packages: + export ANDROID_PYTHONPATH=$T/development/python-packages/adb:$T/development/python-packages/gdbrunner:$T/development/python-packages: if [ -n $VENDOR_PYTHONPATH ]; then ANDROID_PYTHONPATH=$ANDROID_PYTHONPATH$VENDOR_PYTHONPATH fi @@ -365,7 +366,8 @@ function set_global_paths() fi # And in with the new... - ANDROID_GLOBAL_BUILD_PATHS=$T/build/bazel/bin + ANDROID_GLOBAL_BUILD_PATHS=$T/build/soong/bin + ANDROID_GLOBAL_BUILD_PATHS+=:$T/build/bazel/bin ANDROID_GLOBAL_BUILD_PATHS+=:$T/development/scripts ANDROID_GLOBAL_BUILD_PATHS+=:$T/prebuilts/devtools/tools @@ -486,7 +488,7 @@ function addcompletions() function multitree_lunch_help() { - echo "usage: lunch PRODUCT-VARIANT" 1>&2 + echo "usage: lunch PRODUCT-RELEASE-VARIANT" 1>&2 echo " Set up android build environment based on a product short name and variant" 1>&2 echo 1>&2 echo "lunch COMBO_FILE VARIANT" 1>&2 @@ -728,7 +730,7 @@ function print_lunch_menu() { local uname=$(uname) local choices - choices=$(TARGET_BUILD_APPS= TARGET_PRODUCT= TARGET_BUILD_VARIANT= get_build_var COMMON_LUNCH_CHOICES 2>/dev/null) + choices=$(TARGET_BUILD_APPS= TARGET_PRODUCT= TARGET_RELEASE= TARGET_BUILD_VARIANT= get_build_var COMMON_LUNCH_CHOICES 2>/dev/null) local ret=$? echo @@ -774,8 +776,8 @@ function lunch() answer=$1 else print_lunch_menu - echo "Which would you like? [aosp_arm-eng]" - echo -n "Pick from common choices above (e.g. 13) or specify your own (e.g. aosp_barbet-eng): " + echo "Which would you like? [aosp_cf_x86_64_phone-trunk_staging-eng]" + echo -n "Pick from common choices above (e.g. 13) or specify your own (e.g. aosp_barbet-trunk_staging-eng): " read answer used_lunch_menu=1 fi @@ -784,10 +786,10 @@ function lunch() if [ -z "$answer" ] then - selection=aosp_arm-eng + selection=aosp_cf_x86_64_phone-trunk_staging-eng elif (echo -n $answer | grep -q -e "^[0-9][0-9]*$") then - local choices=($(TARGET_BUILD_APPS= get_build_var COMMON_LUNCH_CHOICES)) + local choices=($(TARGET_BUILD_APPS= TARGET_PRODUCT= TARGET_RELEASE= TARGET_BUILD_VARIANT= get_build_var COMMON_LUNCH_CHOICES 2>/dev/null)) if [ $answer -le ${#choices[@]} ] then # array in zsh starts from 1 instead of 0. @@ -804,26 +806,22 @@ function lunch() export TARGET_BUILD_APPS= - local product variant_and_version variant version - product=${selection%%-*} # Trim everything after first dash - variant_and_version=${selection#*-} # Trim everything up to first dash - if [ "$variant_and_version" != "$selection" ]; then - variant=${variant_and_version%%-*} - if [ "$variant" != "$variant_and_version" ]; then - version=${variant_and_version#*-} - fi - fi + # This must be <product>-<release>-<variant> + local product release variant + # Split string on the '-' character. + IFS="-" read -r product release variant <<< "$selection" - if [ -z "$product" ] + if [[ -z "$product" ]] || [[ -z "$release" ]] || [[ -z "$variant" ]] then echo echo "Invalid lunch combo: $selection" + echo "Valid combos must be of the form <product>-<release>-<variant>" return 1 fi TARGET_PRODUCT=$product \ TARGET_BUILD_VARIANT=$variant \ - TARGET_PLATFORM_VERSION=$version \ + TARGET_RELEASE=$release \ build_build_var_cache if [ $? -ne 0 ] then @@ -835,22 +833,25 @@ function lunch() fi export TARGET_PRODUCT=$(get_build_var TARGET_PRODUCT) export TARGET_BUILD_VARIANT=$(get_build_var TARGET_BUILD_VARIANT) - if [ -n "$version" ]; then - export TARGET_PLATFORM_VERSION=$(get_build_var TARGET_PLATFORM_VERSION) - else - unset TARGET_PLATFORM_VERSION - fi + export TARGET_RELEASE=$release + # Note this is the string "release", not the value of the variable. export TARGET_BUILD_TYPE=release + [[ -n "${ANDROID_QUIET_BUILD:-}" ]] || echo + + set_stuff_for_environment + [[ -n "${ANDROID_QUIET_BUILD:-}" ]] || printconfig + + if [ "${TARGET_BUILD_VARIANT}" = "userdebug" ] && [[ -z "${ANDROID_QUIET_BUILD}" ]]; then + echo + echo "Want FASTER LOCAL BUILDS? Use -eng instead of -userdebug (however for" \ + "performance benchmarking continue to use userdebug)" + fi if [ $used_lunch_menu -eq 1 ]; then echo echo "Hint: next time you can simply run 'lunch $selection'" fi - [[ -n "${ANDROID_QUIET_BUILD:-}" ]] || echo - - set_stuff_for_environment - [[ -n "${ANDROID_QUIET_BUILD:-}" ]] || printconfig destroy_build_var_cache if [[ -n "${CHECK_MU_CONFIG:-}" ]]; then @@ -881,6 +882,8 @@ function tapas() { local showHelp="$(echo $* | xargs -n 1 echo | \grep -E '^(help)$' | xargs)" local arch="$(echo $* | xargs -n 1 echo | \grep -E '^(arm|x86|arm64|x86_64)$' | xargs)" + # TODO(b/307975293): Expand tapas to take release arguments (and update hmm() usage). + local release="trunk_staging" local variant="$(echo $* | xargs -n 1 echo | \grep -E '^(user|userdebug|eng)$' | xargs)" local density="$(echo $* | xargs -n 1 echo | \grep -E '^(ldpi|mdpi|tvdpi|hdpi|xhdpi|xxhdpi|xxxhdpi|alldpi)$' | xargs)" local keys="$(echo $* | xargs -n 1 echo | \grep -E '^(devkeys)$' | xargs)" @@ -896,6 +899,10 @@ function tapas() echo "tapas: Error: Multiple build archs supplied: $arch" return fi + if [ $(echo $release | wc -w) -gt 1 ]; then + echo "tapas: Error: Multiple build releases supplied: $release" + return + fi if [ $(echo $variant | wc -w) -gt 1 ]; then echo "tapas: Error: Multiple build variants supplied: $variant" return @@ -930,6 +937,7 @@ function tapas() fi export TARGET_PRODUCT=$product + export TARGET_RELEASE=$release export TARGET_BUILD_VARIANT=$variant export TARGET_BUILD_DENSITY=$density export TARGET_BUILD_TYPE=release @@ -946,9 +954,11 @@ function tapas() function banchan() { local showHelp="$(echo $* | xargs -n 1 echo | \grep -E '^(help)$' | xargs)" - local product="$(echo $* | xargs -n 1 echo | \grep -E '^(.*_)?(arm|x86|arm64|x86_64|arm64only|x86_64only)$' | xargs)" + local product="$(echo $* | xargs -n 1 echo | \grep -E '^(.*_)?(arm|x86|arm64|riscv64|x86_64|arm64only|x86_64only)$' | xargs)" + # TODO: Expand banchan to take release arguments (and update hmm() usage). + local release="trunk_staging" local variant="$(echo $* | xargs -n 1 echo | \grep -E '^(user|userdebug|eng)$' | xargs)" - local apps="$(echo $* | xargs -n 1 echo | \grep -E -v '^(user|userdebug|eng|(.*_)?(arm|x86|arm64|x86_64))$' | xargs)" + local apps="$(echo $* | xargs -n 1 echo | \grep -E -v '^(user|userdebug|eng|(.*_)?(arm|x86|arm64|riscv64|x86_64))$' | xargs)" if [ "$showHelp" != "" ]; then $(gettop)/build/make/banchanHelp.sh @@ -961,6 +971,10 @@ function banchan() echo "banchan: Error: Multiple build archs or products supplied: $products" return fi + if [ $(echo $release | wc -w) -gt 1 ]; then + echo "banchan: Error: Multiple build releases supplied: $release" + return + fi if [ $(echo $variant | wc -w) -gt 1 ]; then echo "banchan: Error: Multiple build variants supplied: $variant" return @@ -974,6 +988,7 @@ function banchan() arm) product=module_arm;; x86) product=module_x86;; arm64) product=module_arm64;; + riscv64) product=module_riscv64;; x86_64) product=module_x86_64;; arm64only) product=module_arm64only;; x86_64only) product=module_x86_64only;; @@ -983,6 +998,7 @@ function banchan() fi export TARGET_PRODUCT=$product + export TARGET_RELEASE=$release export TARGET_BUILD_VARIANT=$variant export TARGET_BUILD_DENSITY=alldpi export TARGET_BUILD_TYPE=release @@ -1068,6 +1084,68 @@ function cproj() echo "can't find Android.mk" } +# Ensure that we're always using the adb in the tree. This works around the fact +# that bash caches $PATH lookups, so if you use adb before lunching/building the +# one in your tree, you'll continue to get /usr/bin/adb or whatever even after +# you have the one from your current tree on your path. Historically this would +# cause confusion because glinux had adb in /usr/bin/ by default, though that +# doesn't appear to be the case on my rodete hosts; it is however still the case +# that my Mac has /usr/local/bin/adb installed by default and on the default +# path. +function adb() { + # We need `command which` because zsh has a built-in `which` that's more + # like `type`. + local ADB=$(command which adb) + if [ -z "$ADB" ]; then + echo "Command adb not found; try lunch (and building) first?" + return 1 + fi + run_tool_with_logging "ADB" $ADB "${@}" +} + +function run_tool_with_logging() { + # Run commands in a subshell for us to handle forced terminations with a trap + # handler. + ( + local tool_tag="$1" + shift + local tool_binary="$1" + shift + + # If the logger is not configured, run the original command and return. + if [[ -z "${ANDROID_TOOL_LOGGER}" ]]; then + "${tool_binary}" "${@}" + return $? + fi + + # Otherwise, run the original command and call the logger when done. + local start_time + start_time=$(date +%s.%N) + local logger=${ANDROID_TOOL_LOGGER} + + # Install a trap to call the logger even when the process terminates abnormally. + # The logger is run in the background and its output suppressed to avoid + # interference with the user flow. + trap ' + exit_code=$?; + # Remove the trap to prevent duplicate log. + trap - EXIT; + "${logger}" \ + --tool_tag "${tool_tag}" \ + --start_timestamp "${start_time}" \ + --end_timestamp "$(date +%s.%N)" \ + --tool_args "$*" \ + --exit_code "${exit_code}" \ + ${ANDROID_TOOL_LOGGER_EXTRA_ARGS} \ + > /dev/null 2>&1 & + exit ${exit_code} + ' SIGINT SIGTERM SIGQUIT EXIT + + # Run the original command. + "${tool_binary}" "${@}" + ) +} + # simplified version of ps; output in the form # <pid> <procname> function qpid() { @@ -1096,12 +1174,12 @@ function qpid() { # # Easy way to make system.img/etc writable function syswrite() { - adb wait-for-device && adb root || return 1 + adb wait-for-device && adb root && adb wait-for-device || return 1 if [[ $(adb disable-verity | grep -i "reboot") ]]; then echo "rebooting" - adb reboot && adb wait-for-device && adb root || return 1 + adb reboot && adb wait-for-device && adb root && adb wait-for-device || return 1 fi - adb wait-for-device && adb remount || return 1 + adb remount || return 1 } # coredump_setup - enable core dumps globally for any process @@ -1351,53 +1429,6 @@ function getprebuilt get_abs_build_var ANDROID_PREBUILTS } -function tracedmdump() -{ - local T=$(gettop) - if [ ! "$T" ]; then - echo "Couldn't locate the top of the tree. Try setting TOP." - return - fi - local prebuiltdir=$(getprebuilt) - local arch=$(gettargetarch) - local KERNEL=$T/prebuilts/qemu-kernel/$arch/vmlinux-qemu - - local TRACE=$1 - if [ ! "$TRACE" ] ; then - echo "usage: tracedmdump tracename" - return - fi - - if [ ! -r "$KERNEL" ] ; then - echo "Error: cannot find kernel: '$KERNEL'" - return - fi - - local BASETRACE=$(basename $TRACE) - if [ "$BASETRACE" = "$TRACE" ] ; then - TRACE=$ANDROID_PRODUCT_OUT/traces/$TRACE - fi - - echo "post-processing traces..." - rm -f $TRACE/qtrace.dexlist - post_trace $TRACE - if [ $? -ne 0 ]; then - echo "***" - echo "*** Error: malformed trace. Did you remember to exit the emulator?" - echo "***" - return - fi - echo "generating dexlist output..." - /bin/ls $ANDROID_PRODUCT_OUT/system/framework/*.jar $ANDROID_PRODUCT_OUT/system/app/*.apk $ANDROID_PRODUCT_OUT/data/app/*.apk 2>/dev/null | xargs dexlist > $TRACE/qtrace.dexlist - echo "generating dmtrace data..." - q2dm -r $ANDROID_PRODUCT_OUT/symbols $TRACE $KERNEL $TRACE/dmtrace || return - echo "generating html file..." - dmtracedump -h $TRACE/dmtrace >| $TRACE/dmtrace.html || return - echo "done, see $TRACE/dmtrace.html for details" - echo "or run:" - echo " traceview $TRACE/dmtrace" -} - # communicate with a running device or emulator, set up necessary state, # and run the hat command. function runhat() @@ -1649,8 +1680,8 @@ function allmod() { # Return the Bazel label of a Soong module if it is converted with bp2build. function bmod() ( - if [ $# -ne 1 ]; then - echo "usage: bmod <module>" >&2 + if [ $# -eq 0 ]; then + echo "usage: bmod <module 1> <module 2> ... <module n>" >&2 return 1 fi @@ -1667,19 +1698,24 @@ function bmod() return 1 fi - local target_label=$(python3 -c "import json -module = '$1' + modules=() + for m in "$@"; do + modules+=("\"$m\",") + done + local res=$(python3 -c "import json +modules = [${modules[*]}] converted_json='$converted_json' bp2build_converted_map = json.load(open(converted_json)) -if module not in bp2build_converted_map: - exit(1) -print(bp2build_converted_map[module] + ':' + module)") - - if [ -z "${target_label}" ]; then - echo "$1 is not converted to Bazel." >&2 - return 1 - else - echo "${target_label}" +for module in modules: + if module not in bp2build_converted_map: + print(module + ' is not converted to Bazel.') + else: + print(bp2build_converted_map[module] + ':' + module)") + + echo "${res}" + unconverted_count=$(echo "${res}" | grep -c "not converted to Bazel") + if [[ ${unconverted_count} -ne 0 ]]; then + return 1 fi ) @@ -1909,6 +1945,11 @@ function _trigger_build() >&2 echo "Couldn't locate the top of the tree. Try setting TOP." return 1 fi + local ret=$? + if [[ ret -eq 0 && -z "${ANDROID_QUIET_BUILD:-}" && -n "${ANDROID_BUILD_BANNER}" ]]; then + echo "${ANDROID_BUILD_BANNER}" + fi + return $ret ) function m() @@ -2042,6 +2083,11 @@ function source_vendorsetup() { fi done done + + if [[ "${PWD}" == /google/cog/* ]]; then + f="build/make/cogsetup.sh" + echo "including $f"; . "$T/$f" + fi } function showcommands() { @@ -2080,6 +2126,16 @@ function avbtool() { "$ANDROID_SOONG_HOST_OUT"/bin/avbtool $@ } +function overrideflags() { + local T="$(gettop)" + (\cd "${T}" && build/make/tools/overrideflags.sh "$@") +} + +function aninja() { + local T="$(gettop)" + (\cd "${T}" && prebuilts/build-tools/linux-x86/bin/ninja -f out/combined-${TARGET_PRODUCT}.ninja "$@") +} + validate_current_shell set_global_paths source_vendorsetup diff --git a/packaging/distdir.mk b/packaging/distdir.mk index 264a8b098c..153ecf65b1 100644 --- a/packaging/distdir.mk +++ b/packaging/distdir.mk @@ -18,16 +18,19 @@ DIST_GOAL_OUTPUT_PAIRS := DIST_SRC_DST_PAIRS := include $(KATI_PACKAGE_MK_DIR)/dist.mk +FILE_NAME_TAG := $(file <$(OUT_DIR)/file_name_tag.txt) +.KATI_READONLY := FILE_NAME_TAG $(foreach pair,$(DIST_GOAL_OUTPUT_PAIRS), \ $(eval goal := $(call word-colon,1,$(pair))) \ - $(eval output := $(call word-colon,2,$(pair))) \ + $(eval output := $(subst FILE_NAME_TAG_PLACEHOLDER,$(FILE_NAME_TAG),$(call word-colon,2,$(pair)))) \ $(eval .PHONY: _dist_$$(goal)) \ $(if $(call streq,$(DIST),true),\ $(eval _dist_$$(goal): $$(DIST_DIR)/$$(output)), \ $(eval _dist_$$(goal):))) define copy-one-dist-file +$(2): .KATI_TAGS += ;rule_name=dist-cp $(2): $(1) @echo "Dist: $$@" rm -f $$@ @@ -37,7 +40,7 @@ endef ifeq ($(DIST),true) $(foreach pair,$(DIST_SRC_DST_PAIRS), \ $(eval src := $(call word-colon,1,$(pair))) \ - $(eval dst := $(DIST_DIR)/$(call word-colon,2,$(pair))) \ + $(eval dst := $(subst FILE_NAME_TAG_PLACEHOLDER,$(FILE_NAME_TAG),$(DIST_DIR)/$(call word-colon,2,$(pair)))) \ $(eval $(call copy-one-dist-file,$(src),$(dst)))) endif diff --git a/rbesetup.sh b/rbesetup.sh index 8386628a26..0da7a57647 100644 --- a/rbesetup.sh +++ b/rbesetup.sh @@ -34,7 +34,7 @@ fi # for the build to be executed with RBE. function use_rbe() { local RBE_BINARIES_DIR="prebuilts/remoteexecution-client/latest" - local DOCKER_IMAGE="gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62" + local DOCKER_IMAGE="gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:1eb7f64b9e17102b970bd7a1af7daaebdb01c3fb777715899ef462d6c6d01a45" # Do not set an invocation-ID and let reproxy auto-generate one. USE_RBE="true" \ diff --git a/target/board/Android.mk b/target/board/Android.mk index 21c0c10ab1..decc345ded 100644 --- a/target/board/Android.mk +++ b/target/board/Android.mk @@ -10,15 +10,29 @@ LOCAL_PATH := $(call my-dir) # device we're building for. This file is typically packaged up # with everything else. # -# If TARGET_BOARD_INFO_FILE (which can be set in BoardConfig.mk) is -# defined, it is used, otherwise board-info.txt is looked for in -# $(TARGET_DEVICE_DIR). +# The following logic is used to find the contents of the info file: +# 1. TARGET_BOARD_INFO_FILES (can be set in BoardConfig.mk) will be combined. +# 2. TARGET_BOARD_INFO_FILE (can be set in BoardConfig.mk) will be used. +# 3. $(TARGET_DEVICE_DIR)/board-info.txt will be used if present. +# +# Specifying both TARGET_BOARD_INFO_FILES and TARGET_BOARD_INFO_FILE is an +# error. # INSTALLED_ANDROID_INFO_TXT_TARGET := $(PRODUCT_OUT)/android-info.txt -board_info_txt := $(TARGET_BOARD_INFO_FILE) -ifndef board_info_txt -board_info_txt := $(wildcard $(TARGET_DEVICE_DIR)/board-info.txt) +ifdef TARGET_BOARD_INFO_FILES + ifdef TARGET_BOARD_INFO_FILE + $(warning Both TARGET_BOARD_INFO_FILES and TARGET_BOARD_INFO_FILE are defined.) + $(warning Using $(TARGET_BOARD_INFO_FILES) rather than $(TARGET_BOARD_INFO_FILE) for android-info.txt) + endif + board_info_txt := $(call intermediates-dir-for,PACKAGING,board-info)/board-info.txt +$(board_info_txt): $(TARGET_BOARD_INFO_FILES) + $(hide) cat $(TARGET_BOARD_INFO_FILES) > $@ +else ifdef TARGET_BOARD_INFO_FILE + board_info_txt := $(TARGET_BOARD_INFO_FILE) +else + board_info_txt := $(wildcard $(TARGET_DEVICE_DIR)/board-info.txt) endif + CHECK_RADIO_VERSIONS := $(HOST_OUT_EXECUTABLES)/check_radio_versions$(HOST_EXECUTABLE_SUFFIX) $(INSTALLED_ANDROID_INFO_TXT_TARGET): $(board_info_txt) $(CHECK_RADIO_VERSIONS) $(hide) $(CHECK_RADIO_VERSIONS) \ diff --git a/target/board/BoardConfigEmuCommon.mk b/target/board/BoardConfigEmuCommon.mk deleted file mode 100644 index 7a07d70bba..0000000000 --- a/target/board/BoardConfigEmuCommon.mk +++ /dev/null @@ -1,91 +0,0 @@ -# BoardConfigEmuCommon.mk -# -# Common compile-time definitions for emulator -# - -HAVE_HTC_AUDIO_DRIVER := true -BOARD_USES_GENERIC_AUDIO := true -TARGET_BOOTLOADER_BOARD_NAME := goldfish_$(TARGET_ARCH) - -# No Kernel -TARGET_NO_KERNEL := true - -# no hardware camera -USE_CAMERA_STUB := true - -NUM_FRAMEBUFFER_SURFACE_BUFFERS := 3 - -# Build OpenGLES emulation guest and host libraries -BUILD_EMULATOR_OPENGL := true -BUILD_QEMU_IMAGES := true - -# Build and enable the OpenGL ES View renderer. When running on the emulator, -# the GLES renderer disables itself if host GL acceleration isn't available. -USE_OPENGL_RENDERER := true - -# Emulator doesn't support sparse image format. -TARGET_USERIMAGES_SPARSE_EXT_DISABLED := true - -ifeq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true) - # emulator is Non-A/B device - AB_OTA_UPDATER := false - - # emulator needs super.img - BOARD_BUILD_SUPER_IMAGE_BY_DEFAULT := true - - # 8G + 8M - BOARD_SUPER_PARTITION_SIZE ?= 8598323200 - BOARD_SUPER_PARTITION_GROUPS := emulator_dynamic_partitions - - ifeq ($(QEMU_USE_SYSTEM_EXT_PARTITIONS),true) - BOARD_EMULATOR_DYNAMIC_PARTITIONS_PARTITION_LIST := \ - system \ - system_ext \ - product \ - vendor - - TARGET_COPY_OUT_PRODUCT := product - BOARD_PRODUCTIMAGE_FILE_SYSTEM_TYPE := ext4 - TARGET_COPY_OUT_SYSTEM_EXT := system_ext - BOARD_SYSTEM_EXTIMAGE_FILE_SYSTEM_TYPE := ext4 - else - TARGET_COPY_OUT_PRODUCT := system/product - TARGET_COPY_OUT_SYSTEM_EXT := system/system_ext - BOARD_EMULATOR_DYNAMIC_PARTITIONS_PARTITION_LIST := \ - system \ - vendor - endif - - # 8G - BOARD_EMULATOR_DYNAMIC_PARTITIONS_SIZE ?= 8589934592 - - # in build environment to speed up make -j - ifeq ($(QEMU_DISABLE_AVB),true) - BOARD_AVB_ENABLE := false - endif -else ifeq ($(PRODUCT_USE_DYNAMIC_PARTITION_SIZE),true) - # Enable dynamic system image size and reserved 64MB in it. - BOARD_SYSTEMIMAGE_PARTITION_RESERVED_SIZE := 67108864 - BOARD_VENDORIMAGE_PARTITION_RESERVED_SIZE := 67108864 -else - BOARD_SYSTEMIMAGE_PARTITION_SIZE := 3221225472 - BOARD_VENDORIMAGE_PARTITION_SIZE := 146800640 -endif - -#vendor boot -BOARD_INCLUDE_DTB_IN_BOOTIMG := false -BOARD_BOOT_HEADER_VERSION := 4 -BOARD_MKBOOTIMG_ARGS += --header_version $(BOARD_BOOT_HEADER_VERSION) -BOARD_VENDOR_BOOTIMAGE_PARTITION_SIZE := 0x06000000 -BOARD_RAMDISK_USE_LZ4 := true - -# Enable chain partition for system. -BOARD_AVB_SYSTEM_KEY_PATH := external/avb/test/data/testkey_rsa2048.pem -BOARD_AVB_SYSTEM_ALGORITHM := SHA256_RSA2048 -BOARD_AVB_SYSTEM_ROLLBACK_INDEX := $(PLATFORM_SECURITY_PATCH_TIMESTAMP) -BOARD_AVB_SYSTEM_ROLLBACK_INDEX_LOCATION := 1 - -BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE := ext4 -BOARD_FLASH_BLOCK_SIZE := 512 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/common diff --git a/target/board/BoardConfigGsiCommon.mk b/target/board/BoardConfigGsiCommon.mk index 4d95b330cb..67e31dfa5f 100644 --- a/target/board/BoardConfigGsiCommon.mk +++ b/target/board/BoardConfigGsiCommon.mk @@ -36,6 +36,7 @@ BOARD_SYSTEMIMAGE_PARTITION_RESERVED_SIZE := 67108864 TARGET_COPY_OUT_PRODUCT := system/product TARGET_COPY_OUT_SYSTEM_EXT := system/system_ext BOARD_PRODUCTIMAGE_FILE_SYSTEM_TYPE := +BOARD_SYSTEM_EXTIMAGE_FILE_SYSTEM_TYPE := # Creates metadata partition mount point under root for # the devices with metadata parition diff --git a/target/board/BoardConfigMainlineCommon.mk b/target/board/BoardConfigMainlineCommon.mk index 00f6e5b0ac..b5e3dc2b1e 100644 --- a/target/board/BoardConfigMainlineCommon.mk +++ b/target/board/BoardConfigMainlineCommon.mk @@ -2,6 +2,9 @@ # # Common compile-time definitions for mainline images. +# Ensure all trunk-stable flags are available. +include build/make/target/product/build_variables.mk + # The generic product target doesn't have any hardware-specific pieces. TARGET_NO_BOOTLOADER := true TARGET_NO_RECOVERY := true @@ -14,17 +17,13 @@ TARGET_USERIMAGES_USE_EXT4 := true TARGET_COPY_OUT_SYSTEM_EXT := system_ext TARGET_COPY_OUT_VENDOR := vendor TARGET_COPY_OUT_PRODUCT := product +BOARD_PRODUCTIMAGE_FILE_SYSTEM_TYPE := ext4 +BOARD_SYSTEM_EXTIMAGE_FILE_SYSTEM_TYPE := ext4 # Creates metadata partition mount point under root for # the devices with metadata parition BOARD_USES_METADATA_PARTITION := true -# Default is current, but allow devices to override vndk version if needed. -BOARD_VNDK_VERSION ?= current - -# Required flag for non-64 bit devices from P. -TARGET_USES_64_BIT_BINDER := true - # 64 bit mediadrmserver TARGET_ENABLE_MEDIADRM_64 := true diff --git a/target/board/BoardConfigModuleCommon.mk b/target/board/BoardConfigModuleCommon.mk deleted file mode 100644 index 24c01a58ef..0000000000 --- a/target/board/BoardConfigModuleCommon.mk +++ /dev/null @@ -1,6 +0,0 @@ -# BoardConfigModuleCommon.mk -# -# Common compile-time settings for module builds. - -# Required for all module devices. -TARGET_USES_64_BIT_BINDER := true diff --git a/target/board/emulator_arm/AndroidBoard.mk b/target/board/emulator_arm/AndroidBoard.mk deleted file mode 100644 index 7911f6105f..0000000000 --- a/target/board/emulator_arm/AndroidBoard.mk +++ /dev/null @@ -1 +0,0 @@ -LOCAL_PATH := $(call my-dir) diff --git a/target/board/emulator_arm/BoardConfig.mk b/target/board/emulator_arm/BoardConfig.mk deleted file mode 100644 index 287824fa7b..0000000000 --- a/target/board/emulator_arm/BoardConfig.mk +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# arm emulator specific definitions -TARGET_ARCH := arm -TARGET_ARCH_VARIANT := armv7-a-neon -TARGET_CPU_VARIANT := generic -TARGET_CPU_ABI := armeabi-v7a -TARGET_CPU_ABI2 := armeabi - -include build/make/target/board/BoardConfigGsiCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/emulator_arm/system_ext.prop b/target/board/emulator_arm/system_ext.prop deleted file mode 100644 index 64829f3ce4..0000000000 --- a/target/board/emulator_arm/system_ext.prop +++ /dev/null @@ -1,5 +0,0 @@ -# -# system.prop for generic sdk -# - -rild.libpath=/vendor/lib/libreference-ril.so diff --git a/target/board/emulator_arm64/BoardConfig.mk b/target/board/emulator_arm64/BoardConfig.mk deleted file mode 100644 index 963e558b6b..0000000000 --- a/target/board/emulator_arm64/BoardConfig.mk +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# arm64 emulator specific definitions -TARGET_ARCH := arm64 -TARGET_ARCH_VARIANT := armv8-a -TARGET_CPU_VARIANT := generic -TARGET_CPU_ABI := arm64-v8a - -ifneq ($(TARGET_BUILD_APPS)$(filter cts sdk,$(MAKECMDGOALS)),) -# DO NOT USE -# DO NOT USE -# -# This architecture / CPU variant must NOT be used for any 64 bit -# platform builds. It is the lowest common denominator required -# to build an unbundled application or cts for all supported 32 and 64 bit -# platforms. -# -# If you're building a 64 bit platform (and not an application) the -# ARM-v8 specification allows you to assume all the features available in an -# armv7-a-neon CPU. You should set the following as 2nd arch/cpu variant: -# -# TARGET_2ND_ARCH_VARIANT := armv8-a -# TARGET_2ND_CPU_VARIANT := generic -# -# DO NOT USE -# DO NOT USE -TARGET_2ND_ARCH_VARIANT := armv7-a-neon -# DO NOT USE -# DO NOT USE -TARGET_2ND_CPU_VARIANT := generic -# DO NOT USE -# DO NOT USE -else -TARGET_2ND_ARCH_VARIANT := armv8-a -TARGET_2ND_CPU_VARIANT := generic -endif - -include build/make/target/board/BoardConfigGsiCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -TARGET_NO_KERNEL := false -BOARD_USES_RECOVERY_AS_BOOT := true - -BOARD_BOOTIMAGE_PARTITION_SIZE := 0x02000000 -BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/emulator_arm64/device.mk b/target/board/emulator_arm64/device.mk deleted file mode 100644 index dc8419246f..0000000000 --- a/target/board/emulator_arm64/device.mk +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. - -# Cuttlefish has GKI kernel prebuilts, so use those for the GKI boot.img. -ifeq ($(TARGET_PREBUILT_KERNEL),) - LOCAL_KERNEL := kernel/prebuilts/5.4/arm64/kernel-5.4-lz4 -else - LOCAL_KERNEL := $(TARGET_PREBUILT_KERNEL) -endif - -PRODUCT_COPY_FILES += \ - $(LOCAL_KERNEL):kernel diff --git a/target/board/emulator_arm64/system_ext.prop b/target/board/emulator_arm64/system_ext.prop deleted file mode 100644 index 2f8f803e4c..0000000000 --- a/target/board/emulator_arm64/system_ext.prop +++ /dev/null @@ -1,5 +0,0 @@ -# -# system.prop for emulator arm64 sdk -# - -rild.libpath=/vendor/lib64/libreference-ril.so diff --git a/target/board/emulator_x86/BoardConfig.mk b/target/board/emulator_x86/BoardConfig.mk deleted file mode 100644 index 8f79166b48..0000000000 --- a/target/board/emulator_x86/BoardConfig.mk +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# x86 emulator specific definitions -TARGET_CPU_ABI := x86 -TARGET_ARCH := x86 -TARGET_ARCH_VARIANT := x86 - -TARGET_PRELINK_MODULE := false - -include build/make/target/board/BoardConfigGsiCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -# Resize to 4G to accommodate ASAN and CTS -BOARD_USERDATAIMAGE_PARTITION_SIZE := 4294967296 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/emulator_x86/system_ext.prop b/target/board/emulator_x86/system_ext.prop deleted file mode 100644 index 64829f3ce4..0000000000 --- a/target/board/emulator_x86/system_ext.prop +++ /dev/null @@ -1,5 +0,0 @@ -# -# system.prop for generic sdk -# - -rild.libpath=/vendor/lib/libreference-ril.so diff --git a/target/board/emulator_x86_64/BoardConfig.mk b/target/board/emulator_x86_64/BoardConfig.mk deleted file mode 100755 index b9cbd8a243..0000000000 --- a/target/board/emulator_x86_64/BoardConfig.mk +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# x86_64 emulator specific definitions -TARGET_CPU_ABI := x86_64 -TARGET_ARCH := x86_64 -TARGET_ARCH_VARIANT := x86_64 - -TARGET_2ND_CPU_ABI := x86 -TARGET_2ND_ARCH := x86 -TARGET_2ND_ARCH_VARIANT := x86_64 - -TARGET_PRELINK_MODULE := false -include build/make/target/board/BoardConfigGsiCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/emulator_x86_64/system_ext.prop b/target/board/emulator_x86_64/system_ext.prop deleted file mode 100644 index ed9d1731c7..0000000000 --- a/target/board/emulator_x86_64/system_ext.prop +++ /dev/null @@ -1,5 +0,0 @@ -# -# system.prop for generic sdk -# - -rild.libpath=/vendor/lib64/libreference-ril.so diff --git a/target/board/emulator_x86_64_arm64/BoardConfig.mk b/target/board/emulator_x86_64_arm64/BoardConfig.mk deleted file mode 100755 index 26b61a6787..0000000000 --- a/target/board/emulator_x86_64_arm64/BoardConfig.mk +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# x86_64 emulator specific definitions -TARGET_CPU_ABI := x86_64 -TARGET_ARCH := x86_64 -TARGET_ARCH_VARIANT := x86_64 - -TARGET_2ND_CPU_ABI := x86 -TARGET_2ND_ARCH := x86 -TARGET_2ND_ARCH_VARIANT := x86_64 - -TARGET_NATIVE_BRIDGE_ARCH := arm64 -TARGET_NATIVE_BRIDGE_ARCH_VARIANT := armv8-a -TARGET_NATIVE_BRIDGE_CPU_VARIANT := generic -TARGET_NATIVE_BRIDGE_ABI := arm64-v8a - -TARGET_NATIVE_BRIDGE_2ND_ARCH := arm -TARGET_NATIVE_BRIDGE_2ND_ARCH_VARIANT := armv7-a-neon -TARGET_NATIVE_BRIDGE_2ND_CPU_VARIANT := generic -TARGET_NATIVE_BRIDGE_2ND_ABI := armeabi-v7a armeabi - -BUILD_BROKEN_DUP_RULES := true - -TARGET_PRELINK_MODULE := false - -include build/make/target/board/BoardConfigMainlineCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -# the settings differ from BoardConfigMainlineCommon.mk -BOARD_USES_SYSTEM_OTHER_ODEX := - -# Resize to 4G to accommodate ASAN and CTS -BOARD_USERDATAIMAGE_PARTITION_SIZE := 4294967296 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/emulator_x86_64_arm64/system_ext.prop b/target/board/emulator_x86_64_arm64/system_ext.prop deleted file mode 100644 index ed9d1731c7..0000000000 --- a/target/board/emulator_x86_64_arm64/system_ext.prop +++ /dev/null @@ -1,5 +0,0 @@ -# -# system.prop for generic sdk -# - -rild.libpath=/vendor/lib64/libreference-ril.so diff --git a/target/board/emulator_x86_arm/BoardConfig.mk b/target/board/emulator_x86_arm/BoardConfig.mk deleted file mode 100644 index 21fdbc8e99..0000000000 --- a/target/board/emulator_x86_arm/BoardConfig.mk +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# x86 emulator specific definitions -TARGET_CPU_ABI := x86 -TARGET_ARCH := x86 -TARGET_ARCH_VARIANT := x86 - -TARGET_NATIVE_BRIDGE_ARCH := arm -TARGET_NATIVE_BRIDGE_ARCH_VARIANT := armv7-a-neon -TARGET_NATIVE_BRIDGE_CPU_VARIANT := generic -TARGET_NATIVE_BRIDGE_ABI := armeabi-v7a armeabi - -BUILD_BROKEN_DUP_RULES := true - -# -# The inclusion order below is important. -# The settings in latter makefiles overwrite those in the former. -# -include build/make/target/board/BoardConfigMainlineCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -# the settings differ from BoardConfigMainlineCommon.mk -BOARD_USES_SYSTEM_OTHER_ODEX := - -# Resize to 4G to accommodate ASAN and CTS -BOARD_USERDATAIMAGE_PARTITION_SIZE := 4294967296 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/emulator_x86_arm/device.mk b/target/board/emulator_x86_arm/device.mk deleted file mode 100644 index af023eb25c..0000000000 --- a/target/board/emulator_x86_arm/device.mk +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. diff --git a/target/board/emulator_x86_arm/system_ext.prop b/target/board/emulator_x86_arm/system_ext.prop deleted file mode 100644 index 64829f3ce4..0000000000 --- a/target/board/emulator_x86_arm/system_ext.prop +++ /dev/null @@ -1,5 +0,0 @@ -# -# system.prop for generic sdk -# - -rild.libpath=/vendor/lib/libreference-ril.so diff --git a/target/board/generic/BoardConfig.mk b/target/board/generic/BoardConfig.mk index 87c16daade..6720ddbe6c 100644 --- a/target/board/generic/BoardConfig.mk +++ b/target/board/generic/BoardConfig.mk @@ -30,20 +30,3 @@ TARGET_CPU_ABI := armeabi-v7a TARGET_CPU_ABI2 := armeabi include build/make/target/board/BoardConfigGsiCommon.mk - -ifndef BUILDING_GSI -include build/make/target/board/BoardConfigEmuCommon.mk - -BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" -endif diff --git a/target/board/generic/device.mk b/target/board/generic/device.mk index 76242c90b2..76edf6b411 100644 --- a/target/board/generic/device.mk +++ b/target/board/generic/device.mk @@ -14,5 +14,3 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. diff --git a/target/board/generic_64bitonly_x86_64/BoardConfig.mk b/target/board/generic_64bitonly_x86_64/BoardConfig.mk index 71c4357ec4..a129ea0fd3 100644 --- a/target/board/generic_64bitonly_x86_64/BoardConfig.mk +++ b/target/board/generic_64bitonly_x86_64/BoardConfig.mk @@ -26,20 +26,5 @@ TARGET_2ND_ARCH := x86 TARGET_2ND_ARCH_VARIANT := x86_64 TARGET_PRELINK_MODULE := false -include build/make/target/board/BoardConfigGsiCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk - -BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800 -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" +include build/make/target/board/BoardConfigGsiCommon.mk diff --git a/target/board/generic_arm64/BoardConfig.mk b/target/board/generic_arm64/BoardConfig.mk index 40be80e413..e2d5fb4df8 100644 --- a/target/board/generic_arm64/BoardConfig.mk +++ b/target/board/generic_arm64/BoardConfig.mk @@ -54,6 +54,8 @@ endif # Include 64-bit mediaserver to support 64-bit only devices TARGET_DYNAMIC_64_32_MEDIASERVER := true +# Include 64-bit drmserver to support 64-bit only devices +TARGET_DYNAMIC_64_32_DRMSERVER := true include build/make/target/board/BoardConfigGsiCommon.mk diff --git a/target/board/generic_x86/BoardConfig.mk b/target/board/generic_x86/BoardConfig.mk index 47fd384c26..26bede8d1f 100644 --- a/target/board/generic_x86/BoardConfig.mk +++ b/target/board/generic_x86/BoardConfig.mk @@ -19,23 +19,3 @@ TARGET_ARCH := x86 TARGET_ARCH_VARIANT := x86 include build/make/target/board/BoardConfigGsiCommon.mk - -ifndef BUILDING_GSI -include build/make/target/board/BoardConfigEmuCommon.mk - -# Resize to 4G to accomodate ASAN and CTS -BOARD_USERDATAIMAGE_PARTITION_SIZE := 4294967296 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" -endif diff --git a/target/board/generic_x86/device.mk b/target/board/generic_x86/device.mk index 5ad008f119..60f0cc33f1 100644 --- a/target/board/generic_x86/device.mk +++ b/target/board/generic_x86/device.mk @@ -14,9 +14,6 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. - ifdef NET_ETH0_STARTONBOOT PRODUCT_VENDOR_PROPERTIES += net.eth0.startonboot=1 endif diff --git a/target/board/generic_x86_64/BoardConfig.mk b/target/board/generic_x86_64/BoardConfig.mk index e7f2ae0072..64da5783db 100755..100644 --- a/target/board/generic_x86_64/BoardConfig.mk +++ b/target/board/generic_x86_64/BoardConfig.mk @@ -24,25 +24,7 @@ TARGET_2ND_ARCH_VARIANT := x86_64 # Include 64-bit mediaserver to support 64-bit only devices TARGET_DYNAMIC_64_32_MEDIASERVER := true +# Include 64-bit drmserver to support 64-bit only devices +TARGET_DYNAMIC_64_32_DRMSERVER := true include build/make/target/board/BoardConfigGsiCommon.mk - -ifndef BUILDING_GSI -include build/make/target/board/BoardConfigEmuCommon.mk - -BOARD_USERDATAIMAGE_PARTITION_SIZE := 576716800 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" - -endif # !BUILDING_GSI diff --git a/target/board/generic_x86_64_arm64/BoardConfig.mk b/target/board/generic_x86_64_arm64/BoardConfig.mk index f528294fc0..818ec44d2b 100755..100644 --- a/target/board/generic_x86_64_arm64/BoardConfig.mk +++ b/target/board/generic_x86_64_arm64/BoardConfig.mk @@ -13,7 +13,6 @@ # limitations under the License. # -# x86_64 emulator specific definitions TARGET_CPU_ABI := x86_64 TARGET_ARCH := x86_64 TARGET_ARCH_VARIANT := x86_64 @@ -37,23 +36,9 @@ BUILD_BROKEN_DUP_RULES := true TARGET_PRELINK_MODULE := false include build/make/target/board/BoardConfigMainlineCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk # the settings differ from BoardConfigMainlineCommon.mk BOARD_USES_SYSTEM_OTHER_ODEX := # Resize to 4G to accommodate ASAN and CTS BOARD_USERDATAIMAGE_PARTITION_SIZE := 4294967296 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/generic_x86_64_arm64/device.mk b/target/board/generic_x86_64_arm64/device.mk index 76242c90b2..76edf6b411 100755..100644 --- a/target/board/generic_x86_64_arm64/device.mk +++ b/target/board/generic_x86_64_arm64/device.mk @@ -14,5 +14,3 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. diff --git a/target/board/generic_x86_arm/BoardConfig.mk b/target/board/generic_x86_arm/BoardConfig.mk index f6589b01e1..62bb5eb1f9 100644 --- a/target/board/generic_x86_arm/BoardConfig.mk +++ b/target/board/generic_x86_arm/BoardConfig.mk @@ -13,7 +13,6 @@ # limitations under the License. # -# x86 emulator specific definitions TARGET_CPU_ABI := x86 TARGET_ARCH := x86 TARGET_ARCH_VARIANT := x86 @@ -30,23 +29,9 @@ BUILD_BROKEN_DUP_RULES := true # The settings in latter makefiles overwrite those in the former. # include build/make/target/board/BoardConfigMainlineCommon.mk -include build/make/target/board/BoardConfigEmuCommon.mk # the settings differ from BoardConfigMainlineCommon.mk BOARD_USES_SYSTEM_OTHER_ODEX := # Resize to 4G to accomodate ASAN and CTS BOARD_USERDATAIMAGE_PARTITION_SIZE := 4294967296 - -BOARD_SEPOLICY_DIRS += device/generic/goldfish/sepolicy/x86 - -# Wifi. -BOARD_WLAN_DEVICE := emulator -BOARD_HOSTAPD_DRIVER := NL80211 -BOARD_WPA_SUPPLICANT_DRIVER := NL80211 -BOARD_HOSTAPD_PRIVATE_LIB := lib_driver_cmd_simulated -BOARD_WPA_SUPPLICANT_PRIVATE_LIB := lib_driver_cmd_simulated -WPA_SUPPLICANT_VERSION := VER_0_8_X -WIFI_DRIVER_FW_PATH_PARAM := "/dev/null" -WIFI_DRIVER_FW_PATH_STA := "/dev/null" -WIFI_DRIVER_FW_PATH_AP := "/dev/null" diff --git a/target/board/generic_x86_arm/device.mk b/target/board/generic_x86_arm/device.mk index 76242c90b2..76edf6b411 100644 --- a/target/board/generic_x86_arm/device.mk +++ b/target/board/generic_x86_arm/device.mk @@ -14,5 +14,3 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. diff --git a/target/board/gsi_arm64/BoardConfig.mk b/target/board/gsi_arm64/BoardConfig.mk index db95082e5f..7910b1da15 100644 --- a/target/board/gsi_arm64/BoardConfig.mk +++ b/target/board/gsi_arm64/BoardConfig.mk @@ -29,6 +29,8 @@ TARGET_2ND_CPU_VARIANT := generic # Include 64-bit mediaserver to support 64-bit only devices TARGET_DYNAMIC_64_32_MEDIASERVER := true +# Include 64-bit drmserver to support 64-bit only devices +TARGET_DYNAMIC_64_32_DRMSERVER := true # TODO(b/111434759, b/111287060) SoC specific hacks BOARD_ROOT_EXTRA_SYMLINKS += /vendor/lib/dsp:/dsp diff --git a/target/board/linux_bionic/BoardConfig.mk b/target/board/linux_bionic/BoardConfig.mk index 7938bdbd7f..7fca911613 100644 --- a/target/board/linux_bionic/BoardConfig.mk +++ b/target/board/linux_bionic/BoardConfig.mk @@ -17,7 +17,11 @@ # (device) target architectures are irrelevant. However, the build system isn't # prepared to handle no target architectures at all, so pick something # arbitrarily. -TARGET_ARCH_SUITE := ndk +TARGET_ARCH := arm +TARGET_ARCH_VARIANT := armv7-a-neon +TARGET_CPU_VARIANT := generic +TARGET_CPU_ABI := armeabi-v7a +TARGET_CPU_ABI2 := armeabi HOST_CROSS_OS := linux_bionic HOST_CROSS_ARCH := x86_64 diff --git a/target/board/mainline_sdk/BoardConfig.mk b/target/board/mainline_sdk/BoardConfig.mk index f5c2dc6b49..e4c6a8c971 100644 --- a/target/board/mainline_sdk/BoardConfig.mk +++ b/target/board/mainline_sdk/BoardConfig.mk @@ -13,11 +13,11 @@ # limitations under the License. # +# Ensure all trunk-stable flags are available. +include build/make/target/product/build_variables.mk + TARGET_ARCH_SUITE := mainline_sdk HOST_CROSS_OS := linux_bionic HOST_CROSS_ARCH := x86_64 HOST_CROSS_2ND_ARCH := - -# Required flag for non-64 bit devices from P. -TARGET_USES_64_BIT_BINDER := true diff --git a/target/board/module_arm/BoardConfig.mk b/target/board/module_arm/BoardConfig.mk index 3f35c060e1..565efc8425 100644 --- a/target/board/module_arm/BoardConfig.mk +++ b/target/board/module_arm/BoardConfig.mk @@ -13,8 +13,6 @@ # limitations under the License. # -include build/make/target/board/BoardConfigModuleCommon.mk - TARGET_ARCH := arm TARGET_ARCH_VARIANT := armv7-a-neon TARGET_CPU_VARIANT := generic diff --git a/target/board/module_arm64/BoardConfig.mk b/target/board/module_arm64/BoardConfig.mk index 3700056da3..66e3792640 100644 --- a/target/board/module_arm64/BoardConfig.mk +++ b/target/board/module_arm64/BoardConfig.mk @@ -13,8 +13,6 @@ # limitations under the License. # -include build/make/target/board/BoardConfigModuleCommon.mk - TARGET_ARCH := arm64 TARGET_ARCH_VARIANT := armv8-a TARGET_CPU_VARIANT := generic diff --git a/target/board/module_arm64only/BoardConfig.mk b/target/board/module_arm64only/BoardConfig.mk index 3cabf05d94..6c265799e7 100644 --- a/target/board/module_arm64only/BoardConfig.mk +++ b/target/board/module_arm64only/BoardConfig.mk @@ -13,8 +13,6 @@ # limitations under the License. # -include build/make/target/board/BoardConfigModuleCommon.mk - TARGET_ARCH := arm64 TARGET_ARCH_VARIANT := armv8-a TARGET_CPU_VARIANT := generic diff --git a/core/combo/HOST_linux-x86_64.mk b/target/board/module_riscv64/BoardConfig.mk index 845733f74d..8bc1999dfe 100644 --- a/core/combo/HOST_linux-x86_64.mk +++ b/target/board/module_riscv64/BoardConfig.mk @@ -1,5 +1,4 @@ -# -# Copyright (C) 2006 The Android Open Source Project +# Copyright (C) 2023 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +13,10 @@ # limitations under the License. # -# Configuration for builds hosted on linux-x86_64. -# Included by combo/select.mk +TARGET_ARCH := riscv64 +TARGET_ARCH_VARIANT := +TARGET_CPU_VARIANT := generic +TARGET_CPU_ABI := riscv64 -define $(combo_var_prefix)transform-shared-lib-to-toc -$(call _gen_toc_command_for_elf,$(1),$(2)) -endef +# Temporary hack while prebuilt modules are missing riscv64. +ALLOW_MISSING_DEPENDENCIES := true diff --git a/target/board/module_riscv64/README.md b/target/board/module_riscv64/README.md new file mode 100644 index 0000000000..edebaa9f80 --- /dev/null +++ b/target/board/module_riscv64/README.md @@ -0,0 +1,2 @@ +This device is suitable for an unbundled module targeted specifically to a +riscv64 device. This is a 64-bit only device (no 32-bit support). diff --git a/target/board/module_x86/BoardConfig.mk b/target/board/module_x86/BoardConfig.mk index a93ac9779c..af3fffd067 100644 --- a/target/board/module_x86/BoardConfig.mk +++ b/target/board/module_x86/BoardConfig.mk @@ -13,8 +13,6 @@ # limitations under the License. # -include build/make/target/board/BoardConfigModuleCommon.mk - TARGET_CPU_ABI := x86 TARGET_ARCH := x86 TARGET_ARCH_VARIANT := x86 diff --git a/target/board/module_x86_64/BoardConfig.mk b/target/board/module_x86_64/BoardConfig.mk index 1ed3be0ba6..1ada02729e 100644 --- a/target/board/module_x86_64/BoardConfig.mk +++ b/target/board/module_x86_64/BoardConfig.mk @@ -13,8 +13,6 @@ # limitations under the License. # -include build/make/target/board/BoardConfigModuleCommon.mk - TARGET_CPU_ABI := x86_64 TARGET_ARCH := x86_64 TARGET_ARCH_VARIANT := x86_64 diff --git a/target/board/module_x86_64only/BoardConfig.mk b/target/board/module_x86_64only/BoardConfig.mk index b0676cbe22..5b86f0a8a3 100644 --- a/target/board/module_x86_64only/BoardConfig.mk +++ b/target/board/module_x86_64only/BoardConfig.mk @@ -13,8 +13,6 @@ # limitations under the License. # -include build/make/target/board/BoardConfigModuleCommon.mk - TARGET_CPU_ABI := x86_64 TARGET_ARCH := x86_64 TARGET_ARCH_VARIANT := x86_64 diff --git a/target/board/ndk/BoardConfig.mk b/target/board/ndk/BoardConfig.mk index da8b5f3e74..d5399b226b 100644 --- a/target/board/ndk/BoardConfig.mk +++ b/target/board/ndk/BoardConfig.mk @@ -14,8 +14,7 @@ # TARGET_ARCH_SUITE := ndk -TARGET_USES_64_BIT_BINDER := true -MALLOC_SVELTE := true +MALLOC_LOW_MEMORY := true USE_SAFESTACK := false diff --git a/target/product/AndroidProducts.mk b/target/product/AndroidProducts.mk index 1e0ce19926..07eb96db2a 100644 --- a/target/product/AndroidProducts.mk +++ b/target/product/AndroidProducts.mk @@ -35,6 +35,7 @@ ifneq ($(TARGET_BUILD_APPS),) PRODUCT_MAKEFILES := \ $(LOCAL_DIR)/aosp_arm64.mk \ + $(LOCAL_DIR)/aosp_arm64_fullmte.mk \ $(LOCAL_DIR)/aosp_arm.mk \ $(LOCAL_DIR)/aosp_riscv64.mk \ $(LOCAL_DIR)/aosp_x86_64.mk \ @@ -46,6 +47,7 @@ else PRODUCT_MAKEFILES := \ $(LOCAL_DIR)/aosp_64bitonly_x86_64.mk \ $(LOCAL_DIR)/aosp_arm64.mk \ + $(LOCAL_DIR)/aosp_arm64_fullmte.mk \ $(LOCAL_DIR)/aosp_arm.mk \ $(LOCAL_DIR)/aosp_riscv64.mk \ $(LOCAL_DIR)/aosp_x86_64.mk \ @@ -64,14 +66,8 @@ PRODUCT_MAKEFILES := \ $(LOCAL_DIR)/mainline_system_x86_64.mk \ $(LOCAL_DIR)/mainline_system_x86_arm.mk \ $(LOCAL_DIR)/ndk.mk \ - $(LOCAL_DIR)/sdk_arm64.mk \ $(LOCAL_DIR)/sdk.mk \ - $(LOCAL_DIR)/sdk_phone_arm64.mk \ - $(LOCAL_DIR)/sdk_phone_armv7.mk \ - $(LOCAL_DIR)/sdk_phone_x86_64.mk \ - $(LOCAL_DIR)/sdk_phone_x86.mk \ - $(LOCAL_DIR)/sdk_x86_64.mk \ - $(LOCAL_DIR)/sdk_x86.mk \ + $(LOCAL_DIR)/sdk_with_runtime_apis.mk \ endif @@ -81,12 +77,13 @@ PRODUCT_MAKEFILES += \ $(LOCAL_DIR)/module_arm.mk \ $(LOCAL_DIR)/module_arm64.mk \ $(LOCAL_DIR)/module_arm64only.mk \ + $(LOCAL_DIR)/module_riscv64.mk \ $(LOCAL_DIR)/module_x86.mk \ $(LOCAL_DIR)/module_x86_64.mk \ $(LOCAL_DIR)/module_x86_64only.mk \ COMMON_LUNCH_CHOICES := \ - aosp_arm64-eng \ - aosp_arm-eng \ - aosp_x86_64-eng \ - aosp_x86-eng \ + aosp_arm64-trunk_staging-eng \ + aosp_arm-trunk_staging-eng \ + aosp_x86_64-trunk_staging-eng \ + aosp_x86-trunk_staging-eng \ diff --git a/target/product/OWNERS b/target/product/OWNERS index 008e4a2109..48d3f2a33c 100644 --- a/target/product/OWNERS +++ b/target/product/OWNERS @@ -1,4 +1,4 @@ -per-file runtime_libart.mk = calin@google.com, mast@google.com, ngeoffray@google.com, oth@google.com, rpl@google.com, vmarko@google.com +per-file runtime_libart.mk = mast@google.com, ngeoffray@google.com, rpl@google.com, vmarko@google.com # GSI per-file gsi_release.mk = file:/target/product/gsi/OWNERS @@ -7,4 +7,4 @@ per-file developer_gsi_keys.mk = file:/target/product/gsi/OWNERS # Android Go per-file go_defaults.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com per-file go_defaults_512.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com -per-file go_defaults_common.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com
\ No newline at end of file +per-file go_defaults_common.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com diff --git a/target/product/sdk_x86_64.mk b/target/product/angle_default.mk index 5f6553e000..72846d3b02 100644 --- a/target/product/sdk_x86_64.mk +++ b/target/product/angle_default.mk @@ -1,5 +1,5 @@ # -# Copyright (C) 2014 The Android Open Source Project +# Copyright 2023 The Android Open-Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ # limitations under the License. # -# Don't modify this file - It's just an alias! +# To enable ANGLE as the default system GLES drivers, add +# $(call inherit-product, $(SRC_TARGET_DIR)/product/angle_default.mk) to the Makefile. -$(call inherit-product, $(SRC_TARGET_DIR)/product/sdk_phone_x86_64.mk) - -PRODUCT_NAME := sdk_x86_64 +PRODUCT_SYSTEM_PROPERTIES += \ + persist.graphics.egl=angle diff --git a/target/product/aosp_64bitonly_x86_64.mk b/target/product/aosp_64bitonly_x86_64.mk index 4de4e0c897..cf812a22d0 100644 --- a/target/product/aosp_64bitonly_x86_64.mk +++ b/target/product/aosp_64bitonly_x86_64.mk @@ -51,14 +51,15 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # # All components inherited here go to vendor image # -$(call inherit-product-if-exists, device/generic/goldfish/x86_64-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86_64/device.mk) # # Special settings for GSI releasing # ifeq (aosp_64bitonly_x86_64,$(TARGET_PRODUCT)) +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) endif diff --git a/target/product/aosp_arm.mk b/target/product/aosp_arm.mk index 5f200aafb6..d9c362eb56 100644 --- a/target/product/aosp_arm.mk +++ b/target/product/aosp_arm.mk @@ -50,13 +50,15 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # All components inherited here go to vendor image # $(call inherit-product-if-exists, build/make/target/product/ramdisk_stub.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86/device.mk) # # Special settings for GSI releasing # ifeq (aosp_arm,$(TARGET_PRODUCT)) +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) endif diff --git a/target/product/aosp_arm64.mk b/target/product/aosp_arm64.mk index ffc37a9bd2..364fed4efc 100644 --- a/target/product/aosp_arm64.mk +++ b/target/product/aosp_arm64.mk @@ -54,14 +54,17 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # # All components inherited here go to vendor or vendor_boot image # -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_arm64/device.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/non_ab_device.mk) +AB_OTA_UPDATER := true +AB_OTA_PARTITIONS ?= system # # Special settings for GSI releasing # ifeq (aosp_arm64,$(TARGET_PRODUCT)) +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) endif @@ -70,3 +73,5 @@ PRODUCT_NAME := aosp_arm64 PRODUCT_DEVICE := generic_arm64 PRODUCT_BRAND := Android PRODUCT_MODEL := AOSP on ARM64 + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true diff --git a/target/board/emulator_x86/device.mk b/target/product/aosp_arm64_fullmte.mk index 8a9d8da921..ed6bd4a5f0 100644 --- a/target/board/emulator_x86/device.mk +++ b/target/product/aosp_arm64_fullmte.mk @@ -1,5 +1,4 @@ -# -# Copyright (C) 2020 The Android Open Source Project +# Copyright (C) 2023 The Android Open-Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +13,15 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. +include $(SRC_TARGET_DIR)/product/fullmte.mk + +PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS := relaxed + +$(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_arm64.mk) + +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true -ifdef NET_ETH0_STARTONBOOT - PRODUCT_VENDOR_PROPERTIES += net.eth0.startonboot=1 -endif +$(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) -# Ensure we package the BIOS files too. -PRODUCT_HOST_PACKAGES += \ - bios.bin \ - vgabios-cirrus.bin \ +PRODUCT_NAME := aosp_arm64_fullmte diff --git a/target/product/aosp_product.mk b/target/product/aosp_product.mk index e396ad11ff..3a5b622f99 100644 --- a/target/product/aosp_product.mk +++ b/target/product/aosp_product.mk @@ -29,10 +29,11 @@ PRODUCT_PRODUCT_PROPERTIES += \ # More AOSP packages PRODUCT_PACKAGES += \ + initial-package-stopped-states-aosp.xml \ messaging \ PhotoTable \ preinstalled-packages-platform-aosp-product.xml \ - WallpaperPicker \ + ThemePicker \ # Telephony: # Provide a APN configuration to GSI product diff --git a/target/product/aosp_riscv64.mk b/target/product/aosp_riscv64.mk index 1261fb16f3..fa503fffbd 100644 --- a/target/product/aosp_riscv64.mk +++ b/target/product/aosp_riscv64.mk @@ -30,8 +30,7 @@ PRODUCT_USE_DYNAMIC_PARTITIONS := true # GSI for system/product & support 64-bit apps only $(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit_only.mk) -#$(call inherit-product, $(SRC_TARGET_DIR)/product/mainline_system.mk) -TARGET_FLATTEN_APEX := false +$(call inherit-product, $(SRC_TARGET_DIR)/product/mainline_system.mk) # # All components inherited here go to system_ext image @@ -47,38 +46,27 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # # All components inherited here go to vendor image # -$(call inherit-product-if-exists, device/generic/goldfish/riscv64-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_riscv64/device.mk) # # Special settings for GSI releasing # ifeq (aosp_riscv64,$(TARGET_PRODUCT)) +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) endif -# TODO: this list should come via mainline_system.mk, but for now list -# just the modules that work for riscv64. -PRODUCT_PACKAGES := \ - init.environ.rc \ - init_first_stage \ - init_system \ - linker \ - shell_and_utilities \ - com.android.art \ - com.android.conscrypt \ - com.android.i18n \ - com.android.runtime \ - com.android.tzdata \ - com.android.os.statsd \ - -$(call inherit-product, $(SRC_TARGET_DIR)/product/default_art_config.mk) -PRODUCT_USES_DEFAULT_ART_CONFIG := false - PRODUCT_ARTIFACT_PATH_REQUIREMENT_ALLOWED_LIST += \ root/init.zygote64.rc +# TODO(b/206676167): This property can be removed when renderscript is removed. +# Prevents framework from attempting to load renderscript libraries, which are +# not supported on this architecture. +PRODUCT_SYSTEM_PROPERTIES += \ + config.disable_renderscript=1 \ + # This build configuration supports 64-bit apps only PRODUCT_NAME := aosp_riscv64 PRODUCT_DEVICE := generic_riscv64 diff --git a/target/product/aosp_x86.mk b/target/product/aosp_x86.mk index 7db2c0ff0c..c26a8bf45c 100644 --- a/target/product/aosp_x86.mk +++ b/target/product/aosp_x86.mk @@ -47,8 +47,6 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # # All components inherited here go to vendor image # -$(call inherit-product-if-exists, device/generic/goldfish/x86-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86/device.mk) @@ -56,6 +54,9 @@ $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86/device.mk) # Special settings for GSI releasing # ifeq (aosp_x86,$(TARGET_PRODUCT)) +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) endif diff --git a/target/product/aosp_x86_64.mk b/target/product/aosp_x86_64.mk index d55866f4ec..595940d9d1 100644 --- a/target/product/aosp_x86_64.mk +++ b/target/product/aosp_x86_64.mk @@ -56,15 +56,17 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # # All components inherited here go to vendor image # -$(call inherit-product-if-exists, device/generic/goldfish/x86_64-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86_64/device.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/non_ab_device.mk) +AB_OTA_UPDATER := true +AB_OTA_PARTITIONS ?= system # # Special settings for GSI releasing # ifeq (aosp_x86_64,$(TARGET_PRODUCT)) +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) endif @@ -73,3 +75,5 @@ PRODUCT_NAME := aosp_x86_64 PRODUCT_DEVICE := generic_x86_64 PRODUCT_BRAND := Android PRODUCT_MODEL := AOSP on x86_64 + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true diff --git a/target/product/aosp_x86_arm.mk b/target/product/aosp_x86_arm.mk index f96e068b56..a103b1a22c 100644 --- a/target/product/aosp_x86_arm.mk +++ b/target/product/aosp_x86_arm.mk @@ -45,8 +45,6 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) # # All components inherited here go to vendor image # -$(call inherit-product-if-exists, device/generic/goldfish/x86-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86_arm/device.mk) diff --git a/target/product/base_system.mk b/target/product/base_system.mk index 94b5c16965..22284b1c18 100644 --- a/target/product/base_system.mk +++ b/target/product/base_system.mk @@ -17,13 +17,12 @@ # Base modules and settings for the system partition. PRODUCT_PACKAGES += \ abx \ + aconfigd \ adbd_system_api \ + aflags \ am \ - android.hidl.allocator@1.0-service \ android.hidl.base-V1.0-java \ android.hidl.manager-V1.0-java \ - android.hidl.memory@1.0-impl \ - android.hidl.memory@1.0-impl.vendor \ android.system.suspend-service \ android.test.base \ android.test.mock \ @@ -53,8 +52,11 @@ PRODUCT_PACKAGES += \ com.android.adservices \ com.android.appsearch \ com.android.btservices \ + com.android.configinfrastructure \ com.android.conscrypt \ + com.android.devicelock \ com.android.extservices \ + com.android.healthfitness \ com.android.i18n \ com.android.ipsec \ com.android.location.provider \ @@ -70,7 +72,7 @@ PRODUCT_PACKAGES += \ com.android.scheduling \ com.android.sdkext \ com.android.tethering \ - com.android.tzdata \ + $(RELEASE_PACKAGE_TZDATA_MODULE) \ com.android.uwb \ com.android.virt \ com.android.wifi \ @@ -88,19 +90,20 @@ PRODUCT_PACKAGES += \ dump.erofs \ dumpstate \ dumpsys \ - DynamicSystemInstallationService \ e2fsck \ ExtShared \ flags_health_check \ framework-graphics \ + framework-location \ framework-minus-apex \ - framework-res \ + framework-minus-apex-install-dependencies \ framework-sysconfig.xml \ fsck.erofs \ fsck_msdos \ fsverity-release-cert-der \ fs_config_files_system \ fs_config_dirs_system \ + gpu_counter_producer \ group_system \ gsid \ gsi_tool \ @@ -109,7 +112,6 @@ PRODUCT_PACKAGES += \ gatekeeperd \ gpuservice \ hid \ - hwservicemanager \ idmap2 \ idmap2d \ ime \ @@ -120,13 +122,14 @@ PRODUCT_PACKAGES += \ incident-helper-cmd \ init.environ.rc \ init_system \ + initial-package-stopped-states.xml \ input \ installd \ IntentResolver \ ip \ iptables \ - ip-up-vpn \ javax.obex \ + kcmdlinectrl \ keystore2 \ credstore \ ld.mc \ @@ -201,6 +204,7 @@ PRODUCT_PACKAGES += \ libui \ libusbhost \ libutils \ + libvintf_jni \ libvulkan \ libwilhelm \ linker \ @@ -222,26 +226,29 @@ PRODUCT_PACKAGES += \ mke2fs \ mkfs.erofs \ monkey \ + misctrl \ mtectrl \ - mtpd \ ndc \ netd \ - NetworkStackNext \ + NetworkStack \ odsign \ org.apache.http.legacy \ otacerts \ PackageInstaller \ passwd_system \ perfetto \ + perfetto-extras \ ping \ ping6 \ + pintool \ platform.xml \ pm \ - pppd \ + preinstalled-packages-asl-files.xml \ preinstalled-packages-platform.xml \ + preinstalled-packages-strict-signature.xml \ + printflags \ privapp-permissions-platform.xml \ prng_seeder \ - racoon \ recovery-persist \ resize2fs \ rss_hwm_reset \ @@ -259,13 +266,13 @@ PRODUCT_PACKAGES += \ services \ settings \ SettingsProvider \ + sfdo \ sgdisk \ Shell \ shell_and_utilities_system \ sm \ snapshotctl \ snapuserd \ - SoundPicker \ storaged \ surfaceflinger \ svc \ @@ -282,25 +289,69 @@ PRODUCT_PACKAGES += \ uncrypt \ usbd \ vdc \ - viewcompiler \ voip-common \ vold \ - WallpaperBackup \ watchdogd \ wificond \ wifi.rc \ wm \ +# When we release crashrecovery module +ifeq ($(RELEASE_CRASHRECOVERY_MODULE),true) + PRODUCT_PACKAGES += \ + com.android.crashrecovery \ + +endif + +# These packages are not used on Android TV +ifneq ($(PRODUCT_IS_ATV),true) + PRODUCT_PACKAGES += \ + $(RELEASE_PACKAGE_SOUND_PICKER) \ + +endif + +# Product does not support Dynamic System Update +ifneq ($(PRODUCT_NO_DYNAMIC_SYSTEM_UPDATE),true) + PRODUCT_PACKAGES += \ + DynamicSystemInstallationService \ + +endif + +# Check if the build supports NFC apex or not +ifeq ($(RELEASE_PACKAGE_NFC_STACK),NfcNci) + PRODUCT_PACKAGES += \ + framework-nfc \ + NfcNci +else + PRODUCT_PACKAGES += \ + com.android.nfcservices +endif + +ifeq ($(RELEASE_USE_WEBVIEW_BOOTSTRAP_MODULE),true) + PRODUCT_PACKAGES += \ + com.android.webview.bootstrap +endif + # VINTF data for system image PRODUCT_PACKAGES += \ system_manifest.xml \ system_compatibility_matrix.xml \ -# HWASAN runtime for SANITIZE_TARGET=hwaddress builds -ifneq (,$(filter hwaddress,$(SANITIZE_TARGET))) - PRODUCT_PACKAGES += \ - libclang_rt.hwasan.bootstrap -endif +# Base modules when shipping api level is less than or equal to 34 +PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34 += \ + android.hidl.memory@1.0-impl \ + +# hwservicemanager is now installed on system_ext, but apexes might be using +# old libraries that are expecting it to be installed on system. This allows +# those apexes to continue working. The symlink can be removed once we are sure +# there are no devices using hwservicemanager (when Android V launching devices +# are no longer supported for dessert upgrades). +PRODUCT_PACKAGES += \ + hwservicemanager_compat_symlink_module \ + +PRODUCT_PACKAGES_ARM64 := libclang_rt.hwasan \ + libclang_rt.hwasan.bootstrap \ + libc_hwasan \ # Jacoco agent JARS to be built and installed, if any. ifeq ($(EMMA_INSTRUMENT),true) @@ -319,6 +370,16 @@ ifeq ($(EMMA_INSTRUMENT),true) endif # EMMA_INSTRUMENT_STATIC endif # EMMA_INSTRUMENT +ifeq (,$(DISABLE_WALLPAPER_BACKUP)) + PRODUCT_PACKAGES += \ + WallpaperBackup +endif + +PRODUCT_PACKAGES += \ + libEGL_angle \ + libGLESv1_CM_angle \ + libGLESv2_angle + # For testing purposes ifeq ($(FORCE_AUDIO_SILENT), true) PRODUCT_SYSTEM_PROPERTIES += ro.audio.silent=1 @@ -328,6 +389,7 @@ endif PRODUCT_HOST_PACKAGES += \ BugReport \ adb \ + adevice \ art-tools \ atest \ bcc \ @@ -343,7 +405,6 @@ PRODUCT_HOST_PACKAGES += \ incident_report \ ld.mc \ lpdump \ - minigzip \ mke2fs \ mkfs.erofs \ resize2fs \ @@ -354,20 +415,17 @@ PRODUCT_HOST_PACKAGES += \ unwind_info \ unwind_reg_info \ unwind_symbols \ - viewcompiler \ tzdata_host \ tzdata_host_tzdata_apex \ tzlookup.xml_host_tzdata_apex \ tz_version_host \ tz_version_host_tzdata_apex \ +PRODUCT_PACKAGES += init.usb.rc init.usb.configfs.rc -PRODUCT_COPY_FILES += \ - system/core/rootdir/init.usb.rc:system/etc/init/hw/init.usb.rc \ - system/core/rootdir/init.usb.configfs.rc:system/etc/init/hw/init.usb.configfs.rc \ - system/core/rootdir/etc/hosts:system/etc/hosts +PRODUCT_PACKAGES += etc_hosts -PRODUCT_COPY_FILES += system/core/rootdir/init.zygote32.rc:system/etc/init/hw/init.zygote32.rc +PRODUCT_PACKAGES += init.zygote32.rc PRODUCT_VENDOR_PROPERTIES += ro.zygote?=zygote32 PRODUCT_SYSTEM_PROPERTIES += debug.atrace.tags.enableflags=0 @@ -376,16 +434,21 @@ PRODUCT_SYSTEM_PROPERTIES += persist.traced.enable=1 # Packages included only for eng or userdebug builds, previously debug tagged PRODUCT_PACKAGES_DEBUG := \ adb_keys \ + adevice_fingerprint \ arping \ dmuserd \ + evemu-record \ idlcli \ init-debug.rc \ iotop \ iperf3 \ iw \ + layertracegenerator \ libclang_rt.ubsan_standalone \ logpersist.start \ logtagd.rc \ + ot-cli-ftd \ + ot-ctl \ procrank \ profcollectd \ profcollectctl \ @@ -408,7 +471,11 @@ PRODUCT_PACKAGES_DEBUG := \ # The set of packages whose code can be loaded by the system server. PRODUCT_SYSTEM_SERVER_APPS += \ SettingsProvider \ + +ifeq (,$(DISABLE_WALLPAPER_BACKUP)) + PRODUCT_SYSTEM_SERVER_APPS += \ WallpaperBackup +endif PRODUCT_PACKAGES_DEBUG_JAVA_COVERAGE := \ libdumpcoverage @@ -416,9 +483,18 @@ PRODUCT_PACKAGES_DEBUG_JAVA_COVERAGE := \ PRODUCT_COPY_FILES += $(call add-to-product-copy-files-if-exists,\ frameworks/base/config/preloaded-classes:system/etc/preloaded-classes) -# Note: it is acceptable to not have a dirty-image-objects file. In that case, the special bin -# for known dirty objects in the image will be empty. -PRODUCT_COPY_FILES += $(call add-to-product-copy-files-if-exists,\ - frameworks/base/config/dirty-image-objects:system/etc/dirty-image-objects) +# Enable dirty image object binning to reduce dirty pages in the image. +PRODUCT_PACKAGES += dirty-image-objects + +# Enable go/perfetto-persistent-tracing for eng builds +ifneq (,$(filter eng, $(TARGET_BUILD_VARIANT))) + PRODUCT_PRODUCT_PROPERTIES += persist.debug.perfetto.persistent_sysui_tracing_for_bugreport=1 +endif $(call inherit-product, $(SRC_TARGET_DIR)/product/runtime_libart.mk) + +# Ensure all trunk-stable flags are available. +$(call inherit-product, $(SRC_TARGET_DIR)/product/build_variables.mk) + +# Use "image" APEXes always. +$(call inherit-product,$(SRC_TARGET_DIR)/product/updatable_apex.mk) diff --git a/target/product/base_system_ext.mk b/target/product/base_system_ext.mk index 852d7ca8f1..76f008f323 100644 --- a/target/product/base_system_ext.mk +++ b/target/product/base_system_ext.mk @@ -20,5 +20,11 @@ PRODUCT_PACKAGES += \ fs_config_files_system_ext \ group_system_ext \ passwd_system_ext \ + SatelliteClient \ selinux_policy_system_ext \ system_ext_manifest.xml \ + +# Base modules when shipping api level is less than or equal to 34 +PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34 += \ + hwservicemanager \ + android.hidl.allocator@1.0-service \ diff --git a/target/product/base_vendor.mk b/target/product/base_vendor.mk index 3c4d62ea91..ec3de75b90 100644 --- a/target/product/base_vendor.mk +++ b/target/product/base_vendor.mk @@ -46,7 +46,7 @@ PRODUCT_HOST_PACKAGES += \ # Base modules and settings for the vendor partition. PRODUCT_PACKAGES += \ - android.hardware.cas-service.example \ + com.android.hardware.cas \ boringssl_self_test_vendor \ dumpsys_vendor \ fs_config_files_nonsystem \ @@ -73,6 +73,10 @@ PRODUCT_PACKAGES += \ selinux_policy_nonsystem \ shell_and_utilities_vendor \ +# Base modules when shipping api level is less than or equal to 34 +PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34 += \ + android.hidl.memory@1.0-impl.vendor \ + # OMX not supported for 64bit_only builds # Only supported when SHIPPING_API_LEVEL is less than or equal to 33 ifneq ($(TARGET_SUPPORTS_OMX_SERVICE),false) diff --git a/target/product/build_variables.mk b/target/product/build_variables.mk new file mode 100644 index 0000000000..5fe5333f45 --- /dev/null +++ b/target/product/build_variables.mk @@ -0,0 +1,21 @@ +# +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file contains the trunk-stable flags that should be exported to all +# Android targets. + +# Use the configured release of sqlite +$(call soong_config_set, libsqlite3, release_package_libsqlite3, $(RELEASE_PACKAGE_LIBSQLITE3)) diff --git a/target/product/cfi-common.mk b/target/product/cfi-common.mk index 11c01a27b5..559963cd7f 100644 --- a/target/product/cfi-common.mk +++ b/target/product/cfi-common.mk @@ -28,6 +28,7 @@ PRODUCT_CFI_INCLUDE_PATHS := \ hardware/broadcom/wlan/bcmdhd/wpa_supplicant_8_lib \ hardware/synaptics/wlan/synadhd/wpa_supplicant_8_lib \ hardware/interfaces/nfc \ + hardware/qcom/wlan/qcwcn/wpa_supplicant_8_lib \ hardware/qcom/wlan/legacy/qcwcn/wpa_supplicant_8_lib \ hardware/qcom/wlan/wcn6740/qcwcn/wpa_supplicant_8_lib \ hardware/interfaces/keymaster \ diff --git a/target/product/core_64_bit.mk b/target/product/core_64_bit.mk index e0c4d53428..790f57b634 100644 --- a/target/product/core_64_bit.mk +++ b/target/product/core_64_bit.mk @@ -23,9 +23,7 @@ # for 32-bit only. # Copy the 64-bit primary, 32-bit secondary zygote startup script -PRODUCT_COPY_FILES += \ - system/core/rootdir/init.zygote64.rc:system/etc/init/hw/init.zygote64.rc \ - system/core/rootdir/init.zygote64_32.rc:system/etc/init/hw/init.zygote64_32.rc \ +PRODUCT_PACKAGES += init.zygote64.rc init.zygote64_32.rc # Set the zygote property to select the 64-bit primary, 32-bit secondary script # This line must be parsed before the one in core_minimal.mk diff --git a/target/product/core_64_bit_only.mk b/target/product/core_64_bit_only.mk index fc2b8e5498..ffa5567a47 100644 --- a/target/product/core_64_bit_only.mk +++ b/target/product/core_64_bit_only.mk @@ -20,7 +20,7 @@ # to core_minimal.mk. # Copy the 64-bit zygote startup script -PRODUCT_COPY_FILES += system/core/rootdir/init.zygote64.rc:system/etc/init/hw/init.zygote64.rc +PRODUCT_PACKAGES += init.zygote64.rc # Set the zygote property to select the 64-bit script. # This line must be parsed before the one in core_minimal.mk diff --git a/target/product/default_art_config.mk b/target/product/default_art_config.mk index 752b199030..4a968d77c8 100644 --- a/target/product/default_art_config.mk +++ b/target/product/default_art_config.mk @@ -38,16 +38,27 @@ endif PRODUCT_BOOT_JARS := \ $(ART_APEX_JARS) +# List of jars to be included in the ART boot image for testing. +# DO NOT reorder this list. The order must match the one described above. +# Note: We use the host variant of "core-icu4j" and "conscrypt" for testing. +PRODUCT_TEST_ONLY_ART_BOOT_IMAGE_JARS := \ + $(ART_APEX_JARS) \ + platform:core-icu4j-host \ + platform:conscrypt-host \ + # /system and /system_ext boot jars. PRODUCT_BOOT_JARS += \ framework-minus-apex \ framework-graphics \ + framework-location \ ext \ telephony-common \ voip-common \ ims-common # APEX boot jars. Keep the list sorted by module names and then library names. +# Note: If the existing apex introduces the new jar, also add it to +# PRODUCT_APEX_BOOT_JARS_FOR_SOURCE_BUILD_ONLY below. # Note: core-icu4j is moved back to PRODUCT_BOOT_JARS in product_config.mk at a later stage. # Note: For modules available in Q, DO NOT add new entries here. PRODUCT_APEX_BOOT_JARS := \ @@ -55,11 +66,16 @@ PRODUCT_APEX_BOOT_JARS := \ com.android.adservices:framework-sdksandbox \ com.android.appsearch:framework-appsearch \ com.android.btservices:framework-bluetooth \ + com.android.configinfrastructure:framework-configinfrastructure \ com.android.conscrypt:conscrypt \ + com.android.devicelock:framework-devicelock \ + com.android.healthfitness:framework-healthfitness \ com.android.i18n:core-icu4j \ com.android.ipsec:android.net.ipsec.ike \ com.android.media:updatable-media \ com.android.mediaprovider:framework-mediaprovider \ + com.android.mediaprovider:framework-pdf \ + com.android.mediaprovider:framework-pdf-v \ com.android.ondevicepersonalization:framework-ondevicepersonalization \ com.android.os.statsd:framework-statsd \ com.android.permission:framework-permission \ @@ -73,6 +89,22 @@ PRODUCT_APEX_BOOT_JARS := \ com.android.virt:framework-virtualization \ com.android.wifi:framework-wifi \ +# When we release crashrecovery module +ifeq ($(RELEASE_CRASHRECOVERY_MODULE),true) + PRODUCT_APEX_BOOT_JARS += \ + com.android.crashrecovery:framework-crashrecovery \ + +endif + +# Check if the build supports NFC apex or not +ifeq ($(RELEASE_PACKAGE_NFC_STACK),NfcNci) + PRODUCT_BOOT_JARS += \ + framework-nfc +else + PRODUCT_APEX_BOOT_JARS := \ + com.android.nfcservices:framework-nfc +endif + # List of system_server classpath jars delivered via apex. # Keep the list sorted by module names and then library names. # Note: For modules available in Q, DO NOT add new entries here. @@ -81,10 +113,24 @@ PRODUCT_APEX_SYSTEM_SERVER_JARS := \ com.android.adservices:service-sdksandbox \ com.android.appsearch:service-appsearch \ com.android.art:service-art \ + com.android.configinfrastructure:service-configinfrastructure \ + com.android.healthfitness:service-healthfitness \ com.android.media:service-media-s \ + com.android.ondevicepersonalization:service-ondevicepersonalization \ com.android.permission:service-permission \ com.android.rkpd:service-rkp \ +# When we release crashrecovery module +ifeq ($(RELEASE_CRASHRECOVERY_MODULE),true) + PRODUCT_APEX_SYSTEM_SERVER_JARS += \ + com.android.crashrecovery:service-crashrecovery \ + +endif + +ifeq ($(RELEASE_AVF_ENABLE_LLPVM_CHANGES),true) + PRODUCT_APEX_SYSTEM_SERVER_JARS += com.android.virt:service-virtualization +endif + # Use $(wildcard) to avoid referencing the profile in thin manifests that don't have the # art project. ifneq (,$(wildcard art)) @@ -100,12 +146,20 @@ PRODUCT_STANDALONE_SYSTEM_SERVER_JARS := \ # Note: For modules available in Q, DO NOT add new entries here. PRODUCT_APEX_STANDALONE_SYSTEM_SERVER_JARS := \ com.android.btservices:service-bluetooth \ + com.android.devicelock:service-devicelock \ com.android.os.statsd:service-statsd \ com.android.scheduling:service-scheduling \ com.android.tethering:service-connectivity \ com.android.uwb:service-uwb \ com.android.wifi:service-wifi \ +# Overrides the (apex, jar) pairs above when determining the on-device location. The format is: +# <old_apex>:<old_jar>:<new_apex>:<new_jar> +PRODUCT_CONFIGURED_JAR_LOCATION_OVERRIDES := \ + platform:framework-minus-apex:platform:framework \ + platform:core-icu4j-host:com.android.i18n:core-icu4j \ + platform:conscrypt-host:com.android.conscrypt:conscrypt \ + # Minimal configuration for running dex2oat (default argument values). # PRODUCT_USES_DEFAULT_ART_CONFIG must be true to enable boot image compilation. PRODUCT_USES_DEFAULT_ART_CONFIG := true @@ -115,4 +169,4 @@ PRODUCT_SYSTEM_PROPERTIES += \ dalvik.vm.dex2oat-Xms=64m \ dalvik.vm.dex2oat-Xmx=512m \ -PRODUCT_ENABLE_UFFD_GC := false # TODO(jiakaiz): Change this to "default". +PRODUCT_ENABLE_UFFD_GC := default diff --git a/target/product/emulator_system.mk b/target/product/emulator_system.mk deleted file mode 100644 index b7e7cfa19b..0000000000 --- a/target/product/emulator_system.mk +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (C) 2019 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This file lists emulator experimental modules added to PRODUCT_PACKAGES, -# only included by targets sdk_phone_x86/64 and sdk_gphone_x86/64 - -PRODUCT_ARTIFACT_PATH_REQUIREMENT_ALLOWED_LIST := \ - system/lib/libemulator_multidisplay_jni.so \ - system/lib64/libemulator_multidisplay_jni.so \ - system/priv-app/MultiDisplayProvider/MultiDisplayProvider.apk \ - -PRODUCT_PACKAGES += MultiDisplayProvider diff --git a/target/product/emulator_vendor.mk b/target/product/emulator_vendor.mk deleted file mode 100644 index f71b275b0e..0000000000 --- a/target/product/emulator_vendor.mk +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (C) 2012 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# This file is included by other product makefiles to add all the -# emulator-related modules to PRODUCT_PACKAGES. -# - -$(call inherit-product, $(SRC_TARGET_DIR)/product/handheld_vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/telephony_vendor.mk) - -# need this for gles libraries to load properly -# after moving to /vendor/lib/ -PRODUCT_PACKAGES += \ - vndk-sp - -DEVICE_PACKAGE_OVERLAYS := device/generic/goldfish/overlay - -PRODUCT_CHARACTERISTICS := emulator - -PRODUCT_FULL_TREBLE_OVERRIDE := true - -# goldfish vendor partition configurations -$(call inherit-product-if-exists, device/generic/goldfish/vendor.mk) - -#watchdog tiggers reboot because location service is not -#responding, disble it for now. -#still keep it on internal main (master) as it is still working -#once it is fixed in aosp, remove this block of comment. -#PRODUCT_VENDOR_PROPERTIES += \ -#config.disable_location=true - -# enable Google-specific location features, -# like NetworkLocationProvider and LocationCollector -PRODUCT_SYSTEM_EXT_PROPERTIES += \ - ro.com.google.locationfeatures=1 - -# disable setupwizard -PRODUCT_SYSTEM_EXT_PROPERTIES += \ - ro.setupwizard.mode?=DISABLED diff --git a/target/product/full.manifest.xml b/target/product/full.manifest.xml new file mode 100644 index 0000000000..b8b0d37875 --- /dev/null +++ b/target/product/full.manifest.xml @@ -0,0 +1,2 @@ +<manifest version="1.0" type="device" target-level="7"> +</manifest> diff --git a/target/product/full.mk b/target/product/full.mk index 945957ff81..da04f49a19 100644 --- a/target/product/full.mk +++ b/target/product/full.mk @@ -20,10 +20,11 @@ # entirely appropriate to inherit from for on-device configurations. $(call inherit-product-if-exists, build/make/target/product/ramdisk_stub.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_base_telephony.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic/device.mk) +DEVICE_MANIFEST_FILE += build/make/target/product/full.manifest.xml + # Enable dynamic partition size PRODUCT_USE_DYNAMIC_PARTITION_SIZE := true diff --git a/target/product/full_x86.mk b/target/product/full_x86.mk index 0f3be91764..07f6472844 100644 --- a/target/product/full_x86.mk +++ b/target/product/full_x86.mk @@ -23,10 +23,11 @@ # that isn't a wifi connection. This will instruct init.rc to enable the # network connection so that you can use it with ADB -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_base_telephony.mk) $(call inherit-product, $(SRC_TARGET_DIR)/board/generic_x86/device.mk) +DEVICE_MANIFEST_FILE += build/make/target/product/full.manifest.xml + ifdef NET_ETH0_STARTONBOOT PRODUCT_VENDOR_PROPERTIES += net.eth0.startonboot=1 endif diff --git a/target/product/fullmte.mk b/target/product/fullmte.mk new file mode 100644 index 0000000000..b62249601e --- /dev/null +++ b/target/product/fullmte.mk @@ -0,0 +1,27 @@ +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Enables more comprehensive detection of memory errors on hardware that +# supports the ARM Memory Tagging Extension (MTE), by building the image with +# MTE stack instrumentation and forcing MTE on in SYNC mode in all processes. +# For more details, see: +# https://source.android.com/docs/security/test/memory-safety/arm-mte +ifeq ($(filter memtag_heap,$(SANITIZE_TARGET)),) + SANITIZE_TARGET := $(strip $(SANITIZE_TARGET) memtag_heap memtag_stack) + SANITIZE_TARGET_DIAG := $(strip $(SANITIZE_TARGET_DIAG) memtag_heap) +endif +PRODUCT_PRODUCT_PROPERTIES += persist.arm64.memtag.default=sync +PRODUCT_SCUDO_ALLOCATION_RING_BUFFER_SIZE := 131072 diff --git a/target/product/generic.mk b/target/product/generic.mk index fb5b727ba1..fd3b3fb0e7 100644 --- a/target/product/generic.mk +++ b/target/product/generic.mk @@ -14,9 +14,6 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. - # This is a generic phone product that isn't specialized for a specific device. # It includes the base Android platform. diff --git a/target/product/generic_system.mk b/target/product/generic_system.mk index 1a639ef717..9748c7cd46 100644 --- a/target/product/generic_system.mk +++ b/target/product/generic_system.mk @@ -32,9 +32,15 @@ PRODUCT_PACKAGES += \ PRODUCT_PACKAGES += \ LiveWallpapersPicker \ PartnerBookmarksProvider \ + preinstalled-packages-platform-generic-system.xml \ Stk \ Tag \ +ifeq ($(RELEASE_AVATAR_PICKER_APP),true) + PRODUCT_PACKAGES += \ + AvatarPicker +endif + # OTA support PRODUCT_PACKAGES += \ recovery-refresh \ @@ -67,7 +73,6 @@ PRODUCT_PACKAGES += \ android.hardware.radio.config@1.0 \ android.hardware.radio.deprecated@1.0 \ android.hardware.secure_element@1.0 \ - android.hardware.wifi@1.0 \ libaudio-resampler \ libaudiohal \ libdrm \ @@ -102,11 +107,18 @@ PRODUCT_PACKAGES += \ libaudiopolicyengineconfigurable \ libpolicy-subsystem +# Add all of the packages used to support older/upgrading devices +# These can be removed as we drop support for the older API levels +PRODUCT_PACKAGES += \ + $(PRODUCT_PACKAGES_SHIPPING_API_LEVEL_29) \ + $(PRODUCT_PACKAGES_SHIPPING_API_LEVEL_33) \ + $(PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34) + # Include all zygote init scripts. "ro.zygote" will select one of them. -PRODUCT_COPY_FILES += \ - system/core/rootdir/init.zygote32.rc:system/etc/init/hw/init.zygote32.rc \ - system/core/rootdir/init.zygote64.rc:system/etc/init/hw/init.zygote64.rc \ - system/core/rootdir/init.zygote64_32.rc:system/etc/init/hw/init.zygote64_32.rc \ +PRODUCT_PACKAGES += \ + init.zygote32.rc \ + init.zygote64.rc \ + init.zygote64_32.rc # Enable dynamic partition size PRODUCT_USE_DYNAMIC_PARTITION_SIZE := true diff --git a/target/product/go_defaults.mk b/target/product/go_defaults.mk index b7174868ee..a10cfa858f 100644 --- a/target/product/go_defaults.mk +++ b/target/product/go_defaults.mk @@ -17,6 +17,8 @@ # Inherit common Android Go defaults. $(call inherit-product, build/make/target/product/go_defaults_common.mk) +PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google/release/go_devices/release_config_map.mk) + # Add the system properties. TARGET_SYSTEM_PROP += \ build/make/target/board/go_defaults.prop diff --git a/target/product/go_defaults_common.mk b/target/product/go_defaults_common.mk index 51a1ef6ec4..5218f29c0a 100644 --- a/target/product/go_defaults_common.mk +++ b/target/product/go_defaults_common.mk @@ -24,10 +24,6 @@ PRODUCT_VENDOR_PROPERTIES += \ # Speed profile services and wifi-service to reduce RAM and storage. PRODUCT_SYSTEM_SERVER_COMPILER_FILTER := speed-profile -# Always preopt extracted APKs to prevent extracting out of the APK for gms -# modules. -PRODUCT_ALWAYS_PREOPT_EXTRACTED_APK := true - # Use a profile based boot image for this device. Note that this is currently a # generic profile and not Android Go optimized. PRODUCT_USE_PROFILE_FOR_BOOT_IMAGE := true @@ -53,6 +49,3 @@ TARGET_SYSTEM_PROP += \ # use the go specific handheld_core_hardware.xml from frameworks PRODUCT_COPY_FILES += \ frameworks/native/data/etc/go_handheld_core_hardware.xml:$(TARGET_COPY_OUT_VENDOR)/etc/permissions/handheld_core_hardware.xml - -# Dedupe VNDK libraries with identical core variants. -TARGET_VNDK_USE_CORE_VARIANT := true diff --git a/target/product/gsi/33.txt b/target/product/gsi/33.txt index db05f93a36..03a143d02c 100644 --- a/target/product/gsi/33.txt +++ b/target/product/gsi/33.txt @@ -79,6 +79,7 @@ VNDK-core: android.hardware.graphics.allocator@3.0.so VNDK-core: android.hardware.graphics.allocator@4.0.so VNDK-core: android.hardware.graphics.bufferqueue@1.0.so VNDK-core: android.hardware.graphics.bufferqueue@2.0.so +VNDK-core: android.hardware.health-V1-ndk.so VNDK-core: android.hardware.health.storage-V1-ndk.so VNDK-core: android.hardware.identity-V4-ndk.so VNDK-core: android.hardware.ir-V1-ndk.so diff --git a/target/product/gsi/34.txt b/target/product/gsi/34.txt new file mode 100644 index 0000000000..ceb2060c19 --- /dev/null +++ b/target/product/gsi/34.txt @@ -0,0 +1,210 @@ +LLNDK: libEGL.so +LLNDK: libGLESv1_CM.so +LLNDK: libGLESv2.so +LLNDK: libGLESv3.so +LLNDK: libRS.so +LLNDK: libandroid_net.so +LLNDK: libbinder_ndk.so +LLNDK: libc.so +LLNDK: libcgrouprc.so +LLNDK: libcom.android.tethering.connectivity_native.so +LLNDK: libdl.so +LLNDK: libft2.so +LLNDK: liblog.so +LLNDK: libm.so +LLNDK: libmediandk.so +LLNDK: libnativewindow.so +LLNDK: libneuralnetworks.so +LLNDK: libselinux.so +LLNDK: libsync.so +LLNDK: libvndksupport.so +LLNDK: libvulkan.so +VNDK-SP: android.hardware.common-V2-ndk.so +VNDK-SP: android.hardware.common.fmq-V1-ndk.so +VNDK-SP: android.hardware.graphics.common-V4-ndk.so +VNDK-SP: android.hardware.graphics.common@1.0.so +VNDK-SP: android.hardware.graphics.common@1.1.so +VNDK-SP: android.hardware.graphics.common@1.2.so +VNDK-SP: android.hardware.graphics.composer3-V1-ndk.so +VNDK-SP: android.hardware.graphics.mapper@2.0.so +VNDK-SP: android.hardware.graphics.mapper@2.1.so +VNDK-SP: android.hardware.graphics.mapper@3.0.so +VNDK-SP: android.hardware.graphics.mapper@4.0.so +VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so +VNDK-SP: android.hardware.renderscript@1.0.so +VNDK-SP: android.hidl.memory.token@1.0.so +VNDK-SP: android.hidl.memory@1.0-impl.so +VNDK-SP: android.hidl.memory@1.0.so +VNDK-SP: android.hidl.safe_union@1.0.so +VNDK-SP: libRSCpuRef.so +VNDK-SP: libRSDriver.so +VNDK-SP: libRS_internal.so +VNDK-SP: libbase.so +VNDK-SP: libbcinfo.so +VNDK-SP: libblas.so +VNDK-SP: libc++.so +VNDK-SP: libcompiler_rt.so +VNDK-SP: libcutils.so +VNDK-SP: libdmabufheap.so +VNDK-SP: libgralloctypes.so +VNDK-SP: libhardware.so +VNDK-SP: libhidlbase.so +VNDK-SP: libhidlmemory.so +VNDK-SP: libion.so +VNDK-SP: libjsoncpp.so +VNDK-SP: liblzma.so +VNDK-SP: libprocessgroup.so +VNDK-SP: libunwindstack.so +VNDK-SP: libutils.so +VNDK-SP: libutilscallstack.so +VNDK-SP: libz.so +VNDK-core: android.frameworks.cameraservice.common-V1-ndk.so +VNDK-core: android.frameworks.cameraservice.device-V1-ndk.so +VNDK-core: android.frameworks.cameraservice.service-V1-ndk.so +VNDK-core: android.hardware.audio.common@2.0.so +VNDK-core: android.hardware.configstore-utils.so +VNDK-core: android.hardware.configstore@1.0.so +VNDK-core: android.hardware.configstore@1.1.so +VNDK-core: android.hardware.confirmationui-support-lib.so +VNDK-core: android.hardware.graphics.allocator@2.0.so +VNDK-core: android.hardware.graphics.allocator@3.0.so +VNDK-core: android.hardware.graphics.allocator@4.0.so +VNDK-core: android.hardware.graphics.bufferqueue@1.0.so +VNDK-core: android.hardware.graphics.bufferqueue@2.0.so +VNDK-core: android.hardware.media.bufferpool@2.0.so +VNDK-core: android.hardware.media.omx@1.0.so +VNDK-core: android.hardware.media@1.0.so +VNDK-core: android.hardware.memtrack-V1-ndk.so +VNDK-core: android.hardware.memtrack@1.0.so +VNDK-core: android.hardware.soundtrigger@2.0-core.so +VNDK-core: android.hardware.soundtrigger@2.0.so +VNDK-core: android.hidl.token@1.0-utils.so +VNDK-core: android.hidl.token@1.0.so +VNDK-core: android.system.suspend-V1-ndk.so +VNDK-core: android.system.suspend@1.0.so +VNDK-core: libaudioroute.so +VNDK-core: libaudioutils.so +VNDK-core: libbinder.so +VNDK-core: libbufferqueueconverter.so +VNDK-core: libcamera_metadata.so +VNDK-core: libcap.so +VNDK-core: libcn-cbor.so +VNDK-core: libcodec2.so +VNDK-core: libcrypto.so +VNDK-core: libcrypto_utils.so +VNDK-core: libcurl.so +VNDK-core: libdiskconfig.so +VNDK-core: libdumpstateutil.so +VNDK-core: libevent.so +VNDK-core: libexif.so +VNDK-core: libexpat.so +VNDK-core: libfmq.so +VNDK-core: libgatekeeper.so +VNDK-core: libgui.so +VNDK-core: libhardware_legacy.so +VNDK-core: libhidlallocatorutils.so +VNDK-core: libjpeg.so +VNDK-core: libldacBT_abr.so +VNDK-core: libldacBT_enc.so +VNDK-core: liblz4.so +VNDK-core: libmedia_helper.so +VNDK-core: libmedia_omx.so +VNDK-core: libmemtrack.so +VNDK-core: libminijail.so +VNDK-core: libmkbootimg_abi_check.so +VNDK-core: libnetutils.so +VNDK-core: libnl.so +VNDK-core: libpcre2.so +VNDK-core: libpiex.so +VNDK-core: libpng.so +VNDK-core: libpower.so +VNDK-core: libprocinfo.so +VNDK-core: libradio_metadata.so +VNDK-core: libspeexresampler.so +VNDK-core: libsqlite.so +VNDK-core: libssl.so +VNDK-core: libstagefright_bufferpool@2.0.so +VNDK-core: libstagefright_bufferqueue_helper.so +VNDK-core: libstagefright_foundation.so +VNDK-core: libstagefright_omx.so +VNDK-core: libstagefright_omx_utils.so +VNDK-core: libstagefright_xmlparser.so +VNDK-core: libsysutils.so +VNDK-core: libtinyalsa.so +VNDK-core: libtinyxml2.so +VNDK-core: libui.so +VNDK-core: libusbhost.so +VNDK-core: libwifi-system-iface.so +VNDK-core: libxml2.so +VNDK-core: libyuv.so +VNDK-core: libziparchive.so +VNDK-private: libblas.so +VNDK-private: libcompiler_rt.so +VNDK-private: libft2.so +VNDK-private: libgui.so +VNDK-product: android.hardware.audio.common@2.0.so +VNDK-product: android.hardware.configstore@1.0.so +VNDK-product: android.hardware.configstore@1.1.so +VNDK-product: android.hardware.graphics.allocator@2.0.so +VNDK-product: android.hardware.graphics.allocator@3.0.so +VNDK-product: android.hardware.graphics.allocator@4.0.so +VNDK-product: android.hardware.graphics.bufferqueue@1.0.so +VNDK-product: android.hardware.graphics.bufferqueue@2.0.so +VNDK-product: android.hardware.graphics.common@1.0.so +VNDK-product: android.hardware.graphics.common@1.1.so +VNDK-product: android.hardware.graphics.common@1.2.so +VNDK-product: android.hardware.graphics.mapper@2.0.so +VNDK-product: android.hardware.graphics.mapper@2.1.so +VNDK-product: android.hardware.graphics.mapper@3.0.so +VNDK-product: android.hardware.graphics.mapper@4.0.so +VNDK-product: android.hardware.media.bufferpool@2.0.so +VNDK-product: android.hardware.media.omx@1.0.so +VNDK-product: android.hardware.media@1.0.so +VNDK-product: android.hardware.memtrack@1.0.so +VNDK-product: android.hardware.renderscript@1.0.so +VNDK-product: android.hardware.soundtrigger@2.0.so +VNDK-product: android.hidl.memory.token@1.0.so +VNDK-product: android.hidl.memory@1.0.so +VNDK-product: android.hidl.safe_union@1.0.so +VNDK-product: android.hidl.token@1.0.so +VNDK-product: android.system.suspend@1.0.so +VNDK-product: libaudioutils.so +VNDK-product: libbase.so +VNDK-product: libc++.so +VNDK-product: libcamera_metadata.so +VNDK-product: libcap.so +VNDK-product: libcompiler_rt.so +VNDK-product: libcrypto.so +VNDK-product: libcurl.so +VNDK-product: libcutils.so +VNDK-product: libevent.so +VNDK-product: libexpat.so +VNDK-product: libfmq.so +VNDK-product: libhidlbase.so +VNDK-product: libhidlmemory.so +VNDK-product: libion.so +VNDK-product: libjpeg.so +VNDK-product: libjsoncpp.so +VNDK-product: libldacBT_abr.so +VNDK-product: libldacBT_enc.so +VNDK-product: liblz4.so +VNDK-product: liblzma.so +VNDK-product: libminijail.so +VNDK-product: libnl.so +VNDK-product: libpcre2.so +VNDK-product: libpiex.so +VNDK-product: libpng.so +VNDK-product: libprocessgroup.so +VNDK-product: libprocinfo.so +VNDK-product: libspeexresampler.so +VNDK-product: libssl.so +VNDK-product: libtinyalsa.so +VNDK-product: libtinyxml2.so +VNDK-product: libunwindstack.so +VNDK-product: libutils.so +VNDK-product: libutilscallstack.so +VNDK-product: libwifi-system-iface.so +VNDK-product: libxml2.so +VNDK-product: libyuv.so +VNDK-product: libz.so +VNDK-product: libziparchive.so diff --git a/target/product/gsi/Android.bp b/target/product/gsi/Android.bp index a8af9c4eca..45ba14331b 100644 --- a/target/product/gsi/Android.bp +++ b/target/product/gsi/Android.bp @@ -23,3 +23,26 @@ filegroup { "*.txt", ], } + +prebuilt_etc { + name: "gsi_skip_mount.cfg", + filename: "skip_mount.cfg", + src: "gsi_skip_mount.cfg", + + system_ext_specific: true, + relative_install_path: "init/config", + + required: ["gsi_skip_mount_compat_symlink"], +} + +// Adds a symlink under /system/etc/init/config pointing to /system/system_ext/etc/init/config +// because first-stage init in Android 10.0 will read the skip_mount.cfg from /system/etc/* after +// chroot /system. +// TODO: remove this symlink when no need to support new GSI on Android 10. +// The actual file needs to be under /system/system_ext because it's GSI-specific and does not +// belong to core CSI. +install_symlink { + name: "gsi_skip_mount_compat_symlink", + installed_location: "etc/init/config", + symlink_target: "/system/system_ext/etc/init/config", +} diff --git a/target/product/gsi/Android.mk b/target/product/gsi/Android.mk index 107c94f685..36897fef8e 100644 --- a/target/product/gsi/Android.mk +++ b/target/product/gsi/Android.mk @@ -1,113 +1,60 @@ LOCAL_PATH:= $(call my-dir) ##################################################################### -# list of vndk libraries from the source code. -INTERNAL_VNDK_LIB_LIST := $(SOONG_VNDK_LIBRARIES_FILE) - -##################################################################### -# This is the up-to-date list of vndk libs. -# TODO(b/62012285): the lib list should be stored somewhere under -# /prebuilts/vndk -ifeq (REL,$(PLATFORM_VERSION_CODENAME)) -LATEST_VNDK_LIB_LIST := $(LOCAL_PATH)/$(PLATFORM_VNDK_VERSION).txt -ifeq ($(wildcard $(LATEST_VNDK_LIB_LIST)),) -$(error $(LATEST_VNDK_LIB_LIST) file not found. Please copy "$(LOCAL_PATH)/current.txt" to "$(LATEST_VNDK_LIB_LIST)" and commit a CL for release branch) -endif -else -LATEST_VNDK_LIB_LIST := $(LOCAL_PATH)/current.txt -endif - -##################################################################### # Check the generate list against the latest list stored in the # source tree -.PHONY: check-vndk-list +.PHONY: check-abi-dump-list # Check if vndk list is changed -droidcore: check-vndk-list - -check-vndk-list-timestamp := $(call intermediates-dir-for,PACKAGING,vndk)/check-list-timestamp -check-vndk-abi-dump-list-timestamp := $(call intermediates-dir-for,PACKAGING,vndk)/check-abi-dump-list-timestamp - -ifeq ($(TARGET_IS_64_BIT)|$(TARGET_2ND_ARCH),true|) -# TODO(b/110429754) remove this condition when we support 64-bit-only device -check-vndk-list: ; -else ifeq ($(TARGET_SKIP_CURRENT_VNDK),true) -check-vndk-list: ; -else ifeq ($(BOARD_VNDK_VERSION),) -# b/143233626 do not check vndk-list when vndk libs are not built -check-vndk-list: ; -else -check-vndk-list: $(check-vndk-list-timestamp) -ifneq ($(SKIP_ABI_CHECKS),true) -check-vndk-list: $(check-vndk-abi-dump-list-timestamp) +droidcore: check-abi-dump-list + +check-abi-dump-list-timestamp := $(call intermediates-dir-for,PACKAGING,vndk)/check-abi-dump-list-timestamp + +# The ABI tool does not support sanitizer and coverage builds. +ifeq (,$(filter true,$(SKIP_ABI_CHECKS) $(CLANG_COVERAGE))) +ifeq (,$(SANITIZE_TARGET)) +check-abi-dump-list: $(check-abi-dump-list-timestamp) endif endif -_vndk_check_failure_message := " error: VNDK library list has been changed.\n" -ifeq (REL,$(PLATFORM_VERSION_CODENAME)) -_vndk_check_failure_message += " Changing the VNDK library list is not allowed in API locked branches." -else -_vndk_check_failure_message += " Run \`update-vndk-list.sh\` to update $(LATEST_VNDK_LIB_LIST)" -endif +##################################################################### +# ABI reference dumps. -# The *-ndk_platform.so libraries no longer exist and are removed from the VNDK set. However, they -# can exist if NEED_AIDL_NDK_PLATFORM_BACKEND is set to true for legacy devices. Don't be bothered -# with the extraneous libraries. -ifeq ($(NEED_AIDL_NDK_PLATFORM_BACKEND),true) - _READ_INTERNAL_VNDK_LIB_LIST := sed /ndk_platform.so/d $(INTERNAL_VNDK_LIB_LIST) -else - _READ_INTERNAL_VNDK_LIB_LIST := cat $(INTERNAL_VNDK_LIB_LIST) -endif +# LSDUMP_PATHS is a list of tag:path. They are written to LSDUMP_PATHS_FILE. +LSDUMP_PATHS_FILE := $(PRODUCT_OUT)/lsdump_paths.txt -$(check-vndk-list-timestamp): $(INTERNAL_VNDK_LIB_LIST) $(LATEST_VNDK_LIB_LIST) $(HOST_OUT_EXECUTABLES)/update-vndk-list.sh - $(hide) ($(_READ_INTERNAL_VNDK_LIB_LIST) | sort | \ - diff --old-line-format="Removed %L" \ - --new-line-format="Added %L" \ - --unchanged-line-format="" \ - <(cat $(LATEST_VNDK_LIB_LIST) | sort) - \ - || ( echo -e $(_vndk_check_failure_message); exit 1 )) - $(hide) mkdir -p $(dir $@) - $(hide) touch $@ +$(LSDUMP_PATHS_FILE): PRIVATE_LSDUMP_PATHS := $(LSDUMP_PATHS) +$(LSDUMP_PATHS_FILE): + @echo "Generate $@" + @rm -rf $@ && echo -e "$(subst :,:$(space),$(subst $(space),\n,$(PRIVATE_LSDUMP_PATHS)))" > $@ -##################################################################### -# Script to update the latest VNDK lib list -include $(CLEAR_VARS) -LOCAL_MODULE := update-vndk-list.sh -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := EXECUTABLES -LOCAL_MODULE_STEM := $(LOCAL_MODULE) -LOCAL_IS_HOST_MODULE := true -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_INTERNAL_VNDK_LIB_LIST := $(INTERNAL_VNDK_LIB_LIST) -$(LOCAL_BUILT_MODULE): PRIVATE_LATEST_VNDK_LIB_LIST := $(LATEST_VNDK_LIB_LIST) -$(LOCAL_BUILT_MODULE): - @echo "Generate: $@" - @mkdir -p $(dir $@) - @rm -f $@ - $(hide) echo "#!/bin/bash" > $@ -ifeq (REL,$(PLATFORM_VERSION_CODENAME)) - $(hide) echo "echo Updating VNDK library list is NOT allowed in API locked branches." >> $@; \ - echo "exit 1" >> $@ -else - $(hide) echo "if [ -z \"\$${ANDROID_BUILD_TOP}\" ]; then" >> $@; \ - echo " echo Run lunch or choosecombo first" >> $@; \ - echo " exit 1" >> $@; \ - echo "fi" >> $@; \ - echo "cd \$${ANDROID_BUILD_TOP}" >> $@ -ifeq ($(NEED_AIDL_NDK_PLATFORM_BACKEND),true) - $(hide) echo "sed /ndk_platform.so/d $(PRIVATE_INTERNAL_VNDK_LIB_LIST) > $(PRIVATE_LATEST_VNDK_LIB_LIST)" >> $@ -else - $(hide) echo "cp $(PRIVATE_INTERNAL_VNDK_LIB_LIST) $(PRIVATE_LATEST_VNDK_LIB_LIST)" >> $@ -endif - $(hide) echo "echo $(PRIVATE_LATEST_VNDK_LIB_LIST) updated." >> $@ -endif - @chmod a+x $@ +# $(1): A list of tags. +# $(2): A list of tag:path. +# Return the file paths of the ABI dumps that match the tags. +define filter-abi-dump-paths +$(eval tag_patterns := $(addsuffix :%,$(1))) +$(patsubst $(tag_patterns),%,$(filter $(tag_patterns),$(2))) +endef + +# Subsets of LSDUMP_PATHS. +.PHONY: findlsdumps_APEX +findlsdumps_APEX: $(LSDUMP_PATHS_FILE) $(call filter-abi-dump-paths,APEX,$(LSDUMP_PATHS)) + +.PHONY: findlsdumps_LLNDK +findlsdumps_LLNDK: $(LSDUMP_PATHS_FILE) $(call filter-abi-dump-paths,LLNDK,$(LSDUMP_PATHS)) + +.PHONY: findlsdumps_NDK +findlsdumps_NDK: $(LSDUMP_PATHS_FILE) $(call filter-abi-dump-paths,NDK,$(LSDUMP_PATHS)) + +.PHONY: findlsdumps_PLATFORM +findlsdumps_PLATFORM: $(LSDUMP_PATHS_FILE) $(call filter-abi-dump-paths,PLATFORM,$(LSDUMP_PATHS)) + +.PHONY: findlsdumps +findlsdumps: $(LSDUMP_PATHS_FILE) $(foreach p,$(LSDUMP_PATHS),$(call word-colon,2,$(p))) ##################################################################### # Check that all ABI reference dumps have corresponding -# NDK/VNDK/PLATFORM libraries. +# APEX/LLNDK/PLATFORM libraries. # $(1): The directory containing ABI dumps. # Return a list of ABI dump paths ending with .so.lsdump. @@ -119,128 +66,78 @@ endef # $(1): A list of tags. # $(2): A list of tag:path. -# Return the file names of the ABI dumps that match the tags. -define filter-abi-dump-paths -$(eval tag_patterns := $(foreach tag,$(1),$(tag):%)) -$(notdir $(patsubst $(tag_patterns),%,$(filter $(tag_patterns),$(2)))) +# Return the file names of the ABI dumps that match the tags, and replace the +# file name extensions with .so.lsdump. +define filter-abi-dump-names +$(patsubst %.so.llndk.lsdump,%.so.lsdump, \ + $(patsubst %.so.apex.lsdump,%.so.lsdump, \ + $(notdir $(call filter-abi-dump-paths,$(1),$(2))))) endef -VNDK_ABI_DUMP_DIR := prebuilts/abi-dumps/vndk/$(PLATFORM_VNDK_VERSION) +VNDK_ABI_DUMP_DIR := prebuilts/abi-dumps/vndk/$(RELEASE_BOARD_API_LEVEL) ifeq (REL,$(PLATFORM_VERSION_CODENAME)) - NDK_ABI_DUMP_DIR := prebuilts/abi-dumps/ndk/$(PLATFORM_SDK_VERSION) PLATFORM_ABI_DUMP_DIR := prebuilts/abi-dumps/platform/$(PLATFORM_SDK_VERSION) else - NDK_ABI_DUMP_DIR := prebuilts/abi-dumps/ndk/current PLATFORM_ABI_DUMP_DIR := prebuilts/abi-dumps/platform/current endif VNDK_ABI_DUMPS := $(call find-abi-dump-paths,$(VNDK_ABI_DUMP_DIR)) -NDK_ABI_DUMPS := $(call find-abi-dump-paths,$(NDK_ABI_DUMP_DIR)) PLATFORM_ABI_DUMPS := $(call find-abi-dump-paths,$(PLATFORM_ABI_DUMP_DIR)) # Check for superfluous lsdump files. Since LSDUMP_PATHS only covers the # libraries that can be built from source in the current build, and prebuilts of # Mainline modules may be in use, we also allow the libs in STUB_LIBRARIES for -# NDK and platform ABIs. +# platform ABIs. +# In addition, libRS is allowed because it's disabled for RISC-V. -$(check-vndk-abi-dump-list-timestamp): PRIVATE_LSDUMP_PATHS := $(LSDUMP_PATHS) -$(check-vndk-abi-dump-list-timestamp): PRIVATE_STUB_LIBRARIES := $(STUB_LIBRARIES) -$(check-vndk-abi-dump-list-timestamp): +$(check-abi-dump-list-timestamp): PRIVATE_LSDUMP_PATHS := $(LSDUMP_PATHS) +$(check-abi-dump-list-timestamp): PRIVATE_STUB_LIBRARIES := $(STUB_LIBRARIES) +$(check-abi-dump-list-timestamp): $(eval added_vndk_abi_dumps := $(strip $(sort $(filter-out \ - $(call filter-abi-dump-paths,VNDK-SP VNDK-core,$(PRIVATE_LSDUMP_PATHS)), \ + $(call filter-abi-dump-names,LLNDK,$(PRIVATE_LSDUMP_PATHS)) libRS.so.lsdump, \ $(notdir $(VNDK_ABI_DUMPS)))))) $(if $(added_vndk_abi_dumps), \ echo -e "Found unexpected ABI reference dump files under $(VNDK_ABI_DUMP_DIR). It is caused by mismatch between Android.bp and the dump files. Run \`find \$${ANDROID_BUILD_TOP}/$(VNDK_ABI_DUMP_DIR) '(' -name $(subst $(space), -or -name ,$(added_vndk_abi_dumps)) ')' -delete\` to delete the dump files.") - $(eval added_ndk_abi_dumps := $(strip $(sort $(filter-out \ - $(call filter-abi-dump-paths,NDK,$(PRIVATE_LSDUMP_PATHS)) \ - $(addsuffix .lsdump,$(PRIVATE_STUB_LIBRARIES)), \ - $(notdir $(NDK_ABI_DUMPS)))))) - $(if $(added_ndk_abi_dumps), \ - echo -e "Found unexpected ABI reference dump files under $(NDK_ABI_DUMP_DIR). It is caused by mismatch between Android.bp and the dump files. Run \`find \$${ANDROID_BUILD_TOP}/$(NDK_ABI_DUMP_DIR) '(' -name $(subst $(space), -or -name ,$(added_ndk_abi_dumps)) ')' -delete\` to delete the dump files.") - + # TODO(b/314010764): Remove LLNDK tag after PLATFORM_SDK_VERSION is upgraded to 35. $(eval added_platform_abi_dumps := $(strip $(sort $(filter-out \ - $(call filter-abi-dump-paths,LLNDK PLATFORM,$(PRIVATE_LSDUMP_PATHS)) \ - $(addsuffix .lsdump,$(PRIVATE_STUB_LIBRARIES)), \ + $(call filter-abi-dump-names,APEX LLNDK PLATFORM,$(PRIVATE_LSDUMP_PATHS)) \ + $(addsuffix .lsdump,$(PRIVATE_STUB_LIBRARIES)) libRS.so.lsdump, \ $(notdir $(PLATFORM_ABI_DUMPS)))))) $(if $(added_platform_abi_dumps), \ echo -e "Found unexpected ABI reference dump files under $(PLATFORM_ABI_DUMP_DIR). It is caused by mismatch between Android.bp and the dump files. Run \`find \$${ANDROID_BUILD_TOP}/$(PLATFORM_ABI_DUMP_DIR) '(' -name $(subst $(space), -or -name ,$(added_platform_abi_dumps)) ')' -delete\` to delete the dump files.") - $(if $(added_vndk_abi_dumps)$(added_ndk_abi_dumps)$(added_platform_abi_dumps),exit 1) + $(if $(added_vndk_abi_dumps)$(added_platform_abi_dumps),exit 1) $(hide) mkdir -p $(dir $@) $(hide) touch $@ ##################################################################### # VNDK package and snapshot. -ifneq ($(BOARD_VNDK_VERSION),) - include $(CLEAR_VARS) -LOCAL_MODULE := vndk_package -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -# Filter LLNDK libs moved to APEX to avoid pulling them into /system/LIB -LOCAL_REQUIRED_MODULES := \ - $(filter-out $(LLNDK_MOVED_TO_APEX_LIBRARIES),$(LLNDK_LIBRARIES)) -ifneq ($(TARGET_SKIP_CURRENT_VNDK),true) -LOCAL_REQUIRED_MODULES += \ - vndkcorevariant.libraries.txt \ - $(addsuffix .vendor,$(VNDK_CORE_LIBRARIES)) \ - $(addsuffix .vendor,$(VNDK_SAMEPROCESS_LIBRARIES)) \ - $(VNDK_USING_CORE_VARIANT_LIBRARIES) \ - com.android.vndk.current - -LOCAL_ADDITIONAL_DEPENDENCIES += $(call module-built-files,\ - $(addsuffix .vendor,$(VNDK_CORE_LIBRARIES) $(VNDK_SAMEPROCESS_LIBRARIES))) - -endif -include $(BUILD_PHONY_PACKAGE) - -include $(CLEAR_VARS) -_vndk_versions := -ifeq ($(filter com.android.vndk.current.on_vendor, $(PRODUCT_PACKAGES)),) - _vndk_versions += $(PRODUCT_EXTRA_VNDK_VERSIONS) -endif -ifneq ($(BOARD_VNDK_VERSION),current) - _vndk_versions += $(BOARD_VNDK_VERSION) -endif LOCAL_MODULE := vndk_apex_snapshot_package LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(foreach vndk_ver,$(_vndk_versions),com.android.vndk.v$(vndk_ver)) +LOCAL_REQUIRED_MODULES := $(foreach vndk_ver,$(PRODUCT_EXTRA_VNDK_VERSIONS),com.android.vndk.v$(vndk_ver)) include $(BUILD_PHONY_PACKAGE) -_vndk_versions := - -endif # BOARD_VNDK_VERSION is set - ##################################################################### -# skip_mount.cfg, read by init to skip mounting some partitions when GSI is used. - +# Define Phony module to install LLNDK modules which are installed in +# the system image include $(CLEAR_VARS) -LOCAL_MODULE := gsi_skip_mount.cfg +LOCAL_MODULE := llndk_in_system LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_STEM := skip_mount.cfg -LOCAL_SRC_FILES := $(LOCAL_MODULE) -LOCAL_MODULE_CLASS := ETC -LOCAL_SYSTEM_EXT_MODULE := true -LOCAL_MODULE_RELATIVE_PATH := init/config - -# Adds a symlink under /system/etc/init/config pointing to /system/system_ext/etc/init/config -# because first-stage init in Android 10.0 will read the skip_mount.cfg from /system/etc/* after -# chroot /system. -# TODO: remove this symlink when no need to support new GSI on Android 10. -# The actual file needs to be under /system/system_ext because it's GSI-specific and does not -# belong to core CSI. -LOCAL_POST_INSTALL_CMD := \ - mkdir -p $(TARGET_OUT)/etc/init; \ - ln -sf /system/system_ext/etc/init/config $(TARGET_OUT)/etc/init/config -include $(BUILD_PREBUILT) +# Filter LLNDK libs moved to APEX to avoid pulling them into /system/LIB +LOCAL_REQUIRED_MODULES := \ + $(filter-out $(LLNDK_MOVED_TO_APEX_LIBRARIES),$(LLNDK_LIBRARIES)) \ + llndk.libraries.txt + + +include $(BUILD_PHONY_PACKAGE) ##################################################################### # init.gsi.rc, GSI-specific init script. diff --git a/target/product/gsi/current.txt b/target/product/gsi/current.txt index 474cb20239..f771916f7a 100644 --- a/target/product/gsi/current.txt +++ b/target/product/gsi/current.txt @@ -4,6 +4,7 @@ LLNDK: libGLESv2.so LLNDK: libGLESv3.so LLNDK: libRS.so LLNDK: libandroid_net.so +LLNDK: libapexsupport.so LLNDK: libbinder_ndk.so LLNDK: libc.so LLNDK: libcgrouprc.so @@ -17,11 +18,13 @@ LLNDK: libnativewindow.so LLNDK: libneuralnetworks.so LLNDK: libselinux.so LLNDK: libsync.so +LLNDK: libvendorsupport.so LLNDK: libvndksupport.so LLNDK: libvulkan.so VNDK-SP: android.hardware.common-V2-ndk.so VNDK-SP: android.hardware.common.fmq-V1-ndk.so -VNDK-SP: android.hardware.graphics.common-V4-ndk.so +VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so +VNDK-SP: android.hardware.graphics.common-V5-ndk.so VNDK-SP: android.hardware.graphics.common@1.0.so VNDK-SP: android.hardware.graphics.common@1.1.so VNDK-SP: android.hardware.graphics.common@1.2.so @@ -30,10 +33,8 @@ VNDK-SP: android.hardware.graphics.mapper@2.0.so VNDK-SP: android.hardware.graphics.mapper@2.1.so VNDK-SP: android.hardware.graphics.mapper@3.0.so VNDK-SP: android.hardware.graphics.mapper@4.0.so -VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so VNDK-SP: android.hardware.renderscript@1.0.so VNDK-SP: android.hidl.memory.token@1.0.so -VNDK-SP: android.hidl.memory@1.0-impl.so VNDK-SP: android.hidl.memory@1.0.so VNDK-SP: android.hidl.safe_union@1.0.so VNDK-SP: libRSCpuRef.so @@ -58,6 +59,9 @@ VNDK-SP: libunwindstack.so VNDK-SP: libutils.so VNDK-SP: libutilscallstack.so VNDK-SP: libz.so +VNDK-core: android.frameworks.cameraservice.common-V1-ndk.so +VNDK-core: android.frameworks.cameraservice.device-V2-ndk.so +VNDK-core: android.frameworks.cameraservice.service-V2-ndk.so VNDK-core: android.hardware.audio.common@2.0.so VNDK-core: android.hardware.configstore-utils.so VNDK-core: android.hardware.configstore@1.0.so @@ -90,7 +94,6 @@ VNDK-core: libcodec2.so VNDK-core: libcrypto.so VNDK-core: libcrypto_utils.so VNDK-core: libcurl.so -VNDK-core: libdiskconfig.so VNDK-core: libdumpstateutil.so VNDK-core: libevent.so VNDK-core: libexif.so @@ -135,6 +138,7 @@ VNDK-core: libwifi-system-iface.so VNDK-core: libxml2.so VNDK-core: libyuv.so VNDK-core: libziparchive.so +VNDK-core: server_configurable_flags.so VNDK-private: libblas.so VNDK-private: libcompiler_rt.so VNDK-private: libft2.so @@ -205,3 +209,4 @@ VNDK-product: libxml2.so VNDK-product: libyuv.so VNDK-product: libz.so VNDK-product: libziparchive.so +VNDK-product: server_configurable_flags.so diff --git a/target/product/gsi_release.mk b/target/product/gsi_release.mk index 09d4bc9a6b..884b419868 100644 --- a/target/product/gsi_release.mk +++ b/target/product/gsi_release.mk @@ -28,15 +28,15 @@ BUILDING_GSI := true -# Exclude all files under system/product and system/system_ext PRODUCT_ARTIFACT_PATH_REQUIREMENT_ALLOWED_LIST += \ + system/etc/init/config \ system/product/% \ system/system_ext/% # GSI should always support up-to-date platform features. # Keep this value at the latest API level to ensure latest build system # default configs are applied. -PRODUCT_SHIPPING_API_LEVEL := 31 +PRODUCT_SHIPPING_API_LEVEL := 34 # Enable dynamic partitions to facilitate mixing onto Cuttlefish PRODUCT_USE_DYNAMIC_PARTITIONS := true @@ -44,20 +44,6 @@ PRODUCT_USE_DYNAMIC_PARTITIONS := true # Enable dynamic partition size PRODUCT_USE_DYNAMIC_PARTITION_SIZE := true -# Disable the build-time debugfs restrictions on GSI builds -PRODUCT_SET_DEBUGFS_RESTRICTIONS := false - -# GSI targets should install "unflattened" APEXes in /system -TARGET_FLATTEN_APEX := false - -# GSI targets should install "flattened" APEXes in /system_ext as well -PRODUCT_INSTALL_EXTRA_FLATTENED_APEXES := true - -# The flattened version of com.android.apex.cts.shim.v1 should be explicitly installed -# because the shim apex is prebuilt one and PRODUCT_INSTALL_EXTRA_FLATTENED_APEXES is not -# supported for prebuilt_apex modules yet. -PRODUCT_PACKAGES += com.android.apex.cts.shim.v1_with_prebuilts.flattened - # GSI specific tasks on boot PRODUCT_PACKAGES += \ gsi_skip_mount.cfg \ @@ -69,13 +55,16 @@ PRODUCT_PACKAGES += gsi_overlay_systemui PRODUCT_COPY_FILES += \ device/generic/common/overlays/overlay-config.xml:$(TARGET_COPY_OUT_SYSTEM_EXT)/overlay/config/config.xml +# b/308878144 no more VNDK on 24Q1 and beyond +KEEP_VNDK ?= false + # Support additional VNDK snapshots PRODUCT_EXTRA_VNDK_VERSIONS := \ - 29 \ 30 \ 31 \ 32 \ 33 \ + 34 \ # Do not build non-GSI partition images. PRODUCT_BUILD_CACHE_IMAGE := false @@ -88,9 +77,18 @@ PRODUCT_BUILD_SUPER_EMPTY_IMAGE := false PRODUCT_BUILD_SYSTEM_DLKM_IMAGE := false PRODUCT_EXPORT_BOOT_IMAGE_TO_DIST := true -# Always build modules from source -MODULE_BUILD_FROM_SOURCE := true - # Additional settings used in all GSI builds PRODUCT_PRODUCT_PROPERTIES += \ ro.crypto.metadata_init_delete_all_keys.enabled=false \ + +# Window Extensions +ifneq ($(PRODUCT_IS_ATV),true) +$(call inherit-product, $(SRC_TARGET_DIR)/product/window_extensions.mk) +endif + +# A GSI is to be mixed with different boot images. That means we can't determine +# the kernel version when building a GSI. +# Assume the device supports UFFD. If it doesn't, the ART runtime will fall back +# to CC, and odrefresh will regenerate core dexopt artifacts on the first boot, +# so this is okay. +PRODUCT_ENABLE_UFFD_GC := true diff --git a/target/product/handheld_system.mk b/target/product/handheld_system.mk index 41233b2c0a..3c401f399b 100644 --- a/target/product/handheld_system.mk +++ b/target/product/handheld_system.mk @@ -27,6 +27,7 @@ $(call inherit-product-if-exists, external/google-fonts/cutive-mono/fonts.mk) $(call inherit-product-if-exists, external/google-fonts/source-sans-pro/fonts.mk) $(call inherit-product-if-exists, external/noto-fonts/fonts.mk) $(call inherit-product-if-exists, external/roboto-fonts/fonts.mk) +$(call inherit-product-if-exists, external/roboto-flex-fonts/fonts.mk) $(call inherit-product-if-exists, external/hyphenation-patterns/patterns.mk) $(call inherit-product-if-exists, frameworks/base/data/keyboards/keyboards.mk) $(call inherit-product-if-exists, frameworks/webview/chromium/chromium.mk) @@ -42,6 +43,8 @@ PRODUCT_PACKAGES += \ CameraExtensionsProxy \ CaptivePortalLogin \ CertInstaller \ + CredentialManager \ + DeviceAsWebcam \ DocumentsUI \ DownloadProviderUi \ EasterEgg \ @@ -54,8 +57,8 @@ PRODUCT_PACKAGES += \ MmsService \ MtpService \ MusicFX \ - NfcNci \ PacProcessor \ + preinstalled-packages-platform-handheld-system.xml \ PrintRecommendationService \ PrintSpooler \ ProxyHandler \ @@ -71,6 +74,7 @@ PRODUCT_PACKAGES += \ VpnDialogs \ vr \ +PRODUCT_PACKAGES += $(RELEASE_PACKAGE_VIRTUAL_CAMERA) PRODUCT_SYSTEM_SERVER_APPS += \ FusedLocation \ @@ -78,8 +82,10 @@ PRODUCT_SYSTEM_SERVER_APPS += \ KeyChain \ Telecom \ +PRODUCT_PACKAGES += framework-audio_effects.xml + PRODUCT_COPY_FILES += \ - frameworks/av/media/libeffects/data/audio_effects.conf:system/etc/audio_effects.conf + frameworks/native/data/etc/android.software.window_magnification.xml:$(TARGET_COPY_OUT_SYSTEM)/etc/permissions/android.software.window_magnification.xml \ PRODUCT_VENDOR_PROPERTIES += \ ro.carrier?=unknown \ diff --git a/target/product/handheld_system_ext.mk b/target/product/handheld_system_ext.mk index d935fbfddf..187b6275bb 100644 --- a/target/product/handheld_system_ext.mk +++ b/target/product/handheld_system_ext.mk @@ -22,6 +22,7 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/media_system_ext.mk) # /system_ext packages PRODUCT_PACKAGES += \ + AccessibilityMenu \ Launcher3QuickStep \ Provision \ Settings \ diff --git a/target/product/mainline_sdk.mk b/target/product/mainline_sdk.mk index 343aed6f12..10bb0a0224 100644 --- a/target/product/mainline_sdk.mk +++ b/target/product/mainline_sdk.mk @@ -16,3 +16,5 @@ PRODUCT_NAME := mainline_sdk PRODUCT_BRAND := Android PRODUCT_DEVICE := mainline_sdk + +PRODUCT_BUILD_FROM_SOURCE_STUB := true
\ No newline at end of file diff --git a/target/product/media_system.mk b/target/product/media_system.mk index 79bd74a01c..503c9b3531 100644 --- a/target/product/media_system.mk +++ b/target/product/media_system.mk @@ -59,10 +59,6 @@ PRODUCT_COPY_FILES += \ PRODUCT_COPY_FILES += $(call add-to-product-copy-files-if-exists,\ frameworks/base/config/compiled-classes-phone:system/etc/compiled-classes) -# Enable dirty image object binning to reduce dirty pages in the image. -PRODUCT_COPY_FILES += $(call add-to-product-copy-files-if-exists,\ - frameworks/base/dirty-image-objects-phone:system/etc/dirty-image-objects) - # On userdebug builds, collect more tombstones by default. ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT))) PRODUCT_VENDOR_PROPERTIES += \ @@ -76,3 +72,7 @@ PRODUCT_VENDOR_PROPERTIES += \ # Enable CFI for security-sensitive components $(call inherit-product, $(SRC_TARGET_DIR)/product/cfi-common.mk) $(call inherit-product-if-exists, vendor/google/products/cfi-vendor.mk) + +# Enable MTE for security-sensitive components +$(call inherit-product, $(SRC_TARGET_DIR)/product/memtag-common.mk) +$(call inherit-product-if-exists, vendor/google/products/memtag-vendor.mk) diff --git a/target/board/emulator_x86_64/device.mk b/target/product/memtag-common.mk index 8a9d8da921..829cb41c93 100755..100644 --- a/target/board/emulator_x86_64/device.mk +++ b/target/product/memtag-common.mk @@ -1,5 +1,4 @@ -# -# Copyright (C) 2020 The Android Open Source Project +# Copyright (C) 2023 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +13,18 @@ # limitations under the License. # -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish # for libwifi-hal-emu -PRODUCT_SOONG_NAMESPACES += device/generic/goldfish-opengl # for goldfish deps. - -ifdef NET_ETH0_STARTONBOOT - PRODUCT_VENDOR_PROPERTIES += net.eth0.startonboot=1 -endif +# This is a recommended set of common components to enable MTE for. -# Ensure we package the BIOS files too. -PRODUCT_HOST_PACKAGES += \ - bios.bin \ - vgabios-cirrus.bin \ +PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS := \ + external/android-clat \ + external/iproute2 \ + external/iptables \ + external/mtpd \ + external/ppp \ + hardware/st/nfc \ + hardware/st/secure_element \ + hardware/st/secure_element2 \ + packages/modules/StatsD \ + system/bpf \ + system/netd/netutil_wrappers \ + system/netd/server diff --git a/target/product/module_arm.mk b/target/product/module_arm.mk index d99dce8b2b..434f7ad097 100644 --- a/target/product/module_arm.mk +++ b/target/product/module_arm.mk @@ -17,5 +17,4 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) PRODUCT_NAME := module_arm -PRODUCT_BRAND := Android PRODUCT_DEVICE := module_arm diff --git a/target/product/module_arm64.mk b/target/product/module_arm64.mk index fc9529ce39..d6487ca56b 100644 --- a/target/product/module_arm64.mk +++ b/target/product/module_arm64.mk @@ -18,5 +18,7 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit.mk) PRODUCT_NAME := module_arm64 -PRODUCT_BRAND := Android PRODUCT_DEVICE := module_arm64 + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true +PRODUCT_MAX_PAGE_SIZE_SUPPORTED := 16384 diff --git a/target/product/module_arm64only.mk b/target/product/module_arm64only.mk index 4e8d53eb75..137701a441 100644 --- a/target/product/module_arm64only.mk +++ b/target/product/module_arm64only.mk @@ -18,5 +18,7 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit_only.mk) PRODUCT_NAME := module_arm64only -PRODUCT_BRAND := Android PRODUCT_DEVICE := module_arm64only + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true +PRODUCT_MAX_PAGE_SIZE_SUPPORTED := 16384 diff --git a/target/product/module_common.mk b/target/product/module_common.mk index ec670ee4c9..da4ea23ad9 100644 --- a/target/product/module_common.mk +++ b/target/product/module_common.mk @@ -14,19 +14,24 @@ # limitations under the License. # +$(call inherit-product, $(SRC_TARGET_DIR)/product/build_variables.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/default_art_config.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/languages_default.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/cfi-common.mk) +$(call inherit-product, $(SRC_TARGET_DIR)/product/memtag-common.mk) # Enables treble, which enabled certain -D compilation flags. In particular, libhidlbase # uses -DENFORCE_VINTF_MANIFEST. See b/185759877 PRODUCT_SHIPPING_API_LEVEL := 29 -# Builds using a module product should build modules from source, even if -# BRANCH_DEFAULT_MODULE_BUILD_FROM_SOURCE says otherwise. +# If true, this builds the mainline modules from source. This overrides any +# prebuilts selected via RELEASE_APEX_CONTRIBUTIONS_* build flags for the +# current release config. PRODUCT_MODULE_BUILD_FROM_SOURCE := true # Build sdk from source if the branch is not using slim manifests. ifneq (,$(strip $(wildcard frameworks/base/Android.bp))) UNBUNDLED_BUILD_SDKS_FROM_SOURCE := true endif + +PRODUCT_BRAND := Android diff --git a/target/product/module_riscv64.mk b/target/product/module_riscv64.mk new file mode 100644 index 0000000000..4fd38c063c --- /dev/null +++ b/target/product/module_riscv64.mk @@ -0,0 +1,21 @@ +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +$(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) +$(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit_only.mk) + +PRODUCT_NAME := module_riscv64 +PRODUCT_DEVICE := module_riscv64 diff --git a/target/product/module_x86.mk b/target/product/module_x86.mk index b852e7a6de..f38e2b9793 100644 --- a/target/product/module_x86.mk +++ b/target/product/module_x86.mk @@ -17,5 +17,4 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) PRODUCT_NAME := module_x86 -PRODUCT_BRAND := Android PRODUCT_DEVICE := module_x86 diff --git a/target/product/module_x86_64.mk b/target/product/module_x86_64.mk index f6bc1fc307..e182bf6578 100644 --- a/target/product/module_x86_64.mk +++ b/target/product/module_x86_64.mk @@ -18,5 +18,7 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit.mk) PRODUCT_NAME := module_x86_64 -PRODUCT_BRAND := Android PRODUCT_DEVICE := module_x86_64 + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true +PRODUCT_MAX_PAGE_SIZE_SUPPORTED := 16384 diff --git a/target/product/module_x86_64only.mk b/target/product/module_x86_64only.mk index bca4541efe..fa4a04d7e8 100644 --- a/target/product/module_x86_64only.mk +++ b/target/product/module_x86_64only.mk @@ -18,5 +18,7 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/module_common.mk) $(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit_only.mk) PRODUCT_NAME := module_x86_64only -PRODUCT_BRAND := Android PRODUCT_DEVICE := module_x86_64only + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true +PRODUCT_MAX_PAGE_SIZE_SUPPORTED := 16384 diff --git a/target/product/runtime_libart.mk b/target/product/runtime_libart.mk index 39666ead68..d9c3c9a0a8 100644 --- a/target/product/runtime_libart.mk +++ b/target/product/runtime_libart.mk @@ -102,39 +102,48 @@ PRODUCT_SYSTEM_PROPERTIES += \ PRODUCT_SYSTEM_PROPERTIES += \ ro.dalvik.vm.native.bridge?=0 -# Different dexopt types for different package update/install times. -# On eng builds, make "boot" reasons only extract for faster turnaround. -ifeq (eng,$(TARGET_BUILD_VARIANT)) - PRODUCT_SYSTEM_PROPERTIES += \ - pm.dexopt.first-boot?=extract \ - pm.dexopt.boot-after-ota?=extract -else - PRODUCT_SYSTEM_PROPERTIES += \ - pm.dexopt.first-boot?=verify \ - pm.dexopt.boot-after-ota?=verify -endif - # The install filter is speed-profile in order to enable the use of # profiles from the dex metadata files. Note that if a profile is not provided # or if it is empty speed-profile is equivalent to (quicken + empty app image). # Note that `cmdline` is not strictly needed but it simplifies the management # of compilation reason in the platform (as we have a unified, single path, # without exceptions). +# TODO(b/243646876): Remove `pm.dexopt.post-boot`. PRODUCT_SYSTEM_PROPERTIES += \ - pm.dexopt.post-boot?=extract \ + pm.dexopt.post-boot?=verify \ + pm.dexopt.first-boot?=verify \ + pm.dexopt.boot-after-ota?=verify \ pm.dexopt.boot-after-mainline-update?=verify \ pm.dexopt.install?=speed-profile \ pm.dexopt.install-fast?=skip \ pm.dexopt.install-bulk?=speed-profile \ pm.dexopt.install-bulk-secondary?=verify \ pm.dexopt.install-bulk-downgraded?=verify \ - pm.dexopt.install-bulk-secondary-downgraded?=extract \ + pm.dexopt.install-bulk-secondary-downgraded?=verify \ pm.dexopt.bg-dexopt?=speed-profile \ pm.dexopt.ab-ota?=speed-profile \ pm.dexopt.inactive?=verify \ pm.dexopt.cmdline?=verify \ pm.dexopt.shared?=speed +ifneq (,$(filter eng,$(TARGET_BUILD_VARIANT))) + OVERRIDE_DISABLE_DEXOPT_ALL ?= true +endif + +# OVERRIDE_DISABLE_DEXOPT_ALL disables all dexpreopt (build-time) and dexopt (on-device) activities. +# This option is for faster iteration during development and should never be enabled for production. +ifneq (,$(filter true,$(OVERRIDE_DISABLE_DEXOPT_ALL))) + PRODUCT_SYSTEM_PROPERTIES += \ + dalvik.vm.disable-art-service-dexopt=true \ + dalvik.vm.disable-odrefresh=true + + # Disable all dexpreopt activities except for the ART boot image. + # We have to dexpreopt the ART boot image because they are used by ART tests. This should not + # be too much of a problem for platform developers because a change to framework code should not + # trigger dexpreopt for the ART boot image. + WITH_DEXPREOPT_ART_BOOT_IMG_ONLY := true +endif + # Enable resolution of startup const strings. PRODUCT_SYSTEM_PROPERTIES += \ dalvik.vm.dex2oat-resolve-startup-strings=true @@ -166,12 +175,5 @@ PRODUCT_SYSTEM_PROPERTIES += \ dalvik.vm.usap_pool_size_min?=1 \ dalvik.vm.usap_pool_refill_delay_ms?=3000 -# Allow dexopt files that are side-effects of already allowlisted files. -# This is only necessary when ART is prebuilt. -ifeq (false,$(ART_MODULE_BUILD_FROM_SOURCE)) - PRODUCT_ARTIFACT_PATH_REQUIREMENT_ALLOWED_LIST += \ - system/framework/%.art \ - system/framework/%.oat \ - system/framework/%.odex \ - system/framework/%.vdex -endif +PRODUCT_SYSTEM_PROPERTIES += \ + dalvik.vm.useartservice=true diff --git a/target/product/sdk.mk b/target/product/sdk.mk index fa7e1ad381..1a073636ef 100644 --- a/target/product/sdk.mk +++ b/target/product/sdk.mk @@ -17,8 +17,26 @@ # This is a simple product that uses configures the minimum amount # needed to build the SDK (without the emulator). +# Ensure all trunk-stable flags are available. +$(call inherit-product, $(SRC_TARGET_DIR)/product/build_variables.mk) + +# In order to build the bootclasspath sources, the bootclasspath needs to +# be setup via default_art_config.mk. The sources only really make sense +# together with a device (e.g. the emulator). So if the SDK sources change +# to be built with the device, this could be removed. +$(call inherit-product, $(SRC_TARGET_DIR)/product/default_art_config.mk) + $(call inherit-product, $(SRC_TARGET_DIR)/product/languages_default.mk) PRODUCT_NAME := sdk PRODUCT_BRAND := Android PRODUCT_DEVICE := mainline_x86 + +PRODUCT_BUILD_FROM_SOURCE_STUB := true + +# Use sources of mainline modules +PRODUCT_MODULE_BUILD_FROM_SOURCE := true + +ifeq ($(WITHOUT_CHECK_API),true) + $(error WITHOUT_CHECK_API cannot be set to true for SDK product builds) +endif diff --git a/target/product/sdk_phone_arm64.mk b/target/product/sdk_phone_arm64.mk deleted file mode 100644 index 4203d45c5e..0000000000 --- a/target/product/sdk_phone_arm64.mk +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (C) 2009 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -QEMU_USE_SYSTEM_EXT_PARTITIONS := true -PRODUCT_USE_DYNAMIC_PARTITIONS := true - -# This is a build configuration for a full-featured build of the -# Open-Source part of the tree. It's geared toward a US-centric -# build quite specifically for the emulator, and might not be -# entirely appropriate to inherit from for on-device configurations. - -# Enable mainline checking for exact this product name -ifeq (sdk_phone_arm64,$(TARGET_PRODUCT)) -PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS := relaxed -endif - -# -# All components inherited here go to system image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/generic_system.mk) - -# -# All components inherited here go to system_ext image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/handheld_system_ext.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/telephony_system_ext.mk) - -# -# All components inherited here go to product image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) - -# -# All components inherited here go to vendor or vendor_boot image -# -$(call inherit-product-if-exists, device/generic/goldfish/arm64-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/board/emulator_arm64/device.mk) - -# keep this apk for sdk targets for now -PRODUCT_PACKAGES += \ - EmulatorSmokeTests - -# Overrides -PRODUCT_BRAND := Android -PRODUCT_NAME := sdk_phone_arm64 -PRODUCT_DEVICE := emulator_arm64 -PRODUCT_MODEL := Android SDK built for arm64 -# Disable <uses-library> checks for SDK product. It lacks some libraries (e.g. -# RadioConfigLib), which makes it impossible to translate their module names to -# library name, so the check fails. -PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true - - diff --git a/target/product/sdk_phone_armv7.mk b/target/product/sdk_phone_armv7.mk deleted file mode 100644 index 888505b665..0000000000 --- a/target/product/sdk_phone_armv7.mk +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (C) 2007 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -QEMU_USE_SYSTEM_EXT_PARTITIONS := true -PRODUCT_USE_DYNAMIC_PARTITIONS := true - -# This is a build configuration for a full-featured build of the -# Open-Source part of the tree. It's geared toward a US-centric -# build quite specifically for the emulator, and might not be -# entirely appropriate to inherit from for on-device configurations. - -# Enable mainline checking for exact this product name -ifeq (sdk_phone_armv7,$(TARGET_PRODUCT)) -PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS := relaxed -endif - -# -# All components inherited here go to system image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/generic_system.mk) - -# -# All components inherited here go to system_ext image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/handheld_system_ext.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/telephony_system_ext.mk) - -# -# All components inherited here go to product image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) - -# -# All components inherited here go to vendor image -# -$(call inherit-product-if-exists, build/make/target/product/ramdisk_stub.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/board/emulator_arm/device.mk) - -# keep this apk for sdk targets for now -PRODUCT_PACKAGES += \ - EmulatorSmokeTests - - -# Overrides -PRODUCT_BRAND := Android -PRODUCT_NAME := sdk_phone_armv7 -PRODUCT_DEVICE := emulator_arm -PRODUCT_MODEL := Android SDK built for arm -# Disable <uses-library> checks for SDK product. It lacks some libraries (e.g. -# RadioConfigLib), which makes it impossible to translate their module names to -# library name, so the check fails. -PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true diff --git a/target/product/sdk_phone_x86.mk b/target/product/sdk_phone_x86.mk deleted file mode 100644 index a324e5f023..0000000000 --- a/target/product/sdk_phone_x86.mk +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (C) 2009 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -QEMU_USE_SYSTEM_EXT_PARTITIONS := true -PRODUCT_USE_DYNAMIC_PARTITIONS := true - -# This is a build configuration for a full-featured build of the -# Open-Source part of the tree. It's geared toward a US-centric -# build quite specifically for the emulator, and might not be -# entirely appropriate to inherit from for on-device configurations. - -# -# All components inherited here go to system image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/generic_system.mk) - -# Enable mainline checking for exact this product name -ifeq (sdk_phone_x86,$(TARGET_PRODUCT)) -PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS := relaxed -endif - -# -# All components inherited here go to system_ext image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/handheld_system_ext.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/telephony_system_ext.mk) - -# -# All components inherited here go to product image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) - -# -# All components inherited here go to vendor image -# -$(call inherit-product-if-exists, device/generic/goldfish/x86-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/board/emulator_x86/device.mk) - -# Overrides -PRODUCT_BRAND := Android -PRODUCT_NAME := sdk_phone_x86 -PRODUCT_DEVICE := emulator_x86 -PRODUCT_MODEL := Android SDK built for x86 -# Disable <uses-library> checks for SDK product. It lacks some libraries (e.g. -# RadioConfigLib), which makes it impossible to translate their module names to -# library name, so the check fails. -PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true diff --git a/target/product/sdk_phone_x86_64.mk b/target/product/sdk_phone_x86_64.mk deleted file mode 100644 index ff9018d93f..0000000000 --- a/target/product/sdk_phone_x86_64.mk +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (C) 2009 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -QEMU_USE_SYSTEM_EXT_PARTITIONS := true -PRODUCT_USE_DYNAMIC_PARTITIONS := true - -# This is a build configuration for a full-featured build of the -# Open-Source part of the tree. It's geared toward a US-centric -# build quite specifically for the emulator, and might not be -# entirely appropriate to inherit from for on-device configurations. - -# -# All components inherited here go to system image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/generic_system.mk) - -# Enable mainline checking for exact this product name -ifeq (sdk_phone_x86_64,$(TARGET_PRODUCT)) -PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS := relaxed -endif - -# -# All components inherited here go to system_ext image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/handheld_system_ext.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/telephony_system_ext.mk) - -# -# All components inherited here go to product image -# -$(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) - -# -# All components inherited here go to vendor image -# -$(call inherit-product-if-exists, device/generic/goldfish/x86_64-vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/emulator_vendor.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/board/emulator_x86_64/device.mk) - -# Overrides -PRODUCT_BRAND := Android -PRODUCT_NAME := sdk_phone_x86_64 -PRODUCT_DEVICE := emulator_x86_64 -PRODUCT_MODEL := Android SDK built for x86_64 -# Disable <uses-library> checks for SDK product. It lacks some libraries (e.g. -# RadioConfigLib), which makes it impossible to translate their module names to -# library name, so the check fails. -PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true diff --git a/target/product/sdk_x86.mk b/target/product/sdk_with_runtime_apis.mk index 13ee57d805..e80b4fb600 100644 --- a/target/product/sdk_x86.mk +++ b/target/product/sdk_with_runtime_apis.mk @@ -1,5 +1,5 @@ # -# Copyright (C) 2014 The Android Open Source Project +# Copyright (C) 2024 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ # limitations under the License. # -# Don't modify this file - It's just an alias! +$(call inherit-product, $(SRC_TARGET_DIR)/product/sdk.mk) -$(call inherit-product, $(SRC_TARGET_DIR)/product/sdk_phone_x86.mk) +PRODUCT_NAME := sdk_with_runtime_apis -PRODUCT_NAME := sdk_x86 +PRODUCT_HIDDEN_API_EXPORTABLE_STUBS := true +PRODUCT_EXPORT_RUNTIME_APIS := true
\ No newline at end of file diff --git a/target/product/security/BUILD.bazel b/target/product/security/BUILD.bazel new file mode 100644 index 0000000000..c12be79833 --- /dev/null +++ b/target/product/security/BUILD.bazel @@ -0,0 +1,8 @@ +filegroup( + name = "android_certificate_directory", + srcs = glob([ + "*.pk8", + "*.pem", + ]), + visibility = ["//visibility:public"], +) diff --git a/target/product/security/nfc.pk8 b/target/product/security/nfc.pk8 Binary files differnew file mode 100644 index 0000000000..4a5e1b733c --- /dev/null +++ b/target/product/security/nfc.pk8 diff --git a/target/product/security/nfc.x509.pem b/target/product/security/nfc.x509.pem new file mode 100644 index 0000000000..e6bff6a3dc --- /dev/null +++ b/target/product/security/nfc.x509.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE-----
+MIIF2DCCA8CgAwIBAgIUC94q348hFaPm2jow3R84ZjNFc3EwDQYJKoZIhvcNAQELBQAwfDELMAkG
+A1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDAS
+BgNVBAoTC0dvb2dsZSBJbmMuMRAwDgYDVQQLEwdBbmRyb2lkMRgwFgYDVQQDDA9jb21fYW5kcm9p
+ZF9uZmMwIBcNMjMxMTAxMjEzNzE1WhgPMjA1MzExMDEyMTM3MTVaMHwxCzAJBgNVBAYTAlVTMRMw
+EQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtHb29n
+bGUgSW5jLjEQMA4GA1UECxMHQW5kcm9pZDEYMBYGA1UEAwwPY29tX2FuZHJvaWRfbmZjMIICIjAN
+BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArpgwKyLDl8M3KRb1Fxs3P2mnt81sB3uGZs44R6C6
+CwFhiiACOmEQBcm79BUKBMrE9dUbyOL/GKluNzD026UsDE+N2wDQ8siTxaljDAwhZBpurhOu4uH8
+BKJOzoczAlJFMHpFIMCKQXwotMjT93BuhlSo024Q5QDd2j7Gajk21TkkyQNlBOiyEpKkrRPBuArw
+2knqhuX+nLYkJ5roANaJVDsiKMDG/mKnjwAndrgVbBiKaOdfuRd+pJleN3LUkAfYHHBqlOJnPGSI
+jfYK+9TjsIEYVEOb4SMI3CbWwHfOdEIBgz3IPqMtamEnbZHNlfVWURTNGAF2co+DF3TDGDEjraK4
+R5pXDk/W+4Ex77wQPCIT+d981zkbTpgsPXvZmsBzCYMw6tYksPj86fSVJUrJhlibDk4YHVFsF7OK
+arNf044yPhZ+WUIDqWJ6GB0GU8LWGbbe8iaP0ro9Q1DYgYc6buYWIcX81XZO+hHgWtUb2rNqIUsp
+/4DmT1vgz7TiMWcY7pjrHlNHtVf4jC+OU2c+p8u4XUGQxdIKGgZSoHldtAcnwqGuIpat9lS+gtVl
+vJUp8w3Z2gv4q/bBVZ3NNasA1d3HXVQUWiwszcjiVvoSRa/AlMVUGureGRbsiKsyHisYp9rxk1DB
+dPS9h7tMs/5rV6RM2nZfdfQr71zX9ieSoz0CAwEAAaNQME4wDAYDVR0TBAUwAwEB/zAdBgNVHQ4E
+FgQU9v9SL0QIU9fq7aB70/jqVBLmZJUwHwYDVR0jBBgwFoAU9v9SL0QIU9fq7aB70/jqVBLmZJUw
+DQYJKoZIhvcNAQELBQADggIBAExt2/NFt+2IhC5+/cgi8hzvwZKQyml1MQ9pjrkfQy0JGzGTOPDr
++NPuT5jh/SOfGzdBsGVs3hvK7hFvXsFEWwVQaDOkKhcruks+g7FxldhXC2z9iKgjNrOeUXoE7SiE
+zXA/p1KiBcRL3MSMbg/iQjQVxlJky4BDo39SoEgDeL9+i7L4cBwZQ2LBBLPIOdE7G/cG1Q6UE+KN
+/mnz0kk3+FIg0q4szXDxH+o2V4ZYHSOy8P6TBW8gEQ71RGOnh0wtaTIx2RD/zqJAi+BzWMLsC636
+TNMmqKassG4MH445ul2w0ZzOClJ4gkl1e7dtK7/Kes4kcLOI/i4JHLOcydEqum+t8gQtMYyGM1Kv
+mVpC3hEv2pYwFfhg8hg31MljZmLD761kLOLfw98N350h6dNdQ0jci/3rqbbjVinVQoQSVEzJcA9Q
+gQhRLKHiO7oRmht6ilRLFtGZd/PwIMWMNqksTfVM5frMIIZXdfew+efHIJ7X+ZgJu3tGWcbFYFte
+K/BbmPLnp3aAGg/wwU1dqwCANf53oUc4ZzqRm9eovlVsrFiRM/DGt2/t4ujorU6Uwwt2+n05QU7b
+7PXhc7bTP6adUWMNMxSNIPo6wHmsTb2pCg+K5LuNMFJzXcoI3uBW9Qu4M/tLRv4kRKZzphqUbX+e
+/5hW2myw2BvbdwWFrz6XBgkz
+-----END CERTIFICATE-----
diff --git a/target/product/sysconfig/Android.bp b/target/product/sysconfig/Android.bp index 29122e4af2..95042a707e 100644 --- a/target/product/sysconfig/Android.bp +++ b/target/product/sysconfig/Android.bp @@ -30,8 +30,34 @@ prebuilt_etc { } prebuilt_etc { + name: "preinstalled-packages-platform-generic-system.xml", + sub_dir: "sysconfig", + src: "preinstalled-packages-platform-generic-system.xml", +} + +prebuilt_etc { name: "preinstalled-packages-platform-handheld-product.xml", product_specific: true, sub_dir: "sysconfig", src: "preinstalled-packages-platform-handheld-product.xml", } + +prebuilt_etc { + name: "preinstalled-packages-platform-handheld-system.xml", + sub_dir: "sysconfig", + src: "preinstalled-packages-platform-handheld-system.xml", +} + +prebuilt_etc { + name: "preinstalled-packages-platform-telephony-product.xml", + product_specific: true, + sub_dir: "sysconfig", + src: "preinstalled-packages-platform-telephony-product.xml", +} + +prebuilt_etc { + name: "initial-package-stopped-states-aosp.xml", + product_specific: true, + sub_dir: "sysconfig", + src: "initial-package-stopped-states-aosp.xml", +} diff --git a/target/product/sysconfig/initial-package-stopped-states-aosp.xml b/target/product/sysconfig/initial-package-stopped-states-aosp.xml new file mode 100644 index 0000000000..1704ff2c44 --- /dev/null +++ b/target/product/sysconfig/initial-package-stopped-states-aosp.xml @@ -0,0 +1,47 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + ~ Copyright (C) 2023 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!-- +This XML defines an allowlist for packages that should not be scanned in a "stopped" state. +When this feature is turned on (indicated by the config config_stopSystemPackagesByDefault in +core/res/res/values/config.xml) packages on the system partition that are encountered by +the PackageManagerService for the first time are scanned in the "stopped" state. This allowlist +is also considered while creating new users on the device. Stopped state is not set during +subsequent reboots. + +Example usage + 1. <initial-package-state package="com.example.app" stopped="false"/> + Indicates that a system package - com.example.app's initial stopped state should not be set + by the Package Manager. By default, system apps are marked as stopped. + 2. <initial-package-state package="com.example.app" stopped="true"/> + Indicates that a system package - com.example.app's initial state should be set by the + Package Manager to "stopped=true". It will have the same effect on the + package's stopped state even if this package was not included in the allow list. + 3. <initial-package-state package="com.example.app"/> + Invalid usage. +--> + +<config> + <initial-package-state package="com.android.calendar" stopped="false"/> + <initial-package-state package="com.android.camera2" stopped="false"/> + <initial-package-state package="com.android.contacts" stopped="false"/> + <initial-package-state package="com.android.documentsui" stopped="false"/> + <initial-package-state package="com.android.messaging" stopped="false"/> + <initial-package-state package="com.android.quicksearchbox" stopped="false"/> + <initial-package-state package="com.android.settings" stopped="false"/> + <initial-package-state package="com.android.stk" stopped="false"/> +</config> diff --git a/target/product/sysconfig/preinstalled-packages-platform-aosp-product.xml b/target/product/sysconfig/preinstalled-packages-platform-aosp-product.xml index eec1326833..d3e2808ada 100644 --- a/target/product/sysconfig/preinstalled-packages-platform-aosp-product.xml +++ b/target/product/sysconfig/preinstalled-packages-platform-aosp-product.xml @@ -20,4 +20,13 @@ <install-in-user-type package="com.android.wallpaperpicker"> <install-in user-type="FULL" /> </install-in-user-type> + + <!-- System packages that should not be pre-installed on the CLONE profile. --> + <!-- Messages --> + <install-in-user-type package="com.android.messaging"> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> </config> diff --git a/target/product/sysconfig/preinstalled-packages-platform-generic-system.xml b/target/product/sysconfig/preinstalled-packages-platform-generic-system.xml new file mode 100644 index 0000000000..ef8056f16b --- /dev/null +++ b/target/product/sysconfig/preinstalled-packages-platform-generic-system.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2022 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- System packages to preinstall on all devices with generic_system, per user type. + Documentation at frameworks/base/data/etc/preinstalled-packages-platform.xml +--> +<config> + <!-- Stk (SIM ToolKit) + TODO(b/258055479): Check if this should be preinstalled on SYSTEM user --> + <install-in-user-type package="com.android.stk"> + <install-in user-type="SYSTEM" /> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> +</config> + diff --git a/target/product/sysconfig/preinstalled-packages-platform-handheld-product.xml b/target/product/sysconfig/preinstalled-packages-platform-handheld-product.xml index a5d9ba21dd..536c35b671 100644 --- a/target/product/sysconfig/preinstalled-packages-platform-handheld-product.xml +++ b/target/product/sysconfig/preinstalled-packages-platform-handheld-product.xml @@ -17,6 +17,60 @@ Documentation at frameworks/base/data/etc/preinstalled-packages-platform.xml --> <config> + <!-- Android Keyboard (AOSP) (LatinIME) TODO(b/258055479) --> + <install-in-user-type package="com.android.inputmethod.latin"> + <install-in user-type="SYSTEM" /> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + </install-in-user-type> + + <!-- Calendar --> + <install-in-user-type package="com.android.calendar"> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> + + <!-- Camera (Camera2) --> + <install-in-user-type package="com.android.camera2"> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + </install-in-user-type> + + <!-- Clock (DeskClock) --> + <install-in-user-type package="com.android.deskclock"> + <install-in user-type="FULL" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> + + <!-- Contacts --> + <install-in-user-type package="com.android.contacts"> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + </install-in-user-type> + + <!-- Gallery (Gallery2) --> + <install-in-user-type package="com.android.gallery3d"> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> + + <!-- Search (QuickSearchBox) TODO(b/258055479) --> + <install-in-user-type package="com.android.quicksearchbox"> + <install-in user-type="SYSTEM" /> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> + + <!-- WallpaperCropper --> <install-in-user-type package="com.android.wallpapercropper"> <install-in user-type="FULL" /> </install-in-user-type> diff --git a/target/product/sysconfig/preinstalled-packages-platform-handheld-system.xml b/target/product/sysconfig/preinstalled-packages-platform-handheld-system.xml new file mode 100644 index 0000000000..02b03f1aaa --- /dev/null +++ b/target/product/sysconfig/preinstalled-packages-platform-handheld-system.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2022 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- System packages to preinstall on all devices with handheld_system, per user type. + Documentation at frameworks/base/data/etc/preinstalled-packages-platform.xml +--> +<config> + <!-- Files (DocumentsUI) TODO(b/258055479) --> + <install-in-user-type package="com.android.documentsui"> + <install-in user-type="SYSTEM" /> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + </install-in-user-type> + + <!-- Printer (BuiltInPrintService) (Does not show on launcher but shows on the share sheet) --> + <install-in-user-type package="com.android.bips"> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + </install-in-user-type> +</config> diff --git a/target/product/sysconfig/preinstalled-packages-platform-telephony-product.xml b/target/product/sysconfig/preinstalled-packages-platform-telephony-product.xml new file mode 100644 index 0000000000..67a2a01c1c --- /dev/null +++ b/target/product/sysconfig/preinstalled-packages-platform-telephony-product.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2022 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- System packages to preinstall on all devices with telephony_product, per user type. + Documentation at frameworks/base/data/etc/preinstalled-packages-platform.xml +--> +<config> + <!-- Phone + TODO(b/258055373): Check if this should be preinstalled on SYSTEM user --> + <install-in-user-type package="com.android.dialer"> + <install-in user-type="SYSTEM" /> + <install-in user-type="FULL" /> + <install-in user-type="PROFILE" /> + <do-not-install-in user-type="android.os.usertype.profile.CLONE" /> + <do-not-install-in user-type="android.os.usertype.profile.PRIVATE" /> + </install-in-user-type> +</config> + diff --git a/target/product/telephony_product.mk b/target/product/telephony_product.mk index 18374d4b23..aa70f46797 100644 --- a/target/product/telephony_product.mk +++ b/target/product/telephony_product.mk @@ -21,3 +21,4 @@ PRODUCT_PACKAGES += \ Dialer \ ImsServiceEntitlement \ + preinstalled-packages-platform-telephony-product.xml diff --git a/target/product/telephony_system_ext.mk b/target/product/telephony_system_ext.mk index f81a607f63..f821381076 100644 --- a/target/product/telephony_system_ext.mk +++ b/target/product/telephony_system_ext.mk @@ -21,3 +21,7 @@ PRODUCT_PACKAGES += \ CarrierConfig \ EmergencyInfo \ + +PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34 += \ + hwservicemanager \ + android.hidl.allocator@1.0-service \ diff --git a/target/product/updatable_apex.mk b/target/product/updatable_apex.mk index d606e0037d..8357fdf7fd 100644 --- a/target/product/updatable_apex.mk +++ b/target/product/updatable_apex.mk @@ -14,17 +14,13 @@ # limitations under the License. # -# Inherit this when the target needs to support updating APEXes +#Â com.android.apex.cts.shim.v1_prebuilt overrides CtsShimPrebuilt +# and CtsShimPrivPrebuilt since they are packaged inside the APEX. +PRODUCT_PACKAGES += com.android.apex.cts.shim.v1_prebuilt +PRODUCT_SYSTEM_PROPERTIES := ro.apex.updatable=true -ifneq ($(OVERRIDE_TARGET_FLATTEN_APEX),true) - #Â com.android.apex.cts.shim.v1_prebuilt overrides CtsShimPrebuilt - # and CtsShimPrivPrebuilt since they are packaged inside the APEX. - PRODUCT_PACKAGES += com.android.apex.cts.shim.v1_prebuilt - PRODUCT_VENDOR_PROPERTIES := ro.apex.updatable=true - TARGET_FLATTEN_APEX := false - # Use compressed apexes in pre-installed partitions. - # Note: this doesn't mean that all pre-installed apexes will be compressed. - # Whether an apex is compressed or not is controlled at apex Soong module - # via compresible property. - PRODUCT_COMPRESSED_APEX := true -endif +# Use compressed apexes in pre-installed partitions. +# Note: this doesn't mean that all pre-installed apexes will be compressed. +# Whether an apex is compressed or not is controlled at apex Soong module +# via compresible property. +PRODUCT_COMPRESSED_APEX := true diff --git a/target/product/virtual_ab_ota/compression.mk b/target/product/virtual_ab_ota/compression.mk index dc1ee3e028..c964860740 100644 --- a/target/product/virtual_ab_ota/compression.mk +++ b/target/product/virtual_ab_ota/compression.mk @@ -28,5 +28,4 @@ PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.batch_writes=true PRODUCT_VIRTUAL_AB_COMPRESSION := true PRODUCT_PACKAGES += \ snapuserd.vendor_ramdisk \ - snapuserd \ - snapuserd.recovery + snapuserd diff --git a/target/product/virtual_ab_ota/compression_retrofit.mk b/target/product/virtual_ab_ota/compression_retrofit.mk index 6c29cba6e1..118d3f2b7b 100644 --- a/target/product/virtual_ab_ota/compression_retrofit.mk +++ b/target/product/virtual_ab_ota/compression_retrofit.mk @@ -24,5 +24,4 @@ PRODUCT_VIRTUAL_AB_COMPRESSION := true # as well. PRODUCT_PACKAGES += \ snapuserd.ramdisk \ - snapuserd \ - snapuserd.recovery + snapuserd diff --git a/target/product/virtual_ab_ota/vabc_features.mk b/target/product/virtual_ab_ota/vabc_features.mk index 874eb9cd83..3f484e4f3e 100644 --- a/target/product/virtual_ab_ota/vabc_features.mk +++ b/target/product/virtual_ab_ota/vabc_features.mk @@ -38,6 +38,9 @@ PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.batch_writes=true # Enabling this property, will improve OTA install time # but will use an additional CPU core # PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.compression.threads=true +ifndef PRODUCT_VIRTUAL_AB_COMPRESSION_FACTOR + PRODUCT_VIRTUAL_AB_COMPRESSION_FACTOR := 65536 +endif PRODUCT_VIRTUAL_AB_COMPRESSION := true PRODUCT_VIRTUAL_AB_COMPRESSION_METHOD ?= none diff --git a/target/product/sdk_arm64.mk b/target/product/window_extensions.mk index 8bb38f41a1..5f5431f677 100644 --- a/target/product/sdk_arm64.mk +++ b/target/product/window_extensions.mk @@ -1,5 +1,5 @@ # -# Copyright (C) 2014 The Android Open Source Project +# Copyright (C) 2022 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,11 @@ # limitations under the License. # -# Don't modify this file - It's just an alias! +# /system_ext packages +PRODUCT_PACKAGES += \ + androidx.window.extensions \ + androidx.window.sidecar -$(call inherit-product, $(SRC_TARGET_DIR)/product/sdk_phone_arm64.mk) - -PRODUCT_NAME := sdk_arm64 +# properties +PRODUCT_PRODUCT_PROPERTIES += \ + persist.wm.extensions.enabled=true diff --git a/teams/Android.bp b/teams/Android.bp new file mode 100644 index 0000000000..b3a5752749 --- /dev/null +++ b/teams/Android.bp @@ -0,0 +1,4393 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +team { + name: "trendy_team_qmc_pss", + + // go/trendy/manage/engineers/6342544841375744 + trendy_team_id: "6342544841375744", +} + +team { + name: "trendy_team_cpu_team", + + // go/trendy/manage/engineers/5119059747307520 + trendy_team_id: "5119059747307520", +} + +team { + name: "trendy_team_pwg_mobile", + + // go/trendy/manage/engineers/4869274588315648 + trendy_team_id: "4869274588315648", +} + +team { + name: "trendy_team_pce_weu", + + // go/trendy/manage/engineers/5205725968891904 + trendy_team_id: "5205725968891904", +} + +team { + name: "trendy_team_peeps_t_pgm_android_engprod", + + // go/trendy/manage/engineers/6288284960358400 + trendy_team_id: "6288284960358400", +} + +team { + name: "trendy_team_appsearch", + + // go/trendy/manage/engineers/5075661716815872 + trendy_team_id: "5075661716815872", +} + +team { + name: "trendy_team_shayba_team", + + // go/trendy/manage/engineers/6213135020228608 + trendy_team_id: "6213135020228608", +} + +team { + name: "trendy_team_wear_wear_cloud_platform", + + // go/trendy/manage/engineers/5917762526281728 + trendy_team_id: "5917762526281728", +} + +team { + name: "trendy_team_pixel_system_software", + + // go/trendy/manage/engineers/4856005120622592 + trendy_team_id: "4856005120622592", +} + +team { + name: "trendy_team_platform_enabler_framework_make_pixel_", + + // go/trendy/manage/engineers/5893944097243136 + trendy_team_id: "5893944097243136", +} + +team { + name: "trendy_team_pixel_system_sw_power_sw", + + // go/trendy/manage/engineers/6703184655286272 + trendy_team_id: "6703184655286272", +} + +team { + name: "trendy_team_marvinpaul_team", + + // go/trendy/manage/engineers/4800689692901376 + trendy_team_id: "4800689692901376", +} + +team { + name: "trendy_team_interactive_tv", + + // go/trendy/manage/engineers/6150577853661184 + trendy_team_id: "6150577853661184", +} + +team { + name: "trendy_team_pixel_system_sw_sensor_framework", + + // go/trendy/manage/engineers/5005310567284736 + trendy_team_id: "5005310567284736", +} + +team { + name: "trendy_team_wsd_arch", + + // go/trendy/manage/engineers/6173769806512128 + trendy_team_id: "6173769806512128", +} + +team { + name: "trendy_team_lanechr_team", + + // go/trendy/manage/engineers/5674594204811264 + trendy_team_id: "5674594204811264", +} + +team { + name: "trendy_team_test_eng_android_for_work", + + // go/trendy/manage/engineers/5909887015845888 + trendy_team_id: "5909887015845888", +} + +team { + name: "trendy_team_camera_app", + + // go/trendy/manage/engineers/5216644934533120 + trendy_team_id: "5216644934533120", +} + +team { + name: "trendy_team_vamaraju_team", + + // go/trendy/manage/engineers/5150510960771072 + trendy_team_id: "5150510960771072", +} + +team { + name: "trendy_team_android_media_audio_framework", + + // go/trendy/manage/engineers/5823575353065472 + trendy_team_id: "5823575353065472", +} + +team { + name: "trendy_team_superglue", + + // go/trendy/manage/engineers/5211667882999808 + trendy_team_id: "5211667882999808", +} + +team { + name: "trendy_team_display_framework", + + // go/trendy/manage/engineers/6035600925163520 + trendy_team_id: "6035600925163520", +} + +team { + name: "trendy_team_ananthak_team", + + // go/trendy/manage/engineers/6706043301298176 + trendy_team_id: "6706043301298176", +} + +team { + name: "trendy_team_qmc_pqm", + + // go/trendy/manage/engineers/4715267632267264 + trendy_team_id: "4715267632267264", +} + +team { + name: "trendy_team_search_allapps", + + // go/trendy/manage/engineers/4926160670195712 + trendy_team_id: "4926160670195712", +} + +team { + name: "trendy_team_communal", + + // go/trendy/manage/engineers/6380669942530048 + trendy_team_id: "6380669942530048", +} + +team { + name: "trendy_team_nova", + + // go/trendy/manage/engineers/5418955074043904 + trendy_team_id: "5418955074043904", +} + +team { + name: "trendy_team_deprecated_framework_o_o", + + // go/trendy/manage/engineers/5999497213509632 + trendy_team_id: "5999497213509632", +} + +team { + name: "trendy_team_android_go", + + // go/trendy/manage/engineers/6543205713444864 + trendy_team_id: "6543205713444864", +} + +team { + name: "trendy_team_wear_wear_frameworks", + + // go/trendy/manage/engineers/5138392408817664 + trendy_team_id: "5138392408817664", +} + +team { + name: "trendy_team_ssd_sensor", + + // go/trendy/manage/engineers/5084703539200000 + trendy_team_id: "5084703539200000", +} + +team { + name: "trendy_team_dontek_team", + + // go/trendy/manage/engineers/5746076285042688 + trendy_team_id: "5746076285042688", +} + +team { + name: "trendy_team_carrier_field_test", + + // go/trendy/manage/engineers/6409766640975872 + trendy_team_id: "6409766640975872", +} + +team { + name: "trendy_team_pmw_standards", + + // go/trendy/manage/engineers/6428806822526976 + trendy_team_id: "6428806822526976", +} + +team { + name: "trendy_team_build_infra", + + // go/trendy/manage/engineers/4516184164433920 + trendy_team_id: "4516184164433920", +} + +team { + name: "trendy_team_qmc_gft", + + // go/trendy/manage/engineers/5454139446132736 + trendy_team_id: "5454139446132736", +} + +team { + name: "trendy_team_android_storage", + + // go/trendy/manage/engineers/6301594936049664 + trendy_team_id: "6301594936049664", +} + +team { + name: "trendy_team_pixel_mobile_wireless", + + // go/trendy/manage/engineers/4821918175887360 + trendy_team_id: "4821918175887360", +} + +team { + name: "trendy_team_camera_from_google", + + // go/trendy/manage/engineers/4799694104854528 + trendy_team_id: "4799694104854528", +} + +team { + name: "trendy_team_pixel_connectivity_settings", + + // go/trendy/manage/engineers/5622496450871296 + trendy_team_id: "5622496450871296", +} + +team { + name: "trendy_team_androidbugtool_abt_", + + // go/trendy/manage/engineers/6531817781493760 + trendy_team_id: "6531817781493760", +} + +team { + name: "trendy_team_wear_wear_security", + + // go/trendy/manage/engineers/6325699325362176 + trendy_team_id: "6325699325362176", +} + +team { + name: "trendy_team_pascallouis_team", + + // go/trendy/manage/engineers/5111238276317184 + trendy_team_id: "5111238276317184", +} + +team { + name: "trendy_team_android_camera_ecosystem_enabling", + + // go/trendy/manage/engineers/4529290269327360 + trendy_team_id: "4529290269327360", +} + +team { + name: "trendy_team_calendar", + + // go/trendy/manage/engineers/6719127573889024 + trendy_team_id: "6719127573889024", +} + +team { + name: "trendy_team_cgc", + + // go/trendy/manage/engineers/4590315499061248 + trendy_team_id: "4590315499061248", +} + +team { + name: "trendy_team_diagnostic_tool", + + // go/trendy/manage/engineers/4689924564746240 + trendy_team_id: "4689924564746240", +} + +team { + name: "trendy_team_pixel_camera_system_software", + + // go/trendy/manage/engineers/6386525306486784 + trendy_team_id: "6386525306486784", +} + +team { + name: "trendy_team_credential_manager", + + // go/trendy/manage/engineers/5276403428655104 + trendy_team_id: "5276403428655104", +} + +team { + name: "trendy_team_wear_wti_wear_tools_and_infra_", + + // go/trendy/manage/engineers/6225571306438656 + trendy_team_id: "6225571306438656", +} + +team { + name: "trendy_team_pixel_biometrics_face", + + // go/trendy/manage/engineers/5028705926742016 + trendy_team_id: "5028705926742016", +} + +team { + name: "trendy_team_location_time", + + // go/trendy/manage/engineers/4883807600017408 + trendy_team_id: "4883807600017408", +} + +team { + name: "trendy_team_android_hardware_backed_security", + + // go/trendy/manage/engineers/6398595556343808 + trendy_team_id: "6398595556343808", +} + +team { + name: "trendy_team_play_newsstand", + + // go/trendy/manage/engineers/5171015201980416 + trendy_team_id: "5171015201980416", +} + +team { + name: "trendy_team_deprecated_framework_jaggies", + + // go/trendy/manage/engineers/5753206608887808 + trendy_team_id: "5753206608887808", +} + +team { + name: "trendy_team_make_pixel_tpgm", + + // go/trendy/manage/engineers/6061069864665088 + trendy_team_id: "6061069864665088", +} + +team { + name: "trendy_team_make_transformer", + + // go/trendy/manage/engineers/6224539427438592 + trendy_team_id: "6224539427438592", +} + +team { + name: "trendy_team_wittrock_team", + + // go/trendy/manage/engineers/5707412083474432 + trendy_team_id: "5707412083474432", +} + +team { + name: "trendy_team_wear_wear_android_companion_sdk", + + // go/trendy/manage/engineers/4864923637022720 + trendy_team_id: "4864923637022720", +} + +team { + name: "trendy_team_assistant_sysui_integration", + + // go/trendy/manage/engineers/4884282575060992 + trendy_team_id: "4884282575060992", +} + +team { + name: "trendy_team_things", + + // go/trendy/manage/engineers/5206199574069248 + trendy_team_id: "5206199574069248", +} + +team { + name: "trendy_team_wsd_w13", + + // go/trendy/manage/engineers/5612469120532480 + trendy_team_id: "5612469120532480", +} + +team { + name: "trendy_team_iqbalasif_team", + + // go/trendy/manage/engineers/4912049813094400 + trendy_team_id: "4912049813094400", +} + +team { + name: "trendy_team_biometric_security", + + // go/trendy/manage/engineers/5797911960649728 + trendy_team_id: "5797911960649728", +} + +team { + name: "trendy_team_silberst_team", + + // go/trendy/manage/engineers/5710892584042496 + trendy_team_id: "5710892584042496", +} + +team { + name: "trendy_team_pmw_telephony", + + // go/trendy/manage/engineers/6029121444151296 + trendy_team_id: "6029121444151296", +} + +team { + name: "trendy_team_zzz", + + // go/trendy/manage/engineers/6351340934397952 + trendy_team_id: "6351340934397952", +} + +team { + name: "trendy_team_lite_team", + + // go/trendy/manage/engineers/5647925813346304 + trendy_team_id: "5647925813346304", +} + +team { + name: "trendy_team_gms_core", + + // go/trendy/manage/engineers/5735614422843392 + trendy_team_id: "5735614422843392", +} + +team { + name: "trendy_team_dialer_make_pixel_", + + // go/trendy/manage/engineers/5126396509978624 + trendy_team_id: "5126396509978624", +} + +team { + name: "trendy_team_pixel_system_sw_color", + + // go/trendy/manage/engineers/5489236125581312 + trendy_team_id: "5489236125581312", +} + +team { + name: "trendy_team_wear_wear_notifications_alerts_attention_management", + + // go/trendy/manage/engineers/6267643681996800 + trendy_team_id: "6267643681996800", +} + +team { + name: "trendy_team_fwk_nfc", + + // go/trendy/manage/engineers/5962312512864256 + trendy_team_id: "5962312512864256", +} + +team { + name: "trendy_team_wear_personalization_developer_surfaces", + + // go/trendy/manage/engineers/4819890988810240 + trendy_team_id: "4819890988810240", +} + +team { + name: "trendy_team_srajkumar_team", + + // go/trendy/manage/engineers/5170053894012928 + trendy_team_id: "5170053894012928", +} + +team { + name: "trendy_team_in_market_tpm", + + // go/trendy/manage/engineers/5352549888196608 + trendy_team_id: "5352549888196608", +} + +team { + name: "trendy_team_leannogasawara_team", + + // go/trendy/manage/engineers/4905467198472192 + trendy_team_id: "4905467198472192", +} + +team { + name: "trendy_team_zurikemp_team", + + // go/trendy/manage/engineers/4559796603879424 + trendy_team_id: "4559796603879424", +} + +team { + name: "trendy_team_android_telemetry_infra", + + // go/trendy/manage/engineers/5295809771732992 + trendy_team_id: "5295809771732992", +} + +team { + name: "trendy_team_system_ui_sensors", + + // go/trendy/manage/engineers/5647653492621312 + trendy_team_id: "5647653492621312", +} + +team { + name: "trendy_team_windowing_animations_transitions", + + // go/trendy/manage/engineers/4803040337362944 + trendy_team_id: "4803040337362944", +} + +team { + name: "trendy_team_deprecated_framework_jjaggi", + + // go/trendy/manage/engineers/6471742270898176 + trendy_team_id: "6471742270898176", +} + +team { + name: "trendy_team_accessibility_hearing_aids", + + // go/trendy/manage/engineers/4661226340253696 + trendy_team_id: "4661226340253696", +} + +team { + name: "trendy_team_performance", + + // go/trendy/manage/engineers/5842000521625600 + trendy_team_id: "5842000521625600", +} + +team { + name: "trendy_team_cloud_android", + + // go/trendy/manage/engineers/5980255760023552 + trendy_team_id: "5980255760023552", +} + +team { + name: "trendy_team_visual_design", + + // go/trendy/manage/engineers/4504161399734272 + trendy_team_id: "4504161399734272", +} + +team { + name: "trendy_team_wilkinsonclay_team", + + // go/trendy/manage/engineers/5680997128634368 + trendy_team_id: "5680997128634368", +} + +team { + name: "trendy_team_tccyp_nadiae", + + // go/trendy/manage/engineers/6556518831652864 + trendy_team_id: "6556518831652864", +} + +team { + name: "trendy_team_accessibility_settings", + + // go/trendy/manage/engineers/5641806510587904 + trendy_team_id: "5641806510587904", +} + +team { + name: "trendy_team_hansmuller_team", + + // go/trendy/manage/engineers/5069192257765376 + trendy_team_id: "5069192257765376", +} + +team { + name: "trendy_team_retail_demo_mode", + + // go/trendy/manage/engineers/6520787531235328 + trendy_team_id: "6520787531235328", +} + +team { + name: "trendy_team_lse_dreams", + + // go/trendy/manage/engineers/6317558842097664 + trendy_team_id: "6317558842097664", +} + +team { + name: "trendy_team_android_usb", + + // go/trendy/manage/engineers/5090707854426112 + trendy_team_id: "5090707854426112", +} + +team { + name: "trendy_team_curtisgalloway_team", + + // go/trendy/manage/engineers/5706857730703360 + trendy_team_id: "5706857730703360", +} + +team { + name: "trendy_team_camera_algorithms", + + // go/trendy/manage/engineers/6544854980886528 + trendy_team_id: "6544854980886528", +} + +team { + name: "trendy_team_cast_3p", + + // go/trendy/manage/engineers/6585564972875776 + trendy_team_id: "6585564972875776", +} + +team { + name: "trendy_team_test_eng_android_wear", + + // go/trendy/manage/engineers/4979150422933504 + trendy_team_id: "4979150422933504", +} + +team { + name: "trendy_team_mesch_team", + + // go/trendy/manage/engineers/5205465899368448 + trendy_team_id: "5205465899368448", +} + +team { + name: "trendy_team_pixel_system_sw_audio_arch", + + // go/trendy/manage/engineers/5560501377073152 + trendy_team_id: "5560501377073152", +} + +team { + name: "trendy_team_defunct_use_controls_quick_settings", + + // go/trendy/manage/engineers/4667861043412992 + trendy_team_id: "4667861043412992", +} + +team { + name: "trendy_team_wear_wear_developer_devx", + + // go/trendy/manage/engineers/4894890764697600 + trendy_team_id: "4894890764697600", +} + +team { + name: "trendy_team_android_rust", + + // go/trendy/manage/engineers/4844600586305536 + trendy_team_id: "4844600586305536", +} + +team { + name: "trendy_team_ailabs", + + // go/trendy/manage/engineers/6673470538285056 + trendy_team_id: "6673470538285056", +} + +team { + name: "trendy_team_wear_wear_connectivity", + + // go/trendy/manage/engineers/6245149466263552 + trendy_team_id: "6245149466263552", +} + +team { + name: "trendy_team_android_core_experiments", + + // go/trendy/manage/engineers/5709654965780480 + trendy_team_id: "5709654965780480", +} + +team { + name: "trendy_team_native_tools_libraries", + + // go/trendy/manage/engineers/5920332376309760 + trendy_team_id: "5920332376309760", +} + +team { + name: "trendy_team_app_compat", + + // go/trendy/manage/engineers/4907132411314176 + trendy_team_id: "4907132411314176", +} + +team { + name: "trendy_team_zra_team", + + // go/trendy/manage/engineers/6227615267586048 + trendy_team_id: "6227615267586048", +} + +team { + name: "trendy_team_pixel_watch_system_software", + + // go/trendy/manage/engineers/5295994500972544 + trendy_team_id: "5295994500972544", +} + +team { + name: "trendy_team_surfaces_engprod", + + // go/trendy/manage/engineers/6154478176600064 + trendy_team_id: "6154478176600064", +} + +team { + name: "trendy_team_android_permissions", + + // go/trendy/manage/engineers/5533977340313600 + trendy_team_id: "5533977340313600", +} + +team { + name: "trendy_team_platform_program_mgrs", + + // go/trendy/manage/engineers/4766394922958848 + trendy_team_id: "4766394922958848", +} + +team { + name: "trendy_team_deprecated_system_health", + + // go/trendy/manage/engineers/4864801213644800 + trendy_team_id: "4864801213644800", +} + +team { + name: "trendy_team_messages", + + // go/trendy/manage/engineers/5137480097333248 + trendy_team_id: "5137480097333248", +} + +team { + name: "trendy_team_palmer_team", + + // go/trendy/manage/engineers/5643570052235264 + trendy_team_id: "5643570052235264", +} + +team { + name: "trendy_team_android_video_image_codecs", + + // go/trendy/manage/engineers/5733246110433280 + trendy_team_id: "5733246110433280", +} + +team { + name: "trendy_team_play_music", + + // go/trendy/manage/engineers/6015440132112384 + trendy_team_id: "6015440132112384", +} + +team { + name: "trendy_team_system_clockwork_internal_", + + // go/trendy/manage/engineers/6509670608797696 + trendy_team_id: "6509670608797696", +} + +team { + name: "trendy_team_multitasking_windowing", + + // go/trendy/manage/engineers/5149185436975104 + trendy_team_id: "5149185436975104", +} + +team { + name: "trendy_team_vr", + + // go/trendy/manage/engineers/4854355853180928 + trendy_team_id: "4854355853180928", +} + +team { + name: "trendy_team_maruel_team", + + // go/trendy/manage/engineers/6302551810146304 + trendy_team_id: "6302551810146304", +} + +team { + name: "trendy_team_tv_os", + + // go/trendy/manage/engineers/4662491074134016 + trendy_team_id: "4662491074134016", +} + +team { + name: "trendy_team_auto_engprod", + + // go/trendy/manage/engineers/6199949475479552 + trendy_team_id: "6199949475479552", +} + +team { + name: "trendy_team_sarahcobb_team", + + // go/trendy/manage/engineers/5755692179947520 + trendy_team_id: "5755692179947520", +} + +team { + name: "trendy_team_accessibility_services", + + // go/trendy/manage/engineers/6367283853000704 + trendy_team_id: "6367283853000704", +} + +team { + name: "trendy_team_documentsui", + + // go/trendy/manage/engineers/5805983167021056 + trendy_team_id: "5805983167021056", +} + +team { + name: "trendy_team_carrier_cert_follow_up", + + // go/trendy/manage/engineers/6751912099741696 + trendy_team_id: "6751912099741696", +} + +team { + name: "trendy_team_mobile_device_partners", + + // go/trendy/manage/engineers/5833057717092352 + trendy_team_id: "5833057717092352", +} + +team { + name: "trendy_team_activity_recognition", + + // go/trendy/manage/engineers/6304701268000768 + trendy_team_id: "6304701268000768", +} + +team { + name: "trendy_team_jasoncampbell_team", + + // go/trendy/manage/engineers/4834972524511232 + trendy_team_id: "4834972524511232", +} + +team { + name: "trendy_team_wear_wallet_on_wear", + + // go/trendy/manage/engineers/5724960437731328 + trendy_team_id: "5724960437731328", +} + +team { + name: "trendy_team_glanceables", + + // go/trendy/manage/engineers/4658222004600832 + trendy_team_id: "4658222004600832", +} + +team { + name: "trendy_team_android_safe_browsing", + + // go/trendy/manage/engineers/6685713244782592 + trendy_team_id: "6685713244782592", +} + +team { + name: "trendy_team_android_input", + + // go/trendy/manage/engineers/5141994775805952 + trendy_team_id: "5141994775805952", +} + +team { + name: "trendy_team_android_rust_toolchain", + + // go/trendy/manage/engineers/6530590989975552 + trendy_team_id: "6530590989975552", +} + +team { + name: "trendy_team_exo", + + // go/trendy/manage/engineers/5631545248088064 + trendy_team_id: "5631545248088064", +} + +team { + name: "trendy_team_camerax", + + // go/trendy/manage/engineers/5272590669479936 + trendy_team_id: "5272590669479936", +} + +team { + name: "trendy_team_accessibility_sound_amplifier", + + // go/trendy/manage/engineers/5674840312020992 + trendy_team_id: "5674840312020992", +} + +team { + name: "trendy_team_android_printing", + + // go/trendy/manage/engineers/6257528146067456 + trendy_team_id: "6257528146067456", +} + +team { + name: "trendy_team_dtiselice_team", + + // go/trendy/manage/engineers/5177934253031424 + trendy_team_id: "5177934253031424", +} + +team { + name: "trendy_team_personal_safety", + + // go/trendy/manage/engineers/6222285147111424 + trendy_team_id: "6222285147111424", +} + +team { + name: "trendy_team_notifications", + + // go/trendy/manage/engineers/5993521355587584 + trendy_team_id: "5993521355587584", +} + +team { + name: "trendy_team_java_core_libraries", + + // go/trendy/manage/engineers/4768044190400512 + trendy_team_id: "4768044190400512", +} + +team { + name: "trendy_team_updatable_sdk_apis", + + // go/trendy/manage/engineers/4840215139483648 + trendy_team_id: "4840215139483648", +} + +team { + name: "trendy_team_wear_low_power_mcu_experiences", + + // go/trendy/manage/engineers/6172878013628416 + trendy_team_id: "6172878013628416", +} + +team { + name: "trendy_team_biometrics_framework", + + // go/trendy/manage/engineers/6205415425998848 + trendy_team_id: "6205415425998848", +} + +team { + name: "trendy_team_pesto", + + // go/trendy/manage/engineers/5551098528825344 + trendy_team_id: "5551098528825344", +} + +team { + name: "trendy_team_wear_engineering_foundations", + + // go/trendy/manage/engineers/5366936275681280 + trendy_team_id: "5366936275681280", +} + +team { + name: "trendy_team_wear_wcs_developer", + + // go/trendy/manage/engineers/5114199579459584 + trendy_team_id: "5114199579459584", +} + +team { + name: "trendy_team_aaos_framework", + + // go/trendy/manage/engineers/6547794223333376 + trendy_team_id: "6547794223333376", +} + +team { + name: "trendy_team_wear_3xp", + + // go/trendy/manage/engineers/5692317612539904 + trendy_team_id: "5692317612539904", +} + +team { + name: "trendy_team_clockwork", + + // go/trendy/manage/engineers/4908781678755840 + trendy_team_id: "4908781678755840", +} + +team { + name: "trendy_team_pixel_connectivity_wifi_drivers_firmware", + + // go/trendy/manage/engineers/4583326236934144 + trendy_team_id: "4583326236934144", +} + +team { + name: "trendy_team_games", + + // go/trendy/manage/engineers/6736719759933440 + trendy_team_id: "6736719759933440", +} + +team { + name: "trendy_team_systems_n4_", + + // go/trendy/manage/engineers/6474486236708864 + trendy_team_id: "6474486236708864", +} + +team { + name: "trendy_team_android_kvm", + + // go/trendy/manage/engineers/6529318184714240 + trendy_team_id: "6529318184714240", +} + +team { + name: "trendy_team_wsd_w22", + + // go/trendy/manage/engineers/6580039352975360 + trendy_team_id: "6580039352975360", +} + +team { + name: "trendy_team_android_sudo", + + // go/trendy/manage/engineers/5329344876380160 + trendy_team_id: "5329344876380160", +} + +team { + name: "trendy_team_wear_wear_system_health_and_power", + + // go/trendy/manage/engineers/5219147457658880 + trendy_team_id: "5219147457658880", +} + +team { + name: "trendy_team_android_build_release_tools", + + // go/trendy/manage/engineers/6260558107803648 + trendy_team_id: "6260558107803648", +} + +team { + name: "trendy_team_fused_presence_provider", + + // go/trendy/manage/engineers/6536344753307648 + trendy_team_id: "6536344753307648", +} + +team { + name: "trendy_team_agsa", + + // go/trendy/manage/engineers/6157826887909376 + trendy_team_id: "6157826887909376", +} + +team { + name: "trendy_team_wear_wear_developer_tiles", + + // go/trendy/manage/engineers/6633777396613120 + trendy_team_id: "6633777396613120", +} + +team { + name: "trendy_team_essential_applications", + + // go/trendy/manage/engineers/4926373864800256 + trendy_team_id: "4926373864800256", +} + +team { + name: "trendy_team_pixel_mobile_data", + + // go/trendy/manage/engineers/4996742608977920 + trendy_team_id: "4996742608977920", +} + +team { + name: "trendy_team_wsd_w52", + + // go/trendy/manage/engineers/6280972190220288 + trendy_team_id: "6280972190220288", +} + +team { + name: "trendy_team_pixel_mobile_connectivity", + + // go/trendy/manage/engineers/6754311945977856 + trendy_team_id: "6754311945977856", +} + +team { + name: "trendy_team_essentialapps_clock_calculator", + + // go/trendy/manage/engineers/5270363728674816 + trendy_team_id: "5270363728674816", +} + +team { + name: "trendy_team_ssd_system_health", + + // go/trendy/manage/engineers/6456894050664448 + trendy_team_id: "6456894050664448", +} + +team { + name: "trendy_team_pixel_continuity", + + // go/trendy/manage/engineers/4786635551309824 + trendy_team_id: "4786635551309824", +} + +team { + name: "trendy_team_wear_software_nti", + + // go/trendy/manage/engineers/5164973558759424 + trendy_team_id: "5164973558759424", +} + +team { + name: "trendy_team_machine_learning", + + // go/trendy/manage/engineers/5276568318246912 + trendy_team_id: "5276568318246912", +} + +team { + name: "trendy_team_pixel_ml", + + // go/trendy/manage/engineers/5339883108990976 + trendy_team_id: "5339883108990976", +} + +team { + name: "trendy_team_ex_enterprise", + + // go/trendy/manage/engineers/6738369027375104 + trendy_team_id: "6738369027375104", +} + +team { + name: "trendy_team_pixel_system_sw_aoc", + + // go/trendy/manage/engineers/4712464983425024 + trendy_team_id: "4712464983425024", +} + +team { + name: "trendy_team_android_platform_communications", + + // go/trendy/manage/engineers/6577505415102464 + trendy_team_id: "6577505415102464", +} + +team { + name: "trendy_team_sunshine", + + // go/trendy/manage/engineers/6105050329776128 + trendy_team_id: "6105050329776128", +} + +team { + name: "trendy_team_qmc_iqt_tao", + + // go/trendy/manage/engineers/5065462085713920 + trendy_team_id: "5065462085713920", +} + +team { + name: "trendy_team_mckillop_team", + + // go/trendy/manage/engineers/5926589599744000 + trendy_team_id: "5926589599744000", +} + +team { + name: "trendy_team_pixel_process_experience", + + // go/trendy/manage/engineers/5745436633235456 + trendy_team_id: "5745436633235456", +} + +team { + name: "trendy_team_wsd_l1", + + // go/trendy/manage/engineers/5119887911288832 + trendy_team_id: "5119887911288832", +} + +team { + name: "trendy_team_foldables", + + // go/trendy/manage/engineers/5149421392920576 + trendy_team_id: "5149421392920576", +} + +team { + name: "trendy_team_arc_next", + + // go/trendy/manage/engineers/6238917659361280 + trendy_team_id: "6238917659361280", +} + +team { + name: "trendy_team_android_rubidium", + + // go/trendy/manage/engineers/5098012529295360 + trendy_team_id: "5098012529295360", +} + +team { + name: "trendy_team_wear_wear_power_emulator", + + // go/trendy/manage/engineers/5160338936725504 + trendy_team_id: "5160338936725504", +} + +team { + name: "trendy_team_deprecated_framework_svetoslavganov", + + // go/trendy/manage/engineers/6404117492531200 + trendy_team_id: "6404117492531200", +} + +team { + name: "trendy_team_gregsimon_team", + + // go/trendy/manage/engineers/5702018510520320 + trendy_team_id: "5702018510520320", +} + +team { + name: "trendy_team_wear_opus", + + // go/trendy/manage/engineers/5098351636676608 + trendy_team_id: "5098351636676608", +} + +team { + name: "trendy_team_text_to_speech", + + // go/trendy/manage/engineers/6368933120442368 + trendy_team_id: "6368933120442368", +} + +team { + name: "trendy_team_pixel_system_sw_audio", + + // go/trendy/manage/engineers/6492078422753280 + trendy_team_id: "6492078422753280", +} + +team { + name: "trendy_team_transformer", + + // go/trendy/manage/engineers/5964312841420800 + trendy_team_id: "5964312841420800", +} + +team { + name: "trendy_team_pixel_system_sw_video", + + // go/trendy/manage/engineers/6442361728696320 + trendy_team_id: "6442361728696320", +} + +team { + name: "trendy_team_lse_app_compat", + + // go/trendy/manage/engineers/5180827749154816 + trendy_team_id: "5180827749154816", +} + +team { + name: "trendy_team_android_media_leads", + + // go/trendy/manage/engineers/5487674550779904 + trendy_team_id: "5487674550779904", +} + +team { + name: "trendy_team_kousha_team", + + // go/trendy/manage/engineers/5157338676887552 + trendy_team_id: "5157338676887552", +} + +team { + name: "trendy_team_security", + + // go/trendy/manage/engineers/5241383946158080 + trendy_team_id: "5241383946158080", +} + +team { + name: "trendy_team_pixel_system_sw_battery_life_system_power_", + + // go/trendy/manage/engineers/4512957492756480 + trendy_team_id: "4512957492756480", +} + +team { + name: "trendy_team_eggs", + + // go/trendy/manage/engineers/4568929198309376 + trendy_team_id: "4568929198309376", +} + +team { + name: "trendy_team_jeremymanson_team", + + // go/trendy/manage/engineers/5095869749297152 + trendy_team_id: "5095869749297152", +} + +team { + name: "trendy_team_exchange_active_sync_in_gmail", + + // go/trendy/manage/engineers/5382121434513408 + trendy_team_id: "5382121434513408", +} + +team { + name: "trendy_team_ios_backup_restore_make_pixel_", + + // go/trendy/manage/engineers/5752160863420416 + trendy_team_id: "5752160863420416", +} + +team { + name: "trendy_team_deprecated_location", + + // go/trendy/manage/engineers/6228195632087040 + trendy_team_id: "6228195632087040", +} + +team { + name: "trendy_team_input_framework", + + // go/trendy/manage/engineers/4999436357238784 + trendy_team_id: "4999436357238784", +} + +team { + name: "trendy_team_wear_developer_foundation", + + // go/trendy/manage/engineers/5239127108648960 + trendy_team_id: "5239127108648960", +} + +team { + name: "trendy_team_tpm_tvc", + + // go/trendy/manage/engineers/5390683333230592 + trendy_team_id: "5390683333230592", +} + +team { + name: "trendy_team_wear_wear_ux", + + // go/trendy/manage/engineers/5782097411080192 + trendy_team_id: "5782097411080192", +} + +team { + name: "trendy_team_lse_desktop_os_experience", + + // go/trendy/manage/engineers/5125234900434944 + trendy_team_id: "5125234900434944", +} + +team { + name: "trendy_team_android_for_india_device_experiences", + + // go/trendy/manage/engineers/5395413652111360 + trendy_team_id: "5395413652111360", +} + +team { + name: "trendy_team_pixel_zombie", + + // go/trendy/manage/engineers/5074646910533632 + trendy_team_id: "5074646910533632", +} + +team { + name: "trendy_team_android_onboarding", + + // go/trendy/manage/engineers/5152271974367232 + trendy_team_id: "5152271974367232", +} + +team { + name: "trendy_team_pixel_audio", + + // go/trendy/manage/engineers/5436547260088320 + trendy_team_id: "5436547260088320", +} + +team { + name: "trendy_team_pixel_connectivity_bt", + + // go/trendy/manage/engineers/6328035423453184 + trendy_team_id: "6328035423453184", +} + +team { + name: "trendy_team_wsd_w12", + + // go/trendy/manage/engineers/6333748748353536 + trendy_team_id: "6333748748353536", +} + +team { + name: "trendy_team_qmc_mvt", + + // go/trendy/manage/engineers/4572880876470272 + trendy_team_id: "4572880876470272", +} + +team { + name: "trendy_team_switch_access_voice_access", + + // go/trendy/manage/engineers/4794432469467136 + trendy_team_id: "4794432469467136", +} + +team { + name: "trendy_team_mainline_modularization", + + // go/trendy/manage/engineers/5845084143386624 + trendy_team_id: "5845084143386624", +} + +team { + name: "trendy_team_fwk_telecom", + + // go/trendy/manage/engineers/5330994143821824 + trendy_team_id: "5330994143821824", +} + +team { + name: "trendy_team_deprecated_framework_akulian", + + // go/trendy/manage/engineers/5323210872750080 + trendy_team_id: "5323210872750080", +} + +team { + name: "trendy_team_wear_wear_identity", + + // go/trendy/manage/engineers/6017732386390016 + trendy_team_id: "6017732386390016", +} + +team { + name: "trendy_team_android_pdf", + + // go/trendy/manage/engineers/5175136433045504 + trendy_team_id: "5175136433045504", +} + +team { + name: "trendy_team_developer_relations", + + // go/trendy/manage/engineers/5709226143776768 + trendy_team_id: "5709226143776768", +} + +team { + name: "trendy_team_system_intelligence", + + // go/trendy/manage/engineers/5849675995709440 + trendy_team_id: "5849675995709440", +} + +team { + name: "trendy_team_mainline_updates", + + // go/trendy/manage/engineers/4845810809995264 + trendy_team_id: "4845810809995264", +} + +team { + name: "trendy_team_n_a_1", + + // go/trendy/manage/engineers/5946720655376384 + trendy_team_id: "5946720655376384", +} + +team { + name: "trendy_team_google_drive_docs_sheets_and_slides", + + // go/trendy/manage/engineers/6613574457622528 + trendy_team_id: "6613574457622528", +} + +team { + name: "trendy_team_deprecated_awareness_health_experiences", + + // go/trendy/manage/engineers/6627866395967488 + trendy_team_id: "6627866395967488", +} + +team { + name: "trendy_team_context_infrastructure", + + // go/trendy/manage/engineers/4701268040646656 + trendy_team_id: "4701268040646656", +} + +team { + name: "trendy_team_android_media_solutions", + + // go/trendy/manage/engineers/4750452004356096 + trendy_team_id: "4750452004356096", +} + +team { + name: "trendy_team_wear_device_and_infrastructure", + + // go/trendy/manage/engineers/6358069369798656 + trendy_team_id: "6358069369798656", +} + +team { + name: "trendy_team_pixel_biometrics", + + // go/trendy/manage/engineers/5780875748737024 + trendy_team_id: "5780875748737024", +} + +team { + name: "trendy_team_app_knowledge_platform", + + // go/trendy/manage/engineers/6272266390634496 + trendy_team_id: "6272266390634496", +} + +team { + name: "trendy_team_wsd", + + // go/trendy/manage/engineers/4680083260178432 + trendy_team_id: "4680083260178432", +} + +team { + name: "trendy_team_seg", + + // go/trendy/manage/engineers/5067111353155584 + trendy_team_id: "5067111353155584", +} + +team { + name: "trendy_team_devinlawson_team", + + // go/trendy/manage/engineers/4805900971442176 + trendy_team_id: "4805900971442176", +} + +team { + name: "trendy_team_camera_hardware", + + // go/trendy/manage/engineers/6087458143731712 + trendy_team_id: "6087458143731712", +} + +team { + name: "trendy_team_camera_image_quality", + + // go/trendy/manage/engineers/5401362887999488 + trendy_team_id: "5401362887999488", +} + +team { + name: "trendy_team_wear_wear_assistant", + + // go/trendy/manage/engineers/5848075306172416 + trendy_team_id: "5848075306172416", +} + +team { + name: "trendy_team_android_power_and_comms_infra", + + // go/trendy/manage/engineers/5325547653332992 + trendy_team_id: "5325547653332992", +} + +team { + name: "trendy_team_pmw_pmo", + + // go/trendy/manage/engineers/4656299270504448 + trendy_team_id: "4656299270504448", +} + +team { + name: "trendy_team_filament", + + // go/trendy/manage/engineers/6031425915486208 + trendy_team_id: "6031425915486208", +} + +team { + name: "trendy_team_pixel_system_sw_bspcore", + + // go/trendy/manage/engineers/6508021341356032 + trendy_team_id: "6508021341356032", +} + +team { + name: "trendy_team_powermanager_framework", + + // go/trendy/manage/engineers/5116162121564160 + trendy_team_id: "5116162121564160", +} + +team { + name: "trendy_team_wear_romanesco", + + // go/trendy/manage/engineers/5112520062697472 + trendy_team_id: "5112520062697472", +} + +team { + name: "trendy_team_deprecated_theming", + + // go/trendy/manage/engineers/5179308179881984 + trendy_team_id: "5179308179881984", +} + +team { + name: "trendy_team_recorder", + + // go/trendy/manage/engineers/5085035337383936 + trendy_team_id: "5085035337383936", +} + +team { + name: "trendy_team_framework_accessibility", + + // go/trendy/manage/engineers/5474751170019328 + trendy_team_id: "5474751170019328", +} + +team { + name: "trendy_team_windowing_infra_", + + // go/trendy/manage/engineers/4578440609431552 + trendy_team_id: "4578440609431552", +} + +team { + name: "trendy_team_pmw_mcs", + + // go/trendy/manage/engineers/5864733550608384 + trendy_team_id: "5864733550608384", +} + +team { + name: "trendy_team_wear_wear_sysui_ctrl_carousel_tiles_recents_launcher_", + + // go/trendy/manage/engineers/4820131976740864 + trendy_team_id: "4820131976740864", +} + +team { + name: "trendy_team_wear_wear_accessibility_compliance", + + // go/trendy/manage/engineers/5381719553114112 + trendy_team_id: "5381719553114112", +} + +team { + name: "trendy_team_pixel_system_sw_performance_thermal", + + // go/trendy/manage/engineers/5146276190355456 + trendy_team_id: "5146276190355456", +} + +team { + name: "trendy_team_neelsa_team", + + // go/trendy/manage/engineers/5736750978334720 + trendy_team_id: "5736750978334720", +} + +team { + name: "trendy_team_pixel_camera_engineering_experience", + + // go/trendy/manage/engineers/5190256655466496 + trendy_team_id: "5190256655466496", +} + +team { + name: "trendy_team_embedded_web_on_android", + + // go/trendy/manage/engineers/630061306576896 + trendy_team_id: "630061306576896", +} + +team { + name: "trendy_team_gantry", + + // go/trendy/manage/engineers/5677019153858560 + trendy_team_id: "5677019153858560", +} + +team { + name: "trendy_team_pixel_system_sw_battery", + + // go/trendy/manage/engineers/6052273771642880 + trendy_team_id: "6052273771642880", +} + +team { + name: "trendy_team_enigma", + + // go/trendy/manage/engineers/5396338361597952 + trendy_team_id: "5396338361597952", +} + +team { + name: "trendy_team_pixel_gps_power", + + // go/trendy/manage/engineers/5075907446177792 + trendy_team_id: "5075907446177792", +} + +team { + name: "trendy_team_tool_frank", + + // go/trendy/manage/engineers/6200209976360960 + trendy_team_id: "6200209976360960", +} + +team { + name: "trendy_team_rginda_team", + + // go/trendy/manage/engineers/6031367105314816 + trendy_team_id: "6031367105314816", +} + +team { + name: "trendy_team_pixel_system_sw_soc_power", + + // go/trendy/manage/engineers/5400771358785536 + trendy_team_id: "5400771358785536", +} + +team { + name: "trendy_team_android_crumpet", + + // go/trendy/manage/engineers/5199704478351360 + trendy_team_id: "5199704478351360", +} + +team { + name: "trendy_team_wallpapers", + + // go/trendy/manage/engineers/5125411306373120 + trendy_team_id: "5125411306373120", +} + +team { + name: "trendy_team_deprecated_volta", + + // go/trendy/manage/engineers/6316156562309120 + trendy_team_id: "6316156562309120", +} + +team { + name: "trendy_team_camera_machine_intelligence", + + // go/trendy/manage/engineers/6578390085533696 + trendy_team_id: "6578390085533696", +} + +team { + name: "trendy_team_sheepo_team", + + // go/trendy/manage/engineers/5068061372743680 + trendy_team_id: "5068061372743680", +} + +team { + name: "trendy_team_android_profile_experiences", + + // go/trendy/manage/engineers/5914919462404096 + trendy_team_id: "5914919462404096", +} + +team { + name: "trendy_team_review_platform", + + // go/trendy/manage/engineers/5952905574514688 + trendy_team_id: "5952905574514688", +} + +team { + name: "trendy_team_abarth_team", + + // go/trendy/manage/engineers/4857528786780160 + trendy_team_id: "4857528786780160", +} + +team { + name: "trendy_team_treble", + + // go/trendy/manage/engineers/5452490178691072 + trendy_team_id: "5452490178691072", +} + +team { + name: "trendy_team_jsasinowski_team", + + // go/trendy/manage/engineers/6239259762786304 + trendy_team_id: "6239259762786304", +} + +team { + name: "trendy_team_vaas_team", + + // go/trendy/manage/engineers/5106754296905728 + trendy_team_id: "5106754296905728", +} + +team { + name: "trendy_team_internationalization", + + // go/trendy/manage/engineers/5911536283287552 + trendy_team_id: "5911536283287552", +} + +team { + name: "trendy_team_android_safer_apps", + + // go/trendy/manage/engineers/5943179005034496 + trendy_team_id: "5943179005034496", +} + +team { + name: "trendy_team_connectivity_telemetry", + + // go/trendy/manage/engineers/5084491349393408 + trendy_team_id: "5084491349393408", +} + +team { + name: "trendy_team_eseidel_team", + + // go/trendy/manage/engineers/5453997738721280 + trendy_team_id: "5453997738721280", +} + +team { + name: "trendy_team_test_eng_infrastructure", + + // go/trendy/manage/engineers/5981905027465216 + trendy_team_id: "5981905027465216", +} + +team { + name: "trendy_team_wear_wear_wcs_notification", + + // go/trendy/manage/engineers/4805871527690240 + trendy_team_id: "4805871527690240", +} + +team { + name: "trendy_team_konkers_team", + + // go/trendy/manage/engineers/5751147701895168 + trendy_team_id: "5751147701895168", +} + +team { + name: "trendy_team_mkearney_team", + + // go/trendy/manage/engineers/5082590844452864 + trendy_team_id: "5082590844452864", +} + +team { + name: "trendy_team_android_kernel", + + // go/trendy/manage/engineers/5014334795022336 + trendy_team_id: "5014334795022336", +} + +team { + name: "trendy_team_chrome", + + // go/trendy/manage/engineers/6439301864620032 + trendy_team_id: "6439301864620032", +} + +team { + name: "trendy_team_wear_wear_dialer_messages", + + // go/trendy/manage/engineers/4906732878725120 + trendy_team_id: "4906732878725120", +} + +team { + name: "trendy_team_android_tv_engprod", + + // go/trendy/manage/engineers/5538081157185536 + trendy_team_id: "5538081157185536", +} + +team { + name: "trendy_team_nicoh_team", + + // go/trendy/manage/engineers/5662292009189376 + trendy_team_id: "5662292009189376", +} + +team { + name: "trendy_team_wear_apps_ecosystem", + + // go/trendy/manage/engineers/4908413871882240 + trendy_team_id: "4908413871882240", +} + +team { + name: "trendy_team_cellular_security", + + // go/trendy/manage/engineers/6529004011683840 + trendy_team_id: "6529004011683840", +} + +team { + name: "trendy_team_dhaloni_team", + + // go/trendy/manage/engineers/6213556497448960 + trendy_team_id: "6213556497448960", +} + +team { + name: "trendy_team_applications_google_wide_", + + // go/trendy/manage/engineers/6120993248378880 + trendy_team_id: "6120993248378880", +} + +team { + name: "trendy_team_defunct_system_ui_intelligence_dfeng", + + // go/trendy/manage/engineers/5577284748443648 + trendy_team_id: "5577284748443648", +} + +team { + name: "trendy_team_oslo", + + // go/trendy/manage/engineers/5779594887954432 + trendy_team_id: "5779594887954432", +} + +team { + name: "trendy_team_bluetooth", + + // go/trendy/manage/engineers/6226546364645376 + trendy_team_id: "6226546364645376", +} + +team { + name: "trendy_team_localization", + + // go/trendy/manage/engineers/5751557341446144 + trendy_team_id: "5751557341446144", +} + +team { + name: "trendy_team_ssd_security", + + // go/trendy/manage/engineers/4608065248559104 + trendy_team_id: "4608065248559104", +} + +team { + name: "trendy_team_pixel_system_sw_usb", + + // go/trendy/manage/engineers/5100646457802752 + trendy_team_id: "5100646457802752", +} + +team { + name: "trendy_team_hiroshi_team", + + // go/trendy/manage/engineers/5756564662288384 + trendy_team_id: "5756564662288384", +} + +team { + name: "trendy_team_contacts", + + // go/trendy/manage/engineers/4732859818311680 + trendy_team_id: "4732859818311680", +} + +team { + name: "trendy_team_wsd_w2", + + // go/trendy/manage/engineers/5945071387934720 + trendy_team_id: "5945071387934720", +} + +team { + name: "trendy_team_wsd_w51", + + // go/trendy/manage/engineers/5698780783312896 + trendy_team_id: "5698780783312896", +} + +team { + name: "trendy_team_erahm_team", + + // go/trendy/manage/engineers/5666347807309824 + trendy_team_id: "5666347807309824", +} + +team { + name: "trendy_team_wsd_w53", + + // go/trendy/manage/engineers/5841167539109888 + trendy_team_id: "5841167539109888", +} + +team { + name: "trendy_team_framework_overground", + + // go/trendy/manage/engineers/5135830829891584 + trendy_team_id: "5135830829891584", +} + +team { + name: "trendy_team_android_performance_console", + + // go/trendy/manage/engineers/5761662355963904 + trendy_team_id: "5761662355963904", +} + +team { + name: "trendy_team_partner_modem", + + // go/trendy/manage/engineers/6502710329376768 + trendy_team_id: "6502710329376768", +} + +team { + name: "trendy_team_scd_tool", + + // go/trendy/manage/engineers/5225441027555328 + trendy_team_id: "5225441027555328", +} + +team { + name: "trendy_team_gtw_sw", + + // go/trendy/manage/engineers/6069865957687296 + trendy_team_id: "6069865957687296", +} + +team { + name: "trendy_team_wearables", + + // go/trendy/manage/engineers/6122642515820544 + trendy_team_id: "6122642515820544", +} + +team { + name: "trendy_team_android_text", + + // go/trendy/manage/engineers/5194085585289216 + trendy_team_id: "5194085585289216", +} + +team { + name: "trendy_team_android_health", + + // go/trendy/manage/engineers/5177772706004992 + trendy_team_id: "5177772706004992", +} + +team { + name: "trendy_team_wsd_w23", + + // go/trendy/manage/engineers/6191361992556544 + trendy_team_id: "6191361992556544", +} + +team { + name: "trendy_team_pixel_connectivity_networking", + + // go/trendy/manage/engineers/6685592469241856 + trendy_team_id: "6685592469241856", +} + +team { + name: "trendy_team_ppi_team", + + // go/trendy/manage/engineers/5171933646848000 + trendy_team_id: "5171933646848000", +} + +team { + name: "trendy_team_ssd_mm_peripheral", + + // go/trendy/manage/engineers/6624019818086400 + trendy_team_id: "6624019818086400", +} + +team { + name: "trendy_team_n_a", + + // go/trendy/manage/engineers/4891189492711424 + trendy_team_id: "4891189492711424", +} + +team { + name: "trendy_team_pixel_system_sw_hid_driver", + + // go/trendy/manage/engineers/4534130425102336 + trendy_team_id: "4534130425102336", +} + +team { + name: "trendy_team_wear_wearable_motion_algorithms", + + // go/trendy/manage/engineers/5397550198587392 + trendy_team_id: "5397550198587392", +} + +team { + name: "trendy_team_wear_wear_sysui_notifications", + + // go/trendy/manage/engineers/5256257183055872 + trendy_team_id: "5256257183055872", +} + +team { + name: "trendy_team_android_camera_engprod", + + // go/trendy/manage/engineers/5594382843281408 + trendy_team_id: "5594382843281408", +} + +team { + name: "trendy_team_lockscreen_aod", + + // go/trendy/manage/engineers/5503979641012224 + trendy_team_id: "5503979641012224", +} + +team { + name: "trendy_team_windowing_sdk", + + // go/trendy/manage/engineers/5683037008723968 + trendy_team_id: "5683037008723968", +} + +team { + name: "trendy_team_pixel_system_sw_inmarket_power", + + // go/trendy/manage/engineers/6675891331170304 + trendy_team_id: "6675891331170304", +} + +team { + name: "trendy_team_betterbug", + + // go/trendy/manage/engineers/4910400652607488 + trendy_team_id: "4910400652607488", +} + +team { + name: "trendy_team_pixel_system_sw_security", + + // go/trendy/manage/engineers/5030277713625088 + trendy_team_id: "5030277713625088", +} + +team { + name: "trendy_team_pixel_energizer", + + // go/trendy/manage/engineers/4970605270302720 + trendy_team_id: "4970605270302720", +} + +team { + name: "trendy_team_fwk_core_networking", + + // go/trendy/manage/engineers/5559692562399232 + trendy_team_id: "5559692562399232", +} + +team { + name: "trendy_team_chromium_webview", + + // go/trendy/manage/engineers/5630061306576896 + trendy_team_id: "5630061306576896", +} + +team { + name: "trendy_team_framework_cdm", + + // go/trendy/manage/engineers/4793721887031296 + trendy_team_id: "4793721887031296", +} + +team { + name: "trendy_team_system_walleye_", + + // go/trendy/manage/engineers/5665245678665728 + trendy_team_id: "5665245678665728", +} + +team { + name: "trendy_team_system_marlin_sailfish_", + + // go/trendy/manage/engineers/4713618364825600 + trendy_team_id: "4713618364825600", +} + +team { + name: "trendy_team_qmc", + + // go/trendy/manage/engineers/5207848841510912 + trendy_team_id: "5207848841510912", +} + +team { + name: "trendy_team_android_wallet_integration", + + // go/trendy/manage/engineers/5785777995153408 + trendy_team_id: "5785777995153408", +} + +team { + name: "trendy_team_noreent_team", + + // go/trendy/manage/engineers/5766299843198976 + trendy_team_id: "5766299843198976", +} + +team { + name: "trendy_team_ink", + + // go/trendy/manage/engineers/6620225162608640 + trendy_team_id: "6620225162608640", +} + +team { + name: "trendy_team_make_pixel", + + // go/trendy/manage/engineers/6140234701864960 + trendy_team_id: "6140234701864960", +} + +team { + name: "trendy_team_chillers_team", + + // go/trendy/manage/engineers/5631647887294464 + trendy_team_id: "5631647887294464", +} + +team { + name: "trendy_team_system_experience", + + // go/trendy/manage/engineers/5083633521950720 + trendy_team_id: "5083633521950720", +} + +team { + name: "trendy_team_deprecated_framework_roosa", + + // go/trendy/manage/engineers/6708067074998272 + trendy_team_id: "6708067074998272", +} + +team { + name: "trendy_team_build", + + // go/trendy/manage/engineers/5542100376354816 + trendy_team_id: "5542100376354816", +} + +team { + name: "trendy_team_play_store", + + // go/trendy/manage/engineers/4803228562489344 + trendy_team_id: "4803228562489344", +} + +team { + name: "trendy_team_clocks", + + // go/trendy/manage/engineers/6327058391007232 + trendy_team_id: "6327058391007232", +} + +team { + name: "trendy_team_asafi_team", + + // go/trendy/manage/engineers/6217735399964672 + trendy_team_id: "6217735399964672", +} + +team { + name: "trendy_team_pixel_system_sw_storage", + + // go/trendy/manage/engineers/4644898888089600 + trendy_team_id: "4644898888089600", +} + +team { + name: "trendy_team_play_movies", + + // go/trendy/manage/engineers/4838412934578176 + trendy_team_id: "4838412934578176", +} + +team { + name: "trendy_team_system_hammerhead_camera_", + + // go/trendy/manage/engineers/6597631539019776 + trendy_team_id: "6597631539019776", +} + +team { + name: "trendy_team_wear_wear_sysui_applications", + + // go/trendy/manage/engineers/4929833494544384 + trendy_team_id: "4929833494544384", +} + +team { + name: "trendy_team_pixel_system_sw_tpm", + + // go/trendy/manage/engineers/4612922981122048 + trendy_team_id: "4612922981122048", +} + +team { + name: "trendy_team_qmc_script_automation", + + // go/trendy/manage/engineers/5047869899669504 + trendy_team_id: "5047869899669504", +} + +team { + name: "trendy_team_pixel_sw_tpm", + + // go/trendy/manage/engineers/5506916004265984 + trendy_team_id: "5506916004265984", +} + +team { + name: "trendy_team_device_and_factory_tpm", + + // go/trendy/manage/engineers/4574530143911936 + trendy_team_id: "4574530143911936", +} + +team { + name: "trendy_team_pmw_mss", + + // go/trendy/manage/engineers/4525262032896000 + trendy_team_id: "4525262032896000", +} + +team { + name: "trendy_team_wear_wear_esim_and_carriers", + + // go/trendy/manage/engineers/5045168113614848 + trendy_team_id: "5045168113614848", +} + +team { + name: "trendy_team_android_pixel_context_hub", + + // go/trendy/manage/engineers/5375970200944640 + trendy_team_id: "5375970200944640", +} + +team { + name: "trendy_team_pixel_setting_exp", + + // go/trendy/manage/engineers/5758010936295424 + trendy_team_id: "5758010936295424", +} + +team { + name: "trendy_team_defunct_system_ui_intelligence_praveenj", + + // go/trendy/manage/engineers/6648758829711360 + trendy_team_id: "6648758829711360", +} + +team { + name: "trendy_team_system_bullhead_", + + // go/trendy/manage/engineers/4592122329956352 + trendy_team_id: "4592122329956352", +} + +team { + name: "trendy_team_pixel_system_sw_fingerprint", + + // go/trendy/manage/engineers/5380181285568512 + trendy_team_id: "5380181285568512", +} + +team { + name: "trendy_team_android_sdlc", + + // go/trendy/manage/engineers/6492896504152064 + trendy_team_id: "6492896504152064", +} + +team { + name: "trendy_team_android_core_graphics_stack", + + // go/trendy/manage/engineers/5260625399644160 + trendy_team_id: "5260625399644160", +} + +team { + name: "trendy_team_accessibility_switch_access", + + // go/trendy/manage/engineers/6026869039857664 + trendy_team_id: "6026869039857664", +} + +team { + name: "trendy_team_sqa_make_pixel_", + + // go/trendy/manage/engineers/5610819853090816 + trendy_team_id: "5610819853090816", +} + +team { + name: "trendy_team_controls", + + // go/trendy/manage/engineers/5005994102259712 + trendy_team_id: "5005994102259712", +} + +team { + name: "trendy_team_renderscript_nnapi", + + // go/trendy/manage/engineers/6527262794842112 + trendy_team_id: "6527262794842112", +} + +team { + name: "trendy_team_test_infrastructure", + + // go/trendy/manage/engineers/5130189115654144 + trendy_team_id: "5130189115654144", +} + +team { + name: "trendy_team_ssd_peripheral", + + // go/trendy/manage/engineers/6314507294867456 + trendy_team_id: "6314507294867456", +} + +team { + name: "trendy_team_device_connectivity_experiences_make_pixel_", + + // go/trendy/manage/engineers/5348586329866240 + trendy_team_id: "5348586329866240", +} + +team { + name: "trendy_team_nandunair_team", + + // go/trendy/manage/engineers/4874500384129024 + trendy_team_id: "4874500384129024", +} + +team { + name: "trendy_team_godofredoc_team", + + // go/trendy/manage/engineers/4892528710156288 + trendy_team_id: "4892528710156288", +} + +team { + name: "trendy_team_gtw_misc", + + // go/trendy/manage/engineers/6437652597178368 + trendy_team_id: "6437652597178368", +} + +team { + name: "trendy_team_perception_virtualization", + + // go/trendy/manage/engineers/5133931925897216 + trendy_team_id: "5133931925897216", +} + +team { + name: "trendy_team_safety_els_earthquake", + + // go/trendy/manage/engineers/6508498165071872 + trendy_team_id: "6508498165071872", +} + +team { + name: "trendy_team_xr_framework", + + // go/trendy/manage/engineers/4798040542445568 + trendy_team_id: "4798040542445568", +} + +team { + name: "trendy_team_system_ui_please_use_a_more_specific_subteam_if_possible_", + + // go/trendy/manage/engineers/6402468225089536 + trendy_team_id: "6402468225089536", +} + +team { + name: "trendy_team_android_media_reliability", + + // go/trendy/manage/engineers/5489323818221568 + trendy_team_id: "5489323818221568", +} + +team { + name: "trendy_team_wear_wear_services", + + // go/trendy/manage/engineers/5140566757179392 + trendy_team_id: "5140566757179392", +} + +team { + name: "trendy_team_qmc_pda", + + // go/trendy/manage/engineers/5155072283377664 + trendy_team_id: "5155072283377664", +} + +team { + name: "trendy_team_vsl", + + // go/trendy/manage/engineers/6562447166930944 + trendy_team_id: "6562447166930944", +} + +team { + name: "trendy_team_android_release_metrics", + + // go/trendy/manage/engineers/6018925759201280 + trendy_team_id: "6018925759201280", +} + +team { + name: "trendy_team_testing", + + // go/trendy/manage/engineers/5892294829801472 + trendy_team_id: "5892294829801472", +} + +team { + name: "trendy_team_deprecated_wallet_integration", + + // go/trendy/manage/engineers/5286726042288128 + trendy_team_id: "5286726042288128", +} + +team { + name: "trendy_team_leonardchan_team", + + // go/trendy/manage/engineers/6260994579005440 + trendy_team_id: "6260994579005440", +} + +team { + name: "trendy_team_system_performance", + + // go/trendy/manage/engineers/5188607388024832 + trendy_team_id: "5188607388024832", +} + +team { + name: "trendy_team_system_power", + + // go/trendy/manage/engineers/4820820748533760 + trendy_team_id: "4820820748533760", +} + +team { + name: "trendy_team_deprecated_gdm_location_ads_marketplaces", + + // go/trendy/manage/engineers/5261636812570624 + trendy_team_id: "5261636812570624", +} + +team { + name: "trendy_team_pixel_system_sw_digital_key", + + // go/trendy/manage/engineers/6444896766033920 + trendy_team_id: "6444896766033920", +} + +team { + name: "trendy_team_aosp", + + // go/trendy/manage/engineers/4860855378018304 + trendy_team_id: "4860855378018304", +} + +team { + name: "trendy_team_launcher", + + // go/trendy/manage/engineers/5102295725244416 + trendy_team_id: "5102295725244416", +} + +team { + name: "trendy_team_ime", + + // go/trendy/manage/engineers/6085808876290048 + trendy_team_id: "6085808876290048", +} + +team { + name: "trendy_team_jyotiraju_team", + + // go/trendy/manage/engineers/5720977167253504 + trendy_team_id: "5720977167253504", +} + +team { + name: "trendy_team_camera", + + // go/trendy/manage/engineers/5718022236798976 + trendy_team_id: "5718022236798976", +} + +team { + name: "trendy_team_wear_wear_backup_restore", + + // go/trendy/manage/engineers/4875982171176960 + trendy_team_id: "4875982171176960", +} + +team { + name: "trendy_team_wear_wear_developer_watch_faces_complications", + + // go/trendy/manage/engineers/5638213037096960 + trendy_team_id: "5638213037096960", +} + +team { + name: "trendy_team_mainline_reach", + + // go/trendy/manage/engineers/5701386012098560 + trendy_team_id: "5701386012098560", +} + +team { + name: "trendy_team_ssd_bsp", + + // go/trendy/manage/engineers/5876351911198720 + trendy_team_id: "5876351911198720", +} + +team { + name: "trendy_team_ux_design", + + // go/trendy/manage/engineers/4678433992736768 + trendy_team_id: "4678433992736768", +} + +team { + name: "trendy_team_accessibility_fw_make_pixel", + + // go/trendy/manage/engineers/5522858922868736 + trendy_team_id: "5522858922868736", +} + +team { + name: "trendy_team_wear_wear_media", + + // go/trendy/manage/engineers/5365411545513984 + trendy_team_id: "5365411545513984", +} + +team { + name: "trendy_team_system_angler_", + + // go/trendy/manage/engineers/5593227667046400 + trendy_team_id: "5593227667046400", +} + +team { + name: "trendy_team_pixel_system_sw_tools", + + // go/trendy/manage/engineers/6460475572912128 + trendy_team_id: "6460475572912128", +} + +team { + name: "trendy_team_platform_build", + + // go/trendy/manage/engineers/5774403578920960 + trendy_team_id: "5774403578920960", +} + +team { + name: "trendy_team_pchitoor_team", + + // go/trendy/manage/engineers/5962577315266560 + trendy_team_id: "5962577315266560", +} + +team { + name: "trendy_team_multi_device_platform", + + // go/trendy/manage/engineers/5850153090711552 + trendy_team_id: "5850153090711552", +} + +team { + name: "trendy_team_safetynet", + + // go/trendy/manage/engineers/4748802736914432 + trendy_team_id: "4748802736914432", +} + +team { + name: "trendy_team_android_resources", + + // go/trendy/manage/engineers/4678767020703744 + trendy_team_id: "4678767020703744", +} + +team { + name: "trendy_team_joshconner_team", + + // go/trendy/manage/engineers/6226828248383488 + trendy_team_id: "6226828248383488", +} + +team { + name: "trendy_team_qmc_ait", + + // go/trendy/manage/engineers/6175419073953792 + trendy_team_id: "6175419073953792", +} + +team { + name: "trendy_team_pixel_global", + + // go/trendy/manage/engineers/4609714516000768 + trendy_team_id: "4609714516000768", +} + +team { + name: "trendy_team_qa_automation", + + // go/trendy/manage/engineers/6159878303678464 + trendy_team_id: "6159878303678464", +} + +team { + name: "trendy_team_android_gpu", + + // go/trendy/manage/engineers/6105848565104640 + trendy_team_id: "6105848565104640", +} + +team { + name: "trendy_team_qmc_ate", + + // go/trendy/manage/engineers/4819171481092096 + trendy_team_id: "4819171481092096", +} + +team { + name: "trendy_team_hangouts", + + // go/trendy/manage/engineers/6263380004175872 + trendy_team_id: "6263380004175872", +} + +team { + name: "trendy_team_cross_device_control", + + // go/trendy/manage/engineers/5888607197757440 + trendy_team_id: "5888607197757440", +} + +team { + name: "trendy_team_interactions_frameworks", + + // go/trendy/manage/engineers/4795124029489152 + trendy_team_id: "4795124029489152", +} + +team { + name: "trendy_team_deprecated_framework_santoscordon", + + // go/trendy/manage/engineers/6049242537951232 + trendy_team_id: "6049242537951232", +} + +team { + name: "trendy_team_authentication", + + // go/trendy/manage/engineers/5901909380988928 + trendy_team_id: "5901909380988928", +} + +team { + name: "trendy_team_stylus", + + // go/trendy/manage/engineers/5685003218747392 + trendy_team_id: "5685003218747392", +} + +team { + name: "trendy_team_linus_team", + + // go/trendy/manage/engineers/6210035100844032 + trendy_team_id: "6210035100844032", +} + +team { + name: "trendy_team_virtualization", + + // go/trendy/manage/engineers/5117131519787008 + trendy_team_id: "5117131519787008", +} + +team { + name: "trendy_team_billstevenson_team", + + // go/trendy/manage/engineers/5631064744820736 + trendy_team_id: "5631064744820736", +} + +team { + name: "trendy_team_android_smartos", + + // go/trendy/manage/engineers/5637973325414400 + trendy_team_id: "5637973325414400", +} + +team { + name: "trendy_team_art_performance", + + // go/trendy/manage/engineers/6210603446042624 + trendy_team_id: "6210603446042624", +} + +team { + name: "trendy_team_ssd", + + // go/trendy/manage/engineers/5858759725154304 + trendy_team_id: "5858759725154304", +} + +team { + name: "trendy_team_abc_engops", + + // go/trendy/manage/engineers/5273578928504832 + trendy_team_id: "5273578928504832", +} + +team { + name: "trendy_team_wear_weather_android_app", + + // go/trendy/manage/engineers/6496415568003072 + trendy_team_id: "6496415568003072", +} + +team { + name: "trendy_team_rubidium_sdk_runtime", + + // go/trendy/manage/engineers/6286508355911680 + trendy_team_id: "6286508355911680", +} + +team { + name: "trendy_team_borthakur_team", + + // go/trendy/manage/engineers/4962243059023872 + trendy_team_id: "4962243059023872", +} + +team { + name: "trendy_team_pixel_system_sw_graphics", + + // go/trendy/manage/engineers/5791644907110400 + trendy_team_id: "5791644907110400", +} + +team { + name: "trendy_team_make_creative_android_key_experience", + + // go/trendy/manage/engineers/5716738515173376 + trendy_team_id: "5716738515173376", +} + +team { + name: "trendy_team_pmw_l1rf", + + // go/trendy/manage/engineers/5302906915684352 + trendy_team_id: "5302906915684352", +} + +team { + name: "trendy_team_test_eng_comms_power", + + // go/trendy/manage/engineers/6632815911108608 + trendy_team_id: "6632815911108608", +} + +team { + name: "trendy_team_wear_wear_architecture_group", + + // go/trendy/manage/engineers/5609928060207104 + trendy_team_id: "5609928060207104", +} + +team { + name: "trendy_team_wear_wear_esim_carriers", + + // go/trendy/manage/engineers/5928361498935296 + trendy_team_id: "5928361498935296", +} + +team { + name: "trendy_team_pixel_connectivity_gps", + + // go/trendy/manage/engineers/4920660539899904 + trendy_team_id: "4920660539899904", +} + +team { + name: "trendy_team_adversarial_code_ai_arc_ai_", + + // go/trendy/manage/engineers/4850213657673728 + trendy_team_id: "4850213657673728", +} + +team { + name: "trendy_team_android_binary_transparency", + + // go/trendy/manage/engineers/6585365243002880 + trendy_team_id: "6585365243002880", +} + +team { + name: "trendy_team_test_eng_automotive_tv", + + // go/trendy/manage/engineers/6156177620467712 + trendy_team_id: "6156177620467712", +} + +team { + name: "trendy_team_tgosselaar_team", + + // go/trendy/manage/engineers/4897638077071360 + trendy_team_id: "4897638077071360", +} + +team { + name: "trendy_team_pixel_connectivity_wifi", + + // go/trendy/manage/engineers/4776219313340416 + trendy_team_id: "4776219313340416", +} + +team { + name: "trendy_team_setup_wizard", + + // go/trendy/manage/engineers/5417305806602240 + trendy_team_id: "5417305806602240", +} + +team { + name: "trendy_team_security_validation_engineering_sve_", + + // go/trendy/manage/engineers/5850943050907648 + trendy_team_id: "5850943050907648", +} + +team { + name: "trendy_team_wsd_function", + + // go/trendy/manage/engineers/6650408097153024 + trendy_team_id: "6650408097153024", +} + +team { + name: "trendy_team_platform_product_mgrs", + + // go/trendy/manage/engineers/6483282329731072 + trendy_team_id: "6483282329731072", +} + +team { + name: "trendy_team_partner_telephony", + + // go/trendy/manage/engineers/5767882120265728 + trendy_team_id: "5767882120265728", +} + +team { + name: "trendy_team_crjohns_team", + + // go/trendy/manage/engineers/4804101473992704 + trendy_team_id: "4804101473992704", +} + +team { + name: "trendy_team_wsd_ims", + + // go/trendy/manage/engineers/6050624504201216 + trendy_team_id: "6050624504201216", +} + +team { + name: "trendy_team_wear_wear_weather", + + // go/trendy/manage/engineers/5464164419928064 + trendy_team_id: "5464164419928064", +} + +team { + name: "trendy_team_guptaritu_team", + + // go/trendy/manage/engineers/5142679624777728 + trendy_team_id: "5142679624777728", +} + +team { + name: "trendy_team_wear_wear_developer_tools", + + // go/trendy/manage/engineers/6228915878002688 + trendy_team_id: "6228915878002688", +} + +team { + name: "trendy_team_play_books", + + // go/trendy/manage/engineers/5769149527490560 + trendy_team_id: "5769149527490560", +} + +team { + name: "trendy_team_melissadaniels_team", + + // go/trendy/manage/engineers/5715112926281728 + trendy_team_id: "5715112926281728", +} + +team { + name: "trendy_team_wear_shared_context_state", + + // go/trendy/manage/engineers/5329107344588800 + trendy_team_id: "5329107344588800", +} + +team { + name: "trendy_team_motion", + + // go/trendy/manage/engineers/6331351269277696 + trendy_team_id: "6331351269277696", +} + +team { + name: "trendy_team_gpp_on_device", + + // go/trendy/manage/engineers/5181961504980992 + trendy_team_id: "5181961504980992", +} + +team { + name: "trendy_team_android_settings_app", + + // go/trendy/manage/engineers/6204400884154368 + trendy_team_id: "6204400884154368", +} + +team { + name: "trendy_team_l1_inmarket", + + // go/trendy/manage/engineers/5172846450737152 + trendy_team_id: "5172846450737152", +} + +team { + name: "trendy_team_wear_wearflow", + + // go/trendy/manage/engineers/5947250429558784 + trendy_team_id: "5947250429558784", +} + +team { + name: "trendy_team_enterprise", + + // go/trendy/manage/engineers/5366178515910656 + trendy_team_id: "5366178515910656", +} + +team { + name: "trendy_team_deprecated_framework_michaelwr", + + // go/trendy/manage/engineers/4574277124915200 + trendy_team_id: "4574277124915200", +} + +team { + name: "trendy_team_color", + + // go/trendy/manage/engineers/6305208086724608 + trendy_team_id: "6305208086724608", +} + +team { + name: "trendy_team_osbornc_team", + + // go/trendy/manage/engineers/6319504650043392 + trendy_team_id: "6319504650043392", +} + +team { + name: "trendy_team_dialer", + + // go/trendy/manage/engineers/6595982271578112 + trendy_team_id: "6595982271578112", +} + +team { + name: "trendy_team_framework_bpm", + + // go/trendy/manage/engineers/5498119911243776 + trendy_team_id: "5498119911243776", +} + +team { + name: "trendy_team_wsd_core_team", + + // go/trendy/manage/engineers/6683943201800192 + trendy_team_id: "6683943201800192", +} + +team { + name: "trendy_team_pixel_connectivity_ril", + + // go/trendy/manage/engineers/4995093341536256 + trendy_team_id: "4995093341536256", +} + +team { + name: "trendy_team_perfetto", + + // go/trendy/manage/engineers/4783987109003264 + trendy_team_id: "4783987109003264", +} + +team { + name: "trendy_team_partner_devrel", + + // go/trendy/manage/engineers/4537696504381440 + trendy_team_id: "4537696504381440", +} + +team { + name: "trendy_team_fwk_thread_network", + + // go/trendy/manage/engineers/5094685775134720 + trendy_team_id: "5094685775134720", +} + +team { + name: "trendy_team_thatguy_team", + + // go/trendy/manage/engineers/5688369775542272 + trendy_team_id: "5688369775542272", +} + +team { + name: "trendy_team_finder", + + // go/trendy/manage/engineers/6492831025364992 + trendy_team_id: "6492831025364992", +} + +team { + name: "trendy_team_boot_time", + + // go/trendy/manage/engineers/6017089399554048 + trendy_team_id: "6017089399554048", +} + +team { + name: "trendy_team_wear_wcs_beto", + + // go/trendy/manage/engineers/4910945436336128 + trendy_team_id: "4910945436336128", +} + +team { + name: "trendy_team_responsible_apis", + + // go/trendy/manage/engineers/5651962854178816 + trendy_team_id: "5651962854178816", +} + +team { + name: "trendy_team_pixel_system_sw_corebsp", + + // go/trendy/manage/engineers/4626673699553280 + trendy_team_id: "4626673699553280", +} + +team { + name: "trendy_team_education", + + // go/trendy/manage/engineers/4889540225269760 + trendy_team_id: "4889540225269760", +} + +team { + name: "trendy_team_alarm_clock", + + // go/trendy/manage/engineers/6193011259998208 + trendy_team_id: "6193011259998208", +} + +team { + name: "trendy_team_pmw_tvc", + + // go/trendy/manage/engineers/4526288764960768 + trendy_team_id: "4526288764960768", +} + +team { + name: "trendy_team_deprecated_pixel_wifi", + + // go/trendy/manage/engineers/5741405413900288 + trendy_team_id: "5741405413900288", +} + +team { + name: "trendy_team_abdulla_team", + + // go/trendy/manage/engineers/6223585145421824 + trendy_team_id: "6223585145421824", +} + +team { + name: "trendy_team_hardware", + + // go/trendy/manage/engineers/5357382422888448 + trendy_team_id: "5357382422888448", +} + +team { + name: "trendy_team_status_bar", + + // go/trendy/manage/engineers/6329516043173888 + trendy_team_id: "6329516043173888", +} + +team { + name: "trendy_team_wear_wear_sysui_big_picture", + + // go/trendy/manage/engineers/5081356719521792 + trendy_team_id: "5081356719521792", +} + +team { + name: "trendy_team_wear_material_design_for_wearos", + + // go/trendy/manage/engineers/4792942333952000 + trendy_team_id: "4792942333952000", +} + +team { + name: "trendy_team_platform_security", + + // go/trendy/manage/engineers/5243033213599744 + trendy_team_id: "5243033213599744", +} + +team { + name: "trendy_team_llvm_and_toolchains", + + // go/trendy/manage/engineers/5990701120487424 + trendy_team_id: "5990701120487424", +} + +team { + name: "trendy_team_jmccandless_team", + + // go/trendy/manage/engineers/5227794226380800 + trendy_team_id: "5227794226380800", +} + +team { + name: "trendy_team_safety_center", + + // go/trendy/manage/engineers/5930273843609600 + trendy_team_id: "5930273843609600", +} + +team { + name: "trendy_team_wear_wear_systems_engineering_and_devices", + + // go/trendy/manage/engineers/5407847298793472 + trendy_team_id: "5407847298793472", +} + +team { + name: "trendy_team_framework_android_multiuser", + + // go/trendy/manage/engineers/5981525732392960 + trendy_team_id: "5981525732392960", +} + +team { + name: "trendy_team_deprecated_test2", + + // go/trendy/manage/engineers/4590958690074624 + trendy_team_id: "4590958690074624", +} + +team { + name: "trendy_team_qmc_iqt_tpe", + + // go/trendy/manage/engineers/6668000283197440 + trendy_team_id: "6668000283197440", +} + +team { + name: "trendy_team_pixel_system_sw_factory", + + // go/trendy/manage/engineers/6267032573739008 + trendy_team_id: "6267032573739008", +} + +team { + name: "trendy_team_automotive", + + // go/trendy/manage/engineers/5770798794932224 + trendy_team_id: "5770798794932224", +} + +team { + name: "trendy_team_camera_htc_lg_qualcomm", + + // go/trendy/manage/engineers/6332099480911872 + trendy_team_id: "6332099480911872", +} + +team { + name: "trendy_team_rkp_keystore", + + // go/trendy/manage/engineers/5634304374505472 + trendy_team_id: "5634304374505472", +} + +team { + name: "trendy_team_wear_wear_watch_faces", + + // go/trendy/manage/engineers/5885708195495936 + trendy_team_id: "5885708195495936", +} + +team { + name: "trendy_team_arc_app_compat", + + // go/trendy/manage/engineers/4811894441279488 + trendy_team_id: "4811894441279488", +} + +team { + name: "trendy_team_psohn_team", + + // go/trendy/manage/engineers/4852673947009024 + trendy_team_id: "4852673947009024", +} + +team { + name: "trendy_team_hollande_team", + + // go/trendy/manage/engineers/5356533186723840 + trendy_team_id: "5356533186723840", +} + +team { + name: "trendy_team_wear_wear_partner_programs_and_engineering_team", + + // go/trendy/manage/engineers/4934997571960832 + trendy_team_id: "4934997571960832", +} + +team { + name: "trendy_team_pixel_connectivity_nfc", + + // go/trendy/manage/engineers/5631272051965952 + trendy_team_id: "5631272051965952", +} + +team { + name: "trendy_team_wear_wear_ios_connectivity", + + // go/trendy/manage/engineers/4702455644192768 + trendy_team_id: "4702455644192768", +} + +team { + name: "trendy_team_arc_", + + // go/trendy/manage/engineers/4556937957867520 + trendy_team_id: "4556937957867520", +} + +team { + name: "trendy_team_android_one", + + // go/trendy/manage/engineers/6272176097198080 + trendy_team_id: "6272176097198080", +} + +team { + name: "trendy_team_ccherubino_team", + + // go/trendy/manage/engineers/4846471192150016 + trendy_team_id: "4846471192150016", +} + +team { + name: "trendy_team_deprecated_bluetooth_and_nfc", + + // go/trendy/manage/engineers/5259280851435520 + trendy_team_id: "5259280851435520", +} + +team { + name: "trendy_team_test_eng_afw_auth_location_camera_media", + + // go/trendy/manage/engineers/6298564376264704 + trendy_team_id: "6298564376264704", +} + +team { + name: "trendy_team_frousseau_team", + + // go/trendy/manage/engineers/5718296436572160 + trendy_team_id: "5718296436572160", +} + +team { + name: "trendy_team_pixel_system_sw_touch_haptic", + + // go/trendy/manage/engineers/6469264330096640 + trendy_team_id: "6469264330096640", +} + +team { + name: "trendy_team_partner_eng", + + // go/trendy/manage/engineers/5172664469422080 + trendy_team_id: "5172664469422080", +} + +team { + name: "trendy_team_dogfooders", + + // go/trendy/manage/engineers/4643249620647936 + trendy_team_id: "4643249620647936", +} + +team { + name: "trendy_team_system_external_", + + // go/trendy/manage/engineers/5558043294957568 + trendy_team_id: "5558043294957568", +} + +team { + name: "trendy_team_foundations", + + // go/trendy/manage/engineers/6216250952679424 + trendy_team_id: "6216250952679424", +} + +team { + name: "trendy_team_camera_framework", + + // go/trendy/manage/engineers/6455244783222784 + trendy_team_id: "6455244783222784", +} + +team { + name: "trendy_team_bugjuggler", + + // go/trendy/manage/engineers/6472836969267200 + trendy_team_id: "6472836969267200", +} + +team { + name: "trendy_team_cligh_team", + + // go/trendy/manage/engineers/6273778455314432 + trendy_team_id: "6273778455314432", +} + +team { + name: "trendy_team_android_testing_experiences", + + // go/trendy/manage/engineers/5653137056366592 + trendy_team_id: "5653137056366592", +} + +team { + name: "trendy_team_dx", + + // go/trendy/manage/engineers/5677977168281600 + trendy_team_id: "5677977168281600", +} + +team { + name: "trendy_team_framework_backstage_power", + + // go/trendy/manage/engineers/6314066964283392 + trendy_team_id: "6314066964283392", +} + +team { + name: "trendy_team_rlb_team", + + // go/trendy/manage/engineers/5206858878058496 + trendy_team_id: "5206858878058496", +} + +team { + name: "trendy_team_pixel_system_sw_pts", + + // go/trendy/manage/engineers/5553725663510528 + trendy_team_id: "5553725663510528", +} + +team { + name: "trendy_team_keir_team", + + // go/trendy/manage/engineers/5731700089978880 + trendy_team_id: "5731700089978880", +} + +team { + name: "trendy_team_system_taimen_", + + // go/trendy/manage/engineers/6421709678575616 + trendy_team_id: "6421709678575616", +} + +team { + name: "trendy_team_security_response", + + // go/trendy/manage/engineers/5031926981066752 + trendy_team_id: "5031926981066752", +} + +team { + name: "trendy_team_preload_safety", + + // go/trendy/manage/engineers/4584609580744704 + trendy_team_id: "4584609580744704", +} + +team { + name: "trendy_team_pixel_customizations_make_", + + // go/trendy/manage/engineers/5155643836366848 + trendy_team_id: "5155643836366848", +} + +team { + name: "trendy_team_tooltopia", + + // go/trendy/manage/engineers/5456472883101696 + trendy_team_id: "5456472883101696", +} + +team { + name: "trendy_team_accessibility_live_transcribe", + + // go/trendy/manage/engineers/6299695642345472 + trendy_team_id: "6299695642345472", +} + +team { + name: "trendy_team_trusty", + + // go/trendy/manage/engineers/5109319549616128 + trendy_team_id: "5109319549616128", +} + +team { + name: "trendy_team_amathes_team", + + // go/trendy/manage/engineers/5157715862257664 + trendy_team_id: "5157715862257664", +} + +team { + name: "trendy_team_wear_wear_developer_android_devrel_", + + // go/trendy/manage/engineers/5861820594028544 + trendy_team_id: "5861820594028544", +} + +team { + name: "trendy_team_overview", + + // go/trendy/manage/engineers/5071790575550464 + trendy_team_id: "5071790575550464", +} + +team { + name: "trendy_team_android_sensors", + + // go/trendy/manage/engineers/4776371090259968 + trendy_team_id: "4776371090259968", +} + +team { + name: "trendy_team_ui_toolkit", + + // go/trendy/manage/engineers/5638857399599104 + trendy_team_id: "5638857399599104", +} + +team { + name: "trendy_team_gesture_nav", + + // go/trendy/manage/engineers/6304405391310848 + trendy_team_id: "6304405391310848", +} + +team { + name: "trendy_team_qmc_wifi_storage", + + // go/trendy/manage/engineers/4924724597358592 + trendy_team_id: "4924724597358592", +} + +team { + name: "trendy_team_wsd_w11", + + // go/trendy/manage/engineers/5929128469331968 + trendy_team_id: "5929128469331968", +} + +team { + name: "trendy_team_fsamuel_team", + + // go/trendy/manage/engineers/5753514497310720 + trendy_team_id: "5753514497310720", +} + +team { + name: "trendy_team_pixel_haptic", + + // go/trendy/manage/engineers/5919013003493376 + trendy_team_id: "5919013003493376", +} + +team { + name: "trendy_team_pixel_retention", + + // go/trendy/manage/engineers/5647985290805248 + trendy_team_id: "5647985290805248", +} + +team { + name: "trendy_team_pixel_onboarding", + + // go/trendy/manage/engineers/5531340811960320 + trendy_team_id: "5531340811960320", +} + +team { + name: "trendy_team_wsd_standard", + + // go/trendy/manage/engineers/6296915108823040 + trendy_team_id: "6296915108823040", +} + +team { + name: "trendy_team_art_mainline", + + // go/trendy/manage/engineers/5733965155401728 + trendy_team_id: "5733965155401728", +} + +team { + name: "trendy_team_shade", + + // go/trendy/manage/engineers/5646715170226176 + trendy_team_id: "5646715170226176", +} + +team { + name: "trendy_team_gchips_compute_sw", + + // go/trendy/manage/engineers/6245787818131456 + trendy_team_id: "6245787818131456", +} + +team { + name: "trendy_team_haptics_framework", + + // go/trendy/manage/engineers/5895438509441024 + trendy_team_id: "5895438509441024", +} + +team { + name: "trendy_team_accessibility_braille", + + // go/trendy/manage/engineers/4992530205933568 + trendy_team_id: "4992530205933568", +} + +team { + name: "trendy_team_qmc_utd", + + // go/trendy/manage/engineers/5524508190310400 + trendy_team_id: "5524508190310400", +} + +team { + name: "trendy_team_android_on", + + // go/trendy/manage/engineers/4539345771823104 + trendy_team_id: "4539345771823104", +} + +team { + name: "trendy_team_fit", + + // go/trendy/manage/engineers/5628412039135232 + trendy_team_id: "5628412039135232", +} + +team { + name: "trendy_team_fwk_uwb", + + // go/trendy/manage/engineers/5983733408235520 + trendy_team_id: "5983733408235520", +} + +team { + name: "trendy_team_aidroid", + + // go/trendy/manage/engineers/4697675446222848 + trendy_team_id: "4697675446222848", +} + +team { + name: "trendy_team_deprecated_test1", + + // go/trendy/manage/engineers/6212752467460096 + trendy_team_id: "6212752467460096", +} + +team { + name: "trendy_team_wear_wear_releases_telemetry_and_analytics", + + // go/trendy/manage/engineers/5892057298010112 + trendy_team_id: "5892057298010112", +} + +team { + name: "trendy_team_context_hub", + + // go/trendy/manage/engineers/5080704467501056 + trendy_team_id: "5080704467501056", +} + +team { + name: "trendy_team_carrier_follow_up", + + // go/trendy/manage/engineers/6615223725064192 + trendy_team_id: "6615223725064192", +} + +team { + name: "trendy_team_accessibility_live_caption", + + // go/trendy/manage/engineers/4764529665409024 + trendy_team_id: "4764529665409024", +} + +team { + name: "trendy_team_pixel_system_sw_display", + + // go/trendy/manage/engineers/6261730736734208 + trendy_team_id: "6261730736734208", +} + +team { + name: "trendy_team_foundation_security_rust_pkvm_", + + // go/trendy/manage/engineers/5071354421084160 + trendy_team_id: "5071354421084160", +} + +team { + name: "trendy_team_pixel_repair_mode", + + // go/trendy/manage/engineers/6083775867813888 + trendy_team_id: "6083775867813888", +} + +team { + name: "trendy_team_capture_and_share", + + // go/trendy/manage/engineers/5644523746787328 + trendy_team_id: "5644523746787328", +} + +team { + name: "trendy_team_keep", + + // go/trendy/manage/engineers/5839518271668224 + trendy_team_id: "5839518271668224", +} + +team { + name: "trendy_team_aaos_security", + + // go/trendy/manage/engineers/6264394363076608 + trendy_team_id: "6264394363076608", +} + +team { + name: "trendy_team_zero_jank", + + // go/trendy/manage/engineers/4764874133897216 + trendy_team_id: "4764874133897216", +} + +team { + name: "trendy_team_android_unified_core_infrastructure", + + // go/trendy/manage/engineers/5842172961914880 + trendy_team_id: "5842172961914880", +} + +team { + name: "trendy_team_android_media_codec_framework", + + // go/trendy/manage/engineers/4943966050844672 + trendy_team_id: "4943966050844672", +} + +team { + name: "trendy_team_system_gn_", + + // go/trendy/manage/engineers/4785636376444928 + trendy_team_id: "4785636376444928", +} + +team { + name: "trendy_team_deprecated_android_auth_client", + + // go/trendy/manage/engineers/5471731632177152 + trendy_team_id: "5471731632177152", +} + +team { + name: "trendy_team_wear_wear_ios_companion_sdk", + + // go/trendy/manage/engineers/5737044865089536 + trendy_team_id: "5737044865089536", +} + +team { + name: "trendy_team_nearby", + + // go/trendy/manage/engineers/4959908969447424 + trendy_team_id: "4959908969447424", +} + +team { + name: "trendy_team_camerax_make_pixel_", + + // go/trendy/manage/engineers/4521753585778688 + trendy_team_id: "4521753585778688", +} + +team { + name: "trendy_team_wear_wear_health_services", + + // go/trendy/manage/engineers/6526182686097408 + trendy_team_id: "6526182686097408", +} + +team { + name: "trendy_team_media_volume", + + // go/trendy/manage/engineers/6360142070841344 + trendy_team_id: "6360142070841344", +} + +team { + name: "trendy_team_large_screen_experiences_sysui", + + // go/trendy/manage/engineers/5855214130069504 + trendy_team_id: "5855214130069504", +} + +team { + name: "trendy_team_mainline_engprod", + + // go/trendy/manage/engineers/5474634789847040 + trendy_team_id: "5474634789847040", +} + +team { + name: "trendy_team_wear_wear_power_foundations", + + // go/trendy/manage/engineers/6292909196214272 + trendy_team_id: "6292909196214272", +} + +team { + name: "trendy_team_windowing_tools", + + // go/trendy/manage/engineers/6382778382188544 + trendy_team_id: "6382778382188544", +} + +team { + name: "trendy_team_android_framework_appcompat", + + // go/trendy/manage/engineers/5383770701955072 + trendy_team_id: "5383770701955072", +} + +team { + name: "trendy_team_fitbit", + + // go/trendy/manage/engineers/6497885327360000 + trendy_team_id: "6497885327360000", +} + +team { + name: "trendy_team_overdrive", + + // go/trendy/manage/engineers/4961558236889088 + trendy_team_id: "4961558236889088", +} + +team { + name: "trendy_team_framework_android_packages", + + // go/trendy/manage/engineers/5989762407104512 + trendy_team_id: "5989762407104512", +} + +team { + name: "trendy_team_tkilbourn_team", + + // go/trendy/manage/engineers/4856646707871744 + trendy_team_id: "4856646707871744", +} + +team { + name: "trendy_team_large_screen_experiences_platform", + + // go/trendy/manage/engineers/4826462937317376 + trendy_team_id: "4826462937317376", +} + +team { + name: "trendy_team_pixel_system_service", + + // go/trendy/manage/engineers/5802643790135296 + trendy_team_id: "5802643790135296", +} + +team { + name: "trendy_team_android_developer_tools", + + // go/trendy/manage/engineers/6201807353020416 + trendy_team_id: "6201807353020416", +} + +team { + name: "trendy_team_autofill", + + // go/trendy/manage/engineers/4676203460329472 + trendy_team_id: "4676203460329472", +} + +team { + name: "trendy_team_wsd_w5", + + // go/trendy/manage/engineers/4627306702045184 + trendy_team_id: "4627306702045184", +} + +team { + name: "trendy_team_wear_wear_xfood_xwear_and_logistics", + + // go/trendy/manage/engineers/6599089311350784 + trendy_team_id: "6599089311350784", +} + +team { + name: "trendy_team_android_media_drm", + + // go/trendy/manage/engineers/5311752690335744 + trendy_team_id: "5311752690335744", +} + +team { + name: "trendy_team_nsylvain_team", + + // go/trendy/manage/engineers/5129062893912064 + trendy_team_id: "5129062893912064", +} + +team { + name: "trendy_team_fwk_wifi_hal", + + // go/trendy/manage/engineers/5470082364735488 + trendy_team_id: "5470082364735488", +} + +team { + name: "trendy_team_tombergan_team", + + // go/trendy/manage/engineers/5764031194497024 + trendy_team_id: "5764031194497024", +} + +team { + name: "trendy_team_wear_wear_platform_program_partner_eng_", + + // go/trendy/manage/engineers/5472261070815232 + trendy_team_id: "5472261070815232", +} + +team { + name: "trendy_team_masd_pixel_key_experiences", + + // go/trendy/manage/engineers/4873597306667008 + trendy_team_id: "4873597306667008", +} + +team { + name: "trendy_team_external", + + // go/trendy/manage/engineers/6033032318156800 + trendy_team_id: "6033032318156800", +} + +team { + name: "trendy_team_fwk_telephony", + + // go/trendy/manage/engineers/5663596411224064 + trendy_team_id: "5663596411224064", +} + +team { + name: "trendy_team_customization_picker", + + // go/trendy/manage/engineers/6311142173081600 + trendy_team_id: "6311142173081600", +} + +team { + name: "trendy_team_android_test_surfaces", + + // go/trendy/manage/engineers/4879149651099648 + trendy_team_id: "4879149651099648", +} + +team { + name: "trendy_team_instant_apps", + + // go/trendy/manage/engineers/6720776841330688 + trendy_team_id: "6720776841330688", +} + +team { + name: "trendy_team_accessibility_aas", + + // go/trendy/manage/engineers/6043198228299776 + trendy_team_id: "6043198228299776", +} + +team { + name: "trendy_team_android_engprod_lon", + + // go/trendy/manage/engineers/5432243163791360 + trendy_team_id: "5432243163791360", +} + +team { + name: "trendy_team_pmw_mce", + + // go/trendy/manage/engineers/6424723868909568 + trendy_team_id: "6424723868909568", +} + +team { + name: "trendy_team_pixel_system_sw_kernel", + + // go/trendy/manage/engineers/5436802711846912 + trendy_team_id: "5436802711846912", +} + +team { + name: "trendy_team_nga", + + // go/trendy/manage/engineers/5594876934488064 + trendy_team_id: "5594876934488064", +} + +team { + name: "trendy_team_web_on_android_performance", + + // go/trendy/manage/engineers/5864851748847616 + trendy_team_id: "5864851748847616", +} + +team { + name: "trendy_team_reveman_team", + + // go/trendy/manage/engineers/5113274057261056 + trendy_team_id: "5113274057261056", +} + +team { + name: "trendy_team_test_eng_ota_framework_sysui_suw_abvt_cts", + + // go/trendy/manage/engineers/4653694981111808 + trendy_team_id: "4653694981111808", +} + +team { + name: "trendy_team_backup_restore", + + // go/trendy/manage/engineers/5049519167111168 + trendy_team_id: "5049519167111168", +} + +team { + name: "trendy_team_pixel_system_sw_bringup_and_factory", + + // go/trendy/manage/engineers/5999752665268224 + trendy_team_id: "5999752665268224", +} + +team { + name: "trendy_team_wear_wear_calling_messaging", + + // go/trendy/manage/engineers/5401274807648256 + trendy_team_id: "5401274807648256", +} + +team { + name: "trendy_team_android_imaging", + + // go/trendy/manage/engineers/5838538113384448 + trendy_team_id: "5838538113384448", +} + +team { + name: "trendy_team_pixel_system_sw_sensor", + + // go/trendy/manage/engineers/4904417557643264 + trendy_team_id: "4904417557643264", +} + +team { + name: "trendy_team_dogfood_triage", + + // go/trendy/manage/engineers/5130823981236224 + trendy_team_id: "5130823981236224", +} + +team { + name: "trendy_team_input_method_framework", + + // go/trendy/manage/engineers/6394201770459136 + trendy_team_id: "6394201770459136", +} + +team { + name: "trendy_team_wear_wear_platform_dev_lead_device_program_", + + // go/trendy/manage/engineers/4642393194135552 + trendy_team_id: "4642393194135552", +} + +team { + name: "trendy_team_wear_bona_companion", + + // go/trendy/manage/engineers/4721932784009216 + trendy_team_id: "4721932784009216", +} + +team { + name: "trendy_team_wear_wear_compose", + + // go/trendy/manage/engineers/4958404271308800 + trendy_team_id: "4958404271308800", +} + +team { + name: "trendy_team_system_fugu_", + + // go/trendy/manage/engineers/5682837864710144 + trendy_team_id: "5682837864710144", +} + +team { + name: "trendy_team_tvolkert_team", + + // go/trendy/manage/engineers/5093014696525824 + trendy_team_id: "5093014696525824", +} + +team { + name: "trendy_team_media_framework_drm", + + // go/trendy/manage/engineers/5311752690335744 + trendy_team_id: "5311752690335744", +} + +team { + name: "trendy_team_media_framework_audio", + + // go/trendy/manage/engineers/5823575353065472 + trendy_team_id: "5823575353065472", +} + +team { + name: "trendy_team_pixel_pearl", + + // go/trendy/manage/engineers/6326219602231296 + trendy_team_id: "6326219602231296", +} + +team { + name: "trendy_team_ar_sensors_context_hub", + + // go/trendy/manage/engineers/4776371090259968 + trendy_team_id: "4776371090259968", +} + +team { + name: "trendy_team_media_codec_framework", + + // go/trendy/manage/engineers/4943966050844672 + trendy_team_id: "4943966050844672", +} + +team { + name: "trendy_team_android_platform_performance_testing", + + // go/trendy/manage/engineers/5810097836621824 + trendy_team_id: "5810097836621824", +} + +team { + name: "trendy_team_adte", + + // go/trendy/manage/engineers/5551098528825344 + trendy_team_id: "5551098528825344", +} + +team { + name: "trendy_team_incremental", + + // go/trendy/manage/engineers/5955405559201792 + trendy_team_id: "5955405559201792", +} diff --git a/teams/OWNERS b/teams/OWNERS new file mode 100644 index 0000000000..85e69f356b --- /dev/null +++ b/teams/OWNERS @@ -0,0 +1,3 @@ +dariofreni@google.com +ronish@google.com +caditya@google.com diff --git a/tests/Android.bp b/tests/Android.bp new file mode 100644 index 0000000000..39debf5c6d --- /dev/null +++ b/tests/Android.bp @@ -0,0 +1,42 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], + default_team: "trendy_team_build", +} + +python_test_host { + name: "run_tool_with_logging_test", + main: "run_tool_with_logging_test.py", + pkg_path: "testdata", + srcs: [ + "run_tool_with_logging_test.py", + ], + test_options: { + unit_test: true, + }, + data: [ + ":envsetup_minimum.zip", + ":tool_event_logger", + ], + test_suites: [ + "general-tests", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, +} diff --git a/tests/b_tests.sh b/tests/b_tests.sh index 13f156d712..68a13e3b9c 100755 --- a/tests/b_tests.sh +++ b/tests/b_tests.sh @@ -23,7 +23,12 @@ lunch aosp_arm64 test_target=//build/bazel/scripts/difftool:difftool +if b build //build/bazel:nonexistent_module &>/dev/null ; then + echo "b did not fail when building a nonexistent module" >&2 + exit 1 +fi b build "$test_target" +b build -- "$test_target" b build "$test_target" --run-soong-tests b build --run-soong-tests "$test_target" b --run-soong-tests build "$test_target" diff --git a/tests/lunch_tests.sh b/tests/lunch_tests.sh index 4285d13cc4..9b142ee6ab 100755 --- a/tests/lunch_tests.sh +++ b/tests/lunch_tests.sh @@ -28,7 +28,7 @@ function check_lunch [ "$TARGET_PLATFORM_VERSION" = "$4" ] || ( echo "lunch $1: expected TARGET_PLATFORM_VERSION='$4', got '$TARGET_PLATFORM_VERSION'" && exit 1 ) ) -default_version=$(get_build_var DEFAULT_PLATFORM_VERSION) +default_version=$(get_build_var RELEASE_PLATFORM_VERSION) # lunch tests check_lunch "aosp_arm64" "aosp_arm64" "eng" "" diff --git a/tests/product.rbc b/tests/product.rbc index 9ae6393e8e..b4c6d45ced 100644 --- a/tests/product.rbc +++ b/tests/product.rbc @@ -54,6 +54,7 @@ def init(g, handle): rblf.soong_config_append(g, "NS1", "v2", "def") rblf.soong_config_set(g, "NS2", "v3", "abc") rblf.soong_config_set(g, "NS2", "v3", "xyz") + rblf.soong_config_set(g, "NS2", "v4", "xyz ") rblf.mkdist_for_goals(g, "goal", "dir1/file1:out1 dir1/file2:out2") rblf.mkdist_for_goals(g, "goal", "dir2/file2:") diff --git a/tests/run.rbc b/tests/run.rbc index 33583eb3cd..221b40f04d 100644 --- a/tests/run.rbc +++ b/tests/run.rbc @@ -26,6 +26,7 @@ load(":product.rbc", "init") load(":board.rbc", board_init = "init") load(":board_input_vars.rbc", board_input_vars_init = "init") load("//build/make/tests/single_value_inheritance:test.rbc", test_single_value_inheritance = "test") +load("//build/make/tests/single_value_inheritance_2:test.rbc", test_single_value_inheritance_2 = "test") load("//build/make/tests/artifact_path_requirements:test.rbc", test_artifact_path_requirements = "test") load("//build/make/tests/prefixed_sort_order:test.rbc", test_prefixed_sort_order = "test") load("//build/make/tests/inherits_in_regular_variables:test.rbc", test_inherits_in_regular_variables = "test") @@ -144,7 +145,8 @@ assert_eq( "v2": "def" }, "NS2": { - "v3": "xyz" + "v3": "xyz", + "v4": "xyz" } }, {k:v for k, v in sorted(ns.items()) } @@ -180,6 +182,7 @@ assert_eq("f", cfg["BAZ"]) assert_eq("", g.get("NEWVAR")) test_single_value_inheritance() +test_single_value_inheritance_2() test_artifact_path_requirements() test_prefixed_sort_order() test_inherits_in_regular_variables() diff --git a/tests/run_tool_with_logging_test.py b/tests/run_tool_with_logging_test.py new file mode 100644 index 0000000000..18abd8e54f --- /dev/null +++ b/tests/run_tool_with_logging_test.py @@ -0,0 +1,332 @@ +# Copyright 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import glob +from importlib import resources +import logging +import os +from pathlib import Path +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +import zipfile + +EXII_RETURN_CODE = 0 +INTERRUPTED_RETURN_CODE = 130 + + +class RunToolWithLoggingTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Configure to print logging to stdout. + logging.basicConfig(filename=None, level=logging.DEBUG) + console = logging.StreamHandler(sys.stdout) + logging.getLogger("").addHandler(console) + + def setUp(self): + super().setUp() + self.working_dir = tempfile.TemporaryDirectory() + # Run all the tests from working_dir which is our temp Android build top. + os.chdir(self.working_dir.name) + # Extract envsetup.zip which contains the envsetup.sh and other dependent + # scripts required to set up the build environments. + with resources.files("testdata").joinpath("envsetup.zip").open("rb") as p: + with zipfile.ZipFile(p, "r") as zip_f: + zip_f.extractall() + + def tearDown(self): + self.working_dir.cleanup() + super().tearDown() + + def test_does_not_log_when_logger_var_empty(self): + test_tool = TestScript.create(self.working_dir) + + self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + test_tool.assert_called_once_with_args("arg1 arg2") + + def test_does_not_log_with_logger_unset(self): + test_tool = TestScript.create(self.working_dir) + + self._run_script_and_wait(f""" + unset ANDROID_TOOL_LOGGER + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + test_tool.assert_called_once_with_args("arg1 arg2") + + def test_log_success_with_logger_enabled(self): + test_tool = TestScript.create(self.working_dir) + test_logger = TestScript.create(self.working_dir) + + self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + test_tool.assert_called_once_with_args("arg1 arg2") + expected_logger_args = ( + "--tool_tag FAKE_TOOL --start_timestamp \d+\.\d+ --end_timestamp" + " \d+\.\d+ --tool_args arg1 arg2 --exit_code 0" + ) + test_logger.assert_called_once_with_args(expected_logger_args) + + def test_run_tool_output_is_same_with_and_without_logging(self): + test_tool = TestScript.create(self.working_dir, "echo 'tool called'") + test_logger = TestScript.create(self.working_dir) + + run_tool_with_logging_stdout, run_tool_with_logging_stderr = ( + self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + ) + + run_tool_without_logging_stdout, run_tool_without_logging_stderr = ( + self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + {test_tool.executable} arg1 arg2 + """) + ) + + self.assertEqual( + run_tool_with_logging_stdout, run_tool_without_logging_stdout + ) + self.assertEqual( + run_tool_with_logging_stderr, run_tool_without_logging_stderr + ) + + def test_logger_output_is_suppressed(self): + test_tool = TestScript.create(self.working_dir) + test_logger = TestScript.create(self.working_dir, "echo 'logger called'") + + run_tool_with_logging_output, _ = self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + self.assertNotIn("logger called", run_tool_with_logging_output) + + def test_logger_error_is_suppressed(self): + test_tool = TestScript.create(self.working_dir) + test_logger = TestScript.create( + self.working_dir, "echo 'logger failed' > /dev/stderr; exit 1" + ) + + _, err = self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + self.assertNotIn("logger failed", err) + + def test_log_success_when_tool_interrupted(self): + test_tool = TestScript.create(self.working_dir, script_body="sleep 100") + test_logger = TestScript.create(self.working_dir) + + process = self._run_script_in_build_env(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + pgid = os.getpgid(process.pid) + # Give sometime for the subprocess to start. + time.sleep(1) + # Kill the subprocess and any processes created in the same group. + os.killpg(pgid, signal.SIGINT) + + returncode, _, _ = self._wait_for_process(process) + self.assertEqual(returncode, INTERRUPTED_RETURN_CODE) + + expected_logger_args = ( + "--tool_tag FAKE_TOOL --start_timestamp \d+\.\d+ --end_timestamp" + " \d+\.\d+ --tool_args arg1 arg2 --exit_code 130" + ) + test_logger.assert_called_once_with_args(expected_logger_args) + + def test_logger_can_be_toggled_on(self): + test_tool = TestScript.create(self.working_dir) + test_logger = TestScript.create(self.working_dir) + + self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + test_logger.assert_called_with_times(1) + + def test_logger_can_be_toggled_off(self): + test_tool = TestScript.create(self.working_dir) + test_logger = TestScript.create(self.working_dir) + + self._run_script_and_wait(f""" + ANDROID_TOOL_LOGGER="{test_logger.executable}" + ANDROID_TOOL_LOGGER="" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + test_logger.assert_not_called() + + def test_integration_tool_event_logger_dry_run(self): + test_tool = TestScript.create(self.working_dir) + logger_path = self._import_logger() + + self._run_script_and_wait(f""" + TMPDIR="{self.working_dir.name}" + ANDROID_TOOL_LOGGER="{logger_path}" + ANDROID_TOOL_LOGGER_EXTRA_ARGS="--dry_run" + run_tool_with_logging "FAKE_TOOL" {test_tool.executable} arg1 arg2 + """) + + self._assert_logger_dry_run() + + def _import_logger(self) -> Path: + logger = "tool_event_logger" + logger_path = Path(self.working_dir.name).joinpath(logger) + with resources.as_file(resources.files("testdata").joinpath(logger)) as p: + shutil.copy(p, logger_path) + Path.chmod(logger_path, 0o755) + return logger_path + + def _assert_logger_dry_run(self): + log_files = glob.glob(self.working_dir.name + "/tool_event_logger_*/*.log") + self.assertEqual(len(log_files), 1) + + with open(log_files[0], "r") as f: + lines = f.readlines() + self.assertEqual(len(lines), 1) + self.assertIn("dry run", lines[0]) + + def _create_build_env_script(self) -> str: + return f""" + source {Path(self.working_dir.name).joinpath("build/make/envsetup.sh")} + """ + + def _run_script_and_wait(self, test_script: str) -> tuple[str, str]: + process = self._run_script_in_build_env(test_script) + returncode, out, err = self._wait_for_process(process) + logging.debug("script stdout: %s", out) + logging.debug("script stderr: %s", err) + self.assertEqual(returncode, EXII_RETURN_CODE) + return out, err + + def _run_script_in_build_env(self, test_script: str) -> subprocess.Popen: + setup_build_env_script = self._create_build_env_script() + return subprocess.Popen( + setup_build_env_script + test_script, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + executable="/bin/bash", + ) + + def _wait_for_process( + self, process: subprocess.Popen + ) -> tuple[int, str, str]: + pgid = os.getpgid(process.pid) + out, err = process.communicate() + # Wait for all process in the same group to complete since the logger runs + # as a separate detached process. + self._wait_for_process_group(pgid) + return (process.returncode, out, err) + + def _wait_for_process_group(self, pgid: int, timeout: int = 5): + """Waits for all subprocesses within the process group to complete.""" + start_time = time.time() + while True: + if time.time() - start_time > timeout: + raise TimeoutError( + f"Process group did not complete after {timeout} seconds" + ) + for pid in os.listdir("/proc"): + if pid.isdigit(): + try: + if os.getpgid(int(pid)) == pgid: + time.sleep(0.1) + break + except (FileNotFoundError, PermissionError, ProcessLookupError): + pass + else: + # All processes have completed. + break + + +@dataclasses.dataclass +class TestScript: + executable: Path + output_file: Path + + def create(temp_dir: Path, script_body: str = ""): + with tempfile.NamedTemporaryFile(dir=temp_dir.name, delete=False) as f: + output_file = f.name + + with tempfile.NamedTemporaryFile(dir=temp_dir.name, delete=False) as f: + executable = f.name + executable_contents = textwrap.dedent(f""" + #!/bin/bash + + echo "${{@}}" >> {output_file} + {script_body} + """) + f.write(executable_contents.encode("utf-8")) + + Path.chmod(f.name, os.stat(f.name).st_mode | stat.S_IEXEC) + + return TestScript(executable, output_file) + + def assert_called_with_times(self, expected_call_times: int): + lines = self._read_contents_from_output_file() + assert len(lines) == expected_call_times, ( + f"Expect to call {expected_call_times} times, but actually called" + f" {len(lines)} times." + ) + + def assert_called_with_args(self, expected_args: str): + lines = self._read_contents_from_output_file() + assert len(lines) > 0 + assert re.search(expected_args, lines[0]), ( + f"Expect to call with args {expected_args}, but actually called with" + f" args {lines[0]}." + ) + + def assert_not_called(self): + self.assert_called_with_times(0) + + def assert_called_once_with_args(self, expected_args: str): + self.assert_called_with_times(1) + self.assert_called_with_args(expected_args) + + def _read_contents_from_output_file(self) -> list[str]: + with open(self.output_file, "r") as f: + return f.readlines() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/single_value_inheritance_2/a.rbc b/tests/single_value_inheritance_2/a.rbc new file mode 100644 index 0000000000..fe186c7e84 --- /dev/null +++ b/tests/single_value_inheritance_2/a.rbc @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + + cfg["PRODUCT_ENABLE_UFFD_GC"] = "true" diff --git a/tests/single_value_inheritance_2/b.rbc b/tests/single_value_inheritance_2/b.rbc new file mode 100644 index 0000000000..7d95749aa4 --- /dev/null +++ b/tests/single_value_inheritance_2/b.rbc @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + + cfg["PRODUCT_ENABLE_UFFD_GC"] = "default" diff --git a/tests/single_value_inheritance_2/c.rbc b/tests/single_value_inheritance_2/c.rbc new file mode 100644 index 0000000000..e90e37d318 --- /dev/null +++ b/tests/single_value_inheritance_2/c.rbc @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:product_config.rbc", "rblf") +load(":b.rbc", _b_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + + rblf.inherit(handle, "test/b", _b_init) diff --git a/tests/single_value_inheritance_2/d.rbc b/tests/single_value_inheritance_2/d.rbc new file mode 100644 index 0000000000..3a88c2c17f --- /dev/null +++ b/tests/single_value_inheritance_2/d.rbc @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:product_config.rbc", "rblf") +load(":c.rbc", _c_init = "init") +load(":a.rbc", _a_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + + rblf.inherit(handle, "test/a", _a_init) + rblf.inherit(handle, "test/c", _c_init) diff --git a/tests/single_value_inheritance_2/product.rbc b/tests/single_value_inheritance_2/product.rbc new file mode 100644 index 0000000000..c47664db16 --- /dev/null +++ b/tests/single_value_inheritance_2/product.rbc @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:product_config.rbc", "rblf") +load(":b.rbc", _b_init = "init") +load(":d.rbc", _d_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + + rblf.inherit(handle, "test/b", _b_init) + rblf.inherit(handle, "test/d", _d_init) diff --git a/tests/single_value_inheritance_2/test.rbc b/tests/single_value_inheritance_2/test.rbc new file mode 100644 index 0000000000..fa93aaa51e --- /dev/null +++ b/tests/single_value_inheritance_2/test.rbc @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:product_config.rbc", "rblf") +load("//build/make/tests/input_variables.rbc", input_variables_init = "init") +load(":product.rbc", "init") + + +def assert_eq(expected, actual): + if expected != actual: + fail("Expected '%s', got '%s'" % (expected, actual)) + +# This test is testing that single value variables are "stolen" when processing the inheritance +# graph. i.e. if you have a graph like this: +# +# B A +# |\ | +# | C | +# \ \| +# \ D +# \| +# E +# +# The same variable is defined in both A and B. In D, the value from A is chosen because it comes +# alphabetically before C. But then in E, the value from D is chosen instead of the value from B, +# because the value of B was "stolen" and sucked into C, leaving B with no value set. +def test(): + (globals, globals_base) = rblf.product_configuration("test/device", init, input_variables_init) + assert_eq("true", globals["PRODUCTS.test/device.mk.PRODUCT_ENABLE_UFFD_GC"]) diff --git a/tools/Android.bp b/tools/Android.bp index f446973bbc..59831a61ec 100644 --- a/tools/Android.bp +++ b/tools/Android.bp @@ -18,54 +18,108 @@ package { } python_binary_host { - name: "generate-self-extracting-archive", - srcs: ["generate-self-extracting-archive.py"], + name: "generate-self-extracting-archive", + srcs: ["generate-self-extracting-archive.py"], } python_binary_host { - name: "post_process_props", - srcs: ["post_process_props.py"], + name: "post_process_props", + srcs: ["post_process_props.py"], + libs: [ + "uffd_gc_utils", + ], } python_test_host { - name: "post_process_props_unittest", - main: "test_post_process_props.py", - srcs: [ - "post_process_props.py", - "test_post_process_props.py", - ], - test_config: "post_process_props_unittest.xml", - test_suites: ["general-tests"], + name: "post_process_props_unittest", + main: "test_post_process_props.py", + srcs: [ + "post_process_props.py", + "test_post_process_props.py", + ], + libs: [ + "uffd_gc_utils", + ], + test_config: "post_process_props_unittest.xml", + test_suites: ["general-tests"], } python_binary_host { - name: "extract_kernel", - srcs: ["extract_kernel.py"], + name: "extract_kernel", + srcs: ["extract_kernel.py"], } genrule_defaults { - name: "extract_kernel_release_defaults", - tools: ["extract_kernel", "lz4"], - out: ["kernel_release.txt"], - cmd: "$(location) --tools lz4:$(location lz4) --input $(in) --output-release > $(out)" + name: "extract_kernel_release_defaults", + tools: [ + "extract_kernel", + "lz4", + ], + out: ["kernel_release.txt"], + cmd: "$(location) --tools lz4:$(location lz4) --input $(in) --output-release > $(out)", } cc_binary_host { - name: "build-runfiles", - srcs: ["build-runfiles.cc"], + name: "build-runfiles", + srcs: ["build-runfiles.cc"], } python_binary_host { - name: "check_radio_versions", - srcs: ["check_radio_versions.py"], + name: "check_radio_versions", + srcs: ["check_radio_versions.py"], } python_binary_host { - name: "check_elf_file", - srcs: ["check_elf_file.py"], + name: "check_elf_file", + srcs: ["check_elf_file.py"], } python_binary_host { - name: "generate_gts_shared_report", - srcs: ["generate_gts_shared_report.py"], + name: "generate_gts_shared_report", + srcs: ["generate_gts_shared_report.py"], +} + +python_binary_host { + name: "list_files", + main: "list_files.py", + srcs: [ + "list_files.py", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, +} + +python_test_host { + name: "auto_gen_test_config_test", + main: "auto_gen_test_config_test.py", + srcs: [ + "auto_gen_test_config.py", + "auto_gen_test_config_test.py", + ], + auto_gen_config: true, + test_suites: ["general-tests"], + test_options: { + unit_test: true, + }, +} + +python_binary_host { + name: "characteristics_rro_generator", + srcs: ["characteristics_rro_generator.py"], + version: { + py3: { + embedded_launcher: true, + }, + }, +} + +python_binary_host { + name: "merge-event-log-tags", + srcs: [ + "event_log_tags.py", + "merge-event-log-tags.py", + ], } diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 0de178b709..9ec0dcef85 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,6 +1,7 @@ py_library( name = "event_log_tags", srcs = ["event_log_tags.py"], + imports = ["."], ) py_binary( @@ -25,3 +26,10 @@ py_binary( python_version = "PY3", visibility = ["//visibility:public"], ) + +py_binary( + name = "auto_gen_test_config", + srcs = ["auto_gen_test_config.py"], + python_version = "PY3", + visibility = ["//visibility:public"], +) diff --git a/tools/OWNERS b/tools/OWNERS deleted file mode 100644 index 7d666f1687..0000000000 --- a/tools/OWNERS +++ /dev/null @@ -1 +0,0 @@ -per-file warn.py,checkowners.py = chh@google.com diff --git a/tools/aconfig/.gitignore b/tools/aconfig/.gitignore new file mode 100644 index 0000000000..1b72444aef --- /dev/null +++ b/tools/aconfig/.gitignore @@ -0,0 +1,2 @@ +/Cargo.lock +/target diff --git a/tools/aconfig/Cargo.toml b/tools/aconfig/Cargo.toml new file mode 100644 index 0000000000..bf5e1a9bc4 --- /dev/null +++ b/tools/aconfig/Cargo.toml @@ -0,0 +1,14 @@ +[workspace] + +members = [ + "aconfig", + "aconfig_device_paths", + "aconfig_protos", + "aconfig_storage_file", + "aconfig_storage_read_api", + "aconfig_storage_write_api", + "aflags", + "printflags" +] + +resolver = "2" diff --git a/tools/aconfig/MODULE_LICENSE_APACHE2 b/tools/aconfig/MODULE_LICENSE_APACHE2 new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/aconfig/MODULE_LICENSE_APACHE2 diff --git a/tools/aconfig/OWNERS b/tools/aconfig/OWNERS new file mode 100644 index 0000000000..9a76279cce --- /dev/null +++ b/tools/aconfig/OWNERS @@ -0,0 +1,7 @@ +amhk@google.com +dzshen@google.com +jham@google.com +joeo@google.com +opg@google.com +tedbauer@google.com +zhidou@google.com diff --git a/tools/aconfig/PREUPLOAD.cfg b/tools/aconfig/PREUPLOAD.cfg new file mode 100644 index 0000000000..75ed57ce3e --- /dev/null +++ b/tools/aconfig/PREUPLOAD.cfg @@ -0,0 +1,5 @@ +[Builtin Hooks] +rustfmt = true + +[Builtin Hooks Options] +rustfmt = --config-path=rustfmt.toml diff --git a/tools/aconfig/TEST_MAPPING b/tools/aconfig/TEST_MAPPING new file mode 100644 index 0000000000..421e94a8a6 --- /dev/null +++ b/tools/aconfig/TEST_MAPPING @@ -0,0 +1,105 @@ +{ + "presubmit": [ + { + // aconfig unit tests + "name": "aconfig.test" + }, + { + // aconfig Java integration tests (host) + "name": "AconfigJavaHostTest" + }, + { + // aconfig Java integration tests + "name": "aconfig.test.java" + }, + { + // aconfig C++ integration tests (production mode auto-generated code) + "name": "aconfig.test.cpp" + }, + { + // aconfig C++ integration tests (test mode auto-generated code) + "name": "aconfig.test.cpp.test_mode" + }, + // TODO(b/327420679): Enable export mode for native flag library + // { + // // aconfig C++ integration tests (exported mode auto-generated code) + // "name": "aconfig.test.cpp.exported_mode" + // }, + { + // aconfig Rust integration tests (production mode auto-generated code) + "name": "aconfig.prod_mode.test.rust" + }, + { + // aconfig Rust integration tests (test mode auto-generated code) + "name": "aconfig.test_mode.test.rust" + }, + // TODO(b/327420679): Enable export mode for native flag library + // { + // // aconfig Rust integration tests (exported mode auto-generated code) + // "name": "aconfig.exported_mode.test.rust" + // }, + { + // aflags CLI unit tests + "name": "aflags.test" + }, + { + // printflags unit tests + "name": "printflags.test" + }, + { + // aconfig_protos unit tests + "name": "aconfig_protos.test" + }, + { + // aconfig_storage_file unit tests + "name": "aconfig_storage_file.test" + }, + { + // Ensure changes on aconfig auto generated library is compatible with + // test testing filtering logic. Breakage on this test means all tests + // that using the flag annotations to do filtering will get affected. + "name": "FlagAnnotationTests", + "options": [ + { + "include-filter": "android.cts.flags.tests.FlagAnnotationTest" + } + ] + }, + { + // Ensure changes on aconfig auto generated library is compatible with + // test testing filtering logic. Breakage on this test means all tests + // that using the flag macros to do filtering will get affected. + "name": "FlagMacrosTests" + }, + { + // aconfig_storage_write_api unit tests + "name": "aconfig_storage_write_api.test" + }, + { + // aconfig_storage_read_api unit tests + "name": "aconfig_storage_read_api.test" + }, + { + // aconfig_storage write api rust integration tests + "name": "aconfig_storage_write_api.test.rust" + }, + { + // aconfig_storage write api cpp integration tests + "name": "aconfig_storage_write_api.test.cpp" + }, + { + // aconfig_storage read api rust integration tests + "name": "aconfig_storage_read_api.test.rust" + }, + { + // aconfig_storage read api cpp integration tests + "name": "aconfig_storage_read_api.test.cpp" + } + ], + "postsubmit": [ + { + // aconfig_storage file cpp integration tests + "name": "aconfig_storage_file.test.cpp" + } + ] +} diff --git a/tools/aconfig/aconfig/Android.bp b/tools/aconfig/aconfig/Android.bp new file mode 100644 index 0000000000..68521af91f --- /dev/null +++ b/tools/aconfig/aconfig/Android.bp @@ -0,0 +1,305 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "aconfig.defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + srcs: ["src/main.rs"], + rustlibs: [ + "libaconfig_protos", + "libaconfig_storage_file", + "libanyhow", + "libclap", + "libitertools", + "libprotobuf", + "libserde", + "libserde_json", + "libtinytemplate", + ], +} + +rust_binary_host { + name: "aconfig", + defaults: ["aconfig.defaults"], +} + +rust_test_host { + name: "aconfig.test", + defaults: ["aconfig.defaults"], + rustlibs: [ + "libitertools", + ], + test_suites: ["general-tests"], +} + +// integration tests: general + +aconfig_declarations { + name: "aconfig.test.flags", + package: "com.android.aconfig.test", + container: "system", + srcs: ["tests/test.aconfig"], +} + +aconfig_declarations { + name: "aconfig.test.exported.flags", + package: "com.android.aconfig.test.exported", + exportable: true, + container: "system", + srcs: ["tests/test_exported.aconfig"], +} + +aconfig_declarations { + name: "aconfig.test.forcereadonly.flags", + package: "com.android.aconfig.test.forcereadonly", + container: "system", + srcs: ["tests/test_force_read_only.aconfig"], +} + +aconfig_values { + name: "aconfig.test.flag.values", + package: "com.android.aconfig.test", + srcs: [ + "tests/first.values", + "tests/second.values", + ], +} + +aconfig_value_set { + name: "aconfig.test.flag.value_set", + values: [ + "aconfig.test.flag.values", + ], +} + +// integration tests: java + +java_aconfig_library { + name: "aconfig_test_java_library", + aconfig_declarations: "aconfig.test.flags", +} + +java_aconfig_library { + name: "aconfig_test_java_library_exported", + aconfig_declarations: "aconfig.test.exported.flags", + mode: "exported", +} + +java_aconfig_library { + name: "aconfig_test_java_library_forcereadonly", + aconfig_declarations: "aconfig.test.forcereadonly.flags", + mode: "force-read-only", +} + +android_test { + name: "aconfig.test.java", + srcs: [ + "tests/AconfigTest.java", + ], + manifest: "tests/AndroidManifest.xml", + certificate: "platform", + static_libs: [ + "aconfig_test_java_library", + "aconfig_test_java_library_exported", + "aconfig_test_java_library_forcereadonly", + "androidx.test.rules", + "testng", + ], + test_suites: ["general-tests"], +} + +java_aconfig_library { + name: "aconfig_host_test_java_library", + aconfig_declarations: "aconfig.test.flags", + host_supported: true, + mode: "test", +} + +java_test_host { + name: "AconfigJavaHostTest", + srcs: [ + "tests/AconfigHostTest.java", + ], + static_libs: [ + "aconfig_host_test_java_library", + "junit", + ], + test_suites: ["general-tests"], +} + +// integration tests: C++ + +cc_aconfig_library { + name: "aconfig_test_cpp_library", + aconfig_declarations: "aconfig.test.flags", +} + +cc_aconfig_library { + name: "aconfig_test_cpp_library_test_variant", + aconfig_declarations: "aconfig.test.flags", + mode: "test", +} + +cc_aconfig_library { + name: "aconfig_test_cpp_library_force_read_only_variant", + aconfig_declarations: "aconfig.test.flags", + mode: "force-read-only", +} + +cc_test { + name: "aconfig.test.cpp", + srcs: [ + "tests/aconfig_test.cpp", + ], + static_libs: [ + "aconfig_test_cpp_library", + "libgmock", + ], + shared_libs: [ + "server_configurable_flags", + ], + defaults: [ + "aconfig_lib_cc_static_link.defaults", + ], + test_suites: ["general-tests"], +} + +cc_test { + name: "aconfig.test.cpp.test_mode", + srcs: [ + "tests/aconfig_test_test_variant.cpp", + ], + static_libs: [ + "aconfig_test_cpp_library_test_variant", + "libgmock", + ], + shared_libs: [ + "server_configurable_flags", + ], + defaults: [ + "aconfig_lib_cc_static_link.defaults", + ], + test_suites: ["general-tests"], +} + +// TODO(327420679): Enable export mode for native flag library +/* +cc_aconfig_library { + name: "aconfig_test_cpp_library_exported_variant", + aconfig_declarations: "aconfig.test.flags", + mode: "exported", +} + +cc_test { + name: "aconfig.test.cpp.exported_mode", + srcs: [ + "tests/aconfig_exported_mode_test.cpp", + ], + static_libs: [ + "aconfig_test_cpp_library_exported_variant", + "libgmock", + ], + shared_libs: [ + "server_configurable_flags", + ], + defaults: [ + "aconfig_lib_cc_static_link.defaults", + ], + test_suites: ["general-tests"], +} +*/ + +cc_test { + name: "aconfig.test.cpp.force_read_only_mode", + srcs: [ + "tests/aconfig_force_read_only_mode_test.cpp", + ], + static_libs: [ + "aconfig_test_cpp_library_force_read_only_variant", + "libgmock", + ], + shared_libs: [ + "server_configurable_flags", + ], + defaults: [ + "aconfig_lib_cc_static_link.defaults", + ], + test_suites: ["general-tests"], +} + +rust_aconfig_library { + name: "libaconfig_test_rust_library", + crate_name: "aconfig_test_rust_library", + aconfig_declarations: "aconfig.test.flags", +} + +rust_test { + name: "aconfig.prod_mode.test.rust", + srcs: [ + "tests/aconfig_prod_mode_test.rs", + ], + rustlibs: [ + "libaconfig_test_rust_library", + ], + test_suites: ["general-tests"], +} + +rust_aconfig_library { + name: "libaconfig_test_rust_library_with_test_mode", + crate_name: "aconfig_test_rust_library", + aconfig_declarations: "aconfig.test.flags", + mode: "test", +} + +rust_test { + name: "aconfig.test_mode.test.rust", + srcs: [ + "tests/aconfig_test_mode_test.rs", + ], + rustlibs: [ + "libaconfig_test_rust_library_with_test_mode", + ], + test_suites: ["general-tests"], +} + +// TODO(327420679): Enable export mode for native flag library +/* +rust_aconfig_library { + name: "libaconfig_test_rust_library_with_exported_mode", + crate_name: "aconfig_test_rust_library", + aconfig_declarations: "aconfig.test.flags", + mode: "exported", +} + +rust_test { + name: "aconfig.exported_mode.test.rust", + srcs: [ + "tests/aconfig_exported_mode_test.rs", + ], + rustlibs: [ + "libaconfig_test_rust_library_with_exported_mode", + ], + test_suites: ["general-tests"], +} +*/ + +rust_aconfig_library { + name: "libaconfig_test_rust_library_with_force_read_only_mode", + crate_name: "aconfig_test_rust_library", + aconfig_declarations: "aconfig.test.flags", + mode: "force-read-only", +} + +rust_test { + name: "aconfig.force_read_only_mode.test.rust", + srcs: [ + "tests/aconfig_force_read_only_mode_test.rs", + ], + rustlibs: [ + "libaconfig_test_rust_library_with_force_read_only_mode", + ], + test_suites: ["general-tests"], +} diff --git a/tools/aconfig/aconfig/Cargo.toml b/tools/aconfig/aconfig/Cargo.toml new file mode 100644 index 0000000000..abd3ee01e8 --- /dev/null +++ b/tools/aconfig/aconfig/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "aconfig" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +clap = { version = "4.1.8", features = ["derive"] } +itertools = "0.10.5" +protobuf = "3.2.0" +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +tinytemplate = "1.2.1" +aconfig_protos = { path = "../aconfig_protos" } +aconfig_storage_file = { path = "../aconfig_storage_file" } diff --git a/tools/aconfig/aconfig/src/codegen/cpp.rs b/tools/aconfig/aconfig/src/codegen/cpp.rs new file mode 100644 index 0000000000..e743b2fc59 --- /dev/null +++ b/tools/aconfig/aconfig/src/codegen/cpp.rs @@ -0,0 +1,1288 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::{ensure, Result}; +use serde::Serialize; +use std::collections::HashMap; +use std::path::PathBuf; +use tinytemplate::TinyTemplate; + +use aconfig_protos::{ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag}; + +use crate::codegen; +use crate::codegen::CodegenMode; +use crate::commands::OutputFile; + +pub fn generate_cpp_code<I>( + package: &str, + parsed_flags_iter: I, + codegen_mode: CodegenMode, + flag_ids: HashMap<String, u16>, + allow_instrumentation: bool, +) -> Result<Vec<OutputFile>> +where + I: Iterator<Item = ProtoParsedFlag>, +{ + let mut readwrite_count = 0; + let class_elements: Vec<ClassElement> = parsed_flags_iter + .map(|pf| create_class_element(package, &pf, flag_ids.clone(), &mut readwrite_count)) + .collect(); + let readwrite = readwrite_count > 0; + let has_fixed_read_only = class_elements.iter().any(|item| item.is_fixed_read_only); + let header = package.replace('.', "_"); + let package_macro = header.to_uppercase(); + let cpp_namespace = package.replace('.', "::"); + ensure!(codegen::is_valid_name_ident(&header)); + let context = Context { + header: &header, + package_macro: &package_macro, + cpp_namespace: &cpp_namespace, + package, + has_fixed_read_only, + readwrite, + readwrite_count, + is_test_mode: codegen_mode == CodegenMode::Test, + class_elements, + allow_instrumentation, + }; + + let files = [ + FileSpec { + name: &format!("{}.h", header), + template: include_str!("../../templates/cpp_exported_header.template"), + dir: "include", + }, + FileSpec { + name: &format!("{}.cc", header), + template: include_str!("../../templates/cpp_source_file.template"), + dir: "", + }, + ]; + files.iter().map(|file| generate_file(file, &context)).collect() +} + +pub fn generate_file(file: &FileSpec, context: &Context) -> Result<OutputFile> { + let mut template = TinyTemplate::new(); + template.add_template(file.name, file.template)?; + let contents = template.render(file.name, &context)?; + let path: PathBuf = [&file.dir, &file.name].iter().collect(); + Ok(OutputFile { contents: contents.into(), path }) +} + +#[derive(Serialize)] +pub struct FileSpec<'a> { + pub name: &'a str, + pub template: &'a str, + pub dir: &'a str, +} + +#[derive(Serialize)] +pub struct Context<'a> { + pub header: &'a str, + pub package_macro: &'a str, + pub cpp_namespace: &'a str, + pub package: &'a str, + pub has_fixed_read_only: bool, + pub readwrite: bool, + pub readwrite_count: i32, + pub is_test_mode: bool, + pub class_elements: Vec<ClassElement>, + pub allow_instrumentation: bool, +} + +#[derive(Serialize)] +pub struct ClassElement { + pub readwrite_idx: i32, + pub readwrite: bool, + pub is_fixed_read_only: bool, + pub default_value: String, + pub flag_name: String, + pub flag_macro: String, + pub flag_offset: u16, + pub device_config_namespace: String, + pub device_config_flag: String, + pub container: String, +} + +fn create_class_element( + package: &str, + pf: &ProtoParsedFlag, + flag_ids: HashMap<String, u16>, + rw_count: &mut i32, +) -> ClassElement { + ClassElement { + readwrite_idx: if pf.permission() == ProtoFlagPermission::READ_WRITE { + let index = *rw_count; + *rw_count += 1; + index + } else { + -1 + }, + readwrite: pf.permission() == ProtoFlagPermission::READ_WRITE, + is_fixed_read_only: pf.is_fixed_read_only(), + default_value: if pf.state() == ProtoFlagState::ENABLED { + "true".to_string() + } else { + "false".to_string() + }, + flag_name: pf.name().to_string(), + flag_macro: pf.name().to_uppercase(), + flag_offset: *flag_ids.get(pf.name()).expect("values checked at flag parse time"), + device_config_namespace: pf.namespace().to_string(), + device_config_flag: codegen::create_device_config_ident(package, pf.name()) + .expect("values checked at flag parse time"), + container: pf.container().to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_protos::ProtoParsedFlags; + use std::collections::HashMap; + + const EXPORTED_PROD_HEADER_EXPECTED: &str = r#" +#pragma once + +#ifndef COM_ANDROID_ACONFIG_TEST +#define COM_ANDROID_ACONFIG_TEST(FLAG) COM_ANDROID_ACONFIG_TEST_##FLAG +#endif + +#ifndef COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO +#define COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO true +#endif + +#ifndef COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO_EXPORTED +#define COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO_EXPORTED true +#endif + +#ifdef __cplusplus + +#include <memory> + +namespace com::android::aconfig::test { + +class flag_provider_interface { +public: + virtual ~flag_provider_interface() = default; + + virtual bool disabled_ro() = 0; + + virtual bool disabled_rw() = 0; + + virtual bool disabled_rw_exported() = 0; + + virtual bool disabled_rw_in_other_namespace() = 0; + + virtual bool enabled_fixed_ro() = 0; + + virtual bool enabled_fixed_ro_exported() = 0; + + virtual bool enabled_ro() = 0; + + virtual bool enabled_ro_exported() = 0; + + virtual bool enabled_rw() = 0; +}; + +extern std::unique_ptr<flag_provider_interface> provider_; + +inline bool disabled_ro() { + return false; +} + +inline bool disabled_rw() { + return provider_->disabled_rw(); +} + +inline bool disabled_rw_exported() { + return provider_->disabled_rw_exported(); +} + +inline bool disabled_rw_in_other_namespace() { + return provider_->disabled_rw_in_other_namespace(); +} + +inline bool enabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; +} + +inline bool enabled_fixed_ro_exported() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO_EXPORTED; +} + +inline bool enabled_ro() { + return true; +} + +inline bool enabled_ro_exported() { + return true; +} + +inline bool enabled_rw() { + return provider_->enabled_rw(); +} + +} + +extern "C" { +#endif // __cplusplus + +bool com_android_aconfig_test_disabled_ro(); + +bool com_android_aconfig_test_disabled_rw(); + +bool com_android_aconfig_test_disabled_rw_exported(); + +bool com_android_aconfig_test_disabled_rw_in_other_namespace(); + +bool com_android_aconfig_test_enabled_fixed_ro(); + +bool com_android_aconfig_test_enabled_fixed_ro_exported(); + +bool com_android_aconfig_test_enabled_ro(); + +bool com_android_aconfig_test_enabled_ro_exported(); + +bool com_android_aconfig_test_enabled_rw(); + +#ifdef __cplusplus +} // extern "C" +#endif +"#; + + const EXPORTED_TEST_HEADER_EXPECTED: &str = r#" +#pragma once + +#ifdef __cplusplus + +#include <memory> + +namespace com::android::aconfig::test { + +class flag_provider_interface { +public: + + virtual ~flag_provider_interface() = default; + + virtual bool disabled_ro() = 0; + + virtual void disabled_ro(bool val) = 0; + + virtual bool disabled_rw() = 0; + + virtual void disabled_rw(bool val) = 0; + + virtual bool disabled_rw_exported() = 0; + + virtual void disabled_rw_exported(bool val) = 0; + + virtual bool disabled_rw_in_other_namespace() = 0; + + virtual void disabled_rw_in_other_namespace(bool val) = 0; + + virtual bool enabled_fixed_ro() = 0; + + virtual void enabled_fixed_ro(bool val) = 0; + + virtual bool enabled_fixed_ro_exported() = 0; + + virtual void enabled_fixed_ro_exported(bool val) = 0; + + virtual bool enabled_ro() = 0; + + virtual void enabled_ro(bool val) = 0; + + virtual bool enabled_ro_exported() = 0; + + virtual void enabled_ro_exported(bool val) = 0; + + virtual bool enabled_rw() = 0; + + virtual void enabled_rw(bool val) = 0; + + virtual void reset_flags() {} +}; + +extern std::unique_ptr<flag_provider_interface> provider_; + +inline bool disabled_ro() { + return provider_->disabled_ro(); +} + +inline void disabled_ro(bool val) { + provider_->disabled_ro(val); +} + +inline bool disabled_rw() { + return provider_->disabled_rw(); +} + +inline void disabled_rw(bool val) { + provider_->disabled_rw(val); +} + +inline bool disabled_rw_exported() { + return provider_->disabled_rw_exported(); +} + +inline void disabled_rw_exported(bool val) { + provider_->disabled_rw_exported(val); +} + +inline bool disabled_rw_in_other_namespace() { + return provider_->disabled_rw_in_other_namespace(); +} + +inline void disabled_rw_in_other_namespace(bool val) { + provider_->disabled_rw_in_other_namespace(val); +} + +inline bool enabled_fixed_ro() { + return provider_->enabled_fixed_ro(); +} + +inline void enabled_fixed_ro(bool val) { + provider_->enabled_fixed_ro(val); +} + +inline bool enabled_fixed_ro_exported() { + return provider_->enabled_fixed_ro_exported(); +} + +inline void enabled_fixed_ro_exported(bool val) { + provider_->enabled_fixed_ro_exported(val); +} + +inline bool enabled_ro() { + return provider_->enabled_ro(); +} + +inline void enabled_ro(bool val) { + provider_->enabled_ro(val); +} + +inline bool enabled_ro_exported() { + return provider_->enabled_ro_exported(); +} + +inline void enabled_ro_exported(bool val) { + provider_->enabled_ro_exported(val); +} + +inline bool enabled_rw() { + return provider_->enabled_rw(); +} + +inline void enabled_rw(bool val) { + provider_->enabled_rw(val); +} + +inline void reset_flags() { + return provider_->reset_flags(); +} + +} + +extern "C" { +#endif // __cplusplus + +bool com_android_aconfig_test_disabled_ro(); + +void set_com_android_aconfig_test_disabled_ro(bool val); + +bool com_android_aconfig_test_disabled_rw(); + +void set_com_android_aconfig_test_disabled_rw(bool val); + +bool com_android_aconfig_test_disabled_rw_exported(); + +void set_com_android_aconfig_test_disabled_rw_exported(bool val); + +bool com_android_aconfig_test_disabled_rw_in_other_namespace(); + +void set_com_android_aconfig_test_disabled_rw_in_other_namespace(bool val); + +bool com_android_aconfig_test_enabled_fixed_ro(); + +void set_com_android_aconfig_test_enabled_fixed_ro(bool val); + +bool com_android_aconfig_test_enabled_fixed_ro_exported(); + +void set_com_android_aconfig_test_enabled_fixed_ro_exported(bool val); + +bool com_android_aconfig_test_enabled_ro(); + +void set_com_android_aconfig_test_enabled_ro(bool val); + +bool com_android_aconfig_test_enabled_ro_exported(); + +void set_com_android_aconfig_test_enabled_ro_exported(bool val); + +bool com_android_aconfig_test_enabled_rw(); + +void set_com_android_aconfig_test_enabled_rw(bool val); + +void com_android_aconfig_test_reset_flags(); + + +#ifdef __cplusplus +} // extern "C" +#endif + + +"#; + + const EXPORTED_EXPORTED_HEADER_EXPECTED: &str = r#" +#pragma once + +#ifdef __cplusplus + +#include <memory> + +namespace com::android::aconfig::test { + +class flag_provider_interface { +public: + virtual ~flag_provider_interface() = default; + + virtual bool disabled_rw_exported() = 0; + + virtual bool enabled_fixed_ro_exported() = 0; + + virtual bool enabled_ro_exported() = 0; +}; + +extern std::unique_ptr<flag_provider_interface> provider_; + +inline bool disabled_rw_exported() { + return provider_->disabled_rw_exported(); +} + +inline bool enabled_fixed_ro_exported() { + return provider_->enabled_fixed_ro_exported(); +} + +inline bool enabled_ro_exported() { + return provider_->enabled_ro_exported(); +} + +} + +extern "C" { +#endif // __cplusplus + +bool com_android_aconfig_test_disabled_rw_exported(); + +bool com_android_aconfig_test_enabled_fixed_ro_exported(); + +bool com_android_aconfig_test_enabled_ro_exported(); + +#ifdef __cplusplus +} // extern "C" +#endif +"#; + + const EXPORTED_FORCE_READ_ONLY_HEADER_EXPECTED: &str = r#" +#pragma once + +#ifndef COM_ANDROID_ACONFIG_TEST +#define COM_ANDROID_ACONFIG_TEST(FLAG) COM_ANDROID_ACONFIG_TEST_##FLAG +#endif + +#ifndef COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO +#define COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO true +#endif + +#ifdef __cplusplus + +#include <memory> + +namespace com::android::aconfig::test { + +class flag_provider_interface { +public: + virtual ~flag_provider_interface() = default; + + virtual bool disabled_ro() = 0; + + virtual bool disabled_rw() = 0; + + virtual bool disabled_rw_in_other_namespace() = 0; + + virtual bool enabled_fixed_ro() = 0; + + virtual bool enabled_ro() = 0; + + virtual bool enabled_rw() = 0; +}; + +extern std::unique_ptr<flag_provider_interface> provider_; + +inline bool disabled_ro() { + return false; +} + +inline bool disabled_rw() { + return false; +} + +inline bool disabled_rw_in_other_namespace() { + return false; +} + +inline bool enabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; +} + +inline bool enabled_ro() { + return true; +} + +inline bool enabled_rw() { + return true; +} + +} + +extern "C" { +#endif // __cplusplus + +bool com_android_aconfig_test_disabled_ro(); + +bool com_android_aconfig_test_disabled_rw(); + +bool com_android_aconfig_test_disabled_rw_in_other_namespace(); + +bool com_android_aconfig_test_enabled_fixed_ro(); + +bool com_android_aconfig_test_enabled_ro(); + +bool com_android_aconfig_test_enabled_rw(); + +#ifdef __cplusplus +} // extern "C" +#endif +"#; + + const PROD_SOURCE_FILE_EXPECTED: &str = r#" +#include "com_android_aconfig_test.h" +#include <server_configurable_flags/get_flags.h> +#include <vector> + +namespace com::android::aconfig::test { + + class flag_provider : public flag_provider_interface { + public: + + virtual bool disabled_ro() override { + return false; + } + + virtual bool disabled_rw() override { + if (cache_[0] == -1) { + cache_[0] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw", + "false") == "true"; + } + return cache_[0]; + } + + virtual bool disabled_rw_exported() override { + if (cache_[1] == -1) { + cache_[1] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw_exported", + "false") == "true"; + } + return cache_[1]; + } + + virtual bool disabled_rw_in_other_namespace() override { + if (cache_[2] == -1) { + cache_[2] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.other_namespace", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "false") == "true"; + } + return cache_[2]; + } + + virtual bool enabled_fixed_ro() override { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; + } + + virtual bool enabled_fixed_ro_exported() override { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO_EXPORTED; + } + + virtual bool enabled_ro() override { + return true; + } + + virtual bool enabled_ro_exported() override { + return true; + } + + virtual bool enabled_rw() override { + if (cache_[3] == -1) { + cache_[3] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_rw", + "true") == "true"; + } + return cache_[3]; + } + + private: + std::vector<int8_t> cache_ = std::vector<int8_t>(4, -1); + }; + + std::unique_ptr<flag_provider_interface> provider_ = + std::make_unique<flag_provider>(); +} + +bool com_android_aconfig_test_disabled_ro() { + return false; +} + +bool com_android_aconfig_test_disabled_rw() { + return com::android::aconfig::test::disabled_rw(); +} + +bool com_android_aconfig_test_disabled_rw_exported() { + return com::android::aconfig::test::disabled_rw_exported(); +} + +bool com_android_aconfig_test_disabled_rw_in_other_namespace() { + return com::android::aconfig::test::disabled_rw_in_other_namespace(); +} + +bool com_android_aconfig_test_enabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; +} + +bool com_android_aconfig_test_enabled_fixed_ro_exported() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO_EXPORTED; +} + +bool com_android_aconfig_test_enabled_ro() { + return true; +} + +bool com_android_aconfig_test_enabled_ro_exported() { + return true; +} + +bool com_android_aconfig_test_enabled_rw() { + return com::android::aconfig::test::enabled_rw(); +} + +"#; + + const TEST_SOURCE_FILE_EXPECTED: &str = r#" +#include "com_android_aconfig_test.h" +#include <server_configurable_flags/get_flags.h> +#include <unordered_map> +#include <string> + +namespace com::android::aconfig::test { + + class flag_provider : public flag_provider_interface { + private: + std::unordered_map<std::string, bool> overrides_; + + public: + flag_provider() + : overrides_() + {} + + virtual bool disabled_ro() override { + auto it = overrides_.find("disabled_ro"); + if (it != overrides_.end()) { + return it->second; + } else { + return false; + } + } + + virtual void disabled_ro(bool val) override { + overrides_["disabled_ro"] = val; + } + + virtual bool disabled_rw() override { + auto it = overrides_.find("disabled_rw"); + if (it != overrides_.end()) { + return it->second; + } else { + return server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw", + "false") == "true"; + } + } + + virtual void disabled_rw(bool val) override { + overrides_["disabled_rw"] = val; + } + + virtual bool disabled_rw_exported() override { + auto it = overrides_.find("disabled_rw_exported"); + if (it != overrides_.end()) { + return it->second; + } else { + return server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw_exported", + "false") == "true"; + } + } + + virtual void disabled_rw_exported(bool val) override { + overrides_["disabled_rw_exported"] = val; + } + + virtual bool disabled_rw_in_other_namespace() override { + auto it = overrides_.find("disabled_rw_in_other_namespace"); + if (it != overrides_.end()) { + return it->second; + } else { + return server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.other_namespace", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "false") == "true"; + } + } + + virtual void disabled_rw_in_other_namespace(bool val) override { + overrides_["disabled_rw_in_other_namespace"] = val; + } + + virtual bool enabled_fixed_ro() override { + auto it = overrides_.find("enabled_fixed_ro"); + if (it != overrides_.end()) { + return it->second; + } else { + return true; + } + } + + virtual void enabled_fixed_ro(bool val) override { + overrides_["enabled_fixed_ro"] = val; + } + + virtual bool enabled_fixed_ro_exported() override { + auto it = overrides_.find("enabled_fixed_ro_exported"); + if (it != overrides_.end()) { + return it->second; + } else { + return true; + } + } + + virtual void enabled_fixed_ro_exported(bool val) override { + overrides_["enabled_fixed_ro_exported"] = val; + } + + virtual bool enabled_ro() override { + auto it = overrides_.find("enabled_ro"); + if (it != overrides_.end()) { + return it->second; + } else { + return true; + } + } + + virtual void enabled_ro(bool val) override { + overrides_["enabled_ro"] = val; + } + + virtual bool enabled_ro_exported() override { + auto it = overrides_.find("enabled_ro_exported"); + if (it != overrides_.end()) { + return it->second; + } else { + return true; + } + } + + virtual void enabled_ro_exported(bool val) override { + overrides_["enabled_ro_exported"] = val; + } + + virtual bool enabled_rw() override { + auto it = overrides_.find("enabled_rw"); + if (it != overrides_.end()) { + return it->second; + } else { + return server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_rw", + "true") == "true"; + } + } + + virtual void enabled_rw(bool val) override { + overrides_["enabled_rw"] = val; + } + + virtual void reset_flags() override { + overrides_.clear(); + } + }; + + std::unique_ptr<flag_provider_interface> provider_ = + std::make_unique<flag_provider>(); +} + +bool com_android_aconfig_test_disabled_ro() { + return com::android::aconfig::test::disabled_ro(); +} + + +void set_com_android_aconfig_test_disabled_ro(bool val) { + com::android::aconfig::test::disabled_ro(val); +} + +bool com_android_aconfig_test_disabled_rw() { + return com::android::aconfig::test::disabled_rw(); +} + + +void set_com_android_aconfig_test_disabled_rw(bool val) { + com::android::aconfig::test::disabled_rw(val); +} + + +bool com_android_aconfig_test_disabled_rw_exported() { + return com::android::aconfig::test::disabled_rw_exported(); +} + +void set_com_android_aconfig_test_disabled_rw_exported(bool val) { + com::android::aconfig::test::disabled_rw_exported(val); +} + + +bool com_android_aconfig_test_disabled_rw_in_other_namespace() { + return com::android::aconfig::test::disabled_rw_in_other_namespace(); +} + +void set_com_android_aconfig_test_disabled_rw_in_other_namespace(bool val) { + com::android::aconfig::test::disabled_rw_in_other_namespace(val); +} + + +bool com_android_aconfig_test_enabled_fixed_ro() { + return com::android::aconfig::test::enabled_fixed_ro(); +} + +void set_com_android_aconfig_test_enabled_fixed_ro(bool val) { + com::android::aconfig::test::enabled_fixed_ro(val); +} + +bool com_android_aconfig_test_enabled_fixed_ro_exported() { + return com::android::aconfig::test::enabled_fixed_ro_exported(); +} + +void set_com_android_aconfig_test_enabled_fixed_ro_exported(bool val) { + com::android::aconfig::test::enabled_fixed_ro_exported(val); +} + +bool com_android_aconfig_test_enabled_ro() { + return com::android::aconfig::test::enabled_ro(); +} + + +void set_com_android_aconfig_test_enabled_ro(bool val) { + com::android::aconfig::test::enabled_ro(val); +} + + +bool com_android_aconfig_test_enabled_ro_exported() { + return com::android::aconfig::test::enabled_ro_exported(); +} + + +void set_com_android_aconfig_test_enabled_ro_exported(bool val) { + com::android::aconfig::test::enabled_ro_exported(val); +} + + +bool com_android_aconfig_test_enabled_rw() { + return com::android::aconfig::test::enabled_rw(); +} + + +void set_com_android_aconfig_test_enabled_rw(bool val) { + com::android::aconfig::test::enabled_rw(val); +} + +void com_android_aconfig_test_reset_flags() { + com::android::aconfig::test::reset_flags(); +} + +"#; + + const EXPORTED_SOURCE_FILE_EXPECTED: &str = r#" +#include "com_android_aconfig_test.h" +#include <server_configurable_flags/get_flags.h> +#include <vector> + +namespace com::android::aconfig::test { + + class flag_provider : public flag_provider_interface { + public: + virtual bool disabled_rw_exported() override { + if (cache_[0] == -1) { + cache_[0] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw_exported", + "false") == "true"; + } + return cache_[0]; + } + + virtual bool enabled_fixed_ro_exported() override { + if (cache_[1] == -1) { + cache_[1] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "false") == "true"; + } + return cache_[1]; + } + + virtual bool enabled_ro_exported() override { + if (cache_[2] == -1) { + cache_[2] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_ro_exported", + "false") == "true"; + } + return cache_[2]; + } + + private: + std::vector<int8_t> cache_ = std::vector<int8_t>(3, -1); + }; + + std::unique_ptr<flag_provider_interface> provider_ = + std::make_unique<flag_provider>(); +} + +bool com_android_aconfig_test_disabled_rw_exported() { + return com::android::aconfig::test::disabled_rw_exported(); +} + +bool com_android_aconfig_test_enabled_fixed_ro_exported() { + return com::android::aconfig::test::enabled_fixed_ro_exported(); +} + +bool com_android_aconfig_test_enabled_ro_exported() { + return com::android::aconfig::test::enabled_ro_exported(); +} + + +"#; + + const FORCE_READ_ONLY_SOURCE_FILE_EXPECTED: &str = r#" +#include "com_android_aconfig_test.h" + +namespace com::android::aconfig::test { + + class flag_provider : public flag_provider_interface { + public: + + virtual bool disabled_ro() override { + return false; + } + + virtual bool disabled_rw() override { + return false; + } + + virtual bool disabled_rw_in_other_namespace() override { + return false; + } + + virtual bool enabled_fixed_ro() override { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; + } + + virtual bool enabled_ro() override { + return true; + } + + virtual bool enabled_rw() override { + return true; + } + }; + + std::unique_ptr<flag_provider_interface> provider_ = + std::make_unique<flag_provider>(); +} + +bool com_android_aconfig_test_disabled_ro() { + return false; +} + +bool com_android_aconfig_test_disabled_rw() { + return false; +} + +bool com_android_aconfig_test_disabled_rw_in_other_namespace() { + return false; +} + +bool com_android_aconfig_test_enabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; +} + +bool com_android_aconfig_test_enabled_ro() { + return true; +} + +bool com_android_aconfig_test_enabled_rw() { + return true; +} + +"#; + + const READ_ONLY_EXPORTED_PROD_HEADER_EXPECTED: &str = r#" +#pragma once + +#ifndef COM_ANDROID_ACONFIG_TEST +#define COM_ANDROID_ACONFIG_TEST(FLAG) COM_ANDROID_ACONFIG_TEST_##FLAG +#endif + +#ifndef COM_ANDROID_ACONFIG_TEST_DISABLED_FIXED_RO +#define COM_ANDROID_ACONFIG_TEST_DISABLED_FIXED_RO false +#endif + +#ifndef COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO +#define COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO true +#endif + +#ifdef __cplusplus + +#include <memory> + +namespace com::android::aconfig::test { + +class flag_provider_interface { +public: + virtual ~flag_provider_interface() = default; + + virtual bool disabled_fixed_ro() = 0; + + virtual bool disabled_ro() = 0; + + virtual bool enabled_fixed_ro() = 0; + + virtual bool enabled_ro() = 0; +}; + +extern std::unique_ptr<flag_provider_interface> provider_; + +inline bool disabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_DISABLED_FIXED_RO; +} + +inline bool disabled_ro() { + return false; +} + +inline bool enabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; +} + +inline bool enabled_ro() { + return true; +} +} + +extern "C" { +#endif // __cplusplus + +bool com_android_aconfig_test_disabled_fixed_ro(); + +bool com_android_aconfig_test_disabled_ro(); + +bool com_android_aconfig_test_enabled_fixed_ro(); + +bool com_android_aconfig_test_enabled_ro(); + +#ifdef __cplusplus +} // extern "C" +#endif +"#; + + const READ_ONLY_PROD_SOURCE_FILE_EXPECTED: &str = r#" +#include "com_android_aconfig_test.h" + +namespace com::android::aconfig::test { + + class flag_provider : public flag_provider_interface { + public: + + virtual bool disabled_fixed_ro() override { + return COM_ANDROID_ACONFIG_TEST_DISABLED_FIXED_RO; + } + + virtual bool disabled_ro() override { + return false; + } + + virtual bool enabled_fixed_ro() override { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; + } + + virtual bool enabled_ro() override { + return true; + } + }; + + std::unique_ptr<flag_provider_interface> provider_ = + std::make_unique<flag_provider>(); +} + +bool com_android_aconfig_test_disabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_DISABLED_FIXED_RO; +} + +bool com_android_aconfig_test_disabled_ro() { + return false; +} + +bool com_android_aconfig_test_enabled_fixed_ro() { + return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; +} + +bool com_android_aconfig_test_enabled_ro() { + return true; +} +"#; + use crate::commands::assign_flag_ids; + + fn test_generate_cpp_code( + parsed_flags: ProtoParsedFlags, + mode: CodegenMode, + expected_header: &str, + expected_src: &str, + allow_instrumentation: bool, + ) { + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let flag_ids = + assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let generated = generate_cpp_code( + crate::test::TEST_PACKAGE, + modified_parsed_flags.into_iter(), + mode, + flag_ids, + allow_instrumentation, + ) + .unwrap(); + let mut generated_files_map = HashMap::new(); + for file in generated { + generated_files_map.insert( + String::from(file.path.to_str().unwrap()), + String::from_utf8(file.contents).unwrap(), + ); + } + + let mut target_file_path = String::from("include/com_android_aconfig_test.h"); + assert!(generated_files_map.contains_key(&target_file_path)); + assert_eq!( + None, + crate::test::first_significant_code_diff( + expected_header, + generated_files_map.get(&target_file_path).unwrap() + ) + ); + + target_file_path = String::from("com_android_aconfig_test.cc"); + assert!(generated_files_map.contains_key(&target_file_path)); + assert_eq!( + None, + crate::test::first_significant_code_diff( + expected_src, + generated_files_map.get(&target_file_path).unwrap() + ) + ); + } + + #[test] + fn test_generate_cpp_code_for_prod() { + let parsed_flags = crate::test::parse_test_flags(); + test_generate_cpp_code( + parsed_flags, + CodegenMode::Production, + EXPORTED_PROD_HEADER_EXPECTED, + PROD_SOURCE_FILE_EXPECTED, + false, + ); + } + + #[test] + fn test_generate_cpp_code_for_test() { + let parsed_flags = crate::test::parse_test_flags(); + test_generate_cpp_code( + parsed_flags, + CodegenMode::Test, + EXPORTED_TEST_HEADER_EXPECTED, + TEST_SOURCE_FILE_EXPECTED, + false, + ); + } + + #[test] + fn test_generate_cpp_code_for_exported() { + let parsed_flags = crate::test::parse_test_flags(); + test_generate_cpp_code( + parsed_flags, + CodegenMode::Exported, + EXPORTED_EXPORTED_HEADER_EXPECTED, + EXPORTED_SOURCE_FILE_EXPECTED, + false, + ); + } + + #[test] + fn test_generate_cpp_code_for_force_read_only() { + let parsed_flags = crate::test::parse_test_flags(); + test_generate_cpp_code( + parsed_flags, + CodegenMode::ForceReadOnly, + EXPORTED_FORCE_READ_ONLY_HEADER_EXPECTED, + FORCE_READ_ONLY_SOURCE_FILE_EXPECTED, + false, + ); + } + + #[test] + fn test_generate_cpp_code_for_read_only_prod() { + let parsed_flags = crate::test::parse_read_only_test_flags(); + test_generate_cpp_code( + parsed_flags, + CodegenMode::Production, + READ_ONLY_EXPORTED_PROD_HEADER_EXPECTED, + READ_ONLY_PROD_SOURCE_FILE_EXPECTED, + false, + ); + } +} diff --git a/tools/aconfig/aconfig/src/codegen/java.rs b/tools/aconfig/aconfig/src/codegen/java.rs new file mode 100644 index 0000000000..3360ddd68b --- /dev/null +++ b/tools/aconfig/aconfig/src/codegen/java.rs @@ -0,0 +1,1270 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; +use tinytemplate::TinyTemplate; + +use aconfig_protos::{ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag}; + +use crate::codegen; +use crate::codegen::CodegenMode; +use crate::commands::OutputFile; + +pub fn generate_java_code<I>( + package: &str, + parsed_flags_iter: I, + codegen_mode: CodegenMode, +) -> Result<Vec<OutputFile>> +where + I: Iterator<Item = ProtoParsedFlag>, +{ + let flag_elements: Vec<FlagElement> = + parsed_flags_iter.map(|pf| create_flag_element(package, &pf)).collect(); + let namespace_flags = gen_flags_by_namespace(&flag_elements); + let properties_set: BTreeSet<String> = + flag_elements.iter().map(|fe| format_property_name(&fe.device_config_namespace)).collect(); + let is_test_mode = codegen_mode == CodegenMode::Test; + let library_exported = codegen_mode == CodegenMode::Exported; + let runtime_lookup_required = + flag_elements.iter().any(|elem| elem.is_read_write) || library_exported; + + let context = Context { + flag_elements, + namespace_flags, + is_test_mode, + runtime_lookup_required, + properties_set, + package_name: package.to_string(), + library_exported, + }; + let mut template = TinyTemplate::new(); + template.add_template("Flags.java", include_str!("../../templates/Flags.java.template"))?; + template.add_template( + "FeatureFlagsImpl.java", + include_str!("../../templates/FeatureFlagsImpl.java.template"), + )?; + template.add_template( + "FeatureFlags.java", + include_str!("../../templates/FeatureFlags.java.template"), + )?; + template.add_template( + "CustomFeatureFlags.java", + include_str!("../../templates/CustomFeatureFlags.java.template"), + )?; + template.add_template( + "FakeFeatureFlagsImpl.java", + include_str!("../../templates/FakeFeatureFlagsImpl.java.template"), + )?; + + let path: PathBuf = package.split('.').collect(); + [ + "Flags.java", + "FeatureFlags.java", + "FeatureFlagsImpl.java", + "CustomFeatureFlags.java", + "FakeFeatureFlagsImpl.java", + ] + .iter() + .map(|file| { + Ok(OutputFile { contents: template.render(file, &context)?.into(), path: path.join(file) }) + }) + .collect::<Result<Vec<OutputFile>>>() +} + +fn gen_flags_by_namespace(flags: &[FlagElement]) -> Vec<NamespaceFlags> { + let mut namespace_to_flag: BTreeMap<String, Vec<FlagElement>> = BTreeMap::new(); + + for flag in flags { + match namespace_to_flag.get_mut(&flag.device_config_namespace) { + Some(flag_list) => flag_list.push(flag.clone()), + None => { + namespace_to_flag.insert(flag.device_config_namespace.clone(), vec![flag.clone()]); + } + } + } + + namespace_to_flag + .iter() + .map(|(namespace, flags)| NamespaceFlags { + namespace: namespace.to_string(), + flags: flags.clone(), + }) + .collect() +} + +#[derive(Serialize)] +struct Context { + pub flag_elements: Vec<FlagElement>, + pub namespace_flags: Vec<NamespaceFlags>, + pub is_test_mode: bool, + pub runtime_lookup_required: bool, + pub properties_set: BTreeSet<String>, + pub package_name: String, + pub library_exported: bool, +} + +#[derive(Serialize, Debug)] +struct NamespaceFlags { + pub namespace: String, + pub flags: Vec<FlagElement>, +} + +#[derive(Serialize, Clone, Debug)] +struct FlagElement { + pub default_value: bool, + pub device_config_namespace: String, + pub device_config_flag: String, + pub flag_name_constant_suffix: String, + pub is_read_write: bool, + pub method_name: String, + pub properties: String, +} + +fn create_flag_element(package: &str, pf: &ProtoParsedFlag) -> FlagElement { + let device_config_flag = codegen::create_device_config_ident(package, pf.name()) + .expect("values checked at flag parse time"); + FlagElement { + default_value: pf.state() == ProtoFlagState::ENABLED, + device_config_namespace: pf.namespace().to_string(), + device_config_flag, + flag_name_constant_suffix: pf.name().to_ascii_uppercase(), + is_read_write: pf.permission() == ProtoFlagPermission::READ_WRITE, + method_name: format_java_method_name(pf.name()), + properties: format_property_name(pf.namespace()), + } +} + +fn format_java_method_name(flag_name: &str) -> String { + let splits: Vec<&str> = flag_name.split('_').filter(|&word| !word.is_empty()).collect(); + if splits.len() == 1 { + let name = splits[0]; + name[0..1].to_ascii_lowercase() + &name[1..] + } else { + splits + .iter() + .enumerate() + .map(|(index, word)| { + if index == 0 { + word.to_ascii_lowercase() + } else { + word[0..1].to_ascii_uppercase() + &word[1..].to_ascii_lowercase() + } + }) + .collect::<Vec<String>>() + .join("") + } +} + +fn format_property_name(property_name: &str) -> String { + let name = format_java_method_name(property_name); + format!("mProperties{}{}", &name[0..1].to_ascii_uppercase(), &name[1..]) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + const EXPECTED_FEATUREFLAGS_COMMON_CONTENT: &str = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + /** @hide */ + public interface FeatureFlags { + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRo(); + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRw(); + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRwExported(); + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRwInOtherNamespace(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledFixedRo(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledFixedRoExported(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledRo(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledRoExported(); + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledRw(); + } + "#; + + const EXPECTED_FLAG_COMMON_CONTENT: &str = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + /** @hide */ + public final class Flags { + /** @hide */ + public static final String FLAG_DISABLED_RO = "com.android.aconfig.test.disabled_ro"; + /** @hide */ + public static final String FLAG_DISABLED_RW = "com.android.aconfig.test.disabled_rw"; + /** @hide */ + public static final String FLAG_DISABLED_RW_EXPORTED = "com.android.aconfig.test.disabled_rw_exported"; + /** @hide */ + public static final String FLAG_DISABLED_RW_IN_OTHER_NAMESPACE = "com.android.aconfig.test.disabled_rw_in_other_namespace"; + /** @hide */ + public static final String FLAG_ENABLED_FIXED_RO = "com.android.aconfig.test.enabled_fixed_ro"; + /** @hide */ + public static final String FLAG_ENABLED_FIXED_RO_EXPORTED = "com.android.aconfig.test.enabled_fixed_ro_exported"; + /** @hide */ + public static final String FLAG_ENABLED_RO = "com.android.aconfig.test.enabled_ro"; + /** @hide */ + public static final String FLAG_ENABLED_RO_EXPORTED = "com.android.aconfig.test.enabled_ro_exported"; + /** @hide */ + public static final String FLAG_ENABLED_RW = "com.android.aconfig.test.enabled_rw"; + + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRo() { + return FEATURE_FLAGS.disabledRo(); + } + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRw() { + return FEATURE_FLAGS.disabledRw(); + } + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRwExported() { + return FEATURE_FLAGS.disabledRwExported(); + } + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRwInOtherNamespace() { + return FEATURE_FLAGS.disabledRwInOtherNamespace(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledFixedRo() { + return FEATURE_FLAGS.enabledFixedRo(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledFixedRoExported() { + return FEATURE_FLAGS.enabledFixedRoExported(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledRo() { + return FEATURE_FLAGS.enabledRo(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledRoExported() { + return FEATURE_FLAGS.enabledRoExported(); + } + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledRw() { + return FEATURE_FLAGS.enabledRw(); + } + "#; + + const EXPECTED_CUSTOMFEATUREFLAGS_CONTENT: &str = r#" + package com.android.aconfig.test; + + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + import java.util.Arrays; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + import java.util.function.BiPredicate; + import java.util.function.Predicate; + + /** @hide */ + public class CustomFeatureFlags implements FeatureFlags { + + private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; + + public CustomFeatureFlags(BiPredicate<String, Predicate<FeatureFlags>> getValueImpl) { + mGetValueImpl = getValueImpl; + } + + @Override + @UnsupportedAppUsage + public boolean disabledRo() { + return getValue(Flags.FLAG_DISABLED_RO, + FeatureFlags::disabledRo); + } + @Override + @UnsupportedAppUsage + public boolean disabledRw() { + return getValue(Flags.FLAG_DISABLED_RW, + FeatureFlags::disabledRw); + } + @Override + @UnsupportedAppUsage + public boolean disabledRwExported() { + return getValue(Flags.FLAG_DISABLED_RW_EXPORTED, + FeatureFlags::disabledRwExported); + } + @Override + @UnsupportedAppUsage + public boolean disabledRwInOtherNamespace() { + return getValue(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, + FeatureFlags::disabledRwInOtherNamespace); + } + @Override + @UnsupportedAppUsage + public boolean enabledFixedRo() { + return getValue(Flags.FLAG_ENABLED_FIXED_RO, + FeatureFlags::enabledFixedRo); + } + @Override + @UnsupportedAppUsage + public boolean enabledFixedRoExported() { + return getValue(Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + FeatureFlags::enabledFixedRoExported); + } + @Override + @UnsupportedAppUsage + public boolean enabledRo() { + return getValue(Flags.FLAG_ENABLED_RO, + FeatureFlags::enabledRo); + } + @Override + @UnsupportedAppUsage + public boolean enabledRoExported() { + return getValue(Flags.FLAG_ENABLED_RO_EXPORTED, + FeatureFlags::enabledRoExported); + } + @Override + @UnsupportedAppUsage + public boolean enabledRw() { + return getValue(Flags.FLAG_ENABLED_RW, + FeatureFlags::enabledRw); + } + + public boolean isFlagReadOnlyOptimized(String flagName) { + if (mReadOnlyFlagsSet.contains(flagName) && + isOptimizationEnabled()) { + return true; + } + return false; + } + + @com.android.aconfig.annotations.AssumeTrueForR8 + private boolean isOptimizationEnabled() { + return false; + } + + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) { + return mGetValueImpl.test(flagName, getter); + } + + public List<String> getFlagNames() { + return Arrays.asList( + Flags.FLAG_DISABLED_RO, + Flags.FLAG_DISABLED_RW, + Flags.FLAG_DISABLED_RW_EXPORTED, + Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, + Flags.FLAG_ENABLED_FIXED_RO, + Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + Flags.FLAG_ENABLED_RO, + Flags.FLAG_ENABLED_RO_EXPORTED, + Flags.FLAG_ENABLED_RW + ); + } + + private Set<String> mReadOnlyFlagsSet = new HashSet<>( + Arrays.asList( + Flags.FLAG_DISABLED_RO, + Flags.FLAG_ENABLED_FIXED_RO, + Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + Flags.FLAG_ENABLED_RO, + Flags.FLAG_ENABLED_RO_EXPORTED, + "" + ) + ); + } + "#; + + const EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT: &str = r#" + package com.android.aconfig.test; + + import java.util.HashMap; + import java.util.Map; + import java.util.function.Predicate; + + /** @hide */ + public class FakeFeatureFlagsImpl extends CustomFeatureFlags { + private final Map<String, Boolean> mFlagMap = new HashMap<>(); + private final FeatureFlags mDefaults; + + public FakeFeatureFlagsImpl() { + this(null); + } + + public FakeFeatureFlagsImpl(FeatureFlags defaults) { + super(null); + mDefaults = defaults; + // Initialize the map with null values + for (String flagName : getFlagNames()) { + mFlagMap.put(flagName, null); + } + } + + @Override + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) { + Boolean value = this.mFlagMap.get(flagName); + if (value != null) { + return value; + } + if (mDefaults != null) { + return getter.test(mDefaults); + } + throw new IllegalArgumentException(flagName + " is not set"); + } + + public void setFlag(String flagName, boolean value) { + if (!this.mFlagMap.containsKey(flagName)) { + throw new IllegalArgumentException("no such flag " + flagName); + } + this.mFlagMap.put(flagName, value); + } + + public void resetAll() { + for (Map.Entry entry : mFlagMap.entrySet()) { + entry.setValue(null); + } + } + } + "#; + + #[test] + fn test_generate_java_code_production() { + let parsed_flags = crate::test::parse_test_flags(); + let mode = CodegenMode::Production; + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let generated_files = + generate_java_code(crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), mode) + .unwrap(); + let expect_flags_content = EXPECTED_FLAG_COMMON_CONTENT.to_string() + + r#" + private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl(); + }"#; + + let expect_featureflagsimpl_content = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + import android.provider.DeviceConfig; + import android.provider.DeviceConfig.Properties; + /** @hide */ + public final class FeatureFlagsImpl implements FeatureFlags { + private static boolean aconfig_test_is_cached = false; + private static boolean other_namespace_is_cached = false; + private static boolean disabledRw = false; + private static boolean disabledRwExported = false; + private static boolean disabledRwInOtherNamespace = false; + private static boolean enabledRw = true; + + + private void load_overrides_aconfig_test() { + try { + Properties properties = DeviceConfig.getProperties("aconfig_test"); + disabledRw = + properties.getBoolean("com.android.aconfig.test.disabled_rw", false); + disabledRwExported = + properties.getBoolean("com.android.aconfig.test.disabled_rw_exported", false); + enabledRw = + properties.getBoolean("com.android.aconfig.test.enabled_rw", true); + } catch (NullPointerException e) { + throw new RuntimeException( + "Cannot read value from namespace aconfig_test " + + "from DeviceConfig. It could be that the code using flag " + + "executed before SettingsProvider initialization. Please use " + + "fixed read-only flag by adding is_fixed_read_only: true in " + + "flag declaration.", + e + ); + } + aconfig_test_is_cached = true; + } + + private void load_overrides_other_namespace() { + try { + Properties properties = DeviceConfig.getProperties("other_namespace"); + disabledRwInOtherNamespace = + properties.getBoolean("com.android.aconfig.test.disabled_rw_in_other_namespace", false); + } catch (NullPointerException e) { + throw new RuntimeException( + "Cannot read value from namespace other_namespace " + + "from DeviceConfig. It could be that the code using flag " + + "executed before SettingsProvider initialization. Please use " + + "fixed read-only flag by adding is_fixed_read_only: true in " + + "flag declaration.", + e + ); + } + other_namespace_is_cached = true; + } + + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRo() { + return false; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRw() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return disabledRw; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRwExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return disabledRwExported; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRwInOtherNamespace() { + if (!other_namespace_is_cached) { + load_overrides_other_namespace(); + } + return disabledRwInOtherNamespace; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledFixedRo() { + return true; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledFixedRoExported() { + return true; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRo() { + return true; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRoExported() { + return true; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRw() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return enabledRw; + } + } + "#; + let mut file_set = HashMap::from([ + ("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_featureflagsimpl_content), + ("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT), + ( + "com/android/aconfig/test/CustomFeatureFlags.java", + EXPECTED_CUSTOMFEATUREFLAGS_CONTENT, + ), + ( + "com/android/aconfig/test/FakeFeatureFlagsImpl.java", + EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT, + ), + ]); + + for file in generated_files { + let file_path = file.path.to_str().unwrap(); + assert!(file_set.contains_key(file_path), "Cannot find {}", file_path); + assert_eq!( + None, + crate::test::first_significant_code_diff( + file_set.get(file_path).unwrap(), + &String::from_utf8(file.contents).unwrap() + ), + "File {} content is not correct", + file_path + ); + file_set.remove(file_path); + } + + assert!(file_set.is_empty()); + } + + #[test] + fn test_generate_java_code_exported() { + let parsed_flags = crate::test::parse_test_flags(); + let mode = CodegenMode::Exported; + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let generated_files = + generate_java_code(crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), mode) + .unwrap(); + + let expect_flags_content = r#" + package com.android.aconfig.test; + /** @hide */ + public final class Flags { + /** @hide */ + public static final String FLAG_DISABLED_RW_EXPORTED = "com.android.aconfig.test.disabled_rw_exported"; + /** @hide */ + public static final String FLAG_ENABLED_FIXED_RO_EXPORTED = "com.android.aconfig.test.enabled_fixed_ro_exported"; + /** @hide */ + public static final String FLAG_ENABLED_RO_EXPORTED = "com.android.aconfig.test.enabled_ro_exported"; + public static boolean disabledRwExported() { + return FEATURE_FLAGS.disabledRwExported(); + } + public static boolean enabledFixedRoExported() { + return FEATURE_FLAGS.enabledFixedRoExported(); + } + public static boolean enabledRoExported() { + return FEATURE_FLAGS.enabledRoExported(); + } + private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl(); + } + "#; + + let expect_feature_flags_content = r#" + package com.android.aconfig.test; + /** @hide */ + public interface FeatureFlags { + boolean disabledRwExported(); + boolean enabledFixedRoExported(); + boolean enabledRoExported(); + } + "#; + + let expect_feature_flags_impl_content = r#" + package com.android.aconfig.test; + import android.provider.DeviceConfig; + import android.provider.DeviceConfig.Properties; + /** @hide */ + public final class FeatureFlagsImpl implements FeatureFlags { + private static boolean aconfig_test_is_cached = false; + private static boolean disabledRwExported = false; + private static boolean enabledFixedRoExported = false; + private static boolean enabledRoExported = false; + + + private void load_overrides_aconfig_test() { + try { + Properties properties = DeviceConfig.getProperties("aconfig_test"); + disabledRwExported = + properties.getBoolean("com.android.aconfig.test.disabled_rw_exported", false); + enabledFixedRoExported = + properties.getBoolean("com.android.aconfig.test.enabled_fixed_ro_exported", false); + enabledRoExported = + properties.getBoolean("com.android.aconfig.test.enabled_ro_exported", false); + } catch (NullPointerException e) { + throw new RuntimeException( + "Cannot read value from namespace aconfig_test " + + "from DeviceConfig. It could be that the code using flag " + + "executed before SettingsProvider initialization. Please use " + + "fixed read-only flag by adding is_fixed_read_only: true in " + + "flag declaration.", + e + ); + } + aconfig_test_is_cached = true; + } + @Override + public boolean disabledRwExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return disabledRwExported; + } + @Override + public boolean enabledFixedRoExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return enabledFixedRoExported; + } + @Override + public boolean enabledRoExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return enabledRoExported; + } + }"#; + + let expect_custom_feature_flags_content = r#" + package com.android.aconfig.test; + + import java.util.Arrays; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + import java.util.function.BiPredicate; + import java.util.function.Predicate; + + /** @hide */ + public class CustomFeatureFlags implements FeatureFlags { + + private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; + + public CustomFeatureFlags(BiPredicate<String, Predicate<FeatureFlags>> getValueImpl) { + mGetValueImpl = getValueImpl; + } + + @Override + public boolean disabledRwExported() { + return getValue(Flags.FLAG_DISABLED_RW_EXPORTED, + FeatureFlags::disabledRwExported); + } + @Override + public boolean enabledFixedRoExported() { + return getValue(Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + FeatureFlags::enabledFixedRoExported); + } + @Override + public boolean enabledRoExported() { + return getValue(Flags.FLAG_ENABLED_RO_EXPORTED, + FeatureFlags::enabledRoExported); + } + + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) { + return mGetValueImpl.test(flagName, getter); + } + + public List<String> getFlagNames() { + return Arrays.asList( + Flags.FLAG_DISABLED_RW_EXPORTED, + Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + Flags.FLAG_ENABLED_RO_EXPORTED + ); + } + + private Set<String> mReadOnlyFlagsSet = new HashSet<>( + Arrays.asList( + "" + ) + ); + } + "#; + + let mut file_set = HashMap::from([ + ("com/android/aconfig/test/Flags.java", expect_flags_content), + ("com/android/aconfig/test/FeatureFlags.java", expect_feature_flags_content), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_feature_flags_impl_content), + ( + "com/android/aconfig/test/CustomFeatureFlags.java", + expect_custom_feature_flags_content, + ), + ( + "com/android/aconfig/test/FakeFeatureFlagsImpl.java", + EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT, + ), + ]); + + for file in generated_files { + let file_path = file.path.to_str().unwrap(); + assert!(file_set.contains_key(file_path), "Cannot find {}", file_path); + assert_eq!( + None, + crate::test::first_significant_code_diff( + file_set.get(file_path).unwrap(), + &String::from_utf8(file.contents).unwrap() + ), + "File {} content is not correct", + file_path + ); + file_set.remove(file_path); + } + + assert!(file_set.is_empty()); + } + + #[test] + fn test_generate_java_code_test() { + let parsed_flags = crate::test::parse_test_flags(); + let mode = CodegenMode::Test; + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let generated_files = + generate_java_code(crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), mode) + .unwrap(); + + let expect_flags_content = EXPECTED_FLAG_COMMON_CONTENT.to_string() + + r#" + public static void setFeatureFlags(FeatureFlags featureFlags) { + Flags.FEATURE_FLAGS = featureFlags; + } + public static void unsetFeatureFlags() { + Flags.FEATURE_FLAGS = null; + } + private static FeatureFlags FEATURE_FLAGS; + } + "#; + let expect_featureflagsimpl_content = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + /** @hide */ + public final class FeatureFlagsImpl implements FeatureFlags { + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRo() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRw() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRwExported() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRwInOtherNamespace() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledFixedRo() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledFixedRoExported() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRo() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRoExported() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRw() { + throw new UnsupportedOperationException( + "Method is not implemented."); + } + } + "#; + + let mut file_set = HashMap::from([ + ("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()), + ("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_featureflagsimpl_content), + ( + "com/android/aconfig/test/CustomFeatureFlags.java", + EXPECTED_CUSTOMFEATUREFLAGS_CONTENT, + ), + ( + "com/android/aconfig/test/FakeFeatureFlagsImpl.java", + EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT, + ), + ]); + + for file in generated_files { + let file_path = file.path.to_str().unwrap(); + assert!(file_set.contains_key(file_path), "Cannot find {}", file_path); + assert_eq!( + None, + crate::test::first_significant_code_diff( + file_set.get(file_path).unwrap(), + &String::from_utf8(file.contents).unwrap() + ), + "File {} content is not correct", + file_path + ); + file_set.remove(file_path); + } + + assert!(file_set.is_empty()); + } + + #[test] + fn test_generate_java_code_force_read_only() { + let parsed_flags = crate::test::parse_test_flags(); + let mode = CodegenMode::ForceReadOnly; + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let generated_files = + generate_java_code(crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), mode) + .unwrap(); + let expect_featureflags_content = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + /** @hide */ + public interface FeatureFlags { + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRo(); + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRw(); + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean disabledRwInOtherNamespace(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledFixedRo(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledRo(); + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + boolean enabledRw(); + }"#; + + let expect_featureflagsimpl_content = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + /** @hide */ + public final class FeatureFlagsImpl implements FeatureFlags { + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRo() { + return false; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRw() { + return false; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean disabledRwInOtherNamespace() { + return false; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledFixedRo() { + return true; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRo() { + return true; + } + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean enabledRw() { + return true; + } + } + "#; + + let expect_flags_content = r#" + package com.android.aconfig.test; + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + /** @hide */ + public final class Flags { + /** @hide */ + public static final String FLAG_DISABLED_RO = "com.android.aconfig.test.disabled_ro"; + /** @hide */ + public static final String FLAG_DISABLED_RW = "com.android.aconfig.test.disabled_rw"; + /** @hide */ + public static final String FLAG_DISABLED_RW_IN_OTHER_NAMESPACE = "com.android.aconfig.test.disabled_rw_in_other_namespace"; + /** @hide */ + public static final String FLAG_ENABLED_FIXED_RO = "com.android.aconfig.test.enabled_fixed_ro"; + /** @hide */ + public static final String FLAG_ENABLED_RO = "com.android.aconfig.test.enabled_ro"; + /** @hide */ + public static final String FLAG_ENABLED_RW = "com.android.aconfig.test.enabled_rw"; + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRo() { + return FEATURE_FLAGS.disabledRo(); + } + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRw() { + return FEATURE_FLAGS.disabledRw(); + } + @com.android.aconfig.annotations.AssumeFalseForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean disabledRwInOtherNamespace() { + return FEATURE_FLAGS.disabledRwInOtherNamespace(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledFixedRo() { + return FEATURE_FLAGS.enabledFixedRo(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledRo() { + return FEATURE_FLAGS.enabledRo(); + } + @com.android.aconfig.annotations.AssumeTrueForR8 + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public static boolean enabledRw() { + return FEATURE_FLAGS.enabledRw(); + } + private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl(); + }"#; + + let expect_customfeatureflags_content = r#" + package com.android.aconfig.test; + + // TODO(b/303773055): Remove the annotation after access issue is resolved. + import android.compat.annotation.UnsupportedAppUsage; + import java.util.Arrays; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + import java.util.function.BiPredicate; + import java.util.function.Predicate; + + /** @hide */ + public class CustomFeatureFlags implements FeatureFlags { + + private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; + + public CustomFeatureFlags(BiPredicate<String, Predicate<FeatureFlags>> getValueImpl) { + mGetValueImpl = getValueImpl; + } + + @Override + @UnsupportedAppUsage + public boolean disabledRo() { + return getValue(Flags.FLAG_DISABLED_RO, + FeatureFlags::disabledRo); + } + @Override + @UnsupportedAppUsage + public boolean disabledRw() { + return getValue(Flags.FLAG_DISABLED_RW, + FeatureFlags::disabledRw); + } + @Override + @UnsupportedAppUsage + public boolean disabledRwInOtherNamespace() { + return getValue(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, + FeatureFlags::disabledRwInOtherNamespace); + } + @Override + @UnsupportedAppUsage + public boolean enabledFixedRo() { + return getValue(Flags.FLAG_ENABLED_FIXED_RO, + FeatureFlags::enabledFixedRo); + } + @Override + @UnsupportedAppUsage + public boolean enabledRo() { + return getValue(Flags.FLAG_ENABLED_RO, + FeatureFlags::enabledRo); + } + @Override + @UnsupportedAppUsage + public boolean enabledRw() { + return getValue(Flags.FLAG_ENABLED_RW, + FeatureFlags::enabledRw); + } + + public boolean isFlagReadOnlyOptimized(String flagName) { + if (mReadOnlyFlagsSet.contains(flagName) && + isOptimizationEnabled()) { + return true; + } + return false; + } + + @com.android.aconfig.annotations.AssumeTrueForR8 + private boolean isOptimizationEnabled() { + return false; + } + + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) { + return mGetValueImpl.test(flagName, getter); + } + + public List<String> getFlagNames() { + return Arrays.asList( + Flags.FLAG_DISABLED_RO, + Flags.FLAG_DISABLED_RW, + Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, + Flags.FLAG_ENABLED_FIXED_RO, + Flags.FLAG_ENABLED_RO, + Flags.FLAG_ENABLED_RW + ); + } + + private Set<String> mReadOnlyFlagsSet = new HashSet<>( + Arrays.asList( + Flags.FLAG_DISABLED_RO, + Flags.FLAG_DISABLED_RW, + Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, + Flags.FLAG_ENABLED_FIXED_RO, + Flags.FLAG_ENABLED_RO, + Flags.FLAG_ENABLED_RW, + "" + ) + ); + } + "#; + + let mut file_set = HashMap::from([ + ("com/android/aconfig/test/Flags.java", expect_flags_content), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_featureflagsimpl_content), + ("com/android/aconfig/test/FeatureFlags.java", expect_featureflags_content), + ("com/android/aconfig/test/CustomFeatureFlags.java", expect_customfeatureflags_content), + ( + "com/android/aconfig/test/FakeFeatureFlagsImpl.java", + EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT, + ), + ]); + + for file in generated_files { + let file_path = file.path.to_str().unwrap(); + assert!(file_set.contains_key(file_path), "Cannot find {}", file_path); + assert_eq!( + None, + crate::test::first_significant_code_diff( + file_set.get(file_path).unwrap(), + &String::from_utf8(file.contents).unwrap() + ), + "File {} content is not correct", + file_path + ); + file_set.remove(file_path); + } + + assert!(file_set.is_empty()); + } + + #[test] + fn test_format_java_method_name() { + let expected = "someSnakeName"; + let input = "____some_snake___name____"; + let formatted_name = format_java_method_name(input); + assert_eq!(expected, formatted_name); + + let input = "someSnakeName"; + let formatted_name = format_java_method_name(input); + assert_eq!(expected, formatted_name); + + let input = "SomeSnakeName"; + let formatted_name = format_java_method_name(input); + assert_eq!(expected, formatted_name); + + let input = "SomeSnakeName_"; + let formatted_name = format_java_method_name(input); + assert_eq!(expected, formatted_name); + + let input = "_SomeSnakeName"; + let formatted_name = format_java_method_name(input); + assert_eq!(expected, formatted_name); + } + + #[test] + fn test_format_property_name() { + let expected = "mPropertiesSomeSnakeName"; + let input = "____some_snake___name____"; + let formatted_name = format_property_name(input); + assert_eq!(expected, formatted_name); + + let input = "someSnakeName"; + let formatted_name = format_property_name(input); + assert_eq!(expected, formatted_name); + + let input = "SomeSnakeName"; + let formatted_name = format_property_name(input); + assert_eq!(expected, formatted_name); + + let input = "SomeSnakeName_"; + let formatted_name = format_property_name(input); + assert_eq!(expected, formatted_name); + } +} diff --git a/tools/aconfig/aconfig/src/codegen/mod.rs b/tools/aconfig/aconfig/src/codegen/mod.rs new file mode 100644 index 0000000000..1ea3b37849 --- /dev/null +++ b/tools/aconfig/aconfig/src/codegen/mod.rs @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pub mod cpp; +pub mod java; +pub mod rust; + +use aconfig_protos::{is_valid_name_ident, is_valid_package_ident}; +use anyhow::{ensure, Result}; +use clap::ValueEnum; + +pub fn create_device_config_ident(package: &str, flag_name: &str) -> Result<String> { + ensure!(is_valid_package_ident(package), "bad package"); + ensure!(is_valid_name_ident(flag_name), "bad flag name"); + Ok(format!("{}.{}", package, flag_name)) +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)] +pub enum CodegenMode { + Exported, + ForceReadOnly, + Production, + Test, +} + +impl std::fmt::Display for CodegenMode { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + CodegenMode::Exported => write!(f, "exported"), + CodegenMode::ForceReadOnly => write!(f, "force-read-only"), + CodegenMode::Production => write!(f, "production"), + CodegenMode::Test => write!(f, "test"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_protos::is_valid_container_ident; + + #[test] + fn test_is_valid_name_ident() { + assert!(is_valid_name_ident("foo")); + assert!(is_valid_name_ident("foo_bar_123")); + assert!(is_valid_name_ident("foo_")); + + assert!(!is_valid_name_ident("")); + assert!(!is_valid_name_ident("123_foo")); + assert!(!is_valid_name_ident("foo-bar")); + assert!(!is_valid_name_ident("foo-b\u{00e5}r")); + assert!(!is_valid_name_ident("foo__bar")); + assert!(!is_valid_name_ident("_foo")); + } + + #[test] + fn test_is_valid_package_ident() { + assert!(is_valid_package_ident("foo.bar")); + assert!(is_valid_package_ident("foo.bar_baz")); + assert!(is_valid_package_ident("foo.bar.a123")); + + assert!(!is_valid_package_ident("foo_bar_123")); + assert!(!is_valid_package_ident("foo")); + assert!(!is_valid_package_ident("foo._bar")); + assert!(!is_valid_package_ident("")); + assert!(!is_valid_package_ident("123_foo")); + assert!(!is_valid_package_ident("foo-bar")); + assert!(!is_valid_package_ident("foo-b\u{00e5}r")); + assert!(!is_valid_package_ident("foo.bar.123")); + assert!(!is_valid_package_ident(".foo.bar")); + assert!(!is_valid_package_ident("foo.bar.")); + assert!(!is_valid_package_ident(".")); + assert!(!is_valid_package_ident("..")); + assert!(!is_valid_package_ident("foo..bar")); + assert!(!is_valid_package_ident("foo.__bar")); + } + + #[test] + fn test_is_valid_container_ident() { + assert!(is_valid_container_ident("foo.bar")); + assert!(is_valid_container_ident("foo.bar_baz")); + assert!(is_valid_container_ident("foo.bar.a123")); + assert!(is_valid_container_ident("foo")); + assert!(is_valid_container_ident("foo_bar_123")); + + assert!(!is_valid_container_ident("")); + assert!(!is_valid_container_ident("foo._bar")); + assert!(!is_valid_container_ident("_foo")); + assert!(!is_valid_container_ident("123_foo")); + assert!(!is_valid_container_ident("foo-bar")); + assert!(!is_valid_container_ident("foo-b\u{00e5}r")); + assert!(!is_valid_container_ident("foo.bar.123")); + assert!(!is_valid_container_ident(".foo.bar")); + assert!(!is_valid_container_ident("foo.bar.")); + assert!(!is_valid_container_ident(".")); + assert!(!is_valid_container_ident("..")); + assert!(!is_valid_container_ident("foo..bar")); + assert!(!is_valid_container_ident("foo.__bar")); + } + + #[test] + fn test_create_device_config_ident() { + assert_eq!( + "com.foo.bar.some_flag", + create_device_config_ident("com.foo.bar", "some_flag").unwrap() + ); + } +} diff --git a/tools/aconfig/aconfig/src/codegen/rust.rs b/tools/aconfig/aconfig/src/codegen/rust.rs new file mode 100644 index 0000000000..33c3d37633 --- /dev/null +++ b/tools/aconfig/aconfig/src/codegen/rust.rs @@ -0,0 +1,678 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; +use serde::Serialize; +use tinytemplate::TinyTemplate; + +use aconfig_protos::{ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag}; + +use crate::codegen; +use crate::codegen::CodegenMode; +use crate::commands::OutputFile; + +pub fn generate_rust_code<I>( + package: &str, + parsed_flags_iter: I, + codegen_mode: CodegenMode, +) -> Result<OutputFile> +where + I: Iterator<Item = ProtoParsedFlag>, +{ + let template_flags: Vec<TemplateParsedFlag> = + parsed_flags_iter.map(|pf| TemplateParsedFlag::new(package, &pf)).collect(); + let has_readwrite = template_flags.iter().any(|item| item.readwrite); + let context = TemplateContext { + package: package.to_string(), + template_flags, + modules: package.split('.').map(|s| s.to_string()).collect::<Vec<_>>(), + has_readwrite, + }; + let mut template = TinyTemplate::new(); + template.add_template( + "rust_code_gen", + match codegen_mode { + CodegenMode::Test => include_str!("../../templates/rust_test.template"), + CodegenMode::Exported | CodegenMode::ForceReadOnly | CodegenMode::Production => { + include_str!("../../templates/rust.template") + } + }, + )?; + let contents = template.render("rust_code_gen", &context)?; + let path = ["src", "lib.rs"].iter().collect(); + Ok(OutputFile { contents: contents.into(), path }) +} + +#[derive(Serialize)] +struct TemplateContext { + pub package: String, + pub template_flags: Vec<TemplateParsedFlag>, + pub modules: Vec<String>, + pub has_readwrite: bool, +} + +#[derive(Serialize)] +struct TemplateParsedFlag { + pub readwrite: bool, + pub default_value: String, + pub name: String, + pub device_config_namespace: String, + pub device_config_flag: String, +} + +impl TemplateParsedFlag { + #[allow(clippy::nonminimal_bool)] + fn new(package: &str, pf: &ProtoParsedFlag) -> Self { + let template = TemplateParsedFlag { + readwrite: pf.permission() == ProtoFlagPermission::READ_WRITE, + default_value: match pf.state() { + ProtoFlagState::ENABLED => "true".to_string(), + ProtoFlagState::DISABLED => "false".to_string(), + }, + name: pf.name().to_string(), + device_config_namespace: pf.namespace().to_string(), + device_config_flag: codegen::create_device_config_ident(package, pf.name()) + .expect("values checked at flag parse time"), + }; + template + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PROD_EXPECTED: &str = r#" +//! codegenerated rust flag lib + +/// flag provider +pub struct FlagProvider; + +lazy_static::lazy_static! { + /// flag value cache for disabled_rw + static ref CACHED_disabled_rw: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw", + "false") == "true"; + + /// flag value cache for disabled_rw_exported + static ref CACHED_disabled_rw_exported: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw_exported", + "false") == "true"; + + /// flag value cache for disabled_rw_in_other_namespace + static ref CACHED_disabled_rw_in_other_namespace: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.other_namespace", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "false") == "true"; + + /// flag value cache for enabled_rw + static ref CACHED_enabled_rw: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_rw", + "true") == "true"; + +} + +impl FlagProvider { + /// query flag disabled_ro + pub fn disabled_ro(&self) -> bool { + false + } + + /// query flag disabled_rw + pub fn disabled_rw(&self) -> bool { + *CACHED_disabled_rw + } + + /// query flag disabled_rw_exported + pub fn disabled_rw_exported(&self) -> bool { + *CACHED_disabled_rw_exported + } + + /// query flag disabled_rw_in_other_namespace + pub fn disabled_rw_in_other_namespace(&self) -> bool { + *CACHED_disabled_rw_in_other_namespace + } + + /// query flag enabled_fixed_ro + pub fn enabled_fixed_ro(&self) -> bool { + true + } + + /// query flag enabled_fixed_ro_exported + pub fn enabled_fixed_ro_exported(&self) -> bool { + true + } + + /// query flag enabled_ro + pub fn enabled_ro(&self) -> bool { + true + } + + /// query flag enabled_ro_exported + pub fn enabled_ro_exported(&self) -> bool { + true + } + + /// query flag enabled_rw + pub fn enabled_rw(&self) -> bool { + *CACHED_enabled_rw + } +} + +/// flag provider +pub static PROVIDER: FlagProvider = FlagProvider; + +/// query flag disabled_ro +#[inline(always)] +pub fn disabled_ro() -> bool { + false +} + +/// query flag disabled_rw +#[inline(always)] +pub fn disabled_rw() -> bool { + PROVIDER.disabled_rw() +} + +/// query flag disabled_rw_exported +#[inline(always)] +pub fn disabled_rw_exported() -> bool { + PROVIDER.disabled_rw_exported() +} + +/// query flag disabled_rw_in_other_namespace +#[inline(always)] +pub fn disabled_rw_in_other_namespace() -> bool { + PROVIDER.disabled_rw_in_other_namespace() +} + +/// query flag enabled_fixed_ro +#[inline(always)] +pub fn enabled_fixed_ro() -> bool { + true +} + +/// query flag enabled_fixed_ro_exported +#[inline(always)] +pub fn enabled_fixed_ro_exported() -> bool { + true +} + +/// query flag enabled_ro +#[inline(always)] +pub fn enabled_ro() -> bool { + true +} + +/// query flag enabled_ro_exported +#[inline(always)] +pub fn enabled_ro_exported() -> bool { + true +} + +/// query flag enabled_rw +#[inline(always)] +pub fn enabled_rw() -> bool { + PROVIDER.enabled_rw() +} +"#; + + const TEST_EXPECTED: &str = r#" +//! codegenerated rust flag lib + +use std::collections::BTreeMap; +use std::sync::Mutex; + +/// flag provider +pub struct FlagProvider { + overrides: BTreeMap<&'static str, bool>, +} + +impl FlagProvider { + /// query flag disabled_ro + pub fn disabled_ro(&self) -> bool { + self.overrides.get("disabled_ro").copied().unwrap_or( + false + ) + } + + /// set flag disabled_ro + pub fn set_disabled_ro(&mut self, val: bool) { + self.overrides.insert("disabled_ro", val); + } + + /// query flag disabled_rw + pub fn disabled_rw(&self) -> bool { + self.overrides.get("disabled_rw").copied().unwrap_or( + flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw", + "false") == "true" + ) + } + + /// set flag disabled_rw + pub fn set_disabled_rw(&mut self, val: bool) { + self.overrides.insert("disabled_rw", val); + } + + /// query flag disabled_rw_exported + pub fn disabled_rw_exported(&self) -> bool { + self.overrides.get("disabled_rw_exported").copied().unwrap_or( + flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw_exported", + "false") == "true" + ) + } + + /// set flag disabled_rw_exported + pub fn set_disabled_rw_exported(&mut self, val: bool) { + self.overrides.insert("disabled_rw_exported", val); + } + + /// query flag disabled_rw_in_other_namespace + pub fn disabled_rw_in_other_namespace(&self) -> bool { + self.overrides.get("disabled_rw_in_other_namespace").copied().unwrap_or( + flags_rust::GetServerConfigurableFlag( + "aconfig_flags.other_namespace", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "false") == "true" + ) + } + + /// set flag disabled_rw_in_other_namespace + pub fn set_disabled_rw_in_other_namespace(&mut self, val: bool) { + self.overrides.insert("disabled_rw_in_other_namespace", val); + } + + /// query flag enabled_fixed_ro + pub fn enabled_fixed_ro(&self) -> bool { + self.overrides.get("enabled_fixed_ro").copied().unwrap_or( + true + ) + } + + /// set flag enabled_fixed_ro + pub fn set_enabled_fixed_ro(&mut self, val: bool) { + self.overrides.insert("enabled_fixed_ro", val); + } + + /// query flag enabled_fixed_ro_exported + pub fn enabled_fixed_ro_exported(&self) -> bool { + self.overrides.get("enabled_fixed_ro_exported").copied().unwrap_or( + true + ) + } + + /// set flag enabled_fixed_ro_exported + pub fn set_enabled_fixed_ro_exported(&mut self, val: bool) { + self.overrides.insert("enabled_fixed_ro_exported", val); + } + + /// query flag enabled_ro + pub fn enabled_ro(&self) -> bool { + self.overrides.get("enabled_ro").copied().unwrap_or( + true + ) + } + + /// set flag enabled_ro + pub fn set_enabled_ro(&mut self, val: bool) { + self.overrides.insert("enabled_ro", val); + } + + /// query flag enabled_ro_exported + pub fn enabled_ro_exported(&self) -> bool { + self.overrides.get("enabled_ro_exported").copied().unwrap_or( + true + ) + } + + /// set flag enabled_ro_exported + pub fn set_enabled_ro_exported(&mut self, val: bool) { + self.overrides.insert("enabled_ro_exported", val); + } + + /// query flag enabled_rw + pub fn enabled_rw(&self) -> bool { + self.overrides.get("enabled_rw").copied().unwrap_or( + flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_rw", + "true") == "true" + ) + } + + /// set flag enabled_rw + pub fn set_enabled_rw(&mut self, val: bool) { + self.overrides.insert("enabled_rw", val); + } + + /// clear all flag overrides + pub fn reset_flags(&mut self) { + self.overrides.clear(); + } +} + +/// flag provider +pub static PROVIDER: Mutex<FlagProvider> = Mutex::new( + FlagProvider {overrides: BTreeMap::new()} +); + +/// query flag disabled_ro +#[inline(always)] +pub fn disabled_ro() -> bool { + PROVIDER.lock().unwrap().disabled_ro() +} + +/// set flag disabled_ro +#[inline(always)] +pub fn set_disabled_ro(val: bool) { + PROVIDER.lock().unwrap().set_disabled_ro(val); +} + +/// query flag disabled_rw +#[inline(always)] +pub fn disabled_rw() -> bool { + PROVIDER.lock().unwrap().disabled_rw() +} + +/// set flag disabled_rw +#[inline(always)] +pub fn set_disabled_rw(val: bool) { + PROVIDER.lock().unwrap().set_disabled_rw(val); +} + +/// query flag disabled_rw_exported +#[inline(always)] +pub fn disabled_rw_exported() -> bool { + PROVIDER.lock().unwrap().disabled_rw_exported() +} + +/// set flag disabled_rw_exported +#[inline(always)] +pub fn set_disabled_rw_exported(val: bool) { + PROVIDER.lock().unwrap().set_disabled_rw_exported(val); +} + +/// query flag disabled_rw_in_other_namespace +#[inline(always)] +pub fn disabled_rw_in_other_namespace() -> bool { + PROVIDER.lock().unwrap().disabled_rw_in_other_namespace() +} + +/// set flag disabled_rw_in_other_namespace +#[inline(always)] +pub fn set_disabled_rw_in_other_namespace(val: bool) { + PROVIDER.lock().unwrap().set_disabled_rw_in_other_namespace(val); +} + +/// query flag enabled_fixed_ro +#[inline(always)] +pub fn enabled_fixed_ro() -> bool { + PROVIDER.lock().unwrap().enabled_fixed_ro() +} + +/// set flag enabled_fixed_ro +#[inline(always)] +pub fn set_enabled_fixed_ro(val: bool) { + PROVIDER.lock().unwrap().set_enabled_fixed_ro(val); +} + +/// query flag enabled_fixed_ro_exported +#[inline(always)] +pub fn enabled_fixed_ro_exported() -> bool { + PROVIDER.lock().unwrap().enabled_fixed_ro_exported() +} + +/// set flag enabled_fixed_ro_exported +#[inline(always)] +pub fn set_enabled_fixed_ro_exported(val: bool) { + PROVIDER.lock().unwrap().set_enabled_fixed_ro_exported(val); +} + +/// query flag enabled_ro +#[inline(always)] +pub fn enabled_ro() -> bool { + PROVIDER.lock().unwrap().enabled_ro() +} + +/// set flag enabled_ro +#[inline(always)] +pub fn set_enabled_ro(val: bool) { + PROVIDER.lock().unwrap().set_enabled_ro(val); +} + +/// query flag enabled_ro_exported +#[inline(always)] +pub fn enabled_ro_exported() -> bool { + PROVIDER.lock().unwrap().enabled_ro_exported() +} + +/// set flag enabled_ro_exported +#[inline(always)] +pub fn set_enabled_ro_exported(val: bool) { + PROVIDER.lock().unwrap().set_enabled_ro_exported(val); +} + +/// query flag enabled_rw +#[inline(always)] +pub fn enabled_rw() -> bool { + PROVIDER.lock().unwrap().enabled_rw() +} + +/// set flag enabled_rw +#[inline(always)] +pub fn set_enabled_rw(val: bool) { + PROVIDER.lock().unwrap().set_enabled_rw(val); +} + +/// clear all flag override +pub fn reset_flags() { + PROVIDER.lock().unwrap().reset_flags() +} +"#; + + const EXPORTED_EXPECTED: &str = r#" +//! codegenerated rust flag lib + +/// flag provider +pub struct FlagProvider; + +lazy_static::lazy_static! { + /// flag value cache for disabled_rw_exported + static ref CACHED_disabled_rw_exported: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.disabled_rw_exported", + "false") == "true"; + + /// flag value cache for enabled_fixed_ro_exported + static ref CACHED_enabled_fixed_ro_exported: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "false") == "true"; + + /// flag value cache for enabled_ro_exported + static ref CACHED_enabled_ro_exported: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.aconfig_test", + "com.android.aconfig.test.enabled_ro_exported", + "false") == "true"; + +} + +impl FlagProvider { + /// query flag disabled_rw_exported + pub fn disabled_rw_exported(&self) -> bool { + *CACHED_disabled_rw_exported + } + + /// query flag enabled_fixed_ro_exported + pub fn enabled_fixed_ro_exported(&self) -> bool { + *CACHED_enabled_fixed_ro_exported + } + + /// query flag enabled_ro_exported + pub fn enabled_ro_exported(&self) -> bool { + *CACHED_enabled_ro_exported + } +} + +/// flag provider +pub static PROVIDER: FlagProvider = FlagProvider; + +/// query flag disabled_rw_exported +#[inline(always)] +pub fn disabled_rw_exported() -> bool { + PROVIDER.disabled_rw_exported() +} + +/// query flag enabled_fixed_ro_exported +#[inline(always)] +pub fn enabled_fixed_ro_exported() -> bool { + PROVIDER.enabled_fixed_ro_exported() +} + +/// query flag enabled_ro_exported +#[inline(always)] +pub fn enabled_ro_exported() -> bool { + PROVIDER.enabled_ro_exported() +} +"#; + + const FORCE_READ_ONLY_EXPECTED: &str = r#" +//! codegenerated rust flag lib + +/// flag provider +pub struct FlagProvider; + +impl FlagProvider { + /// query flag disabled_ro + pub fn disabled_ro(&self) -> bool { + false + } + + /// query flag disabled_rw + pub fn disabled_rw(&self) -> bool { + false + } + + /// query flag disabled_rw_in_other_namespace + pub fn disabled_rw_in_other_namespace(&self) -> bool { + false + } + + /// query flag enabled_fixed_ro + pub fn enabled_fixed_ro(&self) -> bool { + true + } + + /// query flag enabled_ro + pub fn enabled_ro(&self) -> bool { + true + } + + /// query flag enabled_rw + pub fn enabled_rw(&self) -> bool { + true + } +} + +/// flag provider +pub static PROVIDER: FlagProvider = FlagProvider; + +/// query flag disabled_ro +#[inline(always)] +pub fn disabled_ro() -> bool { + false +} + +/// query flag disabled_rw +#[inline(always)] +pub fn disabled_rw() -> bool { + false +} + +/// query flag disabled_rw_in_other_namespace +#[inline(always)] +pub fn disabled_rw_in_other_namespace() -> bool { + false +} + +/// query flag enabled_fixed_ro +#[inline(always)] +pub fn enabled_fixed_ro() -> bool { + true +} + +/// query flag enabled_ro +#[inline(always)] +pub fn enabled_ro() -> bool { + true +} + +/// query flag enabled_rw +#[inline(always)] +pub fn enabled_rw() -> bool { + true +} +"#; + + fn test_generate_rust_code(mode: CodegenMode) { + let parsed_flags = crate::test::parse_test_flags(); + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let generated = + generate_rust_code(crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), mode) + .unwrap(); + assert_eq!("src/lib.rs", format!("{}", generated.path.display())); + assert_eq!( + None, + crate::test::first_significant_code_diff( + match mode { + CodegenMode::Production => PROD_EXPECTED, + CodegenMode::Test => TEST_EXPECTED, + CodegenMode::Exported => EXPORTED_EXPECTED, + CodegenMode::ForceReadOnly => FORCE_READ_ONLY_EXPECTED, + }, + &String::from_utf8(generated.contents).unwrap() + ) + ); + } + + #[test] + fn test_generate_rust_code_for_prod() { + test_generate_rust_code(CodegenMode::Production); + } + + #[test] + fn test_generate_rust_code_for_test() { + test_generate_rust_code(CodegenMode::Test); + } + + #[test] + fn test_generate_rust_code_for_exported() { + test_generate_rust_code(CodegenMode::Exported); + } + + #[test] + fn test_generate_rust_code_for_force_read_only() { + test_generate_rust_code(CodegenMode::ForceReadOnly); + } +} diff --git a/tools/aconfig/aconfig/src/commands.rs b/tools/aconfig/aconfig/src/commands.rs new file mode 100644 index 0000000000..6945fd4649 --- /dev/null +++ b/tools/aconfig/aconfig/src/commands.rs @@ -0,0 +1,743 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::{bail, ensure, Context, Result}; +use itertools::Itertools; +use protobuf::Message; +use std::collections::HashMap; +use std::io::Read; +use std::path::PathBuf; + +use crate::codegen::cpp::generate_cpp_code; +use crate::codegen::java::generate_java_code; +use crate::codegen::rust::generate_rust_code; +use crate::codegen::CodegenMode; +use crate::dump::{DumpFormat, DumpPredicate}; +use crate::storage::generate_storage_file; +use aconfig_protos::{ + ParsedFlagExt, ProtoFlagMetadata, ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag, + ProtoParsedFlags, ProtoTracepoint, +}; +use aconfig_storage_file::StorageFileType; + +pub struct Input { + pub source: String, + pub reader: Box<dyn Read>, +} + +impl Input { + fn try_parse_flags(&mut self) -> Result<ProtoParsedFlags> { + let mut buffer = Vec::new(); + self.reader + .read_to_end(&mut buffer) + .with_context(|| format!("failed to read {}", self.source))?; + aconfig_protos::parsed_flags::try_from_binary_proto(&buffer) + .with_context(|| self.error_context()) + } + + fn error_context(&self) -> String { + format!("failed to parse {}", self.source) + } +} + +pub struct OutputFile { + pub path: PathBuf, // relative to some root directory only main knows about + pub contents: Vec<u8>, +} + +pub const DEFAULT_FLAG_STATE: ProtoFlagState = ProtoFlagState::DISABLED; +pub const DEFAULT_FLAG_PERMISSION: ProtoFlagPermission = ProtoFlagPermission::READ_WRITE; + +pub fn parse_flags( + package: &str, + container: Option<&str>, + declarations: Vec<Input>, + values: Vec<Input>, + default_permission: ProtoFlagPermission, +) -> Result<Vec<u8>> { + let mut parsed_flags = ProtoParsedFlags::new(); + + for mut input in declarations { + let mut contents = String::new(); + input + .reader + .read_to_string(&mut contents) + .with_context(|| format!("failed to read {}", input.source))?; + + let flag_declarations = aconfig_protos::flag_declarations::try_from_text_proto(&contents) + .with_context(|| input.error_context())?; + ensure!( + package == flag_declarations.package(), + "failed to parse {}: expected package {}, got {}", + input.source, + package, + flag_declarations.package() + ); + if let Some(c) = container { + ensure!( + c == flag_declarations.container(), + "failed to parse {}: expected container {}, got {}", + input.source, + c, + flag_declarations.container() + ); + } + for mut flag_declaration in flag_declarations.flag.into_iter() { + aconfig_protos::flag_declaration::verify_fields(&flag_declaration) + .with_context(|| input.error_context())?; + + // create ParsedFlag using FlagDeclaration and default values + let mut parsed_flag = ProtoParsedFlag::new(); + if let Some(c) = container { + parsed_flag.set_container(c.to_string()); + } + parsed_flag.set_package(package.to_string()); + parsed_flag.set_name(flag_declaration.take_name()); + parsed_flag.set_namespace(flag_declaration.take_namespace()); + parsed_flag.set_description(flag_declaration.take_description()); + parsed_flag.bug.append(&mut flag_declaration.bug); + parsed_flag.set_state(DEFAULT_FLAG_STATE); + let flag_permission = if flag_declaration.is_fixed_read_only() { + ProtoFlagPermission::READ_ONLY + } else { + default_permission + }; + parsed_flag.set_permission(flag_permission); + parsed_flag.set_is_fixed_read_only(flag_declaration.is_fixed_read_only()); + parsed_flag.set_is_exported(flag_declaration.is_exported()); + let mut tracepoint = ProtoTracepoint::new(); + tracepoint.set_source(input.source.clone()); + tracepoint.set_state(DEFAULT_FLAG_STATE); + tracepoint.set_permission(flag_permission); + parsed_flag.trace.push(tracepoint); + + let mut metadata = ProtoFlagMetadata::new(); + let purpose = flag_declaration.metadata.purpose(); + metadata.set_purpose(purpose); + parsed_flag.metadata = Some(metadata).into(); + + // verify ParsedFlag looks reasonable + aconfig_protos::parsed_flag::verify_fields(&parsed_flag)?; + + // verify ParsedFlag can be added + ensure!( + parsed_flags.parsed_flag.iter().all(|other| other.name() != parsed_flag.name()), + "failed to declare flag {} from {}: flag already declared", + parsed_flag.name(), + input.source + ); + + // add ParsedFlag to ParsedFlags + parsed_flags.parsed_flag.push(parsed_flag); + } + } + + for mut input in values { + let mut contents = String::new(); + input + .reader + .read_to_string(&mut contents) + .with_context(|| format!("failed to read {}", input.source))?; + let flag_values = aconfig_protos::flag_values::try_from_text_proto(&contents) + .with_context(|| input.error_context())?; + for flag_value in flag_values.flag_value.into_iter() { + aconfig_protos::flag_value::verify_fields(&flag_value) + .with_context(|| input.error_context())?; + + let Some(parsed_flag) = parsed_flags + .parsed_flag + .iter_mut() + .find(|pf| pf.package() == flag_value.package() && pf.name() == flag_value.name()) + else { + // (silently) skip unknown flags + continue; + }; + + ensure!( + !parsed_flag.is_fixed_read_only() + || flag_value.permission() == ProtoFlagPermission::READ_ONLY, + "failed to set permission of flag {}, since this flag is fixed read only flag", + flag_value.name() + ); + + parsed_flag.set_state(flag_value.state()); + parsed_flag.set_permission(flag_value.permission()); + let mut tracepoint = ProtoTracepoint::new(); + tracepoint.set_source(input.source.clone()); + tracepoint.set_state(flag_value.state()); + tracepoint.set_permission(flag_value.permission()); + parsed_flag.trace.push(tracepoint); + } + } + + // Create a sorted parsed_flags + aconfig_protos::parsed_flags::sort_parsed_flags(&mut parsed_flags); + aconfig_protos::parsed_flags::verify_fields(&parsed_flags)?; + let mut output = Vec::new(); + parsed_flags.write_to_vec(&mut output)?; + Ok(output) +} + +pub fn create_java_lib(mut input: Input, codegen_mode: CodegenMode) -> Result<Vec<OutputFile>> { + let parsed_flags = input.try_parse_flags()?; + let modified_parsed_flags = modify_parsed_flags_based_on_mode(parsed_flags, codegen_mode)?; + let Some(package) = find_unique_package(&modified_parsed_flags) else { + bail!("no parsed flags, or the parsed flags use different packages"); + }; + let package = package.to_string(); + let _flag_ids = assign_flag_ids(&package, modified_parsed_flags.iter())?; + generate_java_code(&package, modified_parsed_flags.into_iter(), codegen_mode) +} + +pub fn create_cpp_lib( + mut input: Input, + codegen_mode: CodegenMode, + allow_instrumentation: bool, +) -> Result<Vec<OutputFile>> { + // TODO(327420679): Enable export mode for native flag library + ensure!( + codegen_mode != CodegenMode::Exported, + "Exported mode for generated c/c++ flag library is disabled" + ); + let parsed_flags = input.try_parse_flags()?; + let modified_parsed_flags = modify_parsed_flags_based_on_mode(parsed_flags, codegen_mode)?; + let Some(package) = find_unique_package(&modified_parsed_flags) else { + bail!("no parsed flags, or the parsed flags use different packages"); + }; + let package = package.to_string(); + let flag_ids = assign_flag_ids(&package, modified_parsed_flags.iter())?; + generate_cpp_code( + &package, + modified_parsed_flags.into_iter(), + codegen_mode, + flag_ids, + allow_instrumentation, + ) +} + +pub fn create_rust_lib(mut input: Input, codegen_mode: CodegenMode) -> Result<OutputFile> { + // // TODO(327420679): Enable export mode for native flag library + ensure!( + codegen_mode != CodegenMode::Exported, + "Exported mode for generated rust flag library is disabled" + ); + let parsed_flags = input.try_parse_flags()?; + let modified_parsed_flags = modify_parsed_flags_based_on_mode(parsed_flags, codegen_mode)?; + let Some(package) = find_unique_package(&modified_parsed_flags) else { + bail!("no parsed flags, or the parsed flags use different packages"); + }; + let package = package.to_string(); + let _flag_ids = assign_flag_ids(&package, modified_parsed_flags.iter())?; + generate_rust_code(&package, modified_parsed_flags.into_iter(), codegen_mode) +} + +pub fn create_storage( + caches: Vec<Input>, + container: &str, + file: &StorageFileType, +) -> Result<Vec<u8>> { + let parsed_flags_vec: Vec<ProtoParsedFlags> = + caches.into_iter().map(|mut input| input.try_parse_flags()).collect::<Result<Vec<_>>>()?; + generate_storage_file(container, parsed_flags_vec.iter(), file) +} + +pub fn create_device_config_defaults(mut input: Input) -> Result<Vec<u8>> { + let parsed_flags = input.try_parse_flags()?; + let mut output = Vec::new(); + for parsed_flag in parsed_flags + .parsed_flag + .into_iter() + .filter(|pf| pf.permission() == ProtoFlagPermission::READ_WRITE) + { + let line = format!( + "{}:{}={}\n", + parsed_flag.namespace(), + parsed_flag.fully_qualified_name(), + match parsed_flag.state() { + ProtoFlagState::ENABLED => "enabled", + ProtoFlagState::DISABLED => "disabled", + } + ); + output.extend_from_slice(line.as_bytes()); + } + Ok(output) +} + +pub fn create_device_config_sysprops(mut input: Input) -> Result<Vec<u8>> { + let parsed_flags = input.try_parse_flags()?; + let mut output = Vec::new(); + for parsed_flag in parsed_flags + .parsed_flag + .into_iter() + .filter(|pf| pf.permission() == ProtoFlagPermission::READ_WRITE) + { + let line = format!( + "persist.device_config.{}={}\n", + parsed_flag.fully_qualified_name(), + match parsed_flag.state() { + ProtoFlagState::ENABLED => "true", + ProtoFlagState::DISABLED => "false", + } + ); + output.extend_from_slice(line.as_bytes()); + } + Ok(output) +} + +pub fn dump_parsed_flags( + mut input: Vec<Input>, + format: DumpFormat, + filters: &[&str], + dedup: bool, +) -> Result<Vec<u8>> { + let individually_parsed_flags: Result<Vec<ProtoParsedFlags>> = + input.iter_mut().map(|i| i.try_parse_flags()).collect(); + let parsed_flags: ProtoParsedFlags = + aconfig_protos::parsed_flags::merge(individually_parsed_flags?, dedup)?; + let filters: Vec<Box<DumpPredicate>> = if filters.is_empty() { + vec![Box::new(|_| true)] + } else { + filters + .iter() + .map(|f| crate::dump::create_filter_predicate(f)) + .collect::<Result<Vec<_>>>()? + }; + crate::dump::dump_parsed_flags( + parsed_flags.parsed_flag.into_iter().filter(|flag| filters.iter().any(|p| p(flag))), + format, + ) +} + +fn find_unique_package(parsed_flags: &[ProtoParsedFlag]) -> Option<&str> { + let package = parsed_flags.first().map(|pf| pf.package())?; + if parsed_flags.iter().any(|pf| pf.package() != package) { + return None; + } + Some(package) +} + +pub fn modify_parsed_flags_based_on_mode( + parsed_flags: ProtoParsedFlags, + codegen_mode: CodegenMode, +) -> Result<Vec<ProtoParsedFlag>> { + fn exported_mode_flag_modifier(mut parsed_flag: ProtoParsedFlag) -> ProtoParsedFlag { + parsed_flag.set_state(ProtoFlagState::DISABLED); + parsed_flag.set_permission(ProtoFlagPermission::READ_WRITE); + parsed_flag.set_is_fixed_read_only(false); + parsed_flag + } + + fn force_read_only_mode_flag_modifier(mut parsed_flag: ProtoParsedFlag) -> ProtoParsedFlag { + parsed_flag.set_permission(ProtoFlagPermission::READ_ONLY); + parsed_flag + } + + let modified_parsed_flags: Vec<_> = match codegen_mode { + CodegenMode::Exported => parsed_flags + .parsed_flag + .into_iter() + .filter(|pf| pf.is_exported()) + .map(exported_mode_flag_modifier) + .collect(), + CodegenMode::ForceReadOnly => parsed_flags + .parsed_flag + .into_iter() + .filter(|pf| !pf.is_exported()) + .map(force_read_only_mode_flag_modifier) + .collect(), + CodegenMode::Production | CodegenMode::Test => { + parsed_flags.parsed_flag.into_iter().collect() + } + }; + if modified_parsed_flags.is_empty() { + bail!("{codegen_mode} library contains no {codegen_mode} flags"); + } + + Ok(modified_parsed_flags) +} + +pub fn assign_flag_ids<'a, I>(package: &str, parsed_flags_iter: I) -> Result<HashMap<String, u16>> +where + I: Iterator<Item = &'a ProtoParsedFlag> + Clone, +{ + assert!(parsed_flags_iter.clone().tuple_windows().all(|(a, b)| a.name() <= b.name())); + let mut flag_ids = HashMap::new(); + for (id_to_assign, pf) in (0_u32..).zip(parsed_flags_iter) { + if package != pf.package() { + return Err(anyhow::anyhow!("encountered a flag not in current package")); + } + + // put a cap on how many flags a package can contain to 65535 + if id_to_assign > u16::MAX as u32 { + return Err(anyhow::anyhow!("the number of flags in a package cannot exceed 65535")); + } + + flag_ids.insert(pf.name().to_string(), id_to_assign as u16); + } + Ok(flag_ids) +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_protos::ProtoFlagPurpose; + + #[test] + fn test_parse_flags() { + let parsed_flags = crate::test::parse_test_flags(); // calls parse_flags + aconfig_protos::parsed_flags::verify_fields(&parsed_flags).unwrap(); + + let enabled_ro = + parsed_flags.parsed_flag.iter().find(|pf| pf.name() == "enabled_ro").unwrap(); + assert!(aconfig_protos::parsed_flag::verify_fields(enabled_ro).is_ok()); + assert_eq!("com.android.aconfig.test", enabled_ro.package()); + assert_eq!("enabled_ro", enabled_ro.name()); + assert_eq!("This flag is ENABLED + READ_ONLY", enabled_ro.description()); + assert_eq!(ProtoFlagState::ENABLED, enabled_ro.state()); + assert_eq!(ProtoFlagPermission::READ_ONLY, enabled_ro.permission()); + assert_eq!(ProtoFlagPurpose::PURPOSE_BUGFIX, enabled_ro.metadata.purpose()); + assert_eq!(3, enabled_ro.trace.len()); + assert!(!enabled_ro.is_fixed_read_only()); + assert_eq!("tests/test.aconfig", enabled_ro.trace[0].source()); + assert_eq!(ProtoFlagState::DISABLED, enabled_ro.trace[0].state()); + assert_eq!(ProtoFlagPermission::READ_WRITE, enabled_ro.trace[0].permission()); + assert_eq!("tests/first.values", enabled_ro.trace[1].source()); + assert_eq!(ProtoFlagState::DISABLED, enabled_ro.trace[1].state()); + assert_eq!(ProtoFlagPermission::READ_WRITE, enabled_ro.trace[1].permission()); + assert_eq!("tests/second.values", enabled_ro.trace[2].source()); + assert_eq!(ProtoFlagState::ENABLED, enabled_ro.trace[2].state()); + assert_eq!(ProtoFlagPermission::READ_ONLY, enabled_ro.trace[2].permission()); + + assert_eq!(9, parsed_flags.parsed_flag.len()); + for pf in parsed_flags.parsed_flag.iter() { + if pf.name().starts_with("enabled_fixed_ro") { + continue; + } + let first = pf.trace.first().unwrap(); + assert_eq!(DEFAULT_FLAG_STATE, first.state()); + assert_eq!(DEFAULT_FLAG_PERMISSION, first.permission()); + + let last = pf.trace.last().unwrap(); + assert_eq!(pf.state(), last.state()); + assert_eq!(pf.permission(), last.permission()); + } + + let enabled_fixed_ro = + parsed_flags.parsed_flag.iter().find(|pf| pf.name() == "enabled_fixed_ro").unwrap(); + assert!(enabled_fixed_ro.is_fixed_read_only()); + assert_eq!(ProtoFlagState::ENABLED, enabled_fixed_ro.state()); + assert_eq!(ProtoFlagPermission::READ_ONLY, enabled_fixed_ro.permission()); + assert_eq!(2, enabled_fixed_ro.trace.len()); + assert_eq!(ProtoFlagPermission::READ_ONLY, enabled_fixed_ro.trace[0].permission()); + assert_eq!(ProtoFlagPermission::READ_ONLY, enabled_fixed_ro.trace[1].permission()); + } + + #[test] + fn test_parse_flags_setting_default() { + let first_flag = r#" + package: "com.first" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + } + "#; + let declaration = + vec![Input { source: "momery".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + let value: Vec<Input> = vec![]; + + let flags_bytes = crate::commands::parse_flags( + "com.first", + None, + declaration, + value, + ProtoFlagPermission::READ_ONLY, + ) + .unwrap(); + let parsed_flags = + aconfig_protos::parsed_flags::try_from_binary_proto(&flags_bytes).unwrap(); + assert_eq!(1, parsed_flags.parsed_flag.len()); + let parsed_flag = parsed_flags.parsed_flag.first().unwrap(); + assert_eq!(ProtoFlagState::DISABLED, parsed_flag.state()); + assert_eq!(ProtoFlagPermission::READ_ONLY, parsed_flag.permission()); + } + + #[test] + fn test_parse_flags_package_mismatch_between_declaration_and_command_line() { + let first_flag = r#" + package: "com.declaration.package" + container: "first.container" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + } + "#; + let declaration = + vec![Input { source: "memory".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + + let value: Vec<Input> = vec![]; + + let error = crate::commands::parse_flags( + "com.argument.package", + Some("first.container"), + declaration, + value, + ProtoFlagPermission::READ_WRITE, + ) + .unwrap_err(); + assert_eq!( + format!("{:?}", error), + "failed to parse memory: expected package com.argument.package, got com.declaration.package" + ); + } + + #[test] + fn test_parse_flags_container_mismatch_between_declaration_and_command_line() { + let first_flag = r#" + package: "com.first" + container: "declaration.container" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + } + "#; + let declaration = + vec![Input { source: "memory".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + + let value: Vec<Input> = vec![]; + + let error = crate::commands::parse_flags( + "com.first", + Some("argument.container"), + declaration, + value, + ProtoFlagPermission::READ_WRITE, + ) + .unwrap_err(); + assert_eq!( + format!("{:?}", error), + "failed to parse memory: expected container argument.container, got declaration.container" + ); + } + + #[test] + fn test_parse_flags_override_fixed_read_only() { + let first_flag = r#" + package: "com.first" + container: "com.first.container" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + is_fixed_read_only: true + } + "#; + let declaration = + vec![Input { source: "memory".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + + let first_flag_value = r#" + flag_value { + package: "com.first" + name: "first" + state: DISABLED + permission: READ_WRITE + } + "#; + let value = vec![Input { + source: "memory".to_string(), + reader: Box::new(first_flag_value.as_bytes()), + }]; + let error = crate::commands::parse_flags( + "com.first", + Some("com.first.container"), + declaration, + value, + ProtoFlagPermission::READ_WRITE, + ) + .unwrap_err(); + assert_eq!( + format!("{:?}", error), + "failed to set permission of flag first, since this flag is fixed read only flag" + ); + } + + #[test] + fn test_parse_flags_metadata() { + let metadata_flag = r#" + package: "com.first" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of this feature flag." + bug: "123" + metadata { + purpose: PURPOSE_FEATURE + } + } + "#; + let declaration = vec![Input { + source: "memory".to_string(), + reader: Box::new(metadata_flag.as_bytes()), + }]; + let value: Vec<Input> = vec![]; + + let flags_bytes = crate::commands::parse_flags( + "com.first", + None, + declaration, + value, + ProtoFlagPermission::READ_ONLY, + ) + .unwrap(); + let parsed_flags = + aconfig_protos::parsed_flags::try_from_binary_proto(&flags_bytes).unwrap(); + assert_eq!(1, parsed_flags.parsed_flag.len()); + let parsed_flag = parsed_flags.parsed_flag.first().unwrap(); + assert_eq!(ProtoFlagPurpose::PURPOSE_FEATURE, parsed_flag.metadata.purpose()); + } + + #[test] + fn test_create_device_config_defaults() { + let input = parse_test_flags_as_input(); + let bytes = create_device_config_defaults(input).unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert_eq!("aconfig_test:com.android.aconfig.test.disabled_rw=disabled\naconfig_test:com.android.aconfig.test.disabled_rw_exported=disabled\nother_namespace:com.android.aconfig.test.disabled_rw_in_other_namespace=disabled\naconfig_test:com.android.aconfig.test.enabled_rw=enabled\n", text); + } + + #[test] + fn test_create_device_config_sysprops() { + let input = parse_test_flags_as_input(); + let bytes = create_device_config_sysprops(input).unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert_eq!("persist.device_config.com.android.aconfig.test.disabled_rw=false\npersist.device_config.com.android.aconfig.test.disabled_rw_exported=false\npersist.device_config.com.android.aconfig.test.disabled_rw_in_other_namespace=false\npersist.device_config.com.android.aconfig.test.enabled_rw=true\n", text); + } + + #[test] + fn test_dump() { + let input = parse_test_flags_as_input(); + let bytes = dump_parsed_flags( + vec![input], + DumpFormat::Custom("{fully_qualified_name}".to_string()), + &[], + false, + ) + .unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert!(text.contains("com.android.aconfig.test.disabled_ro")); + } + + #[test] + fn test_dump_textproto_format_dedup() { + let input = parse_test_flags_as_input(); + let input2 = parse_test_flags_as_input(); + let bytes = + dump_parsed_flags(vec![input, input2], DumpFormat::Textproto, &[], true).unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert_eq!( + None, + crate::test::first_significant_code_diff( + crate::test::TEST_FLAGS_TEXTPROTO.trim(), + text.trim() + ) + ); + } + + fn parse_test_flags_as_input() -> Input { + let parsed_flags = crate::test::parse_test_flags(); + let binary_proto = parsed_flags.write_to_bytes().unwrap(); + let cursor = std::io::Cursor::new(binary_proto); + let reader = Box::new(cursor); + Input { source: "test.data".to_string(), reader } + } + + #[test] + fn test_modify_parsed_flags_based_on_mode_prod() { + let parsed_flags = crate::test::parse_test_flags(); + let p_parsed_flags = + modify_parsed_flags_based_on_mode(parsed_flags.clone(), CodegenMode::Production) + .unwrap(); + assert_eq!(parsed_flags.parsed_flag.len(), p_parsed_flags.len()); + for (i, item) in p_parsed_flags.iter().enumerate() { + assert!(parsed_flags.parsed_flag[i].eq(item)); + } + } + + #[test] + fn test_modify_parsed_flags_based_on_mode_exported() { + let parsed_flags = crate::test::parse_test_flags(); + let p_parsed_flags = + modify_parsed_flags_based_on_mode(parsed_flags, CodegenMode::Exported).unwrap(); + assert_eq!(3, p_parsed_flags.len()); + for flag in p_parsed_flags.iter() { + assert_eq!(ProtoFlagState::DISABLED, flag.state()); + assert_eq!(ProtoFlagPermission::READ_WRITE, flag.permission()); + assert!(!flag.is_fixed_read_only()); + assert!(flag.is_exported()); + } + + let mut parsed_flags = crate::test::parse_test_flags(); + parsed_flags.parsed_flag.retain(|pf| !pf.is_exported()); + let error = + modify_parsed_flags_based_on_mode(parsed_flags, CodegenMode::Exported).unwrap_err(); + assert_eq!("exported library contains no exported flags", format!("{:?}", error)); + } + + #[test] + fn test_assign_flag_ids() { + let parsed_flags = crate::test::parse_test_flags(); + let package = find_unique_package(&parsed_flags.parsed_flag).unwrap().to_string(); + let flag_ids = assign_flag_ids(&package, parsed_flags.parsed_flag.iter()).unwrap(); + let expected_flag_ids = HashMap::from([ + (String::from("disabled_ro"), 0_u16), + (String::from("disabled_rw"), 1_u16), + (String::from("disabled_rw_exported"), 2_u16), + (String::from("disabled_rw_in_other_namespace"), 3_u16), + (String::from("enabled_fixed_ro"), 4_u16), + (String::from("enabled_fixed_ro_exported"), 5_u16), + (String::from("enabled_ro"), 6_u16), + (String::from("enabled_ro_exported"), 7_u16), + (String::from("enabled_rw"), 8_u16), + ]); + assert_eq!(flag_ids, expected_flag_ids); + } + + #[test] + fn test_modify_parsed_flags_based_on_mode_force_read_only() { + let parsed_flags = crate::test::parse_test_flags(); + let p_parsed_flags = + modify_parsed_flags_based_on_mode(parsed_flags.clone(), CodegenMode::ForceReadOnly) + .unwrap(); + assert_eq!(6, p_parsed_flags.len()); + for pf in p_parsed_flags { + assert_eq!(ProtoFlagPermission::READ_ONLY, pf.permission()); + } + + let mut parsed_flags = crate::test::parse_test_flags(); + parsed_flags.parsed_flag.retain_mut(|pf| pf.is_exported()); + let error = modify_parsed_flags_based_on_mode(parsed_flags, CodegenMode::ForceReadOnly) + .unwrap_err(); + assert_eq!( + "force-read-only library contains no force-read-only flags", + format!("{:?}", error) + ); + } +} diff --git a/tools/aconfig/aconfig/src/dump.rs b/tools/aconfig/aconfig/src/dump.rs new file mode 100644 index 0000000000..2a29c2bccf --- /dev/null +++ b/tools/aconfig/aconfig/src/dump.rs @@ -0,0 +1,408 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use aconfig_protos::{ + ParsedFlagExt, ProtoFlagMetadata, ProtoFlagPermission, ProtoFlagState, ProtoTracepoint, +}; +use aconfig_protos::{ProtoParsedFlag, ProtoParsedFlags}; +use anyhow::{anyhow, bail, Context, Result}; +use protobuf::Message; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DumpFormat { + Protobuf, + Textproto, + Custom(String), +} + +impl TryFrom<&str> for DumpFormat { + type Error = anyhow::Error; + + fn try_from(value: &str) -> std::result::Result<Self, Self::Error> { + match value { + // protobuf formats + "protobuf" => Ok(Self::Protobuf), + "textproto" => Ok(Self::Textproto), + // custom format + _ => Ok(Self::Custom(value.to_owned())), + } + } +} + +pub fn dump_parsed_flags<I>(parsed_flags_iter: I, format: DumpFormat) -> Result<Vec<u8>> +where + I: Iterator<Item = ProtoParsedFlag>, +{ + let mut output = Vec::new(); + match format { + DumpFormat::Protobuf => { + let parsed_flags = + ProtoParsedFlags { parsed_flag: parsed_flags_iter.collect(), ..Default::default() }; + parsed_flags.write_to_vec(&mut output)?; + } + DumpFormat::Textproto => { + let parsed_flags = + ProtoParsedFlags { parsed_flag: parsed_flags_iter.collect(), ..Default::default() }; + let s = protobuf::text_format::print_to_string_pretty(&parsed_flags); + output.extend_from_slice(s.as_bytes()); + } + DumpFormat::Custom(format) => { + for flag in parsed_flags_iter { + dump_custom_format(&flag, &format, &mut output); + } + } + } + Ok(output) +} + +fn dump_custom_format(flag: &ProtoParsedFlag, format: &str, output: &mut Vec<u8>) { + fn format_trace(trace: &[ProtoTracepoint]) -> String { + trace + .iter() + .map(|tracepoint| { + format!( + "{}: {:?} + {:?}", + tracepoint.source(), + tracepoint.permission(), + tracepoint.state() + ) + }) + .collect::<Vec<_>>() + .join(", ") + } + + fn format_trace_paths(trace: &[ProtoTracepoint]) -> String { + trace.iter().map(|tracepoint| tracepoint.source()).collect::<Vec<_>>().join(", ") + } + + fn format_metadata(metadata: &ProtoFlagMetadata) -> String { + format!("{:?}", metadata.purpose()) + } + + let mut str = format + // ProtoParsedFlag fields + .replace("{package}", flag.package()) + .replace("{name}", flag.name()) + .replace("{namespace}", flag.namespace()) + .replace("{description}", flag.description()) + .replace("{bug}", &flag.bug.join(", ")) + .replace("{state}", &format!("{:?}", flag.state())) + .replace("{state:bool}", &format!("{}", flag.state() == ProtoFlagState::ENABLED)) + .replace("{permission}", &format!("{:?}", flag.permission())) + .replace("{trace}", &format_trace(&flag.trace)) + .replace("{trace:paths}", &format_trace_paths(&flag.trace)) + .replace("{is_fixed_read_only}", &format!("{}", flag.is_fixed_read_only())) + .replace("{is_exported}", &format!("{}", flag.is_exported())) + .replace("{container}", flag.container()) + .replace("{metadata}", &format_metadata(&flag.metadata)) + // ParsedFlagExt functions + .replace("{fully_qualified_name}", &flag.fully_qualified_name()); + str.push('\n'); + output.extend_from_slice(str.as_bytes()); +} + +pub type DumpPredicate = dyn Fn(&ProtoParsedFlag) -> bool; + +pub fn create_filter_predicate(filter: &str) -> Result<Box<DumpPredicate>> { + let predicates = filter + .split('+') + .map(|sub_filter| create_filter_predicate_single(sub_filter)) + .collect::<Result<Vec<_>>>()?; + Ok(Box::new(move |flag| predicates.iter().all(|p| p(flag)))) +} + +fn create_filter_predicate_single(filter: &str) -> Result<Box<DumpPredicate>> { + fn enum_from_str<T>(expected: &[T], s: &str) -> Result<T> + where + T: std::fmt::Debug + Copy, + { + for candidate in expected.iter() { + if s == format!("{:?}", candidate) { + return Ok(*candidate); + } + } + let expected = + expected.iter().map(|state| format!("{:?}", state)).collect::<Vec<_>>().join(", "); + bail!("\"{s}\": not a valid flag state, expected one of {expected}"); + } + + let error_msg = format!("\"{filter}\": filter syntax error"); + let (what, arg) = filter.split_once(':').ok_or_else(|| anyhow!(error_msg.clone()))?; + match what { + "package" => { + let expected = arg.to_owned(); + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.package() == expected)) + } + "name" => { + let expected = arg.to_owned(); + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.name() == expected)) + } + "namespace" => { + let expected = arg.to_owned(); + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.namespace() == expected)) + } + // description: not supported yet + "bug" => { + let expected = arg.to_owned(); + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.bug.join(", ") == expected)) + } + "state" => { + let expected = enum_from_str(&[ProtoFlagState::ENABLED, ProtoFlagState::DISABLED], arg) + .context(error_msg)?; + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.state() == expected)) + } + "permission" => { + let expected = enum_from_str( + &[ProtoFlagPermission::READ_ONLY, ProtoFlagPermission::READ_WRITE], + arg, + ) + .context(error_msg)?; + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.permission() == expected)) + } + // trace: not supported yet + "is_fixed_read_only" => { + let expected: bool = arg.parse().context(error_msg)?; + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.is_fixed_read_only() == expected)) + } + "is_exported" => { + let expected: bool = arg.parse().context(error_msg)?; + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.is_exported() == expected)) + } + "container" => { + let expected = arg.to_owned(); + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.container() == expected)) + } + // metadata: not supported yet + "fully_qualified_name" => { + let expected = arg.to_owned(); + Ok(Box::new(move |flag: &ProtoParsedFlag| flag.fully_qualified_name() == expected)) + } + _ => Err(anyhow!(error_msg)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test::parse_test_flags; + use aconfig_protos::ProtoParsedFlags; + use protobuf::Message; + + fn parse_enabled_ro_flag() -> ProtoParsedFlag { + parse_test_flags().parsed_flag.into_iter().find(|pf| pf.name() == "enabled_ro").unwrap() + } + + #[test] + fn test_dumpformat_from_str() { + // supported format types + assert_eq!(DumpFormat::try_from("protobuf").unwrap(), DumpFormat::Protobuf); + assert_eq!(DumpFormat::try_from("textproto").unwrap(), DumpFormat::Textproto); + assert_eq!( + DumpFormat::try_from("foobar").unwrap(), + DumpFormat::Custom("foobar".to_owned()) + ); + } + + #[test] + fn test_dump_parsed_flags_protobuf_format() { + let expected = protobuf::text_format::parse_from_str::<ProtoParsedFlags>( + crate::test::TEST_FLAGS_TEXTPROTO, + ) + .unwrap() + .write_to_bytes() + .unwrap(); + let parsed_flags = parse_test_flags(); + let actual = + dump_parsed_flags(parsed_flags.parsed_flag.into_iter(), DumpFormat::Protobuf).unwrap(); + assert_eq!(expected, actual); + } + + #[test] + fn test_dump_parsed_flags_textproto_format() { + let parsed_flags = parse_test_flags(); + let bytes = + dump_parsed_flags(parsed_flags.parsed_flag.into_iter(), DumpFormat::Textproto).unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert_eq!(crate::test::TEST_FLAGS_TEXTPROTO.trim(), text.trim()); + } + + #[test] + fn test_dump_parsed_flags_custom_format() { + macro_rules! assert_dump_parsed_flags_custom_format_contains { + ($format:expr, $expected:expr) => { + let parsed_flags = parse_test_flags(); + let bytes = dump_parsed_flags( + parsed_flags.parsed_flag.into_iter(), + $format.try_into().unwrap(), + ) + .unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert!(text.contains($expected)); + }; + } + + // custom format + assert_dump_parsed_flags_custom_format_contains!( + "{fully_qualified_name}={permission} + {state}", + "com.android.aconfig.test.enabled_ro=READ_ONLY + ENABLED" + ); + } + + #[test] + fn test_dump_custom_format() { + macro_rules! assert_custom_format { + ($format:expr, $expected:expr) => { + let flag = parse_enabled_ro_flag(); + let mut bytes = vec![]; + dump_custom_format(&flag, $format, &mut bytes); + let text = std::str::from_utf8(&bytes).unwrap(); + assert_eq!(text, $expected); + }; + } + + assert_custom_format!("{package}", "com.android.aconfig.test\n"); + assert_custom_format!("{name}", "enabled_ro\n"); + assert_custom_format!("{namespace}", "aconfig_test\n"); + assert_custom_format!("{description}", "This flag is ENABLED + READ_ONLY\n"); + assert_custom_format!("{bug}", "abc\n"); + assert_custom_format!("{state}", "ENABLED\n"); + assert_custom_format!("{state:bool}", "true\n"); + assert_custom_format!("{permission}", "READ_ONLY\n"); + assert_custom_format!("{trace}", "tests/test.aconfig: READ_WRITE + DISABLED, tests/first.values: READ_WRITE + DISABLED, tests/second.values: READ_ONLY + ENABLED\n"); + assert_custom_format!( + "{trace:paths}", + "tests/test.aconfig, tests/first.values, tests/second.values\n" + ); + assert_custom_format!("{is_fixed_read_only}", "false\n"); + assert_custom_format!("{is_exported}", "false\n"); + assert_custom_format!("{container}", "system\n"); + assert_custom_format!("{metadata}", "PURPOSE_BUGFIX\n"); + + assert_custom_format!("name={name}|state={state}", "name=enabled_ro|state=ENABLED\n"); + assert_custom_format!("{state}{state}{state}", "ENABLEDENABLEDENABLED\n"); + } + + #[test] + fn test_create_filter_predicate() { + macro_rules! assert_create_filter_predicate { + ($filter:expr, $expected:expr) => { + let parsed_flags = parse_test_flags(); + let predicate = create_filter_predicate($filter).unwrap(); + let mut filtered_flags: Vec<String> = parsed_flags + .parsed_flag + .into_iter() + .filter(predicate) + .map(|flag| flag.fully_qualified_name()) + .collect(); + filtered_flags.sort(); + assert_eq!(&filtered_flags, $expected); + }; + } + + assert_create_filter_predicate!( + "package:com.android.aconfig.test", + &[ + "com.android.aconfig.test.disabled_ro", + "com.android.aconfig.test.disabled_rw", + "com.android.aconfig.test.disabled_rw_exported", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro", + "com.android.aconfig.test.enabled_ro_exported", + "com.android.aconfig.test.enabled_rw", + ] + ); + assert_create_filter_predicate!( + "name:disabled_rw", + &["com.android.aconfig.test.disabled_rw"] + ); + assert_create_filter_predicate!( + "namespace:other_namespace", + &["com.android.aconfig.test.disabled_rw_in_other_namespace"] + ); + // description: not supported yet + assert_create_filter_predicate!("bug:123", &["com.android.aconfig.test.disabled_ro",]); + assert_create_filter_predicate!( + "state:ENABLED", + &[ + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro", + "com.android.aconfig.test.enabled_ro_exported", + "com.android.aconfig.test.enabled_rw", + ] + ); + assert_create_filter_predicate!( + "permission:READ_ONLY", + &[ + "com.android.aconfig.test.disabled_ro", + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro", + "com.android.aconfig.test.enabled_ro_exported", + ] + ); + // trace: not supported yet + assert_create_filter_predicate!( + "is_fixed_read_only:true", + &[ + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + ] + ); + assert_create_filter_predicate!( + "is_exported:true", + &[ + "com.android.aconfig.test.disabled_rw_exported", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro_exported", + ] + ); + assert_create_filter_predicate!( + "container:system", + &[ + "com.android.aconfig.test.disabled_ro", + "com.android.aconfig.test.disabled_rw", + "com.android.aconfig.test.disabled_rw_exported", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro", + "com.android.aconfig.test.enabled_ro_exported", + "com.android.aconfig.test.enabled_rw", + ] + ); + // metadata: not supported yet + + // synthesized fields + assert_create_filter_predicate!( + "fully_qualified_name:com.android.aconfig.test.disabled_rw", + &["com.android.aconfig.test.disabled_rw"] + ); + + // multiple sub filters + assert_create_filter_predicate!( + "permission:READ_ONLY+state:ENABLED", + &[ + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro", + "com.android.aconfig.test.enabled_ro_exported", + ] + ); + } +} diff --git a/tools/aconfig/aconfig/src/main.rs b/tools/aconfig/aconfig/src/main.rs new file mode 100644 index 0000000000..72be1c9896 --- /dev/null +++ b/tools/aconfig/aconfig/src/main.rs @@ -0,0 +1,308 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aconfig` is a build time tool to manage build time configurations, such as feature flags. + +use anyhow::{anyhow, bail, Context, Result}; +use clap::{builder::ArgAction, builder::EnumValueParser, Arg, ArgMatches, Command}; +use core::any::Any; +use std::fs; +use std::io; +use std::io::Write; +use std::path::{Path, PathBuf}; + +mod codegen; +mod commands; +mod dump; +mod storage; + +use aconfig_storage_file::StorageFileType; +use codegen::CodegenMode; +use dump::DumpFormat; + +#[cfg(test)] +mod test; + +use commands::{Input, OutputFile}; + +const HELP_DUMP_FILTER: &str = r#" +Limit which flags to output. If multiple --filter arguments are provided, the output will be +limited to flags that match any of the filters. +"#; + +fn cli() -> Command { + Command::new("aconfig") + .subcommand_required(true) + .subcommand( + Command::new("create-cache") + .arg(Arg::new("package").long("package").required(true)) + // TODO(b/312769710): Make this argument required. + .arg(Arg::new("container").long("container")) + .arg(Arg::new("declarations").long("declarations").action(ArgAction::Append)) + .arg(Arg::new("values").long("values").action(ArgAction::Append)) + .arg( + Arg::new("default-permission") + .long("default-permission") + .value_parser(aconfig_protos::flag_permission::parse_from_str) + .default_value(aconfig_protos::flag_permission::to_string( + &commands::DEFAULT_FLAG_PERMISSION, + )), + ) + .arg(Arg::new("cache").long("cache").required(true)), + ) + .subcommand( + Command::new("create-java-lib") + .arg(Arg::new("cache").long("cache").required(true)) + .arg(Arg::new("out").long("out").required(true)) + .arg( + Arg::new("mode") + .long("mode") + .value_parser(EnumValueParser::<CodegenMode>::new()) + .default_value("production"), + ), + ) + .subcommand( + Command::new("create-cpp-lib") + .arg(Arg::new("cache").long("cache").required(true)) + .arg(Arg::new("out").long("out").required(true)) + .arg( + Arg::new("mode") + .long("mode") + .value_parser(EnumValueParser::<CodegenMode>::new()) + .default_value("production"), + ) + .arg( + Arg::new("allow-instrumentation") + .long("allow-instrumentation") + .value_parser(clap::value_parser!(bool)) + .default_value("false"), + ), + ) + .subcommand( + Command::new("create-rust-lib") + .arg(Arg::new("cache").long("cache").required(true)) + .arg(Arg::new("out").long("out").required(true)) + .arg( + Arg::new("mode") + .long("mode") + .value_parser(EnumValueParser::<CodegenMode>::new()) + .default_value("production"), + ), + ) + .subcommand( + Command::new("create-device-config-defaults") + .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true)) + .arg(Arg::new("out").long("out").default_value("-")), + ) + .subcommand( + Command::new("create-device-config-sysprops") + .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true)) + .arg(Arg::new("out").long("out").default_value("-")), + ) + .subcommand( + Command::new("dump-cache") + .alias("dump") + .arg(Arg::new("cache").long("cache").action(ArgAction::Append)) + .arg( + Arg::new("format") + .long("format") + .value_parser(|s: &str| DumpFormat::try_from(s)) + .default_value( + "{fully_qualified_name} [{container}]: {permission} + {state}", + ), + ) + .arg( + Arg::new("filter") + .long("filter") + .action(ArgAction::Append) + .help(HELP_DUMP_FILTER.trim()), + ) + .arg(Arg::new("dedup").long("dedup").num_args(0).action(ArgAction::SetTrue)) + .arg(Arg::new("out").long("out").default_value("-")), + ) + .subcommand( + Command::new("create-storage") + .arg( + Arg::new("container") + .long("container") + .required(true) + .help("The target container for the generated storage file."), + ) + .arg( + Arg::new("file") + .long("file") + .value_parser(|s: &str| StorageFileType::try_from(s)), + ) + .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true)) + .arg(Arg::new("out").long("out").required(true)), + ) +} + +fn get_required_arg<'a, T>(matches: &'a ArgMatches, arg_name: &str) -> Result<&'a T> +where + T: Any + Clone + Send + Sync + 'static, +{ + matches + .get_one::<T>(arg_name) + .ok_or(anyhow!("internal error: required argument '{}' not found", arg_name)) +} + +fn get_optional_arg<'a, T>(matches: &'a ArgMatches, arg_name: &str) -> Option<&'a T> +where + T: Any + Clone + Send + Sync + 'static, +{ + matches.get_one::<T>(arg_name) +} + +fn open_zero_or_more_files(matches: &ArgMatches, arg_name: &str) -> Result<Vec<Input>> { + let mut opened_files = vec![]; + for path in matches.get_many::<String>(arg_name).unwrap_or_default() { + let file = Box::new(fs::File::open(path)?); + opened_files.push(Input { source: path.to_string(), reader: file }); + } + Ok(opened_files) +} + +fn open_single_file(matches: &ArgMatches, arg_name: &str) -> Result<Input> { + let Some(path) = matches.get_one::<String>(arg_name) else { + bail!("missing argument {}", arg_name); + }; + let file = Box::new(fs::File::open(path)?); + Ok(Input { source: path.to_string(), reader: file }) +} + +fn write_output_file_realtive_to_dir(root: &Path, output_file: &OutputFile) -> Result<()> { + let path = root.join(&output_file.path); + let parent = path + .parent() + .ok_or(anyhow!("unable to locate parent of output file {}", path.display()))?; + fs::create_dir_all(parent) + .with_context(|| format!("failed to create directory {}", parent.display()))?; + let mut file = + fs::File::create(&path).with_context(|| format!("failed to open {}", path.display()))?; + file.write_all(&output_file.contents) + .with_context(|| format!("failed to write to {}", path.display()))?; + Ok(()) +} + +fn write_output_to_file_or_stdout(path: &str, data: &[u8]) -> Result<()> { + if path == "-" { + io::stdout().write_all(data).context("failed to write to stdout")?; + } else { + fs::File::create(path) + .with_context(|| format!("failed to open {}", path))? + .write_all(data) + .with_context(|| format!("failed to write to {}", path))?; + } + Ok(()) +} + +fn main() -> Result<()> { + let matches = cli().get_matches(); + match matches.subcommand() { + Some(("create-cache", sub_matches)) => { + let package = get_required_arg::<String>(sub_matches, "package")?; + let container = + get_optional_arg::<String>(sub_matches, "container").map(|c| c.as_str()); + let declarations = open_zero_or_more_files(sub_matches, "declarations")?; + let values = open_zero_or_more_files(sub_matches, "values")?; + let default_permission = get_required_arg::<aconfig_protos::ProtoFlagPermission>( + sub_matches, + "default-permission", + )?; + let output = commands::parse_flags( + package, + container, + declarations, + values, + *default_permission, + ) + .context("failed to create cache")?; + let path = get_required_arg::<String>(sub_matches, "cache")?; + write_output_to_file_or_stdout(path, &output)?; + } + Some(("create-java-lib", sub_matches)) => { + let cache = open_single_file(sub_matches, "cache")?; + let mode = get_required_arg::<CodegenMode>(sub_matches, "mode")?; + let generated_files = + commands::create_java_lib(cache, *mode).context("failed to create java lib")?; + let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?); + generated_files + .iter() + .try_for_each(|file| write_output_file_realtive_to_dir(&dir, file))?; + } + Some(("create-cpp-lib", sub_matches)) => { + let cache = open_single_file(sub_matches, "cache")?; + let mode = get_required_arg::<CodegenMode>(sub_matches, "mode")?; + let allow_instrumentation = + get_required_arg::<bool>(sub_matches, "allow-instrumentation")?; + let generated_files = commands::create_cpp_lib(cache, *mode, *allow_instrumentation) + .context("failed to create cpp lib")?; + let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?); + generated_files + .iter() + .try_for_each(|file| write_output_file_realtive_to_dir(&dir, file))?; + } + Some(("create-rust-lib", sub_matches)) => { + let cache = open_single_file(sub_matches, "cache")?; + let mode = get_required_arg::<CodegenMode>(sub_matches, "mode")?; + let generated_file = + commands::create_rust_lib(cache, *mode).context("failed to create rust lib")?; + let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?); + write_output_file_realtive_to_dir(&dir, &generated_file)?; + } + Some(("create-device-config-defaults", sub_matches)) => { + let cache = open_single_file(sub_matches, "cache")?; + let output = commands::create_device_config_defaults(cache) + .context("failed to create device config defaults")?; + let path = get_required_arg::<String>(sub_matches, "out")?; + write_output_to_file_or_stdout(path, &output)?; + } + Some(("create-device-config-sysprops", sub_matches)) => { + let cache = open_single_file(sub_matches, "cache")?; + let output = commands::create_device_config_sysprops(cache) + .context("failed to create device config sysprops")?; + let path = get_required_arg::<String>(sub_matches, "out")?; + write_output_to_file_or_stdout(path, &output)?; + } + Some(("dump-cache", sub_matches)) => { + let input = open_zero_or_more_files(sub_matches, "cache")?; + let format = get_required_arg::<DumpFormat>(sub_matches, "format") + .context("failed to dump previously parsed flags")?; + let filters = sub_matches + .get_many::<String>("filter") + .unwrap_or_default() + .map(String::as_ref) + .collect::<Vec<_>>(); + let dedup = get_required_arg::<bool>(sub_matches, "dedup")?; + let output = commands::dump_parsed_flags(input, format.clone(), &filters, *dedup)?; + let path = get_required_arg::<String>(sub_matches, "out")?; + write_output_to_file_or_stdout(path, &output)?; + } + Some(("create-storage", sub_matches)) => { + let file = get_required_arg::<StorageFileType>(sub_matches, "file") + .context("Invalid storage file selection")?; + let cache = open_zero_or_more_files(sub_matches, "cache")?; + let container = get_required_arg::<String>(sub_matches, "container")?; + let path = get_required_arg::<String>(sub_matches, "out")?; + let output = commands::create_storage(cache, container, file) + .context("failed to create storage files")?; + write_output_to_file_or_stdout(path, &output)?; + } + _ => unreachable!(), + } + Ok(()) +} diff --git a/tools/aconfig/aconfig/src/storage/flag_table.rs b/tools/aconfig/aconfig/src/storage/flag_table.rs new file mode 100644 index 0000000000..a9712119bf --- /dev/null +++ b/tools/aconfig/aconfig/src/storage/flag_table.rs @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::commands::assign_flag_ids; +use crate::storage::FlagPackage; +use aconfig_protos::ProtoFlagPermission; +use aconfig_storage_file::{ + get_table_size, FlagTable, FlagTableHeader, FlagTableNode, StorageFileType, StoredFlagType, + FILE_VERSION, +}; +use anyhow::{anyhow, Result}; + +fn new_header(container: &str, num_flags: u32) -> FlagTableHeader { + FlagTableHeader { + version: FILE_VERSION, + container: String::from(container), + file_type: StorageFileType::FlagMap as u8, + file_size: 0, + num_flags, + bucket_offset: 0, + node_offset: 0, + } +} + +// a struct that contains FlagTableNode and a bunch of other information to help +// flag table creation +#[derive(PartialEq, Debug, Clone)] +struct FlagTableNodeWrapper { + pub node: FlagTableNode, + pub bucket_index: u32, +} + +impl FlagTableNodeWrapper { + fn new( + package_id: u32, + flag_name: &str, + flag_type: StoredFlagType, + flag_index: u16, + num_buckets: u32, + ) -> Self { + let bucket_index = FlagTableNode::find_bucket_index(package_id, flag_name, num_buckets); + let node = FlagTableNode { + package_id, + flag_name: flag_name.to_string(), + flag_type, + flag_index, + next_offset: None, + }; + Self { node, bucket_index } + } + + fn create_nodes(package: &FlagPackage, num_buckets: u32) -> Result<Vec<Self>> { + let flag_ids = + assign_flag_ids(package.package_name, package.boolean_flags.iter().copied())?; + package + .boolean_flags + .iter() + .map(|&pf| { + let fid = flag_ids + .get(pf.name()) + .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; + let flag_type = if pf.is_fixed_read_only() { + StoredFlagType::FixedReadOnlyBoolean + } else { + match pf.permission() { + ProtoFlagPermission::READ_WRITE => StoredFlagType::ReadWriteBoolean, + ProtoFlagPermission::READ_ONLY => StoredFlagType::ReadOnlyBoolean, + } + }; + Ok(Self::new(package.package_id, pf.name(), flag_type, *fid, num_buckets)) + }) + .collect::<Result<Vec<_>>>() + } +} + +pub fn create_flag_table(container: &str, packages: &[FlagPackage]) -> Result<FlagTable> { + // create table + let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); + let num_buckets = get_table_size(num_flags)?; + + let mut header = new_header(container, num_flags); + let mut buckets = vec![None; num_buckets as usize]; + let mut node_wrappers = packages + .iter() + .map(|pkg| FlagTableNodeWrapper::create_nodes(pkg, num_buckets)) + .collect::<Result<Vec<_>>>()? + .concat(); + + // initialize all header fields + header.bucket_offset = header.into_bytes().len() as u32; + header.node_offset = header.bucket_offset + num_buckets * 4; + header.file_size = header.node_offset + + node_wrappers.iter().map(|x| x.node.into_bytes().len()).sum::<usize>() as u32; + + // sort nodes by bucket index for efficiency + node_wrappers.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); + + // fill all node offset + let mut offset = header.node_offset; + for i in 0..node_wrappers.len() { + let node_bucket_idx = node_wrappers[i].bucket_index; + let next_node_bucket_idx = if i + 1 < node_wrappers.len() { + Some(node_wrappers[i + 1].bucket_index) + } else { + None + }; + + if buckets[node_bucket_idx as usize].is_none() { + buckets[node_bucket_idx as usize] = Some(offset); + } + offset += node_wrappers[i].node.into_bytes().len() as u32; + + if let Some(index) = next_node_bucket_idx { + if index == node_bucket_idx { + node_wrappers[i].node.next_offset = Some(offset); + } + } + } + + let table = + FlagTable { header, buckets, nodes: node_wrappers.into_iter().map(|nw| nw.node).collect() }; + + Ok(table) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; + + fn create_test_flag_table_from_source() -> Result<FlagTable> { + let caches = parse_all_test_flags(); + let packages = group_flags_by_package(caches.iter()); + create_flag_table("mockup", &packages) + } + + #[test] + // this test point locks down the table creation and each field + fn test_table_contents() { + let flag_table = create_test_flag_table_from_source(); + assert!(flag_table.is_ok()); + let expected_flag_table = aconfig_storage_file::test_utils::create_test_flag_table(); + assert_eq!(flag_table.unwrap(), expected_flag_table); + } +} diff --git a/tools/aconfig/aconfig/src/storage/flag_value.rs b/tools/aconfig/aconfig/src/storage/flag_value.rs new file mode 100644 index 0000000000..c15ba54112 --- /dev/null +++ b/tools/aconfig/aconfig/src/storage/flag_value.rs @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::commands::assign_flag_ids; +use crate::storage::FlagPackage; +use aconfig_protos::ProtoFlagState; +use aconfig_storage_file::{FlagValueHeader, FlagValueList, StorageFileType, FILE_VERSION}; +use anyhow::{anyhow, Result}; + +fn new_header(container: &str, num_flags: u32) -> FlagValueHeader { + FlagValueHeader { + version: FILE_VERSION, + container: String::from(container), + file_type: StorageFileType::FlagVal as u8, + file_size: 0, + num_flags, + boolean_value_offset: 0, + } +} + +pub fn create_flag_value(container: &str, packages: &[FlagPackage]) -> Result<FlagValueList> { + // create list + let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); + + let mut list = FlagValueList { + header: new_header(container, num_flags), + booleans: vec![false; num_flags as usize], + }; + + for pkg in packages.iter() { + let start_index = pkg.boolean_start_index as usize; + let flag_ids = assign_flag_ids(pkg.package_name, pkg.boolean_flags.iter().copied())?; + for pf in pkg.boolean_flags.iter() { + let fid = flag_ids + .get(pf.name()) + .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; + + list.booleans[start_index + (*fid as usize)] = pf.state() == ProtoFlagState::ENABLED; + } + } + + // initialize all header fields + list.header.boolean_value_offset = list.header.into_bytes().len() as u32; + list.header.file_size = list.header.boolean_value_offset + num_flags; + + Ok(list) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; + + pub fn create_test_flag_value_list_from_source() -> Result<FlagValueList> { + let caches = parse_all_test_flags(); + let packages = group_flags_by_package(caches.iter()); + create_flag_value("mockup", &packages) + } + + #[test] + // this test point locks down the flag value creation and each field + fn test_list_contents() { + let flag_value_list = create_test_flag_value_list_from_source(); + assert!(flag_value_list.is_ok()); + let expected_flag_value_list = + aconfig_storage_file::test_utils::create_test_flag_value_list(); + assert_eq!(flag_value_list.unwrap(), expected_flag_value_list); + } +} diff --git a/tools/aconfig/aconfig/src/storage/mod.rs b/tools/aconfig/aconfig/src/storage/mod.rs new file mode 100644 index 0000000000..73339f24b3 --- /dev/null +++ b/tools/aconfig/aconfig/src/storage/mod.rs @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pub mod flag_table; +pub mod flag_value; +pub mod package_table; + +use anyhow::{anyhow, Result}; +use std::collections::{HashMap, HashSet}; + +use crate::storage::{ + flag_table::create_flag_table, flag_value::create_flag_value, + package_table::create_package_table, +}; +use aconfig_protos::{ProtoParsedFlag, ProtoParsedFlags}; +use aconfig_storage_file::StorageFileType; + +pub struct FlagPackage<'a> { + pub package_name: &'a str, + pub package_id: u32, + pub flag_names: HashSet<&'a str>, + pub boolean_flags: Vec<&'a ProtoParsedFlag>, + // The index of the first boolean flag in this aconfig package among all boolean + // flags in this container. + pub boolean_start_index: u32, +} + +impl<'a> FlagPackage<'a> { + fn new(package_name: &'a str, package_id: u32) -> Self { + FlagPackage { + package_name, + package_id, + flag_names: HashSet::new(), + boolean_flags: vec![], + boolean_start_index: 0, + } + } + + fn insert(&mut self, pf: &'a ProtoParsedFlag) { + if self.flag_names.insert(pf.name()) { + self.boolean_flags.push(pf); + } + } +} + +pub fn group_flags_by_package<'a, I>(parsed_flags_vec_iter: I) -> Vec<FlagPackage<'a>> +where + I: Iterator<Item = &'a ProtoParsedFlags>, +{ + // group flags by package + let mut packages: Vec<FlagPackage<'a>> = Vec::new(); + let mut package_index: HashMap<&str, usize> = HashMap::new(); + for parsed_flags in parsed_flags_vec_iter { + for parsed_flag in parsed_flags.parsed_flag.iter() { + let index = *(package_index.entry(parsed_flag.package()).or_insert(packages.len())); + if index == packages.len() { + packages.push(FlagPackage::new(parsed_flag.package(), index as u32)); + } + packages[index].insert(parsed_flag); + } + } + + // cacluate boolean flag start index for each package + let mut boolean_start_index = 0; + for p in packages.iter_mut() { + p.boolean_start_index = boolean_start_index; + boolean_start_index += p.boolean_flags.len() as u32; + } + + packages +} + +pub fn generate_storage_file<'a, I>( + container: &str, + parsed_flags_vec_iter: I, + file: &StorageFileType, +) -> Result<Vec<u8>> +where + I: Iterator<Item = &'a ProtoParsedFlags>, +{ + let packages = group_flags_by_package(parsed_flags_vec_iter); + + match file { + StorageFileType::PackageMap => { + let package_table = create_package_table(container, &packages)?; + Ok(package_table.into_bytes()) + } + StorageFileType::FlagMap => { + let flag_table = create_flag_table(container, &packages)?; + Ok(flag_table.into_bytes()) + } + StorageFileType::FlagVal => { + let flag_value = create_flag_value(container, &packages)?; + Ok(flag_value.into_bytes()) + } + _ => Err(anyhow!("aconfig does not support the creation of this storage file type")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Input; + + pub fn parse_all_test_flags() -> Vec<ProtoParsedFlags> { + let aconfig_files = [ + ( + "com.android.aconfig.storage.test_1", + "storage_test_1.aconfig", + include_bytes!("../../tests/storage_test_1.aconfig").as_slice(), + "storage_test_1.value", + include_bytes!("../../tests/storage_test_1.values").as_slice(), + ), + ( + "com.android.aconfig.storage.test_2", + "storage_test_2.aconfig", + include_bytes!("../../tests/storage_test_2.aconfig").as_slice(), + "storage_test_2.value", + include_bytes!("../../tests/storage_test_2.values").as_slice(), + ), + ( + "com.android.aconfig.storage.test_4", + "storage_test_4.aconfig", + include_bytes!("../../tests/storage_test_4.aconfig").as_slice(), + "storage_test_4.value", + include_bytes!("../../tests/storage_test_4.values").as_slice(), + ), + ]; + aconfig_files + .into_iter() + .map(|(pkg, aconfig_file, aconfig_content, value_file, value_content)| { + let bytes = crate::commands::parse_flags( + pkg, + Some("system"), + vec![Input { + source: format!("tests/{}", aconfig_file).to_string(), + reader: Box::new(aconfig_content), + }], + vec![Input { + source: format!("tests/{}", value_file).to_string(), + reader: Box::new(value_content), + }], + crate::commands::DEFAULT_FLAG_PERMISSION, + ) + .unwrap(); + aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() + }) + .collect() + } + + #[test] + fn test_flag_package() { + let caches = parse_all_test_flags(); + let packages = group_flags_by_package(caches.iter()); + + for pkg in packages.iter() { + let pkg_name = pkg.package_name; + assert_eq!(pkg.flag_names.len(), pkg.boolean_flags.len()); + for pf in pkg.boolean_flags.iter() { + assert!(pkg.flag_names.contains(pf.name())); + assert_eq!(pf.package(), pkg_name); + } + } + + assert_eq!(packages.len(), 3); + + assert_eq!(packages[0].package_name, "com.android.aconfig.storage.test_1"); + assert_eq!(packages[0].package_id, 0); + assert_eq!(packages[0].flag_names.len(), 3); + assert!(packages[0].flag_names.contains("enabled_rw")); + assert!(packages[0].flag_names.contains("disabled_rw")); + assert!(packages[0].flag_names.contains("enabled_ro")); + assert_eq!(packages[0].boolean_start_index, 0); + + assert_eq!(packages[1].package_name, "com.android.aconfig.storage.test_2"); + assert_eq!(packages[1].package_id, 1); + assert_eq!(packages[1].flag_names.len(), 3); + assert!(packages[1].flag_names.contains("enabled_ro")); + assert!(packages[1].flag_names.contains("disabled_rw")); + assert!(packages[1].flag_names.contains("enabled_fixed_ro")); + assert_eq!(packages[1].boolean_start_index, 3); + + assert_eq!(packages[2].package_name, "com.android.aconfig.storage.test_4"); + assert_eq!(packages[2].package_id, 2); + assert_eq!(packages[2].flag_names.len(), 2); + assert!(packages[2].flag_names.contains("enabled_rw")); + assert!(packages[2].flag_names.contains("enabled_fixed_ro")); + assert_eq!(packages[2].boolean_start_index, 6); + } +} diff --git a/tools/aconfig/aconfig/src/storage/package_table.rs b/tools/aconfig/aconfig/src/storage/package_table.rs new file mode 100644 index 0000000000..c53602f9cb --- /dev/null +++ b/tools/aconfig/aconfig/src/storage/package_table.rs @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; + +use aconfig_storage_file::{ + get_table_size, PackageTable, PackageTableHeader, PackageTableNode, StorageFileType, + FILE_VERSION, +}; + +use crate::storage::FlagPackage; + +fn new_header(container: &str, num_packages: u32) -> PackageTableHeader { + PackageTableHeader { + version: FILE_VERSION, + container: String::from(container), + file_type: StorageFileType::PackageMap as u8, + file_size: 0, + num_packages, + bucket_offset: 0, + node_offset: 0, + } +} + +// a struct that contains PackageTableNode and a bunch of other information to help +// package table creation +#[derive(PartialEq, Debug)] +struct PackageTableNodeWrapper { + pub node: PackageTableNode, + pub bucket_index: u32, +} + +impl PackageTableNodeWrapper { + fn new(package: &FlagPackage, num_buckets: u32) -> Self { + let node = PackageTableNode { + package_name: String::from(package.package_name), + package_id: package.package_id, + boolean_start_index: package.boolean_start_index, + next_offset: None, + }; + let bucket_index = PackageTableNode::find_bucket_index(package.package_name, num_buckets); + Self { node, bucket_index } + } +} + +pub fn create_package_table(container: &str, packages: &[FlagPackage]) -> Result<PackageTable> { + // create table + let num_packages = packages.len() as u32; + let num_buckets = get_table_size(num_packages)?; + let mut header = new_header(container, num_packages); + let mut buckets = vec![None; num_buckets as usize]; + let mut node_wrappers: Vec<_> = + packages.iter().map(|pkg| PackageTableNodeWrapper::new(pkg, num_buckets)).collect(); + + // initialize all header fields + header.bucket_offset = header.into_bytes().len() as u32; + header.node_offset = header.bucket_offset + num_buckets * 4; + header.file_size = header.node_offset + + node_wrappers.iter().map(|x| x.node.into_bytes().len()).sum::<usize>() as u32; + + // sort node_wrappers by bucket index for efficiency + node_wrappers.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); + + // fill all node offset + let mut offset = header.node_offset; + for i in 0..node_wrappers.len() { + let node_bucket_idx = node_wrappers[i].bucket_index; + let next_node_bucket_idx = if i + 1 < node_wrappers.len() { + Some(node_wrappers[i + 1].bucket_index) + } else { + None + }; + + if buckets[node_bucket_idx as usize].is_none() { + buckets[node_bucket_idx as usize] = Some(offset); + } + offset += node_wrappers[i].node.into_bytes().len() as u32; + + if let Some(index) = next_node_bucket_idx { + if index == node_bucket_idx { + node_wrappers[i].node.next_offset = Some(offset); + } + } + } + + let table = PackageTable { + header, + buckets, + nodes: node_wrappers.into_iter().map(|nw| nw.node).collect(), + }; + Ok(table) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; + + pub fn create_test_package_table_from_source() -> Result<PackageTable> { + let caches = parse_all_test_flags(); + let packages = group_flags_by_package(caches.iter()); + create_package_table("mockup", &packages) + } + + #[test] + // this test point locks down the table creation and each field + fn test_table_contents() { + let package_table = create_test_package_table_from_source(); + assert!(package_table.is_ok()); + let expected_package_table = aconfig_storage_file::test_utils::create_test_package_table(); + assert_eq!(package_table.unwrap(), expected_package_table); + } +} diff --git a/tools/aconfig/aconfig/src/test.rs b/tools/aconfig/aconfig/src/test.rs new file mode 100644 index 0000000000..7409cda6e8 --- /dev/null +++ b/tools/aconfig/aconfig/src/test.rs @@ -0,0 +1,345 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[cfg(test)] +pub use test_utils::*; + +#[cfg(test)] +pub mod test_utils { + use crate::commands::Input; + use aconfig_protos::ProtoParsedFlags; + use itertools; + + pub const TEST_PACKAGE: &str = "com.android.aconfig.test"; + + pub const TEST_FLAGS_TEXTPROTO: &str = r#" +parsed_flag { + package: "com.android.aconfig.test" + name: "disabled_ro" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_ONLY" + bug: "123" + state: DISABLED + permission: READ_ONLY + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/first.values" + state: DISABLED + permission: READ_ONLY + } + is_fixed_read_only: false + is_exported: false + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "disabled_rw" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_WRITE" + bug: "456" + state: DISABLED + permission: READ_WRITE + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + is_fixed_read_only: false + is_exported: false + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "disabled_rw_exported" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_WRITE and exported" + bug: "111" + state: DISABLED + permission: READ_WRITE + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/first.values" + state: DISABLED + permission: READ_WRITE + } + is_fixed_read_only: false + is_exported: true + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "disabled_rw_in_other_namespace" + namespace: "other_namespace" + description: "This flag is DISABLED + READ_WRITE, and is defined in another namespace" + bug: "999" + state: DISABLED + permission: READ_WRITE + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/first.values" + state: DISABLED + permission: READ_WRITE + } + is_fixed_read_only: false + is_exported: false + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "enabled_fixed_ro" + namespace: "aconfig_test" + description: "This flag is fixed READ_ONLY + ENABLED" + bug: "" + state: ENABLED + permission: READ_ONLY + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_ONLY + } + trace { + source: "tests/first.values" + state: ENABLED + permission: READ_ONLY + } + is_fixed_read_only: true + is_exported: false + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "enabled_fixed_ro_exported" + namespace: "aconfig_test" + description: "This flag is fixed ENABLED + READ_ONLY and exported" + bug: "111" + state: ENABLED + permission: READ_ONLY + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_ONLY + } + trace { + source: "tests/first.values" + state: ENABLED + permission: READ_ONLY + } + is_fixed_read_only: true + is_exported: true + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "enabled_ro" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY" + bug: "abc" + state: ENABLED + permission: READ_ONLY + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/first.values" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/second.values" + state: ENABLED + permission: READ_ONLY + } + is_fixed_read_only: false + is_exported: false + container: "system" + metadata { + purpose: PURPOSE_BUGFIX + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "enabled_ro_exported" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY and exported" + bug: "111" + state: ENABLED + permission: READ_ONLY + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/first.values" + state: ENABLED + permission: READ_ONLY + } + is_fixed_read_only: false + is_exported: true + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +parsed_flag { + package: "com.android.aconfig.test" + name: "enabled_rw" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_WRITE" + bug: "" + state: ENABLED + permission: READ_WRITE + trace { + source: "tests/test.aconfig" + state: DISABLED + permission: READ_WRITE + } + trace { + source: "tests/first.values" + state: ENABLED + permission: READ_WRITE + } + is_fixed_read_only: false + is_exported: false + container: "system" + metadata { + purpose: PURPOSE_UNSPECIFIED + } +} +"#; + + pub fn parse_read_only_test_flags() -> ProtoParsedFlags { + let bytes = crate::commands::parse_flags( + "com.android.aconfig.test", + Some("system"), + vec![Input { + source: "tests/read_only_test.aconfig".to_string(), + reader: Box::new(include_bytes!("../tests/read_only_test.aconfig").as_slice()), + }], + vec![Input { + source: "tests/read_only_test.values".to_string(), + reader: Box::new(include_bytes!("../tests/read_only_test.values").as_slice()), + }], + crate::commands::DEFAULT_FLAG_PERMISSION, + ) + .unwrap(); + aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() + } + + pub fn parse_test_flags() -> ProtoParsedFlags { + let bytes = crate::commands::parse_flags( + "com.android.aconfig.test", + Some("system"), + vec![Input { + source: "tests/test.aconfig".to_string(), + reader: Box::new(include_bytes!("../tests/test.aconfig").as_slice()), + }], + vec![ + Input { + source: "tests/first.values".to_string(), + reader: Box::new(include_bytes!("../tests/first.values").as_slice()), + }, + Input { + source: "tests/second.values".to_string(), + reader: Box::new(include_bytes!("../tests/second.values").as_slice()), + }, + ], + crate::commands::DEFAULT_FLAG_PERMISSION, + ) + .unwrap(); + aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() + } + + pub fn first_significant_code_diff(a: &str, b: &str) -> Option<String> { + let a = a.lines().map(|line| line.trim_start()).filter(|line| !line.is_empty()); + let b = b.lines().map(|line| line.trim_start()).filter(|line| !line.is_empty()); + match itertools::diff_with(a, b, |left, right| left == right) { + Some(itertools::Diff::FirstMismatch(_, mut left, mut right)) => { + Some(format!("'{}' vs '{}'", left.next().unwrap(), right.next().unwrap())) + } + Some(itertools::Diff::Shorter(_, mut left)) => { + Some(format!("LHS trailing data: '{}'", left.next().unwrap())) + } + Some(itertools::Diff::Longer(_, mut right)) => { + Some(format!("RHS trailing data: '{}'", right.next().unwrap())) + } + None => None, + } + } + + #[test] + fn test_first_significant_code_diff() { + assert!(first_significant_code_diff("", "").is_none()); + assert!(first_significant_code_diff(" a", "\n\na\n").is_none()); + let a = r#" + public class A { + private static final String FOO = "FOO"; + public static void main(String[] args) { + System.out.println("FOO=" + FOO); + } + } + "#; + let b = r#" + public class A { + private static final String FOO = "BAR"; + public static void main(String[] args) { + System.out.println("foo=" + FOO); + } + } + "#; + assert_eq!(Some(r#"'private static final String FOO = "FOO";' vs 'private static final String FOO = "BAR";'"#.to_string()), first_significant_code_diff(a, b)); + assert_eq!( + Some("LHS trailing data: 'b'".to_string()), + first_significant_code_diff("a\nb", "a") + ); + assert_eq!( + Some("RHS trailing data: 'b'".to_string()), + first_significant_code_diff("a", "a\nb") + ); + } +} diff --git a/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template b/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template new file mode 100644 index 0000000000..b82b9cb827 --- /dev/null +++ b/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template @@ -0,0 +1,70 @@ +package {package_name}; + +{{ if not library_exported- }} +// TODO(b/303773055): Remove the annotation after access issue is resolved. +import android.compat.annotation.UnsupportedAppUsage; +{{ -endif }} +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.BiPredicate; +import java.util.function.Predicate; + +/** @hide */ +public class CustomFeatureFlags implements FeatureFlags \{ + + private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; + + public CustomFeatureFlags(BiPredicate<String, Predicate<FeatureFlags>> getValueImpl) \{ + mGetValueImpl = getValueImpl; + } + +{{ -for item in flag_elements}} + @Override +{{ if not library_exported }} @UnsupportedAppUsage{{ -endif }} + public boolean {item.method_name}() \{ + return getValue(Flags.FLAG_{item.flag_name_constant_suffix}, + FeatureFlags::{item.method_name}); + } +{{ endfor }} + +{{ -if not library_exported }} + public boolean isFlagReadOnlyOptimized(String flagName) \{ + if (mReadOnlyFlagsSet.contains(flagName) && + isOptimizationEnabled()) \{ + return true; + } + return false; + } + + @com.android.aconfig.annotations.AssumeTrueForR8 + private boolean isOptimizationEnabled() \{ + return false; + } +{{ -endif }} + + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) \{ + return mGetValueImpl.test(flagName, getter); + } + + public List<String> getFlagNames() \{ + return Arrays.asList( + {{ -for item in flag_elements }} + Flags.FLAG_{item.flag_name_constant_suffix} + {{ -if not @last }},{{ endif }} + {{ -endfor }} + ); + } + + private Set<String> mReadOnlyFlagsSet = new HashSet<>( + Arrays.asList( + {{ -for item in flag_elements }} + {{ -if not item.is_read_write }} + Flags.FLAG_{item.flag_name_constant_suffix}, + {{ -endif }} + {{ -endfor }} + ""{# The empty string here is to resolve the ending comma #} + ) + ); +} diff --git a/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template b/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template new file mode 100644 index 0000000000..290d2c4b24 --- /dev/null +++ b/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template @@ -0,0 +1,49 @@ +package {package_name}; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; + +/** @hide */ +public class FakeFeatureFlagsImpl extends CustomFeatureFlags \{ + private final Map<String, Boolean> mFlagMap = new HashMap<>(); + private final FeatureFlags mDefaults; + + public FakeFeatureFlagsImpl() \{ + this(null); + } + + public FakeFeatureFlagsImpl(FeatureFlags defaults) \{ + super(null); + mDefaults = defaults; + // Initialize the map with null values + for (String flagName : getFlagNames()) \{ + mFlagMap.put(flagName, null); + } + } + + @Override + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) \{ + Boolean value = this.mFlagMap.get(flagName); + if (value != null) \{ + return value; + } + if (mDefaults != null) \{ + return getter.test(mDefaults); + } + throw new IllegalArgumentException(flagName + " is not set"); + } + + public void setFlag(String flagName, boolean value) \{ + if (!this.mFlagMap.containsKey(flagName)) \{ + throw new IllegalArgumentException("no such flag " + flagName); + } + this.mFlagMap.put(flagName, value); + } + + public void resetAll() \{ + for (Map.Entry entry : mFlagMap.entrySet()) \{ + entry.setValue(null); + } + } +} diff --git a/tools/aconfig/aconfig/templates/FeatureFlags.java.template b/tools/aconfig/aconfig/templates/FeatureFlags.java.template new file mode 100644 index 0000000000..38c8f13aaf --- /dev/null +++ b/tools/aconfig/aconfig/templates/FeatureFlags.java.template @@ -0,0 +1,22 @@ +package {package_name}; +{{ if not library_exported- }} +// TODO(b/303773055): Remove the annotation after access issue is resolved. +import android.compat.annotation.UnsupportedAppUsage; +{{ -endif }} +/** @hide */ +public interface FeatureFlags \{ +{{ for item in flag_elements }} +{{ -if not item.is_read_write }} +{{ -if item.default_value }} + @com.android.aconfig.annotations.AssumeTrueForR8 +{{ -else }} + @com.android.aconfig.annotations.AssumeFalseForR8 +{{ -endif- }} +{{ -endif }} +{{ -if not library_exported }} + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage +{{ -endif }} + boolean {item.method_name}(); +{{ -endfor }} +}
\ No newline at end of file diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template new file mode 100644 index 0000000000..6235e6946c --- /dev/null +++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template @@ -0,0 +1,81 @@ +package {package_name}; +{{ if not library_exported- }} +// TODO(b/303773055): Remove the annotation after access issue is resolved. +import android.compat.annotation.UnsupportedAppUsage; +{{ -endif }} +{{ -if not is_test_mode }} +{{ -if runtime_lookup_required }} +import android.provider.DeviceConfig; +import android.provider.DeviceConfig.Properties; +{{ endif }} +/** @hide */ +public final class FeatureFlagsImpl implements FeatureFlags \{ +{{ -if runtime_lookup_required }} +{{ -for namespace_with_flags in namespace_flags }} + private static boolean {namespace_with_flags.namespace}_is_cached = false; +{{ -endfor- }} + +{{ for flag in flag_elements }} +{{- if flag.is_read_write }} + private static boolean {flag.method_name} = {flag.default_value}; +{{ -endif }} +{{ -endfor }} +{{ for namespace_with_flags in namespace_flags }} + private void load_overrides_{namespace_with_flags.namespace}() \{ + try \{ + Properties properties = DeviceConfig.getProperties("{namespace_with_flags.namespace}"); +{{ -for flag in namespace_with_flags.flags }} +{{ -if flag.is_read_write }} + {flag.method_name} = + properties.getBoolean("{flag.device_config_flag}", {flag.default_value}); +{{ -endif }} +{{ -endfor }} + } catch (NullPointerException e) \{ + throw new RuntimeException( + "Cannot read value from namespace {namespace_with_flags.namespace} " + + "from DeviceConfig. It could be that the code using flag " + + "executed before SettingsProvider initialization. Please use " + + "fixed read-only flag by adding is_fixed_read_only: true in " + + "flag declaration.", + e + ); + } + {namespace_with_flags.namespace}_is_cached = true; + } +{{ endfor- }} +{{ -endif }}{#- end of runtime_lookup_required #} +{{ -for flag in flag_elements }} + @Override +{{ -if not library_exported }} + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage +{{ -endif }} + public boolean {flag.method_name}() \{ +{{ -if flag.is_read_write }} + if (!{flag.device_config_namespace}_is_cached) \{ + load_overrides_{flag.device_config_namespace}(); + } + return {flag.method_name}; +{{ -else }} + return {flag.default_value}; +{{ -endif }} + } +{{ endfor }} +} +{{ else }} +{#- Generate only stub if in test mode #} +/** @hide */ +public final class FeatureFlagsImpl implements FeatureFlags \{ +{{ for flag in flag_elements }} + @Override +{{ -if not library_exported }} + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage +{{ -endif }} + public boolean {flag.method_name}() \{ + throw new UnsupportedOperationException( + "Method is not implemented."); + } +{{ endfor- }} +} +{{ endif }} diff --git a/tools/aconfig/aconfig/templates/Flags.java.template b/tools/aconfig/aconfig/templates/Flags.java.template new file mode 100644 index 0000000000..e2f70b95fa --- /dev/null +++ b/tools/aconfig/aconfig/templates/Flags.java.template @@ -0,0 +1,40 @@ +package {package_name}; +{{ if not library_exported- }} +// TODO(b/303773055): Remove the annotation after access issue is resolved. +import android.compat.annotation.UnsupportedAppUsage; +{{ -endif }} +/** @hide */ +public final class Flags \{ +{{ -for item in flag_elements}} + /** @hide */ + public static final String FLAG_{item.flag_name_constant_suffix} = "{item.device_config_flag}"; +{{- endfor }} +{{ -for item in flag_elements}} +{{ -if not item.is_read_write }} +{{ -if item.default_value }} + @com.android.aconfig.annotations.AssumeTrueForR8 +{{ -else }} + @com.android.aconfig.annotations.AssumeFalseForR8 +{{ -endif }} +{{ -endif }} +{{ -if not library_exported }} + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage +{{ -endif }} + public static boolean {item.method_name}() \{ + return FEATURE_FLAGS.{item.method_name}(); + } +{{ -endfor }} +{{ -if is_test_mode }} + public static void setFeatureFlags(FeatureFlags featureFlags) \{ + Flags.FEATURE_FLAGS = featureFlags; + } + + public static void unsetFeatureFlags() \{ + Flags.FEATURE_FLAGS = null; + } +{{ -endif }} + + private static FeatureFlags FEATURE_FLAGS{{ -if not is_test_mode }} = new FeatureFlagsImpl(){{ -endif- }}; + +} diff --git a/tools/aconfig/aconfig/templates/cpp_exported_header.template b/tools/aconfig/aconfig/templates/cpp_exported_header.template new file mode 100644 index 0000000000..0f7853e405 --- /dev/null +++ b/tools/aconfig/aconfig/templates/cpp_exported_header.template @@ -0,0 +1,92 @@ +#pragma once + +{{ if not is_test_mode- }} +{{ if has_fixed_read_only- }} +#ifndef {package_macro} +#define {package_macro}(FLAG) {package_macro}_##FLAG +#endif +{{ for item in class_elements }} +{{ -if item.is_fixed_read_only }} +#ifndef {package_macro}_{item.flag_macro} +#define {package_macro}_{item.flag_macro} {item.default_value} +#endif +{{ -endif }} +{{ -endfor }} +{{ -endif }} +{{ -endif }} + +#ifdef __cplusplus + +#include <memory> + +namespace {cpp_namespace} \{ + +class flag_provider_interface \{ +public: + virtual ~flag_provider_interface() = default; + {{ -for item in class_elements}} + virtual bool {item.flag_name}() = 0; + + {{ -if is_test_mode }} + virtual void {item.flag_name}(bool val) = 0; + {{ -endif }} + {{ -endfor }} + + {{ -if is_test_mode }} + virtual void reset_flags() \{} + {{ -endif }} +}; + +extern std::unique_ptr<flag_provider_interface> provider_; + +{{ for item in class_elements}} +inline bool {item.flag_name}() \{ + {{ -if is_test_mode }} + return provider_->{item.flag_name}(); + {{ -else }} + {{ -if item.readwrite }} + return provider_->{item.flag_name}(); + {{ -else }} + {{ -if item.is_fixed_read_only }} + return {package_macro}_{item.flag_macro}; + {{ -else }} + return {item.default_value}; + {{ -endif }} + {{ -endif }} + {{ -endif }} +} + +{{ -if is_test_mode }} +inline void {item.flag_name}(bool val) \{ + provider_->{item.flag_name}(val); +} +{{ -endif }} +{{ -endfor }} + +{{ -if is_test_mode }} +inline void reset_flags() \{ + return provider_->reset_flags(); +} +{{ -endif }} + +} + +extern "C" \{ +#endif // __cplusplus + +{{ for item in class_elements }} +bool {header}_{item.flag_name}(); + +{{ -if is_test_mode }} +void set_{header}_{item.flag_name}(bool val); +{{ -endif }} +{{ -endfor }} + +{{ -if is_test_mode }} +void {header}_reset_flags(); +{{ -endif }} + +#ifdef __cplusplus +} // extern "C" +#endif + diff --git a/tools/aconfig/aconfig/templates/cpp_source_file.template b/tools/aconfig/aconfig/templates/cpp_source_file.template new file mode 100644 index 0000000000..4c098c5b41 --- /dev/null +++ b/tools/aconfig/aconfig/templates/cpp_source_file.template @@ -0,0 +1,189 @@ +#include "{header}.h" + +{{ if allow_instrumentation }} +#include <sys/stat.h> +#include "aconfig_storage/aconfig_storage_read_api.hpp" +#include <android/log.h> + +#define ALOGI(msg, ...) \ + __android_log_print(ANDROID_LOG_INFO, "AconfigTestMission1", (msg), __VA_ARGS__) + +{{ endif }} + +{{ if readwrite- }} +#include <server_configurable_flags/get_flags.h> +{{ endif }} +{{ if is_test_mode }} +#include <unordered_map> +#include <string> +{{ -else- }} +{{ if readwrite- }} +#include <vector> +{{ -endif }} +{{ -endif }} + +namespace {cpp_namespace} \{ + +{{ if is_test_mode }} + class flag_provider : public flag_provider_interface \{ + private: + std::unordered_map<std::string, bool> overrides_; + + public: + flag_provider() + : overrides_() + \{} + +{{ for item in class_elements }} + virtual bool {item.flag_name}() override \{ + auto it = overrides_.find("{item.flag_name}"); + if (it != overrides_.end()) \{ + return it->second; + } else \{ + {{ if item.readwrite- }} + return server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.{item.device_config_namespace}", + "{item.device_config_flag}", + "{item.default_value}") == "true"; + {{ -else }} + return {item.default_value}; + {{ -endif }} + } + } + + virtual void {item.flag_name}(bool val) override \{ + overrides_["{item.flag_name}"] = val; + } +{{ endfor }} + + virtual void reset_flags() override \{ + overrides_.clear(); + } + }; + +{{ -else- }} + + class flag_provider : public flag_provider_interface \{ + public: + + {{ -for item in class_elements }} + + virtual bool {item.flag_name}() override \{ + {{ -if item.readwrite }} + if (cache_[{item.readwrite_idx}] == -1) \{ + cache_[{item.readwrite_idx}] = server_configurable_flags::GetServerConfigurableFlag( + "aconfig_flags.{item.device_config_namespace}", + "{item.device_config_flag}", + "{item.default_value}") == "true"; + } + return cache_[{item.readwrite_idx}]; + {{ -else }} + {{ -if item.is_fixed_read_only }} + return {package_macro}_{item.flag_macro}; + {{ -else }} + return {item.default_value}; + {{ -endif }} + {{ -endif }} + } + {{ -endfor }} + {{ if readwrite- }} + private: + std::vector<int8_t> cache_ = std::vector<int8_t>({readwrite_count}, -1); + {{ -endif }} + }; + + +{{ -endif }} + +std::unique_ptr<flag_provider_interface> provider_ = + std::make_unique<flag_provider>(); +} + +{{ for item in class_elements }} +bool {header}_{item.flag_name}() \{ + {{ -if is_test_mode }} + return {cpp_namespace}::{item.flag_name}(); + {{ -else }} + {{ -if item.readwrite }} + return {cpp_namespace}::{item.flag_name}(); + {{ -else }} + {{ if allow_instrumentation }} + auto result = + {{ if item.is_fixed_read_only }} + {package_macro}_{item.flag_macro} + {{ else }} + {item.default_value} + {{ endif }}; + + struct stat buffer; + if (stat("/metadata/aconfig_test_missions/mission_1", &buffer) != 0) \{ + return result; + } + + auto package_map_file = aconfig_storage::get_mapped_file( + "{item.container}", + aconfig_storage::StorageFileType::package_map); + if (!package_map_file.ok()) \{ + ALOGI("error: failed to get package map file: %s", package_map_file.error().message().c_str()); + return result; + } + + auto package_read_context = aconfig_storage::get_package_read_context( + **package_map_file, "{package}"); + if (!package_read_context.ok()) \{ + ALOGI("error: failed to get package read context: %s", package_map_file.error().message().c_str()); + return result; + } + + delete *package_map_file; + + auto flag_val_map = aconfig_storage::get_mapped_file( + "{item.container}", + aconfig_storage::StorageFileType::flag_val); + if (!flag_val_map.ok()) \{ + ALOGI("error: failed to get flag val map: %s", package_map_file.error().message().c_str()); + return result; + } + + auto value = aconfig_storage::get_boolean_flag_value( + **flag_val_map, + package_read_context->boolean_start_index + {item.flag_offset}); + if (!value.ok()) \{ + ALOGI("error: failed to get flag val: %s", package_map_file.error().message().c_str()); + return result; + } + + delete *flag_val_map; + + if (*value != result) \{ + ALOGI("error: new storage value '%d' does not match current value '%d'", *value, result); + } else \{ + ALOGI("success: new storage value was '%d, legacy storage was '%d'", *value, result); + } + + return result; + {{ else }} + {{ -if item.is_fixed_read_only }} + return {package_macro}_{item.flag_macro}; + {{ -else }} + return {item.default_value}; + {{ -endif }} + {{ -endif }} + {{ -endif }} + {{ -endif }} +} + +{{ -if is_test_mode }} +void set_{header}_{item.flag_name}(bool val) \{ + {cpp_namespace}::{item.flag_name}(val); +} +{{ -endif }} +{{ endfor }} + +{{ -if is_test_mode }} +void {header}_reset_flags() \{ + {cpp_namespace}::reset_flags(); +} +{{ -endif }} + + diff --git a/tools/aconfig/aconfig/templates/rust.template b/tools/aconfig/aconfig/templates/rust.template new file mode 100644 index 0000000000..f9a2829033 --- /dev/null +++ b/tools/aconfig/aconfig/templates/rust.template @@ -0,0 +1,49 @@ +//! codegenerated rust flag lib + +/// flag provider +pub struct FlagProvider; + +{{ if has_readwrite- }} +lazy_static::lazy_static! \{ +{{ -for flag in template_flags }} + {{ -if flag.readwrite }} + /// flag value cache for {flag.name} + static ref CACHED_{flag.name}: bool = flags_rust::GetServerConfigurableFlag( + "aconfig_flags.{flag.device_config_namespace}", + "{flag.device_config_flag}", + "{flag.default_value}") == "true"; + {{ -endif }} +{{ -endfor }} +} +{{ -endif }} + +impl FlagProvider \{ + +{{ for flag in template_flags }} + /// query flag {flag.name} + pub fn {flag.name}(&self) -> bool \{ + {{ -if flag.readwrite }} + *CACHED_{flag.name} + {{ -else }} + {flag.default_value} + {{ -endif }} + } +{{ endfor }} + +} + +/// flag provider +pub static PROVIDER: FlagProvider = FlagProvider; + +{{ for flag in template_flags }} +/// query flag {flag.name} +#[inline(always)] +{{ -if flag.readwrite }} +pub fn {flag.name}() -> bool \{ + PROVIDER.{flag.name}() +{{ -else }} +pub fn {flag.name}() -> bool \{ + {flag.default_value} +{{ -endif }} +} +{{ endfor }} diff --git a/tools/aconfig/aconfig/templates/rust_test.template b/tools/aconfig/aconfig/templates/rust_test.template new file mode 100644 index 0000000000..d01f40aab7 --- /dev/null +++ b/tools/aconfig/aconfig/templates/rust_test.template @@ -0,0 +1,61 @@ +//! codegenerated rust flag lib + +use std::collections::BTreeMap; +use std::sync::Mutex; + +/// flag provider +pub struct FlagProvider \{ + overrides: BTreeMap<&'static str, bool>, +} + +impl FlagProvider \{ +{{ for flag in template_flags }} + /// query flag {flag.name} + pub fn {flag.name}(&self) -> bool \{ + self.overrides.get("{flag.name}").copied().unwrap_or( + {{ if flag.readwrite -}} + flags_rust::GetServerConfigurableFlag( + "aconfig_flags.{flag.device_config_namespace}", + "{flag.device_config_flag}", + "{flag.default_value}") == "true" + {{ -else- }} + {flag.default_value} + {{ -endif }} + ) + } + + /// set flag {flag.name} + pub fn set_{flag.name}(&mut self, val: bool) \{ + self.overrides.insert("{flag.name}", val); + } +{{ endfor }} + + /// clear all flag overrides + pub fn reset_flags(&mut self) \{ + self.overrides.clear(); + } +} + +/// flag provider +pub static PROVIDER: Mutex<FlagProvider> = Mutex::new( + FlagProvider \{overrides: BTreeMap::new()} +); + +{{ for flag in template_flags }} +/// query flag {flag.name} +#[inline(always)] +pub fn {flag.name}() -> bool \{ + PROVIDER.lock().unwrap().{flag.name}() +} + +/// set flag {flag.name} +#[inline(always)] +pub fn set_{flag.name}(val: bool) \{ + PROVIDER.lock().unwrap().set_{flag.name}(val); +} +{{ endfor }} + +/// clear all flag override +pub fn reset_flags() \{ + PROVIDER.lock().unwrap().reset_flags() +} diff --git a/tools/aconfig/aconfig/tests/AconfigHostTest.java b/tools/aconfig/aconfig/tests/AconfigHostTest.java new file mode 100644 index 0000000000..ea71b7e52a --- /dev/null +++ b/tools/aconfig/aconfig/tests/AconfigHostTest.java @@ -0,0 +1,88 @@ +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + + +import com.android.aconfig.test.FakeFeatureFlagsImpl; +import com.android.aconfig.test.FeatureFlags; +import com.android.aconfig.test.FeatureFlagsImpl; +import com.android.aconfig.test.Flags; + +@RunWith(JUnit4.class) +public final class AconfigHostTest { + @Test + public void testThrowsExceptionIfFlagNotSet() { + assertThrows(NullPointerException.class, () -> Flags.disabledRo()); + FakeFeatureFlagsImpl featureFlags = new FakeFeatureFlagsImpl(); + assertThrows(IllegalArgumentException.class, () -> featureFlags.disabledRo()); + } + + @Test + public void testSetFlagInFakeFeatureFlagsImpl() { + FakeFeatureFlagsImpl featureFlags = new FakeFeatureFlagsImpl(); + featureFlags.setFlag(Flags.FLAG_ENABLED_RW, true); + assertTrue(featureFlags.enabledRw()); + featureFlags.setFlag(Flags.FLAG_ENABLED_RW, false); + assertFalse(featureFlags.enabledRw()); + + //Set Flags + assertThrows(NullPointerException.class, () -> Flags.enabledRw()); + Flags.setFeatureFlags(featureFlags); + featureFlags.setFlag(Flags.FLAG_ENABLED_RW, true); + assertTrue(Flags.enabledRw()); + Flags.unsetFeatureFlags(); + } + + @Test + public void testSetFlagWithRandomName() { + FakeFeatureFlagsImpl featureFlags = new FakeFeatureFlagsImpl(); + assertThrows(IllegalArgumentException.class, + () -> featureFlags.setFlag("Randome_name", true)); + } + + @Test + public void testResetFlagsInFakeFeatureFlagsImpl() { + FakeFeatureFlagsImpl featureFlags = new FakeFeatureFlagsImpl(); + featureFlags.setFlag(Flags.FLAG_ENABLED_RO, true); + assertTrue(featureFlags.enabledRo()); + featureFlags.resetAll(); + assertThrows(IllegalArgumentException.class, () -> featureFlags.enabledRo()); + + // Set value after reset + featureFlags.setFlag(Flags.FLAG_ENABLED_RO, false); + assertFalse(featureFlags.enabledRo()); + } + + @Test + public void testFlagsSetFeatureFlags() { + FakeFeatureFlagsImpl featureFlags = new FakeFeatureFlagsImpl(); + featureFlags.setFlag(Flags.FLAG_ENABLED_RW, true); + assertThrows(NullPointerException.class, () -> Flags.enabledRw()); + Flags.setFeatureFlags(featureFlags); + assertTrue(Flags.enabledRw()); + Flags.unsetFeatureFlags(); + } + + @Test + public void testFlagsUnsetFeatureFlags() { + FakeFeatureFlagsImpl featureFlags = new FakeFeatureFlagsImpl(); + featureFlags.setFlag(Flags.FLAG_ENABLED_RW, true); + assertThrows(NullPointerException.class, () -> Flags.enabledRw()); + Flags.setFeatureFlags(featureFlags); + assertTrue(Flags.enabledRw()); + + Flags.unsetFeatureFlags(); + assertThrows(NullPointerException.class, () -> Flags.enabledRw()); + } + + @Test + public void testFeatureFlagsImplNotImpl() { + FeatureFlags featureFlags = new FeatureFlagsImpl(); + assertThrows(UnsupportedOperationException.class, + () -> featureFlags.enabledRw()); + } +} diff --git a/tools/aconfig/aconfig/tests/AconfigTest.java b/tools/aconfig/aconfig/tests/AconfigTest.java new file mode 100644 index 0000000000..7e76efba3c --- /dev/null +++ b/tools/aconfig/aconfig/tests/AconfigTest.java @@ -0,0 +1,75 @@ +import static com.android.aconfig.test.Flags.FLAG_DISABLED_RO; +import static com.android.aconfig.test.Flags.FLAG_DISABLED_RW; +import static com.android.aconfig.test.Flags.FLAG_ENABLED_FIXED_RO; +import static com.android.aconfig.test.Flags.FLAG_ENABLED_RO; +import static com.android.aconfig.test.Flags.FLAG_ENABLED_RW; +import static com.android.aconfig.test.Flags.disabledRo; +import static com.android.aconfig.test.Flags.disabledRw; +import static com.android.aconfig.test.Flags.enabledFixedRo; +import static com.android.aconfig.test.Flags.enabledRo; +import static com.android.aconfig.test.Flags.enabledRw; +import static com.android.aconfig.test.exported.Flags.exportedFlag; +import static com.android.aconfig.test.exported.Flags.FLAG_EXPORTED_FLAG; +import static com.android.aconfig.test.forcereadonly.Flags.froRw; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import com.android.aconfig.test.FakeFeatureFlagsImpl; +import com.android.aconfig.test.FeatureFlags; + +@RunWith(JUnit4.class) +public final class AconfigTest { + @Test + public void testDisabledReadOnlyFlag() { + assertEquals("com.android.aconfig.test.disabled_ro", FLAG_DISABLED_RO); + assertFalse(disabledRo()); + } + + @Test + public void testEnabledReadOnlyFlag() { + assertEquals("com.android.aconfig.test.disabled_rw", FLAG_DISABLED_RW); + assertTrue(enabledRo()); + } + + @Test + public void testEnabledFixedReadOnlyFlag() { + assertEquals("com.android.aconfig.test.enabled_fixed_ro", FLAG_ENABLED_FIXED_RO); + assertTrue(enabledFixedRo()); + } + + @Test + public void testDisabledReadWriteFlag() { + assertEquals("com.android.aconfig.test.enabled_ro", FLAG_ENABLED_RO); + assertFalse(disabledRw()); + } + + @Test + public void testEnabledReadWriteFlag() { + assertEquals("com.android.aconfig.test.enabled_rw", FLAG_ENABLED_RW); + assertTrue(enabledRw()); + } + + @Test + public void testFakeFeatureFlagsImplImpled() { + FakeFeatureFlagsImpl fakeFeatureFlags = new FakeFeatureFlagsImpl(); + fakeFeatureFlags.setFlag(FLAG_ENABLED_RW, false); + assertFalse(fakeFeatureFlags.enabledRw()); + } + + @Test + public void testExportedFlag() { + assertEquals("com.android.aconfig.test.exported.exported_flag", FLAG_EXPORTED_FLAG); + assertFalse(exportedFlag()); + } + + @Test + public void testForceReadOnly() { + assertFalse(froRw()); + } +} diff --git a/tools/aconfig/aconfig/tests/AndroidManifest.xml b/tools/aconfig/aconfig/tests/AndroidManifest.xml new file mode 100644 index 0000000000..04002e6241 --- /dev/null +++ b/tools/aconfig/aconfig/tests/AndroidManifest.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="aconfig.test.java"> + + <uses-permission android:name="android.permission.READ_DEVICE_CONFIG" /> + + <application> + <uses-library android:name="android.test.runner"/> + </application> + + <instrumentation android:name="androidx.test.runner.AndroidJUnitRunner" + android:targetPackage="aconfig.test.java" + android:label="aconfig integration tests (java)" /> +</manifest> diff --git a/tools/aconfig/aconfig/tests/aconfig_exported_mode_test.cpp b/tools/aconfig/aconfig/tests/aconfig_exported_mode_test.cpp new file mode 100644 index 0000000000..d6eab4336f --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_exported_mode_test.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "com_android_aconfig_test.h" +#include "gtest/gtest.h" + +using namespace com::android::aconfig::test; + +TEST(AconfigTest, TestDisabledRwExportedFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_rw_exported()); + ASSERT_FALSE(provider_->disabled_rw_exported()); + ASSERT_FALSE(disabled_rw_exported()); +} + +TEST(AconfigTest, TestEnabledFixedRoExportedFlag) { + // TODO: change to assertTrue(enabledFixedRoExported()) when the build supports reading tests/*.values + ASSERT_FALSE(com_android_aconfig_test_enabled_fixed_ro_exported()); + ASSERT_FALSE(provider_->enabled_fixed_ro_exported()); + ASSERT_FALSE(enabled_fixed_ro_exported()); +} + +TEST(AconfigTest, TestEnabledRoExportedFlag) { + // TODO: change to assertTrue(enabledRoExported()) when the build supports reading tests/*.values + ASSERT_FALSE(com_android_aconfig_test_enabled_ro_exported()); + ASSERT_FALSE(provider_->enabled_ro_exported()); + ASSERT_FALSE(enabled_ro_exported()); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}
\ No newline at end of file diff --git a/tools/aconfig/aconfig/tests/aconfig_exported_mode_test.rs b/tools/aconfig/aconfig/tests/aconfig_exported_mode_test.rs new file mode 100644 index 0000000000..4b480471b5 --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_exported_mode_test.rs @@ -0,0 +1,7 @@ +#[cfg(not(feature = "cargo"))] +#[test] +fn test_flags() { + assert!(!aconfig_test_rust_library::disabled_rw_exported()); + assert!(!aconfig_test_rust_library::enabled_fixed_ro_exported()); + assert!(!aconfig_test_rust_library::enabled_ro_exported()); +} diff --git a/tools/aconfig/aconfig/tests/aconfig_force_read_only_mode_test.cpp b/tools/aconfig/aconfig/tests/aconfig_force_read_only_mode_test.cpp new file mode 100644 index 0000000000..0dec481ced --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_force_read_only_mode_test.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "com_android_aconfig_test.h" +#include "gtest/gtest.h" + +using namespace com::android::aconfig::test; + +TEST(AconfigTest, TestDisabledReadOnlyFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_ro()); + ASSERT_FALSE(provider_->disabled_ro()); + ASSERT_FALSE(disabled_ro()); +} + +TEST(AconfigTest, TestEnabledReadOnlyFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_ro()); + ASSERT_TRUE(provider_->enabled_ro()); + ASSERT_TRUE(enabled_ro()); +} + +TEST(AconfigTest, TestDisabledReadWriteFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_rw()); + ASSERT_FALSE(provider_->disabled_rw()); + ASSERT_FALSE(disabled_rw()); +} + +TEST(AconfigTest, TestEnabledReadWriteFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_rw()); + ASSERT_TRUE(provider_->enabled_rw()); + ASSERT_TRUE(enabled_rw()); +} + +TEST(AconfigTest, TestEnabledFixedReadOnlyFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_fixed_ro()); + ASSERT_TRUE(provider_->enabled_fixed_ro()); + ASSERT_TRUE(enabled_fixed_ro()); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/aconfig/aconfig/tests/aconfig_force_read_only_mode_test.rs b/tools/aconfig/aconfig/tests/aconfig_force_read_only_mode_test.rs new file mode 100644 index 0000000000..4f05e26add --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_force_read_only_mode_test.rs @@ -0,0 +1,10 @@ +#[cfg(not(feature = "cargo"))] +#[test] +fn test_flags() { + assert!(!aconfig_test_rust_library::disabled_ro()); + assert!(!aconfig_test_rust_library::disabled_rw()); + assert!(!aconfig_test_rust_library::disabled_rw_in_other_namespace()); + assert!(aconfig_test_rust_library::enabled_fixed_ro()); + assert!(aconfig_test_rust_library::enabled_ro()); + assert!(aconfig_test_rust_library::enabled_rw()); +} diff --git a/tools/aconfig/aconfig/tests/aconfig_prod_mode_test.rs b/tools/aconfig/aconfig/tests/aconfig_prod_mode_test.rs new file mode 100644 index 0000000000..e1fb8e5a09 --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_prod_mode_test.rs @@ -0,0 +1,8 @@ +#[cfg(not(feature = "cargo"))] +#[test] +fn test_flags() { + assert!(!aconfig_test_rust_library::disabled_ro()); + assert!(!aconfig_test_rust_library::disabled_rw()); + assert!(aconfig_test_rust_library::enabled_ro()); + assert!(aconfig_test_rust_library::enabled_rw()); +} diff --git a/tools/aconfig/aconfig/tests/aconfig_test.cpp b/tools/aconfig/aconfig/tests/aconfig_test.cpp new file mode 100644 index 0000000000..0dec481ced --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_test.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "com_android_aconfig_test.h" +#include "gtest/gtest.h" + +using namespace com::android::aconfig::test; + +TEST(AconfigTest, TestDisabledReadOnlyFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_ro()); + ASSERT_FALSE(provider_->disabled_ro()); + ASSERT_FALSE(disabled_ro()); +} + +TEST(AconfigTest, TestEnabledReadOnlyFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_ro()); + ASSERT_TRUE(provider_->enabled_ro()); + ASSERT_TRUE(enabled_ro()); +} + +TEST(AconfigTest, TestDisabledReadWriteFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_rw()); + ASSERT_FALSE(provider_->disabled_rw()); + ASSERT_FALSE(disabled_rw()); +} + +TEST(AconfigTest, TestEnabledReadWriteFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_rw()); + ASSERT_TRUE(provider_->enabled_rw()); + ASSERT_TRUE(enabled_rw()); +} + +TEST(AconfigTest, TestEnabledFixedReadOnlyFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_fixed_ro()); + ASSERT_TRUE(provider_->enabled_fixed_ro()); + ASSERT_TRUE(enabled_fixed_ro()); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/aconfig/aconfig/tests/aconfig_test_mode_test.rs b/tools/aconfig/aconfig/tests/aconfig_test_mode_test.rs new file mode 100644 index 0000000000..a08fbab7f6 --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_test_mode_test.rs @@ -0,0 +1,23 @@ +#[cfg(not(feature = "cargo"))] +#[test] +fn test_flags() { + assert!(!aconfig_test_rust_library::disabled_ro()); + assert!(!aconfig_test_rust_library::disabled_rw()); + assert!(aconfig_test_rust_library::enabled_ro()); + assert!(aconfig_test_rust_library::enabled_rw()); + + aconfig_test_rust_library::set_disabled_ro(true); + assert!(aconfig_test_rust_library::disabled_ro()); + aconfig_test_rust_library::set_disabled_rw(true); + assert!(aconfig_test_rust_library::disabled_rw()); + aconfig_test_rust_library::set_enabled_ro(false); + assert!(!aconfig_test_rust_library::enabled_ro()); + aconfig_test_rust_library::set_enabled_rw(false); + assert!(!aconfig_test_rust_library::enabled_rw()); + + aconfig_test_rust_library::reset_flags(); + assert!(!aconfig_test_rust_library::disabled_ro()); + assert!(!aconfig_test_rust_library::disabled_rw()); + assert!(aconfig_test_rust_library::enabled_ro()); + assert!(aconfig_test_rust_library::enabled_rw()); +} diff --git a/tools/aconfig/aconfig/tests/aconfig_test_test_variant.cpp b/tools/aconfig/aconfig/tests/aconfig_test_test_variant.cpp new file mode 100644 index 0000000000..b3cf7104ec --- /dev/null +++ b/tools/aconfig/aconfig/tests/aconfig_test_test_variant.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "com_android_aconfig_test.h" +#include "gtest/gtest.h" + +using namespace com::android::aconfig::test; + +class AconfigTest : public ::testing::Test { + protected: + void SetUp() override { + reset_flags(); + } +}; + +TEST_F(AconfigTest, TestDisabledReadOnlyFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_ro()); + ASSERT_FALSE(provider_->disabled_ro()); + ASSERT_FALSE(disabled_ro()); +} + +TEST_F(AconfigTest, TestEnabledReadOnlyFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_ro()); + ASSERT_TRUE(provider_->enabled_ro()); + ASSERT_TRUE(enabled_ro()); +} + +TEST_F(AconfigTest, TestDisabledReadWriteFlag) { + ASSERT_FALSE(com_android_aconfig_test_disabled_rw()); + ASSERT_FALSE(provider_->disabled_rw()); + ASSERT_FALSE(disabled_rw()); +} + +TEST_F(AconfigTest, TestEnabledReadWriteFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_rw()); + ASSERT_TRUE(provider_->enabled_rw()); + ASSERT_TRUE(enabled_rw()); +} + +TEST_F(AconfigTest, TestEnabledFixedReadOnlyFlag) { + ASSERT_TRUE(com_android_aconfig_test_enabled_fixed_ro()); + ASSERT_TRUE(provider_->enabled_fixed_ro()); + ASSERT_TRUE(enabled_fixed_ro()); +} + +TEST_F(AconfigTest, OverrideFlagValue) { + ASSERT_FALSE(disabled_ro()); + disabled_ro(true); + ASSERT_TRUE(disabled_ro()); +} + +TEST_F(AconfigTest, ResetFlagValue) { + ASSERT_FALSE(disabled_ro()); + ASSERT_TRUE(enabled_ro()); + disabled_ro(true); + enabled_ro(false); + ASSERT_TRUE(disabled_ro()); + ASSERT_FALSE(enabled_ro()); + reset_flags(); + ASSERT_FALSE(disabled_ro()); + ASSERT_TRUE(enabled_ro()); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/aconfig/aconfig/tests/first.values b/tools/aconfig/aconfig/tests/first.values new file mode 100644 index 0000000000..2efb4632d4 --- /dev/null +++ b/tools/aconfig/aconfig/tests/first.values @@ -0,0 +1,48 @@ +flag_value { + package: "com.android.aconfig.test" + name: "disabled_ro" + state: DISABLED + permission: READ_ONLY +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_ro" + state: DISABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_rw" + state: ENABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.test" + name: "disabled_rw_in_other_namespace" + state: DISABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_fixed_ro" + state: ENABLED + permission: READ_ONLY +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_ro_exported" + state: ENABLED + permission: READ_ONLY +} +flag_value { + package: "com.android.aconfig.test" + name: "disabled_rw_exported" + state: DISABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_fixed_ro_exported" + state: ENABLED + permission: READ_ONLY +} diff --git a/tools/aconfig/aconfig/tests/read_only_test.aconfig b/tools/aconfig/aconfig/tests/read_only_test.aconfig new file mode 100644 index 0000000000..5eb505669b --- /dev/null +++ b/tools/aconfig/aconfig/tests/read_only_test.aconfig @@ -0,0 +1,32 @@ +package: "com.android.aconfig.test" +container: "system" + +flag { + name: "enabled_ro" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY" + bug: "abc" +} + +flag { + name: "disabled_ro" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_ONLY" + bug: "123" +} + +flag { + name: "enabled_fixed_ro" + namespace: "aconfig_test" + description: "This flag is fixed READ_ONLY + ENABLED" + bug: "" + is_fixed_read_only: true +} + +flag { + name: "disabled_fixed_ro" + namespace: "aconfig_test" + description: "This flag is fixed READ_ONLY + DISABLED" + bug: "" + is_fixed_read_only: true +} diff --git a/tools/aconfig/aconfig/tests/read_only_test.values b/tools/aconfig/aconfig/tests/read_only_test.values new file mode 100644 index 0000000000..349c7aa666 --- /dev/null +++ b/tools/aconfig/aconfig/tests/read_only_test.values @@ -0,0 +1,18 @@ +flag_value { + package: "com.android.aconfig.test" + name: "disabled_ro" + state: DISABLED + permission: READ_ONLY +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_ro" + state: ENABLED + permission: READ_ONLY +} +flag_value { + package: "com.android.aconfig.test" + name: "enabled_fixed_ro" + state: ENABLED + permission: READ_ONLY +} diff --git a/tools/aconfig/aconfig/tests/second.values b/tools/aconfig/aconfig/tests/second.values new file mode 100644 index 0000000000..aa09cf6a56 --- /dev/null +++ b/tools/aconfig/aconfig/tests/second.values @@ -0,0 +1,6 @@ +flag_value { + package: "com.android.aconfig.test" + name: "enabled_ro" + state: ENABLED + permission: READ_ONLY +} diff --git a/tools/aconfig/aconfig/tests/storage_test_1.aconfig b/tools/aconfig/aconfig/tests/storage_test_1.aconfig new file mode 100644 index 0000000000..a122c57bd2 --- /dev/null +++ b/tools/aconfig/aconfig/tests/storage_test_1.aconfig @@ -0,0 +1,24 @@ +package: "com.android.aconfig.storage.test_1" +container: "system" + +flag { + name: "enabled_rw" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_WRITE" + bug: "" +} + +flag { + name: "disabled_rw" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_WRITE" + bug: "456" + is_exported: true +} + +flag { + name: "enabled_ro" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY" + bug: "abc" +} diff --git a/tools/aconfig/aconfig/tests/storage_test_1.values b/tools/aconfig/aconfig/tests/storage_test_1.values new file mode 100644 index 0000000000..35548ae5ba --- /dev/null +++ b/tools/aconfig/aconfig/tests/storage_test_1.values @@ -0,0 +1,18 @@ +flag_value { + package: "com.android.aconfig.storage.test_1" + name: "enabled_rw" + state: ENABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.storage.test_1" + name: "disabled_rw" + state: DISABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.storage.test_1" + name: "enabled_ro" + state: ENABLED + permission: READ_ONLY +} diff --git a/tools/aconfig/aconfig/tests/storage_test_2.aconfig b/tools/aconfig/aconfig/tests/storage_test_2.aconfig new file mode 100644 index 0000000000..db77f7a35c --- /dev/null +++ b/tools/aconfig/aconfig/tests/storage_test_2.aconfig @@ -0,0 +1,24 @@ +package: "com.android.aconfig.storage.test_2" +container: "system" + +flag { + name: "enabled_ro" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY" + bug: "abc" +} + +flag { + name: "disabled_rw" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_ONLY" + bug: "123" +} + +flag { + name: "enabled_fixed_ro" + namespace: "aconfig_test" + description: "This flag is fixed READ_ONLY + ENABLED" + bug: "" + is_fixed_read_only: true +} diff --git a/tools/aconfig/aconfig/tests/storage_test_2.values b/tools/aconfig/aconfig/tests/storage_test_2.values new file mode 100644 index 0000000000..b6507210da --- /dev/null +++ b/tools/aconfig/aconfig/tests/storage_test_2.values @@ -0,0 +1,18 @@ +flag_value { + package: "com.android.aconfig.storage.test_2" + name: "enabled_ro" + state: ENABLED + permission: READ_ONLY +} +flag_value { + package: "com.android.aconfig.storage.test_2" + name: "disabled_rw" + state: DISABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.storage.test_2" + name: "enabled_fixed_ro" + state: ENABLED + permission: READ_ONLY +} diff --git a/tools/aconfig/aconfig/tests/storage_test_4.aconfig b/tools/aconfig/aconfig/tests/storage_test_4.aconfig new file mode 100644 index 0000000000..5802a73b6d --- /dev/null +++ b/tools/aconfig/aconfig/tests/storage_test_4.aconfig @@ -0,0 +1,17 @@ +package: "com.android.aconfig.storage.test_4" +container: "system" + +flag { + name: "enabled_rw" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY" + bug: "abc" +} + +flag { + name: "enabled_fixed_ro" + namespace: "aconfig_test" + description: "This flag is fixed READ_ONLY + ENABLED" + bug: "" + is_fixed_read_only: true +} diff --git a/tools/aconfig/aconfig/tests/storage_test_4.values b/tools/aconfig/aconfig/tests/storage_test_4.values new file mode 100644 index 0000000000..784b744f62 --- /dev/null +++ b/tools/aconfig/aconfig/tests/storage_test_4.values @@ -0,0 +1,12 @@ +flag_value { + package: "com.android.aconfig.storage.test_4" + name: "enabled_rw" + state: ENABLED + permission: READ_WRITE +} +flag_value { + package: "com.android.aconfig.storage.test_4" + name: "enabled_fixed_ro" + state: ENABLED + permission: READ_ONLY +} diff --git a/tools/aconfig/aconfig/tests/test.aconfig b/tools/aconfig/aconfig/tests/test.aconfig new file mode 100644 index 0000000000..c11508aabc --- /dev/null +++ b/tools/aconfig/aconfig/tests/test.aconfig @@ -0,0 +1,89 @@ +package: "com.android.aconfig.test" +container: "system" + +# This flag's final value is calculated from: +# - test.aconfig: DISABLED + READ_WRITE (default) +# - first.values: DISABLED + READ_WRITE +# - second.values: ENABLED + READ_ONLY +flag { + name: "enabled_ro" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY" + bug: "abc" + metadata { + purpose: PURPOSE_BUGFIX + } +} + +# This flag's final value is calculated from: +# - test.aconfig: DISABLED + READ_WRITE (default) +# - first.values: ENABLED + READ_WRITE +flag { + name: "enabled_rw" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_WRITE" + # for bug fields, the empty string is a discouraged but valid value + bug: "" +} + +# This flag's final value is calculated from: +# - test.aconfig: DISABLED + READ_WRITE (default) +# - first.values: DISABLED + READ_ONLY +flag { + name: "disabled_ro" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_ONLY" + bug: "123" +} + +# This flag's final value is calculated from: +# - test.aconfig: DISABLED + READ_WRITE (default) +flag { + name: "disabled_rw" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_WRITE" + bug: "456" +} + +# This flag's final value calculated from: +# - test.aconfig: DISABLED + READ_ONLY +# - first.values: ENABLED + READ_ONLY +flag { + name: "enabled_fixed_ro" + namespace: "aconfig_test" + description: "This flag is fixed READ_ONLY + ENABLED" + bug: "" + is_fixed_read_only: true +} + +flag { + name: "disabled_rw_in_other_namespace" + namespace: "other_namespace" + description: "This flag is DISABLED + READ_WRITE, and is defined in another namespace" + bug: "999" +} + +flag { + name: "enabled_ro_exported" + namespace: "aconfig_test" + description: "This flag is ENABLED + READ_ONLY and exported" + bug: "111" + is_exported: true +} + +flag { + name: "disabled_rw_exported" + namespace: "aconfig_test" + description: "This flag is DISABLED + READ_WRITE and exported" + bug: "111" + is_exported: true +} + +flag { + name: "enabled_fixed_ro_exported" + namespace: "aconfig_test" + description: "This flag is fixed ENABLED + READ_ONLY and exported" + bug: "111" + is_fixed_read_only: true + is_exported: true +}
\ No newline at end of file diff --git a/tools/aconfig/aconfig/tests/test_exported.aconfig b/tools/aconfig/aconfig/tests/test_exported.aconfig new file mode 100644 index 0000000000..20f23a3182 --- /dev/null +++ b/tools/aconfig/aconfig/tests/test_exported.aconfig @@ -0,0 +1,17 @@ +package: "com.android.aconfig.test.exported" +container: "system" + +flag { + name: "exported_flag" + namespace: "aconfig_test" + description: "This is an exported flag" + is_exported: true + bug: "888" +} + +flag { + name: "not_exported_flag" + namespace: "aconfig_test" + description: "This flag is not exported" + bug: "777" +} diff --git a/tools/aconfig/aconfig/tests/test_force_read_only.aconfig b/tools/aconfig/aconfig/tests/test_force_read_only.aconfig new file mode 100644 index 0000000000..05ab0e27d3 --- /dev/null +++ b/tools/aconfig/aconfig/tests/test_force_read_only.aconfig @@ -0,0 +1,17 @@ +package: "com.android.aconfig.test.forcereadonly" +container: "system" + +flag { + name: "fro_exported" + namespace: "aconfig_test" + description: "This is an exported flag" + is_exported: true + bug: "888" +} + +flag { + name: "fro_rw" + namespace: "aconfig_test" + description: "This flag is not exported" + bug: "777" +} diff --git a/tools/aconfig/aconfig_device_paths/Android.bp b/tools/aconfig/aconfig_device_paths/Android.bp new file mode 100644 index 0000000000..21aa9a9e08 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/Android.bp @@ -0,0 +1,38 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "libaconfig_device_paths.defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + srcs: ["src/lib.rs"], + rustlibs: [ + "libaconfig_protos", + "libanyhow", + "libprotobuf", + "libregex", + ], +} + +rust_library { + name: "libaconfig_device_paths", + crate_name: "aconfig_device_paths", + host_supported: true, + defaults: ["libaconfig_device_paths.defaults"], +} diff --git a/tools/aconfig/aconfig_device_paths/Cargo.toml b/tools/aconfig/aconfig_device_paths/Cargo.toml new file mode 100644 index 0000000000..dbe9b3a111 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "aconfig_device_paths" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.82" diff --git a/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt b/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt new file mode 100644 index 0000000000..3d2deb2375 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt @@ -0,0 +1,6 @@ +[ + "/system/etc/aconfig_flags.pb", + "/system_ext/etc/aconfig_flags.pb", + "/product/etc/aconfig_flags.pb", + "/vendor/etc/aconfig_flags.pb", +] diff --git a/tools/aconfig/aconfig_device_paths/src/lib.rs b/tools/aconfig/aconfig_device_paths/src/lib.rs new file mode 100644 index 0000000000..7bb62f4247 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/src/lib.rs @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Library for finding all aconfig on-device protobuf file paths. + +use anyhow::Result; +use std::path::PathBuf; + +use std::fs; + +/// Determine all paths that contain an aconfig protobuf file. +pub fn parsed_flags_proto_paths() -> Result<Vec<PathBuf>> { + let mut result: Vec<PathBuf> = include!("../partition_aconfig_flags_paths.txt") + .map(|s| PathBuf::from(s.to_string())) + .to_vec(); + for dir in fs::read_dir("/apex")? { + let dir = dir?; + + // Only scan the currently active version of each mainline module; skip the @version dirs. + if dir.file_name().as_encoded_bytes().iter().any(|&b| b == b'@') { + continue; + } + + let mut path = PathBuf::from("/apex"); + path.push(dir.path()); + path.push("etc"); + path.push("aconfig_flags.pb"); + if path.exists() { + result.push(path); + } + } + + Ok(result) +} diff --git a/tools/aconfig/aconfig_protos/Android.bp b/tools/aconfig/aconfig_protos/Android.bp new file mode 100644 index 0000000000..18c545ae96 --- /dev/null +++ b/tools/aconfig/aconfig_protos/Android.bp @@ -0,0 +1,75 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +// proto libraries for consumers of `aconfig dump --format=protobuf` output + +java_library { + name: "libaconfig_java_proto_lite", + host_supported: true, + srcs: ["protos/aconfig.proto"], + static_libs: ["libprotobuf-java-lite"], + proto: { + type: "lite", + }, + sdk_version: "current", + min_sdk_version: "UpsideDownCake", + apex_available: [ + "com.android.configinfrastructure", + "//apex_available:platform", + ] +} + +java_library_host { + name: "libaconfig_java_proto_full", + srcs: ["protos/aconfig.proto"], + static_libs: ["libprotobuf-java-full"], + proto: { + type: "full", + }, +} + +python_library_host { + name: "libaconfig_python_proto", + srcs: ["protos/aconfig.proto"], + proto: { + canonical_path_from_root: false, + }, +} + +rust_protobuf { + name: "libaconfig_rust_proto", + protos: ["protos/aconfig.proto"], + crate_name: "aconfig_rust_proto", + source_stem: "aconfig_rust_proto", + host_supported: true, +} + +rust_defaults { + name: "aconfig_protos.defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + srcs: ["src/lib.rs"], + rustlibs: [ + "libaconfig_rust_proto", + "libanyhow", + "libprotobuf", + ], + proc_macros: [ + "libpaste", + ] +} + +rust_library { + name: "libaconfig_protos", + crate_name: "aconfig_protos", + host_supported: true, + defaults: ["aconfig_protos.defaults"], +} + +rust_test_host { + name: "aconfig_protos.test", + test_suites: ["general-tests"], + defaults: ["aconfig_protos.defaults"], +} diff --git a/tools/aconfig/aconfig_protos/Cargo.toml b/tools/aconfig/aconfig_protos/Cargo.toml new file mode 100644 index 0000000000..114cf80612 --- /dev/null +++ b/tools/aconfig/aconfig_protos/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "aconfig_protos" +version = "0.1.0" +edition = "2021" +build = "build.rs" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +paste = "1.0.11" +protobuf = "3.2.0" + +[build-dependencies] +protobuf-codegen = "3.2.0" diff --git a/tools/aconfig/aconfig_protos/build.rs b/tools/aconfig/aconfig_protos/build.rs new file mode 100644 index 0000000000..5ef5b6090f --- /dev/null +++ b/tools/aconfig/aconfig_protos/build.rs @@ -0,0 +1,17 @@ +use protobuf_codegen::Codegen; + +fn main() { + let proto_files = vec!["protos/aconfig.proto"]; + + // tell cargo to only re-run the build script if any of the proto files has changed + for path in &proto_files { + println!("cargo:rerun-if-changed={}", path); + } + + Codegen::new() + .pure() + .include("protos") + .inputs(proto_files) + .cargo_out_dir("aconfig_proto") + .run_from_script(); +} diff --git a/tools/aconfig/aconfig_protos/protos/aconfig.proto b/tools/aconfig/aconfig_protos/protos/aconfig.proto new file mode 100644 index 0000000000..9d1b8cbfbf --- /dev/null +++ b/tools/aconfig/aconfig_protos/protos/aconfig.proto @@ -0,0 +1,195 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +// This is the schema definition for aconfig files. Modifications need to be +// either backwards compatible, or include updates to all aconfig files in the +// Android tree. + +syntax = "proto2"; + +package android.aconfig; + +// This protobuf file defines messages used to represent and manage flags in the "aconfig" system +// The following format requirements apply across various message fields: +// +// # name: name of the flag +// +// format: a lowercase string in snake_case format, no consecutive underscores, and no leading +// digit. For example adjust_rate is a valid name, while AdjustRate, adjust__rate, and +// adjust_rate are invalid +// +// # namespace: namespace the flag belongs to +// +// format: a lowercase string in snake_case format, no consecutive underscores, and no leading +// digit. For example android_bar_system +// +// # package: package to which the flag belongs +// +// format: lowercase strings in snake_case format, delimited by dots, no consecutive underscores +// and no leading digit in each string. For example com.android.mypackage is a valid name +// while com.android.myPackage, com.android.1mypackage are invalid +// +// # container: container as software built in its entirety using the same build environment and +// always installed as a single unit +// +// For example the following are all separate containers: +// * the system partition +// * the vendor partition +// * apexes: each APEX is its own container +// * APKs: for APKs which are released independently via Play, each APK is its own container. +// If an APK is released as part of a Mainline module, or as part of the system partition +// via OTA, then they are part of the apex or the system partition container +// +// format: lowercase strings in snake_case format, delimited by dots if multiple, no consecutive +// underscores or leading digits in each string. The recommended container values are the +// partition names or the module names + +// messages used in both aconfig input and output + +enum flag_state { + ENABLED = 1; + DISABLED = 2; +} + +enum flag_permission { + READ_ONLY = 1; + READ_WRITE = 2; +} + +// aconfig input messages: flag declarations and values + +message flag_declaration { + // Name of the flag (required) + // See # name for format detail + optional string name = 1; + + // Namespace the flag belongs to (required) + // See # namespace for format detail + optional string namespace = 2; + + // Textual description of the flag's purpose (required) + optional string description = 3; + + // Single bug id related to the flag (required) + repeated string bug = 4; + + // Indicates if the flag is permanently read-only and cannot be changed + // via release configs (optional) + // Default value false + optional bool is_fixed_read_only = 5; + + // Indicates if the flag is exported and accessible beyond its originating container (optional) + // Default value false + optional bool is_exported = 6; + + // Additional information about the flag, including its purpose and form factors (optional) + optional flag_metadata metadata = 7; +}; + +// Optional metadata about the flag, such as its purpose and its intended form factors. +// Can influence the applied policies and testing strategy. +message flag_metadata { + enum flag_purpose { + PURPOSE_UNSPECIFIED = 0; + PURPOSE_FEATURE = 1; + PURPOSE_BUGFIX = 2; + } + + optional flag_purpose purpose = 1; + + // TODO(b/315025930): Add field to designate intended target device form factor(s), such as phone, watch or other. +} + +message flag_declarations { + // Package to which the flag belongs (required) + // See # package for format detail + optional string package = 1; + + // List of flag_declaration objects (required) + repeated flag_declaration flag = 2; + + // Container the flag belongs to (optional) + // See # container for format detail + optional string container = 3; +}; + +message flag_value { + // Package to which the flag belongs (required) + // See # package for format detail + optional string package = 1; + + // Name of the flag (required) + // See # name for format detail + optional string name = 2; + + optional flag_state state = 3; + optional flag_permission permission = 4; +}; + +message flag_values { + repeated flag_value flag_value = 1; +}; + +// aconfig output messages: parsed and verified flag declarations and values + +message tracepoint { + // path to declaration or value file relative to $TOP + optional string source = 1; + optional flag_state state = 2; + optional flag_permission permission = 3; +} + +message parsed_flag { + // Package to which the flag belongs (required) + // See # package for format detail + optional string package = 1; + + // Name of the flag (required) + // See # name for format detail + optional string name = 2; + + // Namespace the flag belongs to (required) + // See # namespace for format detail + optional string namespace = 3; + + // Textual description of the flag's purpose (required) + optional string description = 4; + + // Single bug id related to the flag (required) + repeated string bug = 5; + + optional flag_state state = 6; + optional flag_permission permission = 7; + repeated tracepoint trace = 8; + + // Indicates if the flag is permanently read-only and cannot be changed + // via release configs (optional) + // Default value false + optional bool is_fixed_read_only = 9; + + // Indicates if the flag is exported and accessible beyond its originating container (optional) + // Default value false + optional bool is_exported = 10; + + // Container the flag belongs to (optional) + // See # container for format detail + optional string container = 11; + + // Additional information about the flag, including its purpose and form factors (optional) + optional flag_metadata metadata = 12; +} + +message parsed_flags { + repeated parsed_flag parsed_flag = 1; +} diff --git a/tools/aconfig/aconfig_protos/src/lib.rs b/tools/aconfig/aconfig_protos/src/lib.rs new file mode 100644 index 0000000000..81bbd7e130 --- /dev/null +++ b/tools/aconfig/aconfig_protos/src/lib.rs @@ -0,0 +1,1076 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aconfig_protos` is a crate for the protos defined for aconfig +// When building with the Android tool-chain +// +// - an external crate `aconfig_protos` will be generated +// - the feature "cargo" will be disabled +// +// When building with cargo +// +// - a local sub-module will be generated in OUT_DIR and included in this file +// - the feature "cargo" will be enabled +// +// This module hides these differences from the rest of aconfig. + +// ---- When building with the Android tool-chain ---- +#[cfg(not(feature = "cargo"))] +mod auto_generated { + pub use aconfig_rust_proto::aconfig::flag_metadata::Flag_purpose as ProtoFlagPurpose; + pub use aconfig_rust_proto::aconfig::Flag_declaration as ProtoFlagDeclaration; + pub use aconfig_rust_proto::aconfig::Flag_declarations as ProtoFlagDeclarations; + pub use aconfig_rust_proto::aconfig::Flag_metadata as ProtoFlagMetadata; + pub use aconfig_rust_proto::aconfig::Flag_permission as ProtoFlagPermission; + pub use aconfig_rust_proto::aconfig::Flag_state as ProtoFlagState; + pub use aconfig_rust_proto::aconfig::Flag_value as ProtoFlagValue; + pub use aconfig_rust_proto::aconfig::Flag_values as ProtoFlagValues; + pub use aconfig_rust_proto::aconfig::Parsed_flag as ProtoParsedFlag; + pub use aconfig_rust_proto::aconfig::Parsed_flags as ProtoParsedFlags; + pub use aconfig_rust_proto::aconfig::Tracepoint as ProtoTracepoint; +} + +// ---- When building with cargo ---- +#[cfg(feature = "cargo")] +mod auto_generated { + // include! statements should be avoided (because they import file contents verbatim), but + // because this is only used during local development, and only if using cargo instead of the + // Android tool-chain, we allow it + include!(concat!(env!("OUT_DIR"), "/aconfig_proto/mod.rs")); + pub use aconfig::flag_metadata::Flag_purpose as ProtoFlagPurpose; + pub use aconfig::Flag_declaration as ProtoFlagDeclaration; + pub use aconfig::Flag_declarations as ProtoFlagDeclarations; + pub use aconfig::Flag_metadata as ProtoFlagMetadata; + pub use aconfig::Flag_permission as ProtoFlagPermission; + pub use aconfig::Flag_state as ProtoFlagState; + pub use aconfig::Flag_value as ProtoFlagValue; + pub use aconfig::Flag_values as ProtoFlagValues; + pub use aconfig::Parsed_flag as ProtoParsedFlag; + pub use aconfig::Parsed_flags as ProtoParsedFlags; + pub use aconfig::Tracepoint as ProtoTracepoint; +} + +// ---- Common for both the Android tool-chain and cargo ---- +pub use auto_generated::*; + +use anyhow::Result; +use paste::paste; + +/// Path to proto file +const ACONFIG_PROTO_PATH: &str = "//build/make/tools/aconfig/aconfig_protos/protos/aconfig.proto"; + +/// Check if the name identifier is valid +pub fn is_valid_name_ident(s: &str) -> bool { + // Identifiers must match [a-z][a-z0-9_]*, except consecutive underscores are not allowed + if s.contains("__") { + return false; + } + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !first.is_ascii_lowercase() { + return false; + } + chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} + +/// Check if the package identifier is valid +pub fn is_valid_package_ident(s: &str) -> bool { + if !s.contains('.') { + return false; + } + s.split('.').all(is_valid_name_ident) +} + +/// Check if the container identifier is valid +pub fn is_valid_container_ident(s: &str) -> bool { + s.split('.').all(is_valid_name_ident) +} + +fn try_from_text_proto<T>(s: &str) -> Result<T> +where + T: protobuf::MessageFull, +{ + protobuf::text_format::parse_from_str(s).map_err(|e| e.into()) +} + +macro_rules! ensure_required_fields { + ($type:expr, $struct:expr, $($field:expr),+) => { + $( + paste! { + ensure!($struct.[<has_ $field>](), "bad {}: missing {}", $type, $field); + } + )+ + }; +} + +/// Utility module for flag_declaration proto +pub mod flag_declaration { + use super::*; + use anyhow::ensure; + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(pdf: &ProtoFlagDeclaration) -> Result<()> { + ensure_required_fields!("flag declaration", pdf, "name", "namespace", "description"); + + ensure!( + is_valid_name_ident(pdf.name()), + "bad flag declaration: bad name {} expected snake_case string; \ + see {ACONFIG_PROTO_PATH} for details", + pdf.name() + ); + ensure!( + is_valid_name_ident(pdf.namespace()), + "bad flag declaration: bad namespace {} expected snake_case string; \ + see {ACONFIG_PROTO_PATH} for details", + pdf.namespace() + ); + ensure!(!pdf.description().is_empty(), "bad flag declaration: empty description"); + ensure!(pdf.bug.len() == 1, "bad flag declaration: exactly one bug required"); + + Ok(()) + } +} + +/// Utility module for flag_declarations proto +pub mod flag_declarations { + use super::*; + use anyhow::ensure; + + /// Construct a proto instance from a textproto string content + pub fn try_from_text_proto(s: &str) -> Result<ProtoFlagDeclarations> { + let pdf: ProtoFlagDeclarations = super::try_from_text_proto(s)?; + verify_fields(&pdf)?; + Ok(pdf) + } + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(pdf: &ProtoFlagDeclarations) -> Result<()> { + ensure_required_fields!("flag declarations", pdf, "package"); + // TODO(b/312769710): Make the container field required. + ensure!( + is_valid_package_ident(pdf.package()), + "bad flag declarations: bad package {} expected snake_case strings delimited by dots; \ + see {ACONFIG_PROTO_PATH} for details", + pdf.package() + ); + ensure!( + !pdf.has_container() || is_valid_container_ident(pdf.container()), + "bad flag declarations: bad container" + ); + for flag_declaration in pdf.flag.iter() { + super::flag_declaration::verify_fields(flag_declaration)?; + } + + Ok(()) + } +} + +/// Utility module for flag_value proto +pub mod flag_value { + use super::*; + use anyhow::ensure; + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(fv: &ProtoFlagValue) -> Result<()> { + ensure_required_fields!("flag value", fv, "package", "name", "state", "permission"); + + ensure!( + is_valid_package_ident(fv.package()), + "bad flag value: bad package {} expected snake_case strings delimited by dots; \ + see {ACONFIG_PROTO_PATH} for details", + fv.package() + ); + ensure!( + is_valid_name_ident(fv.name()), + "bad flag value: bad name {} expected snake_case string; \ + see {ACONFIG_PROTO_PATH} for details", + fv.name() + ); + + Ok(()) + } +} + +/// Utility module for flag_values proto +pub mod flag_values { + use super::*; + + /// Construct a proto instance from a textproto string content + pub fn try_from_text_proto(s: &str) -> Result<ProtoFlagValues> { + let pfv: ProtoFlagValues = super::try_from_text_proto(s)?; + verify_fields(&pfv)?; + Ok(pfv) + } + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(pfv: &ProtoFlagValues) -> Result<()> { + for flag_value in pfv.flag_value.iter() { + super::flag_value::verify_fields(flag_value)?; + } + Ok(()) + } +} + +/// Utility module for flag_permission proto enum +pub mod flag_permission { + use super::*; + use anyhow::bail; + + /// Construct a flag permission proto enum from string + pub fn parse_from_str(permission: &str) -> Result<ProtoFlagPermission> { + match permission.to_ascii_lowercase().as_str() { + "read_write" => Ok(ProtoFlagPermission::READ_WRITE), + "read_only" => Ok(ProtoFlagPermission::READ_ONLY), + _ => bail!("Permission needs to be read_only or read_write."), + } + } + + /// Serialize flag permission proto enum to string + pub fn to_string(permission: &ProtoFlagPermission) -> &str { + match permission { + ProtoFlagPermission::READ_WRITE => "read_write", + ProtoFlagPermission::READ_ONLY => "read_only", + } + } +} + +/// Utility module for tracepoint proto +pub mod tracepoint { + use super::*; + use anyhow::ensure; + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(tp: &ProtoTracepoint) -> Result<()> { + ensure_required_fields!("tracepoint", tp, "source", "state", "permission"); + + ensure!(!tp.source().is_empty(), "bad tracepoint: empty source"); + + Ok(()) + } +} + +/// Utility module for parsed_flag proto +pub mod parsed_flag { + use super::*; + use anyhow::ensure; + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(pf: &ProtoParsedFlag) -> Result<()> { + ensure_required_fields!( + "parsed flag", + pf, + "package", + "name", + "namespace", + "description", + "state", + "permission" + ); + + ensure!( + is_valid_package_ident(pf.package()), + "bad parsed flag: bad package {} expected snake_case strings delimited by dots; \ + see {ACONFIG_PROTO_PATH} for details", + pf.package() + ); + ensure!( + !pf.has_container() || is_valid_container_ident(pf.container()), + "bad parsed flag: bad container" + ); + ensure!( + is_valid_name_ident(pf.name()), + "bad parsed flag: bad name {} expected snake_case string; \ + see {ACONFIG_PROTO_PATH} for details", + pf.name() + ); + ensure!( + is_valid_name_ident(pf.namespace()), + "bad parsed flag: bad namespace {} expected snake_case string; \ + see {ACONFIG_PROTO_PATH} for details", + pf.namespace() + ); + ensure!(!pf.description().is_empty(), "bad parsed flag: empty description"); + ensure!(!pf.trace.is_empty(), "bad parsed flag: empty trace"); + for tp in pf.trace.iter() { + super::tracepoint::verify_fields(tp)?; + } + ensure!(pf.bug.len() == 1, "bad flag declaration: exactly one bug required"); + if pf.is_fixed_read_only() { + ensure!( + pf.permission() == ProtoFlagPermission::READ_ONLY, + "bad parsed flag: flag is is_fixed_read_only but permission is not READ_ONLY" + ); + for tp in pf.trace.iter() { + ensure!(tp.permission() == ProtoFlagPermission::READ_ONLY, + "bad parsed flag: flag is is_fixed_read_only but a tracepoint's permission is not READ_ONLY" + ); + } + } + + Ok(()) + } + + /// Get the file path of the corresponding flag declaration + pub fn path_to_declaration(pf: &ProtoParsedFlag) -> &str { + debug_assert!(!pf.trace.is_empty()); + pf.trace[0].source() + } +} + +/// Utility module for parsed_flags proto +pub mod parsed_flags { + use super::*; + use anyhow::bail; + use std::cmp::Ordering; + + /// Construct a proto instance from a binary proto bytes + pub fn try_from_binary_proto(bytes: &[u8]) -> Result<ProtoParsedFlags> { + let message: ProtoParsedFlags = protobuf::Message::parse_from_bytes(bytes)?; + verify_fields(&message)?; + Ok(message) + } + + /// Ensure the proto instance is valid by checking its fields + pub fn verify_fields(pf: &ProtoParsedFlags) -> Result<()> { + use crate::parsed_flag::path_to_declaration; + + let mut previous: Option<&ProtoParsedFlag> = None; + for parsed_flag in pf.parsed_flag.iter() { + if let Some(prev) = previous { + let a = create_sorting_key(prev); + let b = create_sorting_key(parsed_flag); + match a.cmp(&b) { + Ordering::Less => {} + Ordering::Equal => bail!( + "bad parsed flags: duplicate flag {} (defined in {} and {})", + a, + path_to_declaration(prev), + path_to_declaration(parsed_flag) + ), + Ordering::Greater => { + bail!("bad parsed flags: not sorted: {} comes before {}", a, b) + } + } + } + super::parsed_flag::verify_fields(parsed_flag)?; + previous = Some(parsed_flag); + } + Ok(()) + } + + /// Merge multipe parsed_flags proto + pub fn merge(parsed_flags: Vec<ProtoParsedFlags>, dedup: bool) -> Result<ProtoParsedFlags> { + let mut merged = ProtoParsedFlags::new(); + for mut pfs in parsed_flags.into_iter() { + merged.parsed_flag.append(&mut pfs.parsed_flag); + } + merged.parsed_flag.sort_by_cached_key(create_sorting_key); + if dedup { + // Deduplicate identical protobuf messages. Messages with the same sorting key but + // different fields (including the path to the original source file) will not be + // deduplicated and trigger an error in verify_fields. + merged.parsed_flag.dedup(); + } + verify_fields(&merged)?; + Ok(merged) + } + + /// Sort parsed flags + pub fn sort_parsed_flags(pf: &mut ProtoParsedFlags) { + pf.parsed_flag.sort_by_key(create_sorting_key); + } + + fn create_sorting_key(pf: &ProtoParsedFlag) -> String { + pf.fully_qualified_name() + } +} + +/// ParsedFlagExt trait +pub trait ParsedFlagExt { + /// Return the fully qualified name + fn fully_qualified_name(&self) -> String; +} + +impl ParsedFlagExt for ProtoParsedFlag { + fn fully_qualified_name(&self) -> String { + format!("{}.{}", self.package(), self.name()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_flag_declarations_try_from_text_proto() { + // valid input + let flag_declarations = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +container: "system" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + is_exported: true +} +flag { + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." + bug: "abc" + is_fixed_read_only: true +} +"#, + ) + .unwrap(); + assert_eq!(flag_declarations.package(), "com.foo.bar"); + assert_eq!(flag_declarations.container(), "system"); + let first = flag_declarations.flag.iter().find(|pf| pf.name() == "first").unwrap(); + assert_eq!(first.name(), "first"); + assert_eq!(first.namespace(), "first_ns"); + assert_eq!(first.description(), "This is the description of the first flag."); + assert_eq!(first.bug, vec!["123"]); + assert!(!first.is_fixed_read_only()); + assert!(first.is_exported()); + let second = flag_declarations.flag.iter().find(|pf| pf.name() == "second").unwrap(); + assert_eq!(second.name(), "second"); + assert_eq!(second.namespace(), "second_ns"); + assert_eq!(second.description(), "This is the description of the second flag."); + assert_eq!(second.bug, vec!["abc"]); + assert!(second.is_fixed_read_only()); + assert!(!second.is_exported()); + + // valid input: missing container in flag declarations is supported + let flag_declarations = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" +} +"#, + ) + .unwrap(); + assert_eq!(flag_declarations.container(), ""); + assert!(!flag_declarations.has_container()); + + // bad input: missing package in flag declarations + let error = flag_declarations::try_from_text_proto( + r#" +container: "system" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." +} +flag { + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." +} +"#, + ) + .unwrap_err(); + assert_eq!(format!("{:?}", error), "bad flag declarations: missing package"); + + // bad input: missing namespace in flag declaration + let error = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +container: "system" +flag { + name: "first" + description: "This is the description of the first flag." +} +flag { + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." +} +"#, + ) + .unwrap_err(); + assert_eq!(format!("{:?}", error), "bad flag declaration: missing namespace"); + + // bad input: bad package name in flag declarations + let error = flag_declarations::try_from_text_proto( + r#" +package: "_com.FOO__BAR" +container: "system" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." +} +flag { + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag declarations: bad package")); + + // bad input: bad name in flag declaration + let error = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +container: "system" +flag { + name: "FIRST" + namespace: "first_ns" + description: "This is the description of the first flag." +} +flag { + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag declaration: bad name")); + + // bad input: no bug entries in flag declaration + let error = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +container: "system" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag declaration: exactly one bug required")); + + // bad input: multiple bug entries in flag declaration + let error = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +container: "system" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + bug: "abc" +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag declaration: exactly one bug required")); + + // bad input: invalid container name in flag declaration + let error = flag_declarations::try_from_text_proto( + r#" +package: "com.foo.bar" +container: "__bad_bad_container.com" +flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + bug: "abc" +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag declarations: bad container")); + + // TODO(b/312769710): Verify error when container is missing. + } + + #[test] + fn test_flag_values_try_from_text_proto() { + // valid input + let flag_values = flag_values::try_from_text_proto( + r#" +flag_value { + package: "com.first" + name: "first" + state: DISABLED + permission: READ_ONLY +} +flag_value { + package: "com.second" + name: "second" + state: ENABLED + permission: READ_WRITE +} +"#, + ) + .unwrap(); + let first = flag_values.flag_value.iter().find(|fv| fv.name() == "first").unwrap(); + assert_eq!(first.package(), "com.first"); + assert_eq!(first.name(), "first"); + assert_eq!(first.state(), ProtoFlagState::DISABLED); + assert_eq!(first.permission(), ProtoFlagPermission::READ_ONLY); + let second = flag_values.flag_value.iter().find(|fv| fv.name() == "second").unwrap(); + assert_eq!(second.package(), "com.second"); + assert_eq!(second.name(), "second"); + assert_eq!(second.state(), ProtoFlagState::ENABLED); + assert_eq!(second.permission(), ProtoFlagPermission::READ_WRITE); + + // bad input: bad package in flag value + let error = flag_values::try_from_text_proto( + r#" +flag_value { + package: "COM.FIRST" + name: "first" + state: DISABLED + permission: READ_ONLY +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag value: bad package")); + + // bad input: bad name in flag value + let error = flag_values::try_from_text_proto( + r#" +flag_value { + package: "com.first" + name: "FIRST" + state: DISABLED + permission: READ_ONLY +} +"#, + ) + .unwrap_err(); + assert!(format!("{:?}", error).contains("bad flag value: bad name")); + + // bad input: missing state in flag value + let error = flag_values::try_from_text_proto( + r#" +flag_value { + package: "com.first" + name: "first" + permission: READ_ONLY +} +"#, + ) + .unwrap_err(); + assert_eq!(format!("{:?}", error), "bad flag value: missing state"); + + // bad input: missing permission in flag value + let error = flag_values::try_from_text_proto( + r#" +flag_value { + package: "com.first" + name: "first" + state: DISABLED +} +"#, + ) + .unwrap_err(); + assert_eq!(format!("{:?}", error), "bad flag value: missing permission"); + } + + fn try_from_binary_proto_from_text_proto(text_proto: &str) -> Result<ProtoParsedFlags> { + use protobuf::Message; + + let parsed_flags: ProtoParsedFlags = try_from_text_proto(text_proto)?; + let mut binary_proto = Vec::new(); + parsed_flags.write_to_vec(&mut binary_proto)?; + parsed_flags::try_from_binary_proto(&binary_proto) + } + + #[test] + fn test_parsed_flags_try_from_text_proto() { + // valid input + let text_proto = r#" +parsed_flag { + package: "com.first" + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "SOME_BUG" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +parsed_flag { + package: "com.second" + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." + bug: "SOME_BUG" + state: ENABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + trace { + source: "flags.values" + state: ENABLED + permission: READ_ONLY + } + is_fixed_read_only: true + container: "system" +} +"#; + let parsed_flags = try_from_binary_proto_from_text_proto(text_proto).unwrap(); + assert_eq!(parsed_flags.parsed_flag.len(), 2); + let second = parsed_flags.parsed_flag.iter().find(|fv| fv.name() == "second").unwrap(); + assert_eq!(second.package(), "com.second"); + assert_eq!(second.name(), "second"); + assert_eq!(second.namespace(), "second_ns"); + assert_eq!(second.description(), "This is the description of the second flag."); + assert_eq!(second.bug, vec!["SOME_BUG"]); + assert_eq!(second.state(), ProtoFlagState::ENABLED); + assert_eq!(second.permission(), ProtoFlagPermission::READ_ONLY); + assert_eq!(2, second.trace.len()); + assert_eq!(second.trace[0].source(), "flags.declarations"); + assert_eq!(second.trace[0].state(), ProtoFlagState::DISABLED); + assert_eq!(second.trace[0].permission(), ProtoFlagPermission::READ_ONLY); + assert_eq!(second.trace[1].source(), "flags.values"); + assert_eq!(second.trace[1].state(), ProtoFlagState::ENABLED); + assert_eq!(second.trace[1].permission(), ProtoFlagPermission::READ_ONLY); + assert!(second.is_fixed_read_only()); + + // valid input: empty + let parsed_flags = try_from_binary_proto_from_text_proto("").unwrap(); + assert!(parsed_flags.parsed_flag.is_empty()); + + // bad input: empty trace + let text_proto = r#" +parsed_flag { + package: "com.first" + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + state: DISABLED + permission: READ_ONLY + container: "system" +} +"#; + let error = try_from_binary_proto_from_text_proto(text_proto).unwrap_err(); + assert_eq!(format!("{:?}", error), "bad parsed flag: empty trace"); + + // bad input: missing namespace in parsed_flag + let text_proto = r#" +parsed_flag { + package: "com.first" + name: "first" + description: "This is the description of the first flag." + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let error = try_from_binary_proto_from_text_proto(text_proto).unwrap_err(); + assert_eq!(format!("{:?}", error), "bad parsed flag: missing namespace"); + + // bad input: parsed_flag not sorted by package + let text_proto = r#" +parsed_flag { + package: "bbb.bbb" + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +parsed_flag { + package: "aaa.aaa" + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." + bug: "" + state: ENABLED + permission: READ_WRITE + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let error = try_from_binary_proto_from_text_proto(text_proto).unwrap_err(); + assert_eq!( + format!("{:?}", error), + "bad parsed flags: not sorted: bbb.bbb.first comes before aaa.aaa.second" + ); + + // bad input: parsed_flag not sorted by name + let text_proto = r#" +parsed_flag { + package: "com.foo" + name: "bbb" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +parsed_flag { + package: "com.foo" + name: "aaa" + namespace: "second_ns" + description: "This is the description of the second flag." + bug: "" + state: ENABLED + permission: READ_WRITE + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let error = try_from_binary_proto_from_text_proto(text_proto).unwrap_err(); + assert_eq!( + format!("{:?}", error), + "bad parsed flags: not sorted: com.foo.bbb comes before com.foo.aaa" + ); + + // bad input: duplicate flags + let text_proto = r#" +parsed_flag { + package: "com.foo" + name: "bar" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +parsed_flag { + package: "com.foo" + name: "bar" + namespace: "second_ns" + description: "This is the description of the second flag." + bug: "" + state: ENABLED + permission: READ_WRITE + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let error = try_from_binary_proto_from_text_proto(text_proto).unwrap_err(); + assert_eq!(format!("{:?}", error), "bad parsed flags: duplicate flag com.foo.bar (defined in flags.declarations and flags.declarations)"); + } + + #[test] + fn test_parsed_flag_path_to_declaration() { + let text_proto = r#" +parsed_flag { + package: "com.foo" + name: "bar" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "b/12345678" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + trace { + source: "flags.values" + state: ENABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let parsed_flags = try_from_binary_proto_from_text_proto(text_proto).unwrap(); + let parsed_flag = &parsed_flags.parsed_flag[0]; + assert_eq!(crate::parsed_flag::path_to_declaration(parsed_flag), "flags.declarations"); + } + + #[test] + fn test_parsed_flags_merge() { + let text_proto = r#" +parsed_flag { + package: "com.first" + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "a" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +parsed_flag { + package: "com.second" + name: "second" + namespace: "second_ns" + description: "This is the description of the second flag." + bug: "b" + state: ENABLED + permission: READ_WRITE + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let expected = try_from_binary_proto_from_text_proto(text_proto).unwrap(); + + let text_proto = r#" +parsed_flag { + package: "com.first" + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "a" + state: DISABLED + permission: READ_ONLY + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let first = try_from_binary_proto_from_text_proto(text_proto).unwrap(); + + let text_proto = r#" +parsed_flag { + package: "com.second" + name: "second" + namespace: "second_ns" + bug: "b" + description: "This is the description of the second flag." + state: ENABLED + permission: READ_WRITE + trace { + source: "flags.declarations" + state: DISABLED + permission: READ_ONLY + } + container: "system" +} +"#; + let second = try_from_binary_proto_from_text_proto(text_proto).unwrap(); + + let text_proto = r#" +parsed_flag { + package: "com.second" + name: "second" + namespace: "second_ns" + bug: "b" + description: "This is the description of the second flag." + state: ENABLED + permission: READ_WRITE + trace { + source: "duplicate/flags.declarations" + state: DISABLED + permission: READ_ONLY + } +} +"#; + let second_duplicate = try_from_binary_proto_from_text_proto(text_proto).unwrap(); + + // bad cases + + // two of the same flag with dedup disabled + let error = parsed_flags::merge(vec![first.clone(), first.clone()], false).unwrap_err(); + assert_eq!(format!("{:?}", error), "bad parsed flags: duplicate flag com.first.first (defined in flags.declarations and flags.declarations)"); + + // two conflicting flags with dedup disabled + let error = + parsed_flags::merge(vec![second.clone(), second_duplicate.clone()], false).unwrap_err(); + assert_eq!(format!("{:?}", error), "bad parsed flags: duplicate flag com.second.second (defined in flags.declarations and duplicate/flags.declarations)"); + + // two conflicting flags with dedup enabled + let error = + parsed_flags::merge(vec![second.clone(), second_duplicate.clone()], true).unwrap_err(); + assert_eq!(format!("{:?}", error), "bad parsed flags: duplicate flag com.second.second (defined in flags.declarations and duplicate/flags.declarations)"); + + // valid cases + assert!(parsed_flags::merge(vec![], false).unwrap().parsed_flag.is_empty()); + assert!(parsed_flags::merge(vec![], true).unwrap().parsed_flag.is_empty()); + assert_eq!(first, parsed_flags::merge(vec![first.clone()], false).unwrap()); + assert_eq!(first, parsed_flags::merge(vec![first.clone()], true).unwrap()); + assert_eq!( + expected, + parsed_flags::merge(vec![first.clone(), second.clone()], false).unwrap() + ); + assert_eq!( + expected, + parsed_flags::merge(vec![first.clone(), second.clone()], true).unwrap() + ); + assert_eq!( + expected, + parsed_flags::merge(vec![second.clone(), first.clone()], false).unwrap() + ); + assert_eq!( + expected, + parsed_flags::merge(vec![second.clone(), first.clone()], true).unwrap() + ); + + // two identical flags with dedup enabled + assert_eq!(first, parsed_flags::merge(vec![first.clone(), first.clone()], true).unwrap()); + } +} diff --git a/tools/aconfig/aconfig_storage_file/Android.bp b/tools/aconfig/aconfig_storage_file/Android.bp new file mode 100644 index 0000000000..e066e31ac1 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/Android.bp @@ -0,0 +1,139 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "aconfig_storage_file.defaults", + edition: "2021", + lints: "none", + rustlibs: [ + "libanyhow", + "libthiserror", + "libtempfile", + "libprotobuf", + "libclap", + "libcxx", + "libaconfig_storage_protos", + ], +} + +rust_library { + name: "libaconfig_storage_file", + crate_name: "aconfig_storage_file", + host_supported: true, + defaults: ["aconfig_storage_file.defaults"], + srcs: ["src/lib.rs"], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", + vendor_available: true, + product_available: true, +} + +rust_binary_host { + name: "aconfig-storage", + defaults: ["aconfig_storage_file.defaults"], + srcs: ["src/main.rs"], + rustlibs: ["libaconfig_storage_file"], +} + +rust_test_host { + name: "aconfig_storage_file.test", + test_suites: ["general-tests"], + defaults: ["aconfig_storage_file.defaults"], + srcs: ["src/lib.rs"], +} + +rust_protobuf { + name: "libaconfig_storage_protos", + protos: ["protos/aconfig_storage_metadata.proto"], + crate_name: "aconfig_storage_protos", + source_stem: "aconfig_storage_protos", + host_supported: true, + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", + vendor_available: true, + product_available: true, +} + +cc_library { + name: "libaconfig_storage_protos_cc", + proto: { + export_proto_headers: true, + type: "lite", + }, + srcs: ["protos/aconfig_storage_metadata.proto"], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + host_supported: true, + min_sdk_version: "29", + vendor_available: true, + product_available: true, + double_loadable: true, +} + +// cxx source codegen from rust api +genrule { + name: "libcxx_aconfig_storage_file_bridge_code", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) > $(out)", + srcs: ["src/lib.rs"], + out: ["aconfig_storage/lib.rs.cc"], +} + +// cxx header codegen from rust api +genrule { + name: "libcxx_aconfig_storage_file_bridge_header", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) --header > $(out)", + srcs: ["src/lib.rs"], + out: ["aconfig_storage/lib.rs.h"], +} + +// a static cc lib based on generated code +rust_ffi_static { + name: "libaconfig_storage_file_cxx_bridge", + crate_name: "aconfig_storage_file_cxx_bridge", + host_supported: true, + vendor_available: true, + product_available: true, + srcs: ["src/lib.rs"], + defaults: ["aconfig_storage_file.defaults"], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", +} + +// storage file parse api cc interface +cc_library { + name: "libaconfig_storage_file_cc", + srcs: ["aconfig_storage_file.cpp"], + generated_headers: [ + "cxx-bridge-header", + "libcxx_aconfig_storage_file_bridge_header", + ], + generated_sources: ["libcxx_aconfig_storage_file_bridge_code"], + whole_static_libs: ["libaconfig_storage_file_cxx_bridge"], + export_include_dirs: ["include"], + host_supported: true, + vendor_available: true, + product_available: true, + shared_libs: [ + "libbase", + ], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", + double_loadable: true, +} diff --git a/tools/aconfig/aconfig_storage_file/Cargo.toml b/tools/aconfig/aconfig_storage_file/Cargo.toml new file mode 100644 index 0000000000..192dfad40a --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "aconfig_storage_file" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +protobuf = "3.2.0" +tempfile = "3.9.0" +thiserror = "1.0.56" +clap = { version = "4.1.8", features = ["derive"] } +cxx = "1.0" + +[[bin]] +name = "aconfig-storage" +path = "src/main.rs" + +[build-dependencies] +protobuf-codegen = "3.2.0" +cxx-build = "1.0" diff --git a/tools/aconfig/aconfig_storage_file/aconfig_storage_file.cpp b/tools/aconfig/aconfig_storage_file/aconfig_storage_file.cpp new file mode 100644 index 0000000000..548078f48f --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/aconfig_storage_file.cpp @@ -0,0 +1,38 @@ +#include "rust/cxx.h" +#include "aconfig_storage/lib.rs.h" + +#include "aconfig_storage/aconfig_storage_file.hpp" + +using namespace android::base; + +namespace aconfig_storage { + +Result<std::vector<FlagValueAndInfoSummary>> list_flags_with_info( + const std::string& package_map, + const std::string& flag_map, + const std::string& flag_val, + const std::string& flag_info) { + auto flag_list_cxx = list_flags_with_info_cxx(rust::Str(package_map.c_str()), + rust::Str(flag_map.c_str()), + rust::Str(flag_val.c_str()), + rust::Str(flag_info.c_str())); + if (flag_list_cxx.query_success) { + auto flag_list = std::vector<FlagValueAndInfoSummary>(); + for (const auto& flag_cxx : flag_list_cxx.flags) { + auto flag = FlagValueAndInfoSummary(); + flag.package_name = std::string(flag_cxx.package_name); + flag.flag_name = std::string(flag_cxx.flag_name); + flag.flag_value = std::string(flag_cxx.flag_value); + flag.value_type = std::string(flag_cxx.value_type); + flag.is_readwrite = flag_cxx.is_readwrite; + flag.has_server_override = flag_cxx.has_server_override; + flag.has_local_override = flag_cxx.has_local_override; + flag_list.push_back(flag); + } + return flag_list; + } else { + return Error() << flag_list_cxx.error_message; + } +} + +} // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_file/build.rs b/tools/aconfig/aconfig_storage_file/build.rs new file mode 100644 index 0000000000..e0ade2aba0 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/build.rs @@ -0,0 +1,19 @@ +use protobuf_codegen::Codegen; + +fn main() { + let proto_files = vec!["protos/aconfig_storage_metadata.proto"]; + + // tell cargo to only re-run the build script if any of the proto files has changed + for path in &proto_files { + println!("cargo:rerun-if-changed={}", path); + } + + Codegen::new() + .pure() + .include("protos") + .inputs(proto_files) + .cargo_out_dir("aconfig_storage_protos") + .run_from_script(); + + let _ = cxx_build::bridge("src/lib.rs"); +} diff --git a/tools/aconfig/aconfig_storage_file/include/aconfig_storage/aconfig_storage_file.hpp b/tools/aconfig/aconfig_storage_file/include/aconfig_storage/aconfig_storage_file.hpp new file mode 100644 index 0000000000..5044a4dbfc --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/include/aconfig_storage/aconfig_storage_file.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include <vector> +#include <string> +#include <android-base/result.h> + +namespace aconfig_storage { + +/// Flag value and info summary for a flag +struct FlagValueAndInfoSummary { + std::string package_name; + std::string flag_name; + std::string flag_value; + std::string value_type; + bool is_readwrite; + bool has_server_override; + bool has_local_override; +}; + +/// List all flag values with their flag info +/// \input package_map: package map file +/// \input flag_map: flag map file +/// \input flag_val: flag value file +/// \input flag_info: flag info file +android::base::Result<std::vector<FlagValueAndInfoSummary>> list_flags_with_info( + const std::string& package_map, + const std::string& flag_map, + const std::string& flag_val, + const std::string& flag_info); + +}// namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_file/protos/aconfig_storage_metadata.proto b/tools/aconfig/aconfig_storage_file/protos/aconfig_storage_metadata.proto new file mode 100644 index 0000000000..f6bf1a43c4 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/protos/aconfig_storage_metadata.proto @@ -0,0 +1,37 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +// This is the schema definition for aconfig files. Modifications need to be +// either backwards compatible, or include updates to all aconfig files in the +// Android tree. + +syntax = "proto2"; + +package android.aconfig_storage_metadata; + +message storage_file_info { + optional uint32 version = 1; + optional string container = 2; + optional string package_map = 3; + optional string flag_map = 4; + optional string flag_val = 5; + optional string flag_info = 6; + optional string local_overrides = 7; + optional string default_flag_val = 8; + optional int64 timestamp = 9; +} + +message storage_files { + repeated storage_file_info files = 1; +}; diff --git a/tools/aconfig/aconfig_storage_file/src/flag_info.rs b/tools/aconfig/aconfig_storage_file/src/flag_info.rs new file mode 100644 index 0000000000..beac38d156 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/flag_info.rs @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag info module defines the flag info file format and methods for serialization +//! and deserialization + +use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; +use crate::{AconfigStorageError, StorageFileType}; +use anyhow::anyhow; +use std::fmt; + +/// Flag info header struct +#[derive(PartialEq)] +pub struct FlagInfoHeader { + pub version: u32, + pub container: String, + pub file_type: u8, + pub file_size: u32, + pub num_flags: u32, + pub boolean_flag_offset: u32, +} + +/// Implement debug print trait for header +impl fmt::Debug for FlagInfoHeader { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "Version: {}, Container: {}, File Type: {:?}, File Size: {}", + self.version, + self.container, + StorageFileType::try_from(self.file_type), + self.file_size + )?; + writeln!( + f, + "Num of Flags: {}, Boolean Flag Offset:{}", + self.num_flags, self.boolean_flag_offset + )?; + Ok(()) + } +} + +impl FlagInfoHeader { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_type.to_le_bytes()); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_flags.to_le_bytes()); + result.extend_from_slice(&self.boolean_flag_offset.to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let list = Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_type: read_u8_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_flags: read_u32_from_bytes(bytes, &mut head)?, + boolean_flag_offset: read_u32_from_bytes(bytes, &mut head)?, + }; + if list.file_type != StorageFileType::FlagInfo as u8 { + return Err(AconfigStorageError::BytesParseFail(anyhow!( + "binary file is not a flag info file" + ))); + } + Ok(list) + } +} + +/// bit field for flag info +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FlagInfoBit { + HasServerOverride = 1 << 0, + IsReadWrite = 1 << 1, + HasLocalOverride = 1 << 2, +} + +/// Flag info node struct +#[derive(PartialEq, Clone)] +pub struct FlagInfoNode { + pub attributes: u8, +} + +/// Implement debug print trait for node +impl fmt::Debug for FlagInfoNode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "readwrite: {}, server override: {}, local override: {}", + self.attributes & (FlagInfoBit::IsReadWrite as u8) != 0, + self.attributes & (FlagInfoBit::HasServerOverride as u8) != 0, + self.attributes & (FlagInfoBit::HasLocalOverride as u8) != 0, + )?; + Ok(()) + } +} + +impl FlagInfoNode { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + result.extend_from_slice(&self.attributes.to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let node = Self { attributes: read_u8_from_bytes(bytes, &mut head)? }; + Ok(node) + } + + /// Create flag info node + pub fn create(is_flag_rw: bool) -> Self { + Self { attributes: if is_flag_rw { FlagInfoBit::IsReadWrite as u8 } else { 0u8 } } + } +} + +/// Flag info list struct +#[derive(PartialEq)] +pub struct FlagInfoList { + pub header: FlagInfoHeader, + pub nodes: Vec<FlagInfoNode>, +} + +/// Implement debug print trait for flag info list +impl fmt::Debug for FlagInfoList { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Header:")?; + write!(f, "{:?}", self.header)?; + writeln!(f, "Nodes:")?; + for node in self.nodes.iter() { + write!(f, "{:?}", node)?; + } + Ok(()) + } +} + +impl FlagInfoList { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + [ + self.header.into_bytes(), + self.nodes.iter().map(|v| v.into_bytes()).collect::<Vec<_>>().concat(), + ] + .concat() + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let header = FlagInfoHeader::from_bytes(bytes)?; + let num_flags = header.num_flags; + let mut head = header.into_bytes().len(); + let nodes = (0..num_flags) + .map(|_| { + let node = FlagInfoNode::from_bytes(&bytes[head..])?; + head += node.into_bytes().len(); + Ok(node) + }) + .collect::<Result<Vec<_>, AconfigStorageError>>() + .map_err(|errmsg| { + AconfigStorageError::BytesParseFail(anyhow!( + "fail to parse flag info list: {}", + errmsg + )) + })?; + let list = Self { header, nodes }; + Ok(list) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::create_test_flag_info_list; + + #[test] + // this test point locks down the value list serialization + fn test_serialization() { + let flag_info_list = create_test_flag_info_list(); + + let header: &FlagInfoHeader = &flag_info_list.header; + let reinterpreted_header = FlagInfoHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec<FlagInfoNode> = &flag_info_list.nodes; + for node in nodes.iter() { + let reinterpreted_node = FlagInfoNode::from_bytes(&node.into_bytes()).unwrap(); + assert_eq!(node, &reinterpreted_node); + } + + let flag_info_bytes = flag_info_list.into_bytes(); + let reinterpreted_info_list = FlagInfoList::from_bytes(&flag_info_bytes); + assert!(reinterpreted_info_list.is_ok()); + assert_eq!(&flag_info_list, &reinterpreted_info_list.unwrap()); + assert_eq!(flag_info_bytes.len() as u32, header.file_size); + } + + #[test] + // this test point locks down that version number should be at the top of serialized + // bytes + fn test_version_number() { + let flag_info_list = create_test_flag_info_list(); + let bytes = &flag_info_list.into_bytes(); + let mut head = 0; + let version = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version, 1); + } + + #[test] + // this test point locks down file type check + fn test_file_type_check() { + let mut flag_info_list = create_test_flag_info_list(); + flag_info_list.header.file_type = 123u8; + let error = FlagInfoList::from_bytes(&flag_info_list.into_bytes()).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!("BytesParseFail(binary file is not a flag info file)") + ); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/flag_table.rs b/tools/aconfig/aconfig_storage_file/src/flag_table.rs new file mode 100644 index 0000000000..64b90eabfa --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/flag_table.rs @@ -0,0 +1,269 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag table module defines the flag table file format and methods for serialization +//! and deserialization + +use crate::{ + get_bucket_index, read_str_from_bytes, read_u16_from_bytes, read_u32_from_bytes, + read_u8_from_bytes, +}; +use crate::{AconfigStorageError, StorageFileType, StoredFlagType}; +use anyhow::anyhow; +use std::fmt; + +/// Flag table header struct +#[derive(PartialEq)] +pub struct FlagTableHeader { + pub version: u32, + pub container: String, + pub file_type: u8, + pub file_size: u32, + pub num_flags: u32, + pub bucket_offset: u32, + pub node_offset: u32, +} + +/// Implement debug print trait for header +impl fmt::Debug for FlagTableHeader { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "Version: {}, Container: {}, File Type: {:?}, File Size: {}", + self.version, + self.container, + StorageFileType::try_from(self.file_type), + self.file_size + )?; + writeln!( + f, + "Num of Flags: {}, Bucket Offset:{}, Node Offset: {}", + self.num_flags, self.bucket_offset, self.node_offset + )?; + Ok(()) + } +} + +impl FlagTableHeader { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_type.to_le_bytes()); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_flags.to_le_bytes()); + result.extend_from_slice(&self.bucket_offset.to_le_bytes()); + result.extend_from_slice(&self.node_offset.to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let table = Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_type: read_u8_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_flags: read_u32_from_bytes(bytes, &mut head)?, + bucket_offset: read_u32_from_bytes(bytes, &mut head)?, + node_offset: read_u32_from_bytes(bytes, &mut head)?, + }; + if table.file_type != StorageFileType::FlagMap as u8 { + return Err(AconfigStorageError::BytesParseFail(anyhow!( + "binary file is not a flag map" + ))); + } + Ok(table) + } +} + +/// Flag table node struct +#[derive(PartialEq, Clone)] +pub struct FlagTableNode { + pub package_id: u32, + pub flag_name: String, + pub flag_type: StoredFlagType, + // within package flag index of this flag type + pub flag_index: u16, + pub next_offset: Option<u32>, +} + +/// Implement debug print trait for node +impl fmt::Debug for FlagTableNode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "Package Id: {}, Flag: {}, Type: {:?}, Index: {}, Next: {:?}", + self.package_id, self.flag_name, self.flag_type, self.flag_index, self.next_offset + )?; + Ok(()) + } +} + +impl FlagTableNode { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + result.extend_from_slice(&self.package_id.to_le_bytes()); + let name_bytes = self.flag_name.as_bytes(); + result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(name_bytes); + result.extend_from_slice(&(self.flag_type as u16).to_le_bytes()); + result.extend_from_slice(&self.flag_index.to_le_bytes()); + result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let node = Self { + package_id: read_u32_from_bytes(bytes, &mut head)?, + flag_name: read_str_from_bytes(bytes, &mut head)?, + flag_type: StoredFlagType::try_from(read_u16_from_bytes(bytes, &mut head)?)?, + flag_index: read_u16_from_bytes(bytes, &mut head)?, + next_offset: match read_u32_from_bytes(bytes, &mut head)? { + 0 => None, + val => Some(val), + }, + }; + Ok(node) + } + + /// Calculate node bucket index + pub fn find_bucket_index(package_id: u32, flag_name: &str, num_buckets: u32) -> u32 { + let full_flag_name = package_id.to_string() + "/" + flag_name; + get_bucket_index(&full_flag_name, num_buckets) + } +} + +#[derive(PartialEq)] +pub struct FlagTable { + pub header: FlagTableHeader, + pub buckets: Vec<Option<u32>>, + pub nodes: Vec<FlagTableNode>, +} + +/// Implement debug print trait for flag table +impl fmt::Debug for FlagTable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Header:")?; + write!(f, "{:?}", self.header)?; + writeln!(f, "Buckets:")?; + writeln!(f, "{:?}", self.buckets)?; + writeln!(f, "Nodes:")?; + for node in self.nodes.iter() { + write!(f, "{:?}", node)?; + } + Ok(()) + } +} + +/// Flag table struct +impl FlagTable { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + [ + self.header.into_bytes(), + self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::<Vec<_>>().concat(), + self.nodes.iter().map(|v| v.into_bytes()).collect::<Vec<_>>().concat(), + ] + .concat() + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let header = FlagTableHeader::from_bytes(bytes)?; + let num_flags = header.num_flags; + let num_buckets = crate::get_table_size(num_flags)?; + let mut head = header.into_bytes().len(); + let buckets = (0..num_buckets) + .map(|_| match read_u32_from_bytes(bytes, &mut head).unwrap() { + 0 => None, + val => Some(val), + }) + .collect(); + let nodes = (0..num_flags) + .map(|_| { + let node = FlagTableNode::from_bytes(&bytes[head..])?; + head += node.into_bytes().len(); + Ok(node) + }) + .collect::<Result<Vec<_>, AconfigStorageError>>() + .map_err(|errmsg| { + AconfigStorageError::BytesParseFail(anyhow!("fail to parse flag table: {}", errmsg)) + })?; + + let table = Self { header, buckets, nodes }; + Ok(table) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::create_test_flag_table; + + #[test] + // this test point locks down the table serialization + fn test_serialization() { + let flag_table = create_test_flag_table(); + + let header: &FlagTableHeader = &flag_table.header; + let reinterpreted_header = FlagTableHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec<FlagTableNode> = &flag_table.nodes; + for node in nodes.iter() { + let reinterpreted_node = FlagTableNode::from_bytes(&node.into_bytes()).unwrap(); + assert_eq!(node, &reinterpreted_node); + } + + let flag_table_bytes = flag_table.into_bytes(); + let reinterpreted_table = FlagTable::from_bytes(&flag_table_bytes); + assert!(reinterpreted_table.is_ok()); + assert_eq!(&flag_table, &reinterpreted_table.unwrap()); + assert_eq!(flag_table_bytes.len() as u32, header.file_size); + } + + #[test] + // this test point locks down that version number should be at the top of serialized + // bytes + fn test_version_number() { + let flag_table = create_test_flag_table(); + let bytes = &flag_table.into_bytes(); + let mut head = 0; + let version = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version, 1); + } + + #[test] + // this test point locks down file type check + fn test_file_type_check() { + let mut flag_table = create_test_flag_table(); + flag_table.header.file_type = 123u8; + let error = FlagTable::from_bytes(&flag_table.into_bytes()).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!("BytesParseFail(binary file is not a flag map)") + ); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/flag_value.rs b/tools/aconfig/aconfig_storage_file/src/flag_value.rs new file mode 100644 index 0000000000..506924b339 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/flag_value.rs @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag value module defines the flag value file format and methods for serialization +//! and deserialization + +use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; +use crate::{AconfigStorageError, StorageFileType}; +use anyhow::anyhow; +use std::fmt; + +/// Flag value header struct +#[derive(PartialEq)] +pub struct FlagValueHeader { + pub version: u32, + pub container: String, + pub file_type: u8, + pub file_size: u32, + pub num_flags: u32, + pub boolean_value_offset: u32, +} + +/// Implement debug print trait for header +impl fmt::Debug for FlagValueHeader { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "Version: {}, Container: {}, File Type: {:?}, File Size: {}", + self.version, + self.container, + StorageFileType::try_from(self.file_type), + self.file_size + )?; + writeln!( + f, + "Num of Flags: {}, Value Offset:{}", + self.num_flags, self.boolean_value_offset + )?; + Ok(()) + } +} + +impl FlagValueHeader { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_type.to_le_bytes()); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_flags.to_le_bytes()); + result.extend_from_slice(&self.boolean_value_offset.to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let list = Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_type: read_u8_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_flags: read_u32_from_bytes(bytes, &mut head)?, + boolean_value_offset: read_u32_from_bytes(bytes, &mut head)?, + }; + if list.file_type != StorageFileType::FlagVal as u8 { + return Err(AconfigStorageError::BytesParseFail(anyhow!( + "binary file is not a flag value file" + ))); + } + Ok(list) + } +} + +/// Flag value list struct +#[derive(PartialEq)] +pub struct FlagValueList { + pub header: FlagValueHeader, + pub booleans: Vec<bool>, +} + +/// Implement debug print trait for flag value +impl fmt::Debug for FlagValueList { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Header:")?; + write!(f, "{:?}", self.header)?; + writeln!(f, "Values:")?; + writeln!(f, "{:?}", self.booleans)?; + Ok(()) + } +} + +impl FlagValueList { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + [ + self.header.into_bytes(), + self.booleans.iter().map(|&v| u8::from(v).to_le_bytes()).collect::<Vec<_>>().concat(), + ] + .concat() + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let header = FlagValueHeader::from_bytes(bytes)?; + let num_flags = header.num_flags; + let mut head = header.into_bytes().len(); + let booleans = + (0..num_flags).map(|_| read_u8_from_bytes(bytes, &mut head).unwrap() == 1).collect(); + let list = Self { header, booleans }; + Ok(list) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::create_test_flag_value_list; + + #[test] + // this test point locks down the value list serialization + fn test_serialization() { + let flag_value_list = create_test_flag_value_list(); + + let header: &FlagValueHeader = &flag_value_list.header; + let reinterpreted_header = FlagValueHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let flag_value_bytes = flag_value_list.into_bytes(); + let reinterpreted_value_list = FlagValueList::from_bytes(&flag_value_bytes); + assert!(reinterpreted_value_list.is_ok()); + assert_eq!(&flag_value_list, &reinterpreted_value_list.unwrap()); + assert_eq!(flag_value_bytes.len() as u32, header.file_size); + } + + #[test] + // this test point locks down that version number should be at the top of serialized + // bytes + fn test_version_number() { + let flag_value_list = create_test_flag_value_list(); + let bytes = &flag_value_list.into_bytes(); + let mut head = 0; + let version = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version, 1); + } + + #[test] + // this test point locks down file type check + fn test_file_type_check() { + let mut flag_value_list = create_test_flag_value_list(); + flag_value_list.header.file_type = 123u8; + let error = FlagValueList::from_bytes(&flag_value_list.into_bytes()).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!("BytesParseFail(binary file is not a flag value file)") + ); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/lib.rs b/tools/aconfig/aconfig_storage_file/src/lib.rs new file mode 100644 index 0000000000..80602bbfb0 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/lib.rs @@ -0,0 +1,625 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aconfig_storage_file` is a crate that defines aconfig storage file format, it +//! also includes apis to read flags from storage files. It provides three apis to +//! interface with storage files: +//! +//! 1, function to get package flag value start offset +//! pub fn get_package_offset(container: &str, package: &str) -> `Result<Option<PackageOffset>>>` +//! +//! 2, function to get flag offset within a specific package +//! pub fn get_flag_offset(container: &str, package_id: u32, flag: &str) -> `Result<Option<u16>>>` +//! +//! 3, function to get the actual flag value given the global offset (combined package and +//! flag offset). +//! pub fn get_boolean_flag_value(container: &str, offset: u32) -> `Result<bool>` +//! +//! Note these are low level apis that are expected to be only used in auto generated flag +//! apis. DO NOT DIRECTLY USE THESE APIS IN YOUR SOURCE CODE. For auto generated flag apis +//! please refer to the g3doc go/android-flags + +pub mod flag_info; +pub mod flag_table; +pub mod flag_value; +pub mod package_table; +pub mod protos; +pub mod test_utils; + +use anyhow::anyhow; +use std::cmp::Ordering; +use std::collections::hash_map::DefaultHasher; +use std::fs::File; +use std::hash::{Hash, Hasher}; +use std::io::Read; + +pub use crate::flag_info::{FlagInfoBit, FlagInfoHeader, FlagInfoList, FlagInfoNode}; +pub use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode}; +pub use crate::flag_value::{FlagValueHeader, FlagValueList}; +pub use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode}; + +use crate::AconfigStorageError::{ + BytesParseFail, HashTableSizeLimit, InvalidFlagValueType, InvalidStoredFlagType, +}; + +/// Storage file version +pub const FILE_VERSION: u32 = 1; + +/// Good hash table prime number +pub(crate) const HASH_PRIMES: [u32; 29] = [ + 7, 17, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241, + 786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611, + 402653189, 805306457, 1610612741, +]; + +/// Storage file type enum +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageFileType { + PackageMap = 0, + FlagMap = 1, + FlagVal = 2, + FlagInfo = 3, +} + +impl TryFrom<&str> for StorageFileType { + type Error = anyhow::Error; + + fn try_from(value: &str) -> std::result::Result<Self, Self::Error> { + match value { + "package_map" => Ok(Self::PackageMap), + "flag_map" => Ok(Self::FlagMap), + "flag_val" => Ok(Self::FlagVal), + "flag_info" => Ok(Self::FlagInfo), + _ => Err(anyhow!( + "Invalid storage file type, valid types are package_map|flag_map|flag_val|flag_info" + )), + } + } +} + +impl TryFrom<u8> for StorageFileType { + type Error = anyhow::Error; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + match value { + x if x == Self::PackageMap as u8 => Ok(Self::PackageMap), + x if x == Self::FlagMap as u8 => Ok(Self::FlagMap), + x if x == Self::FlagVal as u8 => Ok(Self::FlagVal), + x if x == Self::FlagInfo as u8 => Ok(Self::FlagInfo), + _ => Err(anyhow!("Invalid storage file type")), + } + } +} + +/// Flag type enum as stored by storage file +/// ONLY APPEND, NEVER REMOVE FOR BACKWARD COMPATIBILITY. THE MAX IS U16. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoredFlagType { + ReadWriteBoolean = 0, + ReadOnlyBoolean = 1, + FixedReadOnlyBoolean = 2, +} + +impl TryFrom<u16> for StoredFlagType { + type Error = AconfigStorageError; + + fn try_from(value: u16) -> Result<Self, Self::Error> { + match value { + x if x == Self::ReadWriteBoolean as u16 => Ok(Self::ReadWriteBoolean), + x if x == Self::ReadOnlyBoolean as u16 => Ok(Self::ReadOnlyBoolean), + x if x == Self::FixedReadOnlyBoolean as u16 => Ok(Self::FixedReadOnlyBoolean), + _ => Err(InvalidStoredFlagType(anyhow!("Invalid stored flag type"))), + } + } +} + +/// Flag value type enum, one FlagValueType maps to many StoredFlagType +/// ONLY APPEND, NEVER REMOVE FOR BACKWARD COMPATIBILITY. THE MAX IS U16 +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FlagValueType { + Boolean = 0, +} + +impl TryFrom<StoredFlagType> for FlagValueType { + type Error = AconfigStorageError; + + fn try_from(value: StoredFlagType) -> Result<Self, Self::Error> { + match value { + StoredFlagType::ReadWriteBoolean => Ok(Self::Boolean), + StoredFlagType::ReadOnlyBoolean => Ok(Self::Boolean), + StoredFlagType::FixedReadOnlyBoolean => Ok(Self::Boolean), + } + } +} + +impl TryFrom<u16> for FlagValueType { + type Error = AconfigStorageError; + + fn try_from(value: u16) -> Result<Self, Self::Error> { + match value { + x if x == Self::Boolean as u16 => Ok(Self::Boolean), + _ => Err(InvalidFlagValueType(anyhow!("Invalid flag value type"))), + } + } +} + +/// Storage query api error +#[non_exhaustive] +#[derive(thiserror::Error, Debug)] +pub enum AconfigStorageError { + #[error("failed to read the file")] + FileReadFail(#[source] anyhow::Error), + + #[error("fail to parse protobuf")] + ProtobufParseFail(#[source] anyhow::Error), + + #[error("storage files not found for this container")] + StorageFileNotFound(#[source] anyhow::Error), + + #[error("fail to map storage file")] + MapFileFail(#[source] anyhow::Error), + + #[error("fail to get mapped file")] + ObtainMappedFileFail(#[source] anyhow::Error), + + #[error("fail to flush mapped storage file")] + MapFlushFail(#[source] anyhow::Error), + + #[error("number of items in hash table exceed limit")] + HashTableSizeLimit(#[source] anyhow::Error), + + #[error("failed to parse bytes into data")] + BytesParseFail(#[source] anyhow::Error), + + #[error("cannot parse storage files with a higher version")] + HigherStorageFileVersion(#[source] anyhow::Error), + + #[error("invalid storage file byte offset")] + InvalidStorageFileOffset(#[source] anyhow::Error), + + #[error("failed to create file")] + FileCreationFail(#[source] anyhow::Error), + + #[error("invalid stored flag type")] + InvalidStoredFlagType(#[source] anyhow::Error), + + #[error("invalid flag value type")] + InvalidFlagValueType(#[source] anyhow::Error), +} + +/// Get the right hash table size given number of entries in the table. Use a +/// load factor of 0.5 for performance. +pub fn get_table_size(entries: u32) -> Result<u32, AconfigStorageError> { + HASH_PRIMES + .iter() + .find(|&&num| num >= 2 * entries) + .copied() + .ok_or(HashTableSizeLimit(anyhow!("Number of items in a hash table exceeds limit"))) +} + +/// Get the corresponding bucket index given the key and number of buckets +pub(crate) fn get_bucket_index<T: Hash>(val: &T, num_buckets: u32) -> u32 { + let mut s = DefaultHasher::new(); + val.hash(&mut s); + (s.finish() % num_buckets as u64) as u32 +} + +/// Read and parse bytes as u8 +pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result<u8, AconfigStorageError> { + let val = + u8::from_le_bytes(buf[*head..*head + 1].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u8 from bytes: {}", errmsg)) + })?); + *head += 1; + Ok(val) +} + +/// Read and parse bytes as u16 +pub(crate) fn read_u16_from_bytes( + buf: &[u8], + head: &mut usize, +) -> Result<u16, AconfigStorageError> { + let val = + u16::from_le_bytes(buf[*head..*head + 2].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u16 from bytes: {}", errmsg)) + })?); + *head += 2; + Ok(val) +} + +/// Read and parse bytes as u32 +pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result<u32, AconfigStorageError> { + let val = + u32::from_le_bytes(buf[*head..*head + 4].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u32 from bytes: {}", errmsg)) + })?); + *head += 4; + Ok(val) +} + +/// Read and parse bytes as string +pub(crate) fn read_str_from_bytes( + buf: &[u8], + head: &mut usize, +) -> Result<String, AconfigStorageError> { + let num_bytes = read_u32_from_bytes(buf, head)? as usize; + let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec()) + .map_err(|errmsg| BytesParseFail(anyhow!("fail to parse string from bytes: {}", errmsg)))?; + *head += num_bytes; + Ok(val) +} + +/// Read in storage file as bytes +pub fn read_file_to_bytes(file_path: &str) -> Result<Vec<u8>, AconfigStorageError> { + let mut file = File::open(file_path).map_err(|errmsg| { + AconfigStorageError::FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)) + })?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).map_err(|errmsg| { + AconfigStorageError::FileReadFail(anyhow!( + "Failed to read bytes from file {}: {}", + file_path, + errmsg + )) + })?; + Ok(buffer) +} + +/// Flag value summary +#[derive(Debug, PartialEq)] +pub struct FlagValueSummary { + pub package_name: String, + pub flag_name: String, + pub flag_value: String, + pub value_type: StoredFlagType, +} + +/// List flag values from storage files +pub fn list_flags( + package_map: &str, + flag_map: &str, + flag_val: &str, +) -> Result<Vec<FlagValueSummary>, AconfigStorageError> { + let package_table = PackageTable::from_bytes(&read_file_to_bytes(package_map)?)?; + let flag_table = FlagTable::from_bytes(&read_file_to_bytes(flag_map)?)?; + let flag_value_list = FlagValueList::from_bytes(&read_file_to_bytes(flag_val)?)?; + + let mut package_info = vec![("", 0); package_table.header.num_packages as usize]; + for node in package_table.nodes.iter() { + package_info[node.package_id as usize] = (&node.package_name, node.boolean_start_index); + } + + let mut flags = Vec::new(); + for node in flag_table.nodes.iter() { + let (package_name, boolean_start_index) = package_info[node.package_id as usize]; + let flag_index = boolean_start_index + node.flag_index as u32; + let flag_value = flag_value_list.booleans[flag_index as usize]; + flags.push(FlagValueSummary { + package_name: String::from(package_name), + flag_name: node.flag_name.clone(), + flag_value: flag_value.to_string(), + value_type: node.flag_type, + }); + } + + flags.sort_by(|v1, v2| match v1.package_name.cmp(&v2.package_name) { + Ordering::Equal => v1.flag_name.cmp(&v2.flag_name), + other => other, + }); + Ok(flags) +} + +/// Flag value and info summary +#[derive(Debug, PartialEq)] +pub struct FlagValueAndInfoSummary { + pub package_name: String, + pub flag_name: String, + pub flag_value: String, + pub value_type: StoredFlagType, + pub is_readwrite: bool, + pub has_server_override: bool, + pub has_local_override: bool, +} + +/// List flag values and info from storage files +pub fn list_flags_with_info( + package_map: &str, + flag_map: &str, + flag_val: &str, + flag_info: &str, +) -> Result<Vec<FlagValueAndInfoSummary>, AconfigStorageError> { + let package_table = PackageTable::from_bytes(&read_file_to_bytes(package_map)?)?; + let flag_table = FlagTable::from_bytes(&read_file_to_bytes(flag_map)?)?; + let flag_value_list = FlagValueList::from_bytes(&read_file_to_bytes(flag_val)?)?; + let flag_info = FlagInfoList::from_bytes(&read_file_to_bytes(flag_info)?)?; + + let mut package_info = vec![("", 0); package_table.header.num_packages as usize]; + for node in package_table.nodes.iter() { + package_info[node.package_id as usize] = (&node.package_name, node.boolean_start_index); + } + + let mut flags = Vec::new(); + for node in flag_table.nodes.iter() { + let (package_name, boolean_start_index) = package_info[node.package_id as usize]; + let flag_index = boolean_start_index + node.flag_index as u32; + let flag_value = flag_value_list.booleans[flag_index as usize]; + let flag_attribute = flag_info.nodes[flag_index as usize].attributes; + flags.push(FlagValueAndInfoSummary { + package_name: String::from(package_name), + flag_name: node.flag_name.clone(), + flag_value: flag_value.to_string(), + value_type: node.flag_type, + is_readwrite: flag_attribute & (FlagInfoBit::IsReadWrite as u8) != 0, + has_server_override: flag_attribute & (FlagInfoBit::HasServerOverride as u8) != 0, + has_local_override: flag_attribute & (FlagInfoBit::HasLocalOverride as u8) != 0, + }); + } + + flags.sort_by(|v1, v2| match v1.package_name.cmp(&v2.package_name) { + Ordering::Equal => v1.flag_name.cmp(&v2.flag_name), + other => other, + }); + Ok(flags) +} + +// *************************************** // +// CC INTERLOP +// *************************************** // + +// Exported rust data structure and methods, c++ code will be generated +#[cxx::bridge] +mod ffi { + /// flag value and info summary cxx return + pub struct FlagValueAndInfoSummaryCXX { + pub package_name: String, + pub flag_name: String, + pub flag_value: String, + pub value_type: String, + pub is_readwrite: bool, + pub has_server_override: bool, + pub has_local_override: bool, + } + + /// list flag result cxx return + pub struct ListFlagValueAndInfoResultCXX { + pub query_success: bool, + pub error_message: String, + pub flags: Vec<FlagValueAndInfoSummaryCXX>, + } + + // Rust export to c++ + extern "Rust" { + pub fn list_flags_with_info_cxx( + package_map: &str, + flag_map: &str, + flag_val: &str, + flag_info: &str, + ) -> ListFlagValueAndInfoResultCXX; + } +} + +/// implement flag value and info summary cxx return type +impl ffi::FlagValueAndInfoSummaryCXX { + pub(crate) fn new(summary: FlagValueAndInfoSummary) -> Self { + Self { + package_name: summary.package_name, + flag_name: summary.flag_name, + flag_value: summary.flag_value, + value_type: format!("{:?}", summary.value_type), + is_readwrite: summary.is_readwrite, + has_server_override: summary.has_server_override, + has_local_override: summary.has_local_override, + } + } +} + +/// implement list flag with info cxx interlop +pub fn list_flags_with_info_cxx( + package_map: &str, + flag_map: &str, + flag_val: &str, + flag_info: &str, +) -> ffi::ListFlagValueAndInfoResultCXX { + match list_flags_with_info(package_map, flag_map, flag_val, flag_info) { + Ok(summary) => ffi::ListFlagValueAndInfoResultCXX { + query_success: true, + error_message: String::new(), + flags: summary.into_iter().map(ffi::FlagValueAndInfoSummaryCXX::new).collect(), + }, + Err(errmsg) => ffi::ListFlagValueAndInfoResultCXX { + query_success: false, + error_message: format!("{:?}", errmsg), + flags: Vec::new(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + create_test_flag_info_list, create_test_flag_table, create_test_flag_value_list, + create_test_package_table, write_bytes_to_temp_file, + }; + + #[test] + // this test point locks down the flag list api + fn test_list_flag() { + let package_table = + write_bytes_to_temp_file(&create_test_package_table().into_bytes()).unwrap(); + let flag_table = write_bytes_to_temp_file(&create_test_flag_table().into_bytes()).unwrap(); + let flag_value_list = + write_bytes_to_temp_file(&create_test_flag_value_list().into_bytes()).unwrap(); + + let package_table_path = package_table.path().display().to_string(); + let flag_table_path = flag_table.path().display().to_string(); + let flag_value_list_path = flag_value_list.path().display().to_string(); + + let flags = + list_flags(&package_table_path, &flag_table_path, &flag_value_list_path).unwrap(); + let expected = [ + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_1"), + flag_name: String::from("disabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("false"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_1"), + flag_name: String::from("enabled_ro"), + value_type: StoredFlagType::ReadOnlyBoolean, + flag_value: String::from("true"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_1"), + flag_name: String::from("enabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("true"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_2"), + flag_name: String::from("disabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("false"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_2"), + flag_name: String::from("enabled_fixed_ro"), + value_type: StoredFlagType::FixedReadOnlyBoolean, + flag_value: String::from("true"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_2"), + flag_name: String::from("enabled_ro"), + value_type: StoredFlagType::ReadOnlyBoolean, + flag_value: String::from("true"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_4"), + flag_name: String::from("enabled_fixed_ro"), + value_type: StoredFlagType::FixedReadOnlyBoolean, + flag_value: String::from("true"), + }, + FlagValueSummary { + package_name: String::from("com.android.aconfig.storage.test_4"), + flag_name: String::from("enabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("true"), + }, + ]; + assert_eq!(flags, expected); + } + + #[test] + // this test point locks down the flag list with info api + fn test_list_flag_with_info() { + let package_table = + write_bytes_to_temp_file(&create_test_package_table().into_bytes()).unwrap(); + let flag_table = write_bytes_to_temp_file(&create_test_flag_table().into_bytes()).unwrap(); + let flag_value_list = + write_bytes_to_temp_file(&create_test_flag_value_list().into_bytes()).unwrap(); + let flag_info_list = + write_bytes_to_temp_file(&create_test_flag_info_list().into_bytes()).unwrap(); + + let package_table_path = package_table.path().display().to_string(); + let flag_table_path = flag_table.path().display().to_string(); + let flag_value_list_path = flag_value_list.path().display().to_string(); + let flag_info_list_path = flag_info_list.path().display().to_string(); + + let flags = list_flags_with_info( + &package_table_path, + &flag_table_path, + &flag_value_list_path, + &flag_info_list_path, + ) + .unwrap(); + let expected = [ + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_1"), + flag_name: String::from("disabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("false"), + is_readwrite: true, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_1"), + flag_name: String::from("enabled_ro"), + value_type: StoredFlagType::ReadOnlyBoolean, + flag_value: String::from("true"), + is_readwrite: false, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_1"), + flag_name: String::from("enabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("true"), + is_readwrite: true, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_2"), + flag_name: String::from("disabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("false"), + is_readwrite: true, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_2"), + flag_name: String::from("enabled_fixed_ro"), + value_type: StoredFlagType::FixedReadOnlyBoolean, + flag_value: String::from("true"), + is_readwrite: false, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_2"), + flag_name: String::from("enabled_ro"), + value_type: StoredFlagType::ReadOnlyBoolean, + flag_value: String::from("true"), + is_readwrite: false, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_4"), + flag_name: String::from("enabled_fixed_ro"), + value_type: StoredFlagType::FixedReadOnlyBoolean, + flag_value: String::from("true"), + is_readwrite: false, + has_server_override: false, + has_local_override: false, + }, + FlagValueAndInfoSummary { + package_name: String::from("com.android.aconfig.storage.test_4"), + flag_name: String::from("enabled_rw"), + value_type: StoredFlagType::ReadWriteBoolean, + flag_value: String::from("true"), + is_readwrite: true, + has_server_override: false, + has_local_override: false, + }, + ]; + assert_eq!(flags, expected); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/main.rs b/tools/aconfig/aconfig_storage_file/src/main.rs new file mode 100644 index 0000000000..8b9e38da02 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/main.rs @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aconfig-storage` is a debugging tool to parse storage files + +use aconfig_storage_file::{ + list_flags, list_flags_with_info, read_file_to_bytes, AconfigStorageError, FlagInfoList, + FlagTable, FlagValueList, PackageTable, StorageFileType, +}; + +use clap::{builder::ArgAction, Arg, Command}; + +fn cli() -> Command { + Command::new("aconfig-storage") + .subcommand_required(true) + .subcommand( + Command::new("print") + .arg(Arg::new("file").long("file").required(true).action(ArgAction::Set)) + .arg( + Arg::new("type") + .long("type") + .required(true) + .value_parser(|s: &str| StorageFileType::try_from(s)), + ), + ) + .subcommand( + Command::new("list") + .arg( + Arg::new("package-map") + .long("package-map") + .required(true) + .action(ArgAction::Set), + ) + .arg(Arg::new("flag-map").long("flag-map").required(true).action(ArgAction::Set)) + .arg(Arg::new("flag-val").long("flag-val").required(true).action(ArgAction::Set)) + .arg( + Arg::new("flag-info").long("flag-info").required(false).action(ArgAction::Set), + ), + ) +} + +fn print_storage_file( + file_path: &str, + file_type: &StorageFileType, +) -> Result<(), AconfigStorageError> { + let bytes = read_file_to_bytes(file_path)?; + match file_type { + StorageFileType::PackageMap => { + let package_table = PackageTable::from_bytes(&bytes)?; + println!("{:?}", package_table); + } + StorageFileType::FlagMap => { + let flag_table = FlagTable::from_bytes(&bytes)?; + println!("{:?}", flag_table); + } + StorageFileType::FlagVal => { + let flag_value = FlagValueList::from_bytes(&bytes)?; + println!("{:?}", flag_value); + } + StorageFileType::FlagInfo => { + let flag_info = FlagInfoList::from_bytes(&bytes)?; + println!("{:?}", flag_info); + } + } + Ok(()) +} + +fn main() -> Result<(), AconfigStorageError> { + let matches = cli().get_matches(); + match matches.subcommand() { + Some(("print", sub_matches)) => { + let file_path = sub_matches.get_one::<String>("file").unwrap(); + let file_type = sub_matches.get_one::<StorageFileType>("type").unwrap(); + print_storage_file(file_path, file_type)? + } + Some(("list", sub_matches)) => { + let package_map = sub_matches.get_one::<String>("package-map").unwrap(); + let flag_map = sub_matches.get_one::<String>("flag-map").unwrap(); + let flag_val = sub_matches.get_one::<String>("flag-val").unwrap(); + let flag_info = sub_matches.get_one::<String>("flag-info"); + match flag_info { + Some(info_file) => { + let flags = list_flags_with_info(package_map, flag_map, flag_val, info_file)?; + for flag in flags.iter() { + println!( + "{} {} {} {:?} IsReadWrite: {}, HasServerOverride: {}, HasLocalOverride: {}", + flag.package_name, flag.flag_name, flag.flag_value, flag.value_type, + flag.is_readwrite, flag.has_server_override, flag.has_local_override, + ); + } + } + None => { + let flags = list_flags(package_map, flag_map, flag_val)?; + for flag in flags.iter() { + println!( + "{} {} {} {:?}", + flag.package_name, flag.flag_name, flag.flag_value, flag.value_type, + ); + } + } + } + } + _ => unreachable!(), + } + Ok(()) +} diff --git a/tools/aconfig/aconfig_storage_file/src/package_table.rs b/tools/aconfig/aconfig_storage_file/src/package_table.rs new file mode 100644 index 0000000000..b734972f33 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/package_table.rs @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! package table module defines the package table file format and methods for serialization +//! and deserialization + +use crate::{get_bucket_index, read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; +use crate::{AconfigStorageError, StorageFileType}; +use anyhow::anyhow; +use std::fmt; + +/// Package table header struct +#[derive(PartialEq)] +pub struct PackageTableHeader { + pub version: u32, + pub container: String, + pub file_type: u8, + pub file_size: u32, + pub num_packages: u32, + pub bucket_offset: u32, + pub node_offset: u32, +} + +/// Implement debug print trait for header +impl fmt::Debug for PackageTableHeader { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "Version: {}, Container: {}, File Type: {:?}, File Size: {}", + self.version, + self.container, + StorageFileType::try_from(self.file_type), + self.file_size + )?; + writeln!( + f, + "Num of Packages: {}, Bucket Offset:{}, Node Offset: {}", + self.num_packages, self.bucket_offset, self.node_offset + )?; + Ok(()) + } +} + +impl PackageTableHeader { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_type.to_le_bytes()); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_packages.to_le_bytes()); + result.extend_from_slice(&self.bucket_offset.to_le_bytes()); + result.extend_from_slice(&self.node_offset.to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let table = Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_type: read_u8_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_packages: read_u32_from_bytes(bytes, &mut head)?, + bucket_offset: read_u32_from_bytes(bytes, &mut head)?, + node_offset: read_u32_from_bytes(bytes, &mut head)?, + }; + if table.file_type != StorageFileType::PackageMap as u8 { + return Err(AconfigStorageError::BytesParseFail(anyhow!( + "binary file is not a package map" + ))); + } + Ok(table) + } +} + +/// Package table node struct +#[derive(PartialEq)] +pub struct PackageTableNode { + pub package_name: String, + pub package_id: u32, + // The index of the first boolean flag in this aconfig package among all boolean + // flags in this container. + pub boolean_start_index: u32, + pub next_offset: Option<u32>, +} + +/// Implement debug print trait for node +impl fmt::Debug for PackageTableNode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!( + f, + "Package: {}, Id: {}, Boolean flag start index: {}, Next: {:?}", + self.package_name, self.package_id, self.boolean_start_index, self.next_offset + )?; + Ok(()) + } +} + +impl PackageTableNode { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + let mut result = Vec::new(); + let name_bytes = self.package_name.as_bytes(); + result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(name_bytes); + result.extend_from_slice(&self.package_id.to_le_bytes()); + result.extend_from_slice(&self.boolean_start_index.to_le_bytes()); + result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); + result + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let node = Self { + package_name: read_str_from_bytes(bytes, &mut head)?, + package_id: read_u32_from_bytes(bytes, &mut head)?, + boolean_start_index: read_u32_from_bytes(bytes, &mut head)?, + next_offset: match read_u32_from_bytes(bytes, &mut head)? { + 0 => None, + val => Some(val), + }, + }; + Ok(node) + } + + /// Get the bucket index for a package table node, defined it here so the + /// construction side (aconfig binary) and consumption side (flag read lib) + /// use the same method of hashing + pub fn find_bucket_index(package: &str, num_buckets: u32) -> u32 { + get_bucket_index(&package, num_buckets) + } +} + +/// Package table struct +#[derive(PartialEq)] +pub struct PackageTable { + pub header: PackageTableHeader, + pub buckets: Vec<Option<u32>>, + pub nodes: Vec<PackageTableNode>, +} + +/// Implement debug print trait for package table +impl fmt::Debug for PackageTable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Header:")?; + write!(f, "{:?}", self.header)?; + writeln!(f, "Buckets:")?; + writeln!(f, "{:?}", self.buckets)?; + writeln!(f, "Nodes:")?; + for node in self.nodes.iter() { + write!(f, "{:?}", node)?; + } + Ok(()) + } +} + +impl PackageTable { + /// Serialize to bytes + pub fn into_bytes(&self) -> Vec<u8> { + [ + self.header.into_bytes(), + self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::<Vec<_>>().concat(), + self.nodes.iter().map(|v| v.into_bytes()).collect::<Vec<_>>().concat(), + ] + .concat() + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let header = PackageTableHeader::from_bytes(bytes)?; + let num_packages = header.num_packages; + let num_buckets = crate::get_table_size(num_packages)?; + let mut head = header.into_bytes().len(); + let buckets = (0..num_buckets) + .map(|_| match read_u32_from_bytes(bytes, &mut head).unwrap() { + 0 => None, + val => Some(val), + }) + .collect(); + let nodes = (0..num_packages) + .map(|_| { + let node = PackageTableNode::from_bytes(&bytes[head..])?; + head += node.into_bytes().len(); + Ok(node) + }) + .collect::<Result<Vec<_>, AconfigStorageError>>() + .map_err(|errmsg| { + AconfigStorageError::BytesParseFail(anyhow!( + "fail to parse package table: {}", + errmsg + )) + })?; + + let table = Self { header, buckets, nodes }; + Ok(table) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::create_test_package_table; + + #[test] + // this test point locks down the table serialization + fn test_serialization() { + let package_table = create_test_package_table(); + let header: &PackageTableHeader = &package_table.header; + let reinterpreted_header = PackageTableHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec<PackageTableNode> = &package_table.nodes; + for node in nodes.iter() { + let reinterpreted_node = PackageTableNode::from_bytes(&node.into_bytes()).unwrap(); + assert_eq!(node, &reinterpreted_node); + } + + let package_table_bytes = package_table.into_bytes(); + let reinterpreted_table = PackageTable::from_bytes(&package_table_bytes); + assert!(reinterpreted_table.is_ok()); + assert_eq!(&package_table, &reinterpreted_table.unwrap()); + assert_eq!(package_table_bytes.len() as u32, header.file_size); + } + + #[test] + // this test point locks down that version number should be at the top of serialized + // bytes + fn test_version_number() { + let package_table = create_test_package_table(); + let bytes = &package_table.into_bytes(); + let mut head = 0; + let version = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version, 1); + } + + #[test] + // this test point locks down file type check + fn test_file_type_check() { + let mut package_table = create_test_package_table(); + package_table.header.file_type = 123u8; + let error = PackageTable::from_bytes(&package_table.into_bytes()).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!("BytesParseFail(binary file is not a package map)") + ); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/protos.rs b/tools/aconfig/aconfig_storage_file/src/protos.rs new file mode 100644 index 0000000000..8b862057e7 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/protos.rs @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// When building with the Android tool-chain +// +// - an external crate `aconfig_storage_metadata_protos` will be generated +// - the feature "cargo" will be disabled +// +// When building with cargo +// +// - a local sub-module will be generated in OUT_DIR and included in this file +// - the feature "cargo" will be enabled +// +// This module hides these differences from the rest of the codebase. + +// ---- When building with the Android tool-chain ---- +#[cfg(not(feature = "cargo"))] +mod auto_generated { + pub use aconfig_storage_protos::aconfig_storage_metadata as ProtoStorage; + pub use ProtoStorage::Storage_file_info as ProtoStorageFileInfo; + pub use ProtoStorage::Storage_files as ProtoStorageFiles; +} + +// ---- When building with cargo ---- +#[cfg(feature = "cargo")] +mod auto_generated { + // include! statements should be avoided (because they import file contents verbatim), but + // because this is only used during local development, and only if using cargo instead of the + // Android tool-chain, we allow it + include!(concat!(env!("OUT_DIR"), "/aconfig_storage_protos/mod.rs")); + pub use aconfig_storage_metadata::Storage_file_info as ProtoStorageFileInfo; + pub use aconfig_storage_metadata::Storage_files as ProtoStorageFiles; +} + +// ---- Common for both the Android tool-chain and cargo ---- +pub use auto_generated::*; + +use anyhow::Result; +use protobuf::Message; +use std::io::Write; +use tempfile::NamedTempFile; + +pub mod storage_record_pb { + use super::*; + use anyhow::ensure; + + pub fn try_from_binary_proto(bytes: &[u8]) -> Result<ProtoStorageFiles> { + let message: ProtoStorageFiles = protobuf::Message::parse_from_bytes(bytes)?; + verify_fields(&message)?; + Ok(message) + } + + pub fn verify_fields(storage_files: &ProtoStorageFiles) -> Result<()> { + for storage_file_info in storage_files.files.iter() { + ensure!( + !storage_file_info.package_map().is_empty(), + "invalid storage file record: missing package map file for container {}", + storage_file_info.container() + ); + ensure!( + !storage_file_info.flag_map().is_empty(), + "invalid storage file record: missing flag map file for container {}", + storage_file_info.container() + ); + ensure!( + !storage_file_info.flag_val().is_empty(), + "invalid storage file record: missing flag val file for container {}", + storage_file_info.container() + ); + } + Ok(()) + } + + pub fn get_binary_proto_from_text_proto(text_proto: &str) -> Result<Vec<u8>> { + let storage_files: ProtoStorageFiles = protobuf::text_format::parse_from_str(text_proto)?; + let mut binary_proto = Vec::new(); + storage_files.write_to_vec(&mut binary_proto)?; + Ok(binary_proto) + } + + pub fn write_proto_to_temp_file(text_proto: &str) -> Result<NamedTempFile> { + let bytes = get_binary_proto_from_text_proto(text_proto).unwrap(); + let mut file = NamedTempFile::new()?; + let _ = file.write_all(&bytes); + Ok(file) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_storage_record_pb() { + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "/system/etc/package.map" + flag_map: "/system/etc/flag.map" + flag_val: "/metadata/aconfig/system.val" + timestamp: 12345 +} +files { + version: 1 + container: "product" + package_map: "/product/etc/package.map" + flag_map: "/product/etc/flag.map" + flag_val: "/metadata/aconfig/product.val" + timestamp: 54321 +} +"#; + let binary_proto_bytes = + storage_record_pb::get_binary_proto_from_text_proto(text_proto).unwrap(); + let storage_files = storage_record_pb::try_from_binary_proto(&binary_proto_bytes).unwrap(); + assert_eq!(storage_files.files.len(), 2); + let system_file = &storage_files.files[0]; + assert_eq!(system_file.version(), 0); + assert_eq!(system_file.container(), "system"); + assert_eq!(system_file.package_map(), "/system/etc/package.map"); + assert_eq!(system_file.flag_map(), "/system/etc/flag.map"); + assert_eq!(system_file.flag_val(), "/metadata/aconfig/system.val"); + assert_eq!(system_file.timestamp(), 12345); + let product_file = &storage_files.files[1]; + assert_eq!(product_file.version(), 1); + assert_eq!(product_file.container(), "product"); + assert_eq!(product_file.package_map(), "/product/etc/package.map"); + assert_eq!(product_file.flag_map(), "/product/etc/flag.map"); + assert_eq!(product_file.flag_val(), "/metadata/aconfig/product.val"); + assert_eq!(product_file.timestamp(), 54321); + } + + #[test] + fn test_parse_invalid_storage_record_pb() { + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "" + flag_map: "/system/etc/flag.map" + flag_val: "/metadata/aconfig/system.val" + timestamp: 12345 +} +"#; + let binary_proto_bytes = + storage_record_pb::get_binary_proto_from_text_proto(text_proto).unwrap(); + let err = storage_record_pb::try_from_binary_proto(&binary_proto_bytes).unwrap_err(); + assert_eq!( + format!("{:?}", err), + "invalid storage file record: missing package map file for container system" + ); + + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "/system/etc/package.map" + flag_map: "" + flag_val: "/metadata/aconfig/system.val" + timestamp: 12345 +} +"#; + let binary_proto_bytes = + storage_record_pb::get_binary_proto_from_text_proto(text_proto).unwrap(); + let err = storage_record_pb::try_from_binary_proto(&binary_proto_bytes).unwrap_err(); + assert_eq!( + format!("{:?}", err), + "invalid storage file record: missing flag map file for container system" + ); + + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "/system/etc/package.map" + flag_map: "/system/etc/flag.map" + flag_val: "" + timestamp: 12345 +} +"#; + let binary_proto_bytes = + storage_record_pb::get_binary_proto_from_text_proto(text_proto).unwrap(); + let err = storage_record_pb::try_from_binary_proto(&binary_proto_bytes).unwrap_err(); + assert_eq!( + format!("{:?}", err), + "invalid storage file record: missing flag val file for container system" + ); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/test_utils.rs b/tools/aconfig/aconfig_storage_file/src/test_utils.rs new file mode 100644 index 0000000000..106666c47f --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/test_utils.rs @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::flag_info::{FlagInfoHeader, FlagInfoList, FlagInfoNode}; +use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode}; +use crate::flag_value::{FlagValueHeader, FlagValueList}; +use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode}; +use crate::{AconfigStorageError, StorageFileType, StoredFlagType}; + +use anyhow::anyhow; +use std::io::Write; +use tempfile::NamedTempFile; + +pub fn create_test_package_table() -> PackageTable { + let header = PackageTableHeader { + version: 1, + container: String::from("mockup"), + file_type: StorageFileType::PackageMap as u8, + file_size: 209, + num_packages: 3, + bucket_offset: 31, + node_offset: 59, + }; + let buckets: Vec<Option<u32>> = vec![Some(59), None, None, Some(109), None, None, None]; + let first_node = PackageTableNode { + package_name: String::from("com.android.aconfig.storage.test_2"), + package_id: 1, + boolean_start_index: 3, + next_offset: None, + }; + let second_node = PackageTableNode { + package_name: String::from("com.android.aconfig.storage.test_1"), + package_id: 0, + boolean_start_index: 0, + next_offset: Some(159), + }; + let third_node = PackageTableNode { + package_name: String::from("com.android.aconfig.storage.test_4"), + package_id: 2, + boolean_start_index: 6, + next_offset: None, + }; + let nodes = vec![first_node, second_node, third_node]; + PackageTable { header, buckets, nodes } +} + +impl FlagTableNode { + // create test baseline, syntactic sugar + fn new_expected( + package_id: u32, + flag_name: &str, + flag_type: u16, + flag_index: u16, + next_offset: Option<u32>, + ) -> Self { + Self { + package_id, + flag_name: flag_name.to_string(), + flag_type: StoredFlagType::try_from(flag_type).unwrap(), + flag_index, + next_offset, + } + } +} + +pub fn create_test_flag_table() -> FlagTable { + let header = FlagTableHeader { + version: 1, + container: String::from("mockup"), + file_type: StorageFileType::FlagMap as u8, + file_size: 321, + num_flags: 8, + bucket_offset: 31, + node_offset: 99, + }; + let buckets: Vec<Option<u32>> = vec![ + Some(99), + Some(125), + None, + None, + None, + None, + Some(177), + Some(204), + None, + Some(262), + None, + None, + None, + None, + None, + Some(294), + None, + ]; + let nodes = vec![ + FlagTableNode::new_expected(0, "enabled_ro", 1, 1, None), + FlagTableNode::new_expected(0, "enabled_rw", 0, 2, Some(151)), + FlagTableNode::new_expected(2, "enabled_rw", 0, 1, None), + FlagTableNode::new_expected(1, "disabled_rw", 0, 0, None), + FlagTableNode::new_expected(1, "enabled_fixed_ro", 2, 1, Some(236)), + FlagTableNode::new_expected(1, "enabled_ro", 1, 2, None), + FlagTableNode::new_expected(2, "enabled_fixed_ro", 2, 0, None), + FlagTableNode::new_expected(0, "disabled_rw", 0, 0, None), + ]; + FlagTable { header, buckets, nodes } +} + +pub fn create_test_flag_value_list() -> FlagValueList { + let header = FlagValueHeader { + version: 1, + container: String::from("mockup"), + file_type: StorageFileType::FlagVal as u8, + file_size: 35, + num_flags: 8, + boolean_value_offset: 27, + }; + let booleans: Vec<bool> = vec![false, true, true, false, true, true, true, true]; + FlagValueList { header, booleans } +} + +pub fn create_test_flag_info_list() -> FlagInfoList { + let header = FlagInfoHeader { + version: 1, + container: String::from("mockup"), + file_type: StorageFileType::FlagInfo as u8, + file_size: 35, + num_flags: 8, + boolean_flag_offset: 27, + }; + let is_flag_rw = [true, false, true, true, false, false, false, true]; + let nodes = is_flag_rw.iter().map(|&rw| FlagInfoNode::create(rw)).collect(); + FlagInfoList { header, nodes } +} + +pub fn write_bytes_to_temp_file(bytes: &[u8]) -> Result<NamedTempFile, AconfigStorageError> { + let mut file = NamedTempFile::new().map_err(|_| { + AconfigStorageError::FileCreationFail(anyhow!("Failed to create temp file")) + })?; + let _ = file.write_all(&bytes); + Ok(file) +} diff --git a/tools/aconfig/aconfig_storage_file/tests/Android.bp b/tools/aconfig/aconfig_storage_file/tests/Android.bp new file mode 100644 index 0000000000..26b7800877 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/Android.bp @@ -0,0 +1,23 @@ + +cc_test { + name: "aconfig_storage_file.test.cpp", + team: "trendy_team_android_core_experiments", + srcs: [ + "storage_file_test.cpp", + ], + static_libs: [ + "libgmock", + "libaconfig_storage_file_cc", + "libbase", + ], + data: [ + "package.map", + "flag.map", + "flag.val", + "flag.info", + ], + test_suites: [ + "device-tests", + "general-tests", + ], +} diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.info b/tools/aconfig/aconfig_storage_file/tests/flag.info Binary files differnew file mode 100644 index 0000000000..6223edf369 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/flag.info diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.map b/tools/aconfig/aconfig_storage_file/tests/flag.map Binary files differnew file mode 100644 index 0000000000..e868f53d7e --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/flag.map diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.val b/tools/aconfig/aconfig_storage_file/tests/flag.val Binary files differnew file mode 100644 index 0000000000..ed203d4d13 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/flag.val diff --git a/tools/aconfig/aconfig_storage_file/tests/package.map b/tools/aconfig/aconfig_storage_file/tests/package.map Binary files differnew file mode 100644 index 0000000000..6c46a0339c --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/package.map diff --git a/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp b/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp new file mode 100644 index 0000000000..eccbca53d5 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> +#include <android-base/file.h> +#include <android-base/result.h> +#include <gtest/gtest.h> +#include "aconfig_storage/aconfig_storage_file.hpp" + +using namespace android::base; +using namespace aconfig_storage; + + +void verify_flag(const FlagValueAndInfoSummary& flag, + const std::string& package_name, + const std::string& flag_name, + const std::string& flag_val, + const std::string& value_type, + bool is_readwrite, + bool has_server_override, + bool has_local_override) { + ASSERT_EQ(flag.package_name, package_name); + ASSERT_EQ(flag.flag_name, flag_name); + ASSERT_EQ(flag.flag_value, flag_val); + ASSERT_EQ(flag.value_type, value_type); + ASSERT_EQ(flag.is_readwrite, is_readwrite); + ASSERT_EQ(flag.has_server_override, has_server_override); + ASSERT_EQ(flag.has_local_override, has_local_override); +} + +TEST(AconfigStorageFileTest, test_list_flag_with_info) { + auto const test_dir = GetExecutableDirectory(); + auto const package_map = test_dir + "/package.map"; + auto const flag_map = test_dir + "/flag.map"; + auto const flag_val = test_dir + "/flag.val"; + auto const flag_info = test_dir + "/flag.info"; + auto flag_list_result = aconfig_storage::list_flags_with_info( + package_map, flag_map, flag_val, flag_info); + ASSERT_TRUE(flag_list_result.ok()); + + auto const& flag_list = *flag_list_result; + ASSERT_EQ(flag_list.size(), 8); + verify_flag(flag_list[0], "com.android.aconfig.storage.test_1", "disabled_rw", + "false", "ReadWriteBoolean", true, false, false); + verify_flag(flag_list[1], "com.android.aconfig.storage.test_1", "enabled_ro", + "true", "ReadOnlyBoolean", false, false, false); + verify_flag(flag_list[2], "com.android.aconfig.storage.test_1", "enabled_rw", + "true", "ReadWriteBoolean", true, false, false); + verify_flag(flag_list[3], "com.android.aconfig.storage.test_2", "disabled_rw", + "false", "ReadWriteBoolean", true, false, false); + verify_flag(flag_list[4], "com.android.aconfig.storage.test_2", "enabled_fixed_ro", + "true", "FixedReadOnlyBoolean", false, false, false); + verify_flag(flag_list[5], "com.android.aconfig.storage.test_2", "enabled_ro", + "true", "ReadOnlyBoolean", false, false, false); + verify_flag(flag_list[6], "com.android.aconfig.storage.test_4", "enabled_fixed_ro", + "true", "FixedReadOnlyBoolean", false, false, false); + verify_flag(flag_list[7], "com.android.aconfig.storage.test_4", "enabled_rw", + "true", "ReadWriteBoolean", true, false, false); +} diff --git a/tools/aconfig/aconfig_storage_read_api/Android.bp b/tools/aconfig/aconfig_storage_read_api/Android.bp new file mode 100644 index 0000000000..db362944c5 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/Android.bp @@ -0,0 +1,119 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "aconfig_storage_read_api.defaults", + edition: "2021", + lints: "none", + srcs: ["src/lib.rs"], + rustlibs: [ + "libanyhow", + "libonce_cell", + "libtempfile", + "libmemmap2", + "libcxx", + "libthiserror", + "libaconfig_storage_file", + ], +} + +rust_library { + name: "libaconfig_storage_read_api", + crate_name: "aconfig_storage_read_api", + host_supported: true, + defaults: ["aconfig_storage_read_api.defaults"], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", +} + +rust_test_host { + name: "aconfig_storage_read_api.test", + test_suites: ["general-tests"], + defaults: ["aconfig_storage_read_api.defaults"], + data: [ + "tests/package.map", + "tests/flag.map", + "tests/flag.val", + "tests/flag.info", + ], +} + +// cxx source codegen from rust api +genrule { + name: "libcxx_aconfig_storage_read_api_bridge_code", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) > $(out)", + srcs: ["src/lib.rs"], + out: ["aconfig_storage/lib.rs.cc"], +} + +// cxx header codegen from rust api +genrule { + name: "libcxx_aconfig_storage_read_api_bridge_header", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) --header > $(out)", + srcs: ["src/lib.rs"], + out: ["aconfig_storage/lib.rs.h"], +} + +// a static cc lib based on generated code +rust_ffi_static { + name: "libaconfig_storage_read_api_cxx_bridge", + crate_name: "aconfig_storage_read_api_cxx_bridge", + host_supported: true, + vendor_available: true, + product_available: true, + defaults: ["aconfig_storage_read_api.defaults"], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", +} + +// flag read api cc interface +cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["aconfig_storage_read_api.cpp"], + generated_headers: [ + "cxx-bridge-header", + "libcxx_aconfig_storage_read_api_bridge_header", + ], + generated_sources: ["libcxx_aconfig_storage_read_api_bridge_code"], + whole_static_libs: ["libaconfig_storage_read_api_cxx_bridge"], + export_include_dirs: ["include"], + host_supported: true, + vendor_available: true, + product_available: true, + static_libs: [ + "libaconfig_storage_protos_cc", + "libprotobuf-cpp-lite", + ], + shared_libs: [ + "liblog", + "libbase", + ], + apex_available: [ + "//apex_available:platform", + "//apex_available:anyapex", + ], + min_sdk_version: "29", + target: { + linux: { + version_script: "libaconfig_storage_read_api_cc.map", + }, + }, + double_loadable: true, +} + +cc_defaults { + name: "aconfig_lib_cc_static_link.defaults", + shared_libs: [ + "libaconfig_storage_read_api_cc", + "liblog", + ], +} diff --git a/tools/aconfig/aconfig_storage_read_api/Cargo.toml b/tools/aconfig/aconfig_storage_read_api/Cargo.toml new file mode 100644 index 0000000000..30a4298826 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "aconfig_storage_read_api" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +memmap2 = "0.8.0" +once_cell = "1.19.0" +tempfile = "3.9.0" +cxx = "1.0" +thiserror = "1.0.56" +aconfig_storage_file = { path = "../aconfig_storage_file" } + +[build-dependencies] +protobuf-codegen = "3.2.0" +cxx-build = "1.0" diff --git a/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp b/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp new file mode 100644 index 0000000000..0aa936a9eb --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp @@ -0,0 +1,213 @@ +#include <android-base/file.h> +#include <android-base/logging.h> +#include <protos/aconfig_storage_metadata.pb.h> + +#include <sys/mman.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "rust/cxx.h" +#include "aconfig_storage/lib.rs.h" +#include "aconfig_storage/aconfig_storage_read_api.hpp" + +using storage_records_pb = android::aconfig_storage_metadata::storage_files; +using storage_record_pb = android::aconfig_storage_metadata::storage_file_info; +using namespace android::base; + +namespace aconfig_storage { + +/// Storage location pb file +static constexpr char kAvailableStorageRecordsPb[] = + "/metadata/aconfig/boot/available_storage_file_records.pb"; + +/// destructor +MappedStorageFile::~MappedStorageFile() { + munmap(file_ptr, file_size); +} + +/// Read aconfig storage records pb file +static Result<storage_records_pb> read_storage_records_pb(std::string const& pb_file) { + auto records = storage_records_pb(); + auto content = std::string(); + if (!ReadFileToString(pb_file, &content)) { + return ErrnoError() << "ReadFileToString failed"; + } + + if (!records.ParseFromString(content)) { + return ErrnoError() << "Unable to parse persistent storage records protobuf"; + } + return records; +} + +/// Get storage file path +static Result<std::string> find_storage_file( + std::string const& pb_file, + std::string const& container, + StorageFileType file_type) { + auto records_pb = read_storage_records_pb(pb_file); + if (!records_pb.ok()) { + return Error() << "Unable to read storage records from " << pb_file + << " : " << records_pb.error(); + } + + for (auto& entry : records_pb->files()) { + if (entry.container() == container) { + switch(file_type) { + case StorageFileType::package_map: + return entry.package_map(); + case StorageFileType::flag_map: + return entry.flag_map(); + case StorageFileType::flag_val: + return entry.flag_val(); + case StorageFileType::flag_info: + return entry.flag_info(); + default: + return Error() << "Invalid file type " << file_type; + } + } + } + + return Error() << "Unable to find storage files for container " << container;; +} + +namespace private_internal_api { + +/// Get mapped file implementation. +Result<MappedStorageFile*> get_mapped_file_impl( + std::string const& pb_file, + std::string const& container, + StorageFileType file_type) { + auto file_result = find_storage_file(pb_file, container, file_type); + if (!file_result.ok()) { + return Error() << file_result.error(); + } + return map_storage_file(*file_result); +} + +} // namespace private internal api + +/// Map a storage file +Result<MappedStorageFile*> map_storage_file(std::string const& file) { + int fd = open(file.c_str(), O_CLOEXEC | O_NOFOLLOW | O_RDONLY); + if (fd == -1) { + return ErrnoError() << "failed to open " << file; + }; + + struct stat fd_stat; + if (fstat(fd, &fd_stat) < 0) { + return ErrnoError() << "fstat failed"; + } + size_t file_size = fd_stat.st_size; + + void* const map_result = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + if (map_result == MAP_FAILED) { + return ErrnoError() << "mmap failed"; + } + + auto mapped_file = new MappedStorageFile(); + mapped_file->file_ptr = map_result; + mapped_file->file_size = file_size; + + return mapped_file; +} + +/// Map from StoredFlagType to FlagValueType +android::base::Result<FlagValueType> map_to_flag_value_type( + StoredFlagType stored_type) { + switch (stored_type) { + case StoredFlagType::ReadWriteBoolean: + case StoredFlagType::ReadOnlyBoolean: + case StoredFlagType::FixedReadOnlyBoolean: + return FlagValueType::Boolean; + default: + return Error() << "Unsupported stored flag type"; + } +} + +/// Get mapped storage file +Result<MappedStorageFile*> get_mapped_file( + std::string const& container, + StorageFileType file_type) { + return private_internal_api::get_mapped_file_impl( + kAvailableStorageRecordsPb, container, file_type); +} + +/// Get storage file version number +Result<uint32_t> get_storage_file_version( + std::string const& file_path) { + auto version_cxx = get_storage_file_version_cxx( + rust::Str(file_path.c_str())); + if (version_cxx.query_success) { + return version_cxx.version_number; + } else { + return Error() << version_cxx.error_message; + } +} + +/// Get package context +Result<PackageReadContext> get_package_read_context( + MappedStorageFile const& file, + std::string const& package) { + auto content = rust::Slice<const uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto context_cxx = get_package_read_context_cxx(content, rust::Str(package.c_str())); + if (context_cxx.query_success) { + auto context = PackageReadContext(); + context.package_exists = context_cxx.package_exists; + context.package_id = context_cxx.package_id; + context.boolean_start_index = context_cxx.boolean_start_index; + return context; + } else { + return Error() << context_cxx.error_message; + } +} + +/// Get flag read context +Result<FlagReadContext> get_flag_read_context( + MappedStorageFile const& file, + uint32_t package_id, + std::string const& flag_name){ + auto content = rust::Slice<const uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto context_cxx = get_flag_read_context_cxx(content, package_id, rust::Str(flag_name.c_str())); + if (context_cxx.query_success) { + auto context = FlagReadContext(); + context.flag_exists = context_cxx.flag_exists; + context.flag_type = static_cast<StoredFlagType>(context_cxx.flag_type); + context.flag_index = context_cxx.flag_index; + return context; + } else { + return Error() << context_cxx.error_message; + } +} + +/// Get boolean flag value +Result<bool> get_boolean_flag_value( + MappedStorageFile const& file, + uint32_t index) { + auto content = rust::Slice<const uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto value_cxx = get_boolean_flag_value_cxx(content, index); + if (value_cxx.query_success) { + return value_cxx.flag_value; + } else { + return Error() << value_cxx.error_message; + } +} + +/// Get boolean flag attribute +Result<uint8_t> get_flag_attribute( + MappedStorageFile const& file, + FlagValueType value_type, + uint32_t index) { + auto content = rust::Slice<const uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto info_cxx = get_flag_attribute_cxx( + content, static_cast<uint16_t>(value_type), index); + if (info_cxx.query_success) { + return info_cxx.flag_attribute; + } else { + return Error() << info_cxx.error_message; + } +} +} // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_read_api/build.rs b/tools/aconfig/aconfig_storage_read_api/build.rs new file mode 100644 index 0000000000..7b1aa53b5f --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/build.rs @@ -0,0 +1,4 @@ +fn main() { + let _ = cxx_build::bridge("src/lib.rs"); + println!("cargo:rerun-if-changed=src/lib.rs"); +} diff --git a/tools/aconfig/aconfig_storage_read_api/include/aconfig_storage/aconfig_storage_read_api.hpp b/tools/aconfig/aconfig_storage_read_api/include/aconfig_storage/aconfig_storage_read_api.hpp new file mode 100644 index 0000000000..e6d75373a9 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/include/aconfig_storage/aconfig_storage_read_api.hpp @@ -0,0 +1,131 @@ +#pragma once + +#include <stdint.h> +#include <string> +#include <android-base/result.h> + +namespace aconfig_storage { + +/// Storage file type enum, to be consistent with the one defined in +/// aconfig_storage_file/src/lib.rs +enum StorageFileType { + package_map, + flag_map, + flag_val, + flag_info +}; + +/// Flag type enum, to be consistent with the one defined in +/// aconfig_storage_file/src/lib.rs +enum StoredFlagType { + ReadWriteBoolean = 0, + ReadOnlyBoolean = 1, + FixedReadOnlyBoolean = 2, +}; + +/// Flag value type enum, to be consistent with the one defined in +/// aconfig_storage_file/src/lib.rs +enum FlagValueType { + Boolean = 0, +}; + +/// Flag info enum, to be consistent with the one defined in +/// aconfig_storage_file/src/flag_info.rs +enum FlagInfoBit { + HasServerOverride = 1<<0, + IsReadWrite = 1<<1, + HasLocalOverride = 1<<2, +}; + +/// Mapped storage file +struct MappedStorageFile { + void* file_ptr; + size_t file_size; + virtual ~MappedStorageFile(); +}; + +/// Package read context query result +struct PackageReadContext { + bool package_exists; + uint32_t package_id; + uint32_t boolean_start_index; +}; + +/// Flag read context query result +struct FlagReadContext { + bool flag_exists; + StoredFlagType flag_type; + uint16_t flag_index; +}; + +/// DO NOT USE APIS IN THE FOLLOWING NAMESPACE DIRECTLY +namespace private_internal_api { + +android::base::Result<MappedStorageFile*> get_mapped_file_impl( + std::string const& pb_file, + std::string const& container, + StorageFileType file_type); + +} // namespace private_internal_api + +/// Map a storage file +android::base::Result<MappedStorageFile*> map_storage_file( + std::string const& file); + + +/// Map from StoredFlagType to FlagValueType +/// \input stored_type: stored flag type in the storage file +/// \returns the flag value type enum +android::base::Result<FlagValueType> map_to_flag_value_type( + StoredFlagType stored_type); + +/// Get mapped storage file +/// \input container: stoarge container name +/// \input file_type: storage file type enum +/// \returns a MappedStorageFileQuery +android::base::Result<MappedStorageFile*> get_mapped_file( + std::string const& container, + StorageFileType file_type); + +/// Get storage file version number +/// \input file_path: the path to the storage file +/// \returns the storage file version +android::base::Result<uint32_t> get_storage_file_version( + std::string const& file_path); + +/// Get package read context +/// \input file: mapped storage file +/// \input package: the flag package name +/// \returns a package read context +android::base::Result<PackageReadContext> get_package_read_context( + MappedStorageFile const& file, + std::string const& package); + +/// Get flag read context +/// \input file: mapped storage file +/// \input package_id: the flag package id obtained from package offset query +/// \input flag_name: flag name +/// \returns the flag read context +android::base::Result<FlagReadContext> get_flag_read_context( + MappedStorageFile const& file, + uint32_t package_id, + std::string const& flag_name); + +/// Get boolean flag value +/// \input file: mapped storage file +/// \input index: the boolean flag index in the file +/// \returns the boolean flag value +android::base::Result<bool> get_boolean_flag_value( + MappedStorageFile const& file, + uint32_t index); + +/// Get boolean flag attribute +/// \input file: mapped storage file +/// \input value_type: flag value type +/// \input index: the boolean flag index in the file +/// \returns the boolean flag attribute +android::base::Result<uint8_t> get_flag_attribute( + MappedStorageFile const& file, + FlagValueType value_type, + uint32_t index); +} // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_read_api/libaconfig_storage_read_api_cc.map b/tools/aconfig/aconfig_storage_read_api/libaconfig_storage_read_api_cc.map new file mode 100644 index 0000000000..7d47e0ba0e --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/libaconfig_storage_read_api_cc.map @@ -0,0 +1,11 @@ +LIBACONFIG_STORAGE_READ_API_CC { + # Export everything in the aconfig_storage namespace. This includes both the + # public API and library internals. + global: + extern "C++" { + aconfig_storage::*; + }; + # Hide everything else. + local: + *; +}; diff --git a/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs b/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs new file mode 100644 index 0000000000..6d03377683 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag value query module defines the flag value file read from mapped bytes + +use crate::{AconfigStorageError, FILE_VERSION}; +use aconfig_storage_file::{flag_info::FlagInfoHeader, read_u8_from_bytes, FlagValueType}; +use anyhow::anyhow; + +/// Get flag attribute bitfield +pub fn find_flag_attribute( + buf: &[u8], + flag_type: FlagValueType, + flag_index: u32, +) -> Result<u8, AconfigStorageError> { + let interpreted_header = FlagInfoHeader::from_bytes(buf)?; + if interpreted_header.version > crate::FILE_VERSION { + return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( + "Cannot read storage file with a higher version of {} with lib version {}", + interpreted_header.version, + FILE_VERSION + ))); + } + + // get byte offset to the flag info + let mut head = match flag_type { + FlagValueType::Boolean => (interpreted_header.boolean_flag_offset + flag_index) as usize, + }; + + if head >= interpreted_header.file_size as usize { + return Err(AconfigStorageError::InvalidStorageFileOffset(anyhow!( + "Flag info offset goes beyond the end of the file." + ))); + } + + let val = read_u8_from_bytes(buf, &mut head)?; + Ok(val) +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_storage_file::{test_utils::create_test_flag_info_list, FlagInfoBit}; + + #[test] + // this test point locks down query if flag has server override + fn test_is_flag_sticky() { + let flag_info_list = create_test_flag_info_list().into_bytes(); + for offset in 0..8 { + let attribute = + find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, offset).unwrap(); + assert_eq!((attribute & FlagInfoBit::HasServerOverride as u8) != 0u8, false); + } + } + + #[test] + // this test point locks down query if flag is readwrite + fn test_is_flag_readwrite() { + let flag_info_list = create_test_flag_info_list().into_bytes(); + let baseline: Vec<bool> = vec![true, false, true, true, false, false, false, true]; + for offset in 0..8 { + let attribute = + find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, offset).unwrap(); + assert_eq!( + (attribute & FlagInfoBit::IsReadWrite as u8) != 0u8, + baseline[offset as usize] + ); + } + } + + #[test] + // this test point locks down query if flag has local override + fn test_flag_has_override() { + let flag_info_list = create_test_flag_info_list().into_bytes(); + for offset in 0..8 { + let attribute = + find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, offset).unwrap(); + assert_eq!((attribute & FlagInfoBit::HasLocalOverride as u8) != 0u8, false); + } + } + + #[test] + // this test point locks down query beyond the end of boolean section + fn test_boolean_out_of_range() { + let flag_info_list = create_test_flag_info_list().into_bytes(); + let error = + find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, 8).unwrap_err(); + assert_eq!( + format!("{:?}", error), + "InvalidStorageFileOffset(Flag info offset goes beyond the end of the file.)" + ); + } + + #[test] + // this test point locks down query error when file has a higher version + fn test_higher_version_storage_file() { + let mut info_list = create_test_flag_info_list(); + info_list.header.version = crate::FILE_VERSION + 1; + let flag_info = info_list.into_bytes(); + let error = find_flag_attribute(&flag_info[..], FlagValueType::Boolean, 4).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!( + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", + crate::FILE_VERSION + 1, + crate::FILE_VERSION + ) + ); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs b/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs new file mode 100644 index 0000000000..a1a4793bc2 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag table query module defines the flag table file read from mapped bytes + +use crate::{AconfigStorageError, FILE_VERSION}; +use aconfig_storage_file::{ + flag_table::FlagTableHeader, flag_table::FlagTableNode, read_u32_from_bytes, StoredFlagType, +}; +use anyhow::anyhow; + +/// Flag table query return +#[derive(PartialEq, Debug)] +pub struct FlagReadContext { + pub flag_type: StoredFlagType, + pub flag_index: u16, +} + +/// Query flag read context: flag type and within package flag index +pub fn find_flag_read_context( + buf: &[u8], + package_id: u32, + flag: &str, +) -> Result<Option<FlagReadContext>, AconfigStorageError> { + let interpreted_header = FlagTableHeader::from_bytes(buf)?; + if interpreted_header.version > crate::FILE_VERSION { + return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( + "Cannot read storage file with a higher version of {} with lib version {}", + interpreted_header.version, + FILE_VERSION + ))); + } + + let num_buckets = (interpreted_header.node_offset - interpreted_header.bucket_offset) / 4; + let bucket_index = FlagTableNode::find_bucket_index(package_id, flag, num_buckets); + + let mut pos = (interpreted_header.bucket_offset + 4 * bucket_index) as usize; + let mut flag_node_offset = read_u32_from_bytes(buf, &mut pos)? as usize; + if flag_node_offset < interpreted_header.node_offset as usize + || flag_node_offset >= interpreted_header.file_size as usize + { + return Ok(None); + } + + loop { + let interpreted_node = FlagTableNode::from_bytes(&buf[flag_node_offset..])?; + if interpreted_node.package_id == package_id && interpreted_node.flag_name == flag { + return Ok(Some(FlagReadContext { + flag_type: interpreted_node.flag_type, + flag_index: interpreted_node.flag_index, + })); + } + match interpreted_node.next_offset { + Some(offset) => flag_node_offset = offset as usize, + None => return Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_storage_file::test_utils::create_test_flag_table; + + #[test] + // this test point locks down table query + fn test_flag_query() { + let flag_table = create_test_flag_table().into_bytes(); + let baseline = vec![ + (0, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 1u16), + (0, "enabled_rw", StoredFlagType::ReadWriteBoolean, 2u16), + (2, "enabled_rw", StoredFlagType::ReadWriteBoolean, 1u16), + (1, "disabled_rw", StoredFlagType::ReadWriteBoolean, 0u16), + (1, "enabled_fixed_ro", StoredFlagType::FixedReadOnlyBoolean, 1u16), + (1, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 2u16), + (2, "enabled_fixed_ro", StoredFlagType::FixedReadOnlyBoolean, 0u16), + (0, "disabled_rw", StoredFlagType::ReadWriteBoolean, 0u16), + ]; + for (package_id, flag_name, flag_type, flag_index) in baseline.into_iter() { + let flag_context = + find_flag_read_context(&flag_table[..], package_id, flag_name).unwrap().unwrap(); + assert_eq!(flag_context.flag_type, flag_type); + assert_eq!(flag_context.flag_index, flag_index); + } + } + + #[test] + // this test point locks down table query of a non exist flag + fn test_not_existed_flag_query() { + let flag_table = create_test_flag_table().into_bytes(); + let flag_context = find_flag_read_context(&flag_table[..], 1, "disabled_fixed_ro").unwrap(); + assert_eq!(flag_context, None); + let flag_context = find_flag_read_context(&flag_table[..], 2, "disabled_rw").unwrap(); + assert_eq!(flag_context, None); + } + + #[test] + // this test point locks down query error when file has a higher version + fn test_higher_version_storage_file() { + let mut table = create_test_flag_table(); + table.header.version = crate::FILE_VERSION + 1; + let flag_table = table.into_bytes(); + let error = find_flag_read_context(&flag_table[..], 0, "enabled_ro").unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!( + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", + crate::FILE_VERSION + 1, + crate::FILE_VERSION + ) + ); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs b/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs new file mode 100644 index 0000000000..9d32a16ac8 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag value query module defines the flag value file read from mapped bytes + +use crate::{AconfigStorageError, FILE_VERSION}; +use aconfig_storage_file::{flag_value::FlagValueHeader, read_u8_from_bytes}; +use anyhow::anyhow; + +/// Query flag value +pub fn find_boolean_flag_value(buf: &[u8], flag_index: u32) -> Result<bool, AconfigStorageError> { + let interpreted_header = FlagValueHeader::from_bytes(buf)?; + if interpreted_header.version > crate::FILE_VERSION { + return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( + "Cannot read storage file with a higher version of {} with lib version {}", + interpreted_header.version, + FILE_VERSION + ))); + } + + // Find byte offset to the flag value, each boolean flag cost one byte to store + let mut head = (interpreted_header.boolean_value_offset + flag_index) as usize; + if head >= interpreted_header.file_size as usize { + return Err(AconfigStorageError::InvalidStorageFileOffset(anyhow!( + "Flag value offset goes beyond the end of the file." + ))); + } + + let val = read_u8_from_bytes(buf, &mut head)?; + Ok(val == 1) +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_storage_file::test_utils::create_test_flag_value_list; + + #[test] + // this test point locks down flag value query + fn test_flag_value_query() { + let flag_value_list = create_test_flag_value_list().into_bytes(); + let baseline: Vec<bool> = vec![false, true, true, false, true, true, true, true]; + for (offset, expected_value) in baseline.into_iter().enumerate() { + let flag_value = find_boolean_flag_value(&flag_value_list[..], offset as u32).unwrap(); + assert_eq!(flag_value, expected_value); + } + } + + #[test] + // this test point locks down query beyond the end of boolean section + fn test_boolean_out_of_range() { + let flag_value_list = create_test_flag_value_list().into_bytes(); + let error = find_boolean_flag_value(&flag_value_list[..], 8).unwrap_err(); + assert_eq!( + format!("{:?}", error), + "InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)" + ); + } + + #[test] + // this test point locks down query error when file has a higher version + fn test_higher_version_storage_file() { + let mut value_list = create_test_flag_value_list(); + value_list.header.version = crate::FILE_VERSION + 1; + let flag_value = value_list.into_bytes(); + let error = find_boolean_flag_value(&flag_value[..], 4).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!( + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", + crate::FILE_VERSION + 1, + crate::FILE_VERSION + ) + ); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/src/lib.rs b/tools/aconfig/aconfig_storage_read_api/src/lib.rs new file mode 100644 index 0000000000..e4192066d5 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/lib.rs @@ -0,0 +1,525 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aconfig_storage_read_api` is a crate that defines read apis to read flags from storage +//! files. It provides four apis to interface with storage files: +//! +//! 1, function to get package read context +//! pub fn get_packager_read_context(container: &str, package: &str) +//! -> `Result<Option<PackageReadContext>>>` +//! +//! 2, function to get flag read context +//! pub fn get_flag_read_context(container: &str, package_id: u32, flag: &str) +//! -> `Result<Option<FlagReadContext>>>` +//! +//! 3, function to get the actual flag value given the global index (combined package and +//! flag index). +//! pub fn get_boolean_flag_value(container: &str, offset: u32) -> `Result<bool>` +//! +//! 4, function to get storage file version without mmapping the file. +//! pub fn get_storage_file_version(file_path: &str) -> Result<u32, AconfigStorageError> +//! +//! Note these are low level apis that are expected to be only used in auto generated flag +//! apis. DO NOT DIRECTLY USE THESE APIS IN YOUR SOURCE CODE. For auto generated flag apis +//! please refer to the g3doc go/android-flags + +pub mod flag_info_query; +pub mod flag_table_query; +pub mod flag_value_query; +pub mod mapped_file; +pub mod package_table_query; + +#[cfg(test)] +mod test_utils; + +pub use aconfig_storage_file::{AconfigStorageError, FlagValueType, StorageFileType}; +pub use flag_table_query::FlagReadContext; +pub use package_table_query::PackageReadContext; + +use aconfig_storage_file::{read_u32_from_bytes, FILE_VERSION}; +use flag_info_query::find_flag_attribute; +use flag_table_query::find_flag_read_context; +use flag_value_query::find_boolean_flag_value; +use package_table_query::find_package_read_context; + +use anyhow::anyhow; +use memmap2::Mmap; +use std::fs::File; +use std::io::Read; + +/// Storage file location pb file +pub const STORAGE_LOCATION_FILE: &str = "/metadata/aconfig/boot/available_storage_file_records.pb"; + +/// Get read only mapped storage files. +/// +/// \input container: the flag package container +/// \input file_type: stoarge file type enum +/// \return a result of read only mapped file +/// +/// # Safety +/// +/// The memory mapped file may have undefined behavior if there are writes to this +/// file after being mapped. Ensure no writes can happen to this file while this +/// mapping stays alive. +pub unsafe fn get_mapped_storage_file( + container: &str, + file_type: StorageFileType, +) -> Result<Mmap, AconfigStorageError> { + unsafe { crate::mapped_file::get_mapped_file(STORAGE_LOCATION_FILE, container, file_type) } +} + +/// Get package read context for a specific package. +/// +/// \input file: mapped package file +/// \input package: package name +/// +/// \return +/// If a package is found, it returns Ok(Some(PackageReadContext)) +/// If a package is not found, it returns Ok(None) +/// If errors out, it returns an Err(errmsg) +pub fn get_package_read_context( + file: &Mmap, + package: &str, +) -> Result<Option<PackageReadContext>, AconfigStorageError> { + find_package_read_context(file, package) +} + +/// Get flag read context for a specific flag. +/// +/// \input file: mapped flag file +/// \input package_id: package id obtained from package mapping file +/// \input flag: flag name +/// +/// \return +/// If a flag is found, it returns Ok(Some(FlagReadContext)) +/// If a flag is not found, it returns Ok(None) +/// If errors out, it returns an Err(errmsg) +pub fn get_flag_read_context( + file: &Mmap, + package_id: u32, + flag: &str, +) -> Result<Option<FlagReadContext>, AconfigStorageError> { + find_flag_read_context(file, package_id, flag) +} + +/// Get the boolean flag value. +/// +/// \input file: mapped flag file +/// \input index: boolean flag offset +/// +/// \return +/// If the provide offset is valid, it returns the boolean flag value, otherwise it +/// returns the error message. +pub fn get_boolean_flag_value(file: &Mmap, index: u32) -> Result<bool, AconfigStorageError> { + find_boolean_flag_value(file, index) +} + +/// Get storage file version number +/// +/// This function would read the first four bytes of the file and interpret it as the +/// version number of the file. There are unit tests in aconfig_storage_file crate to +/// lock down that for all storage files, the first four bytes will be the version +/// number of the storage file +pub fn get_storage_file_version(file_path: &str) -> Result<u32, AconfigStorageError> { + let mut file = File::open(file_path).map_err(|errmsg| { + AconfigStorageError::FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)) + })?; + let mut buffer = [0; 4]; + file.read(&mut buffer).map_err(|errmsg| { + AconfigStorageError::FileReadFail(anyhow!( + "Failed to read 4 bytes from file {}: {}", + file_path, + errmsg + )) + })?; + let mut head = 0; + read_u32_from_bytes(&buffer, &mut head) +} + +/// Get the flag attribute. +/// +/// \input file: mapped flag info file +/// \input flag_type: flag value type +/// \input flag_index: flag index +/// +/// \return +/// If the provide offset is valid, it returns the flag attribute bitfiled, otherwise it +/// returns the error message. +pub fn get_flag_attribute( + file: &Mmap, + flag_type: FlagValueType, + flag_index: u32, +) -> Result<u8, AconfigStorageError> { + find_flag_attribute(file, flag_type, flag_index) +} + +// *************************************** // +// CC INTERLOP +// *************************************** // + +// Exported rust data structure and methods, c++ code will be generated +#[cxx::bridge] +mod ffi { + // Storage file version query return for cc interlop + pub struct VersionNumberQueryCXX { + pub query_success: bool, + pub error_message: String, + pub version_number: u32, + } + + // Package table query return for cc interlop + pub struct PackageReadContextQueryCXX { + pub query_success: bool, + pub error_message: String, + pub package_exists: bool, + pub package_id: u32, + pub boolean_start_index: u32, + } + + // Flag table query return for cc interlop + pub struct FlagReadContextQueryCXX { + pub query_success: bool, + pub error_message: String, + pub flag_exists: bool, + pub flag_type: u16, + pub flag_index: u16, + } + + // Flag value query return for cc interlop + pub struct BooleanFlagValueQueryCXX { + pub query_success: bool, + pub error_message: String, + pub flag_value: bool, + } + + // Flag info query return for cc interlop + pub struct FlagAttributeQueryCXX { + pub query_success: bool, + pub error_message: String, + pub flag_attribute: u8, + } + + // Rust export to c++ + extern "Rust" { + pub fn get_storage_file_version_cxx(file_path: &str) -> VersionNumberQueryCXX; + + pub fn get_package_read_context_cxx( + file: &[u8], + package: &str, + ) -> PackageReadContextQueryCXX; + + pub fn get_flag_read_context_cxx( + file: &[u8], + package_id: u32, + flag: &str, + ) -> FlagReadContextQueryCXX; + + pub fn get_boolean_flag_value_cxx(file: &[u8], offset: u32) -> BooleanFlagValueQueryCXX; + + pub fn get_flag_attribute_cxx( + file: &[u8], + flag_type: u16, + flag_index: u32, + ) -> FlagAttributeQueryCXX; + } +} + +/// Implement the package offset interlop return type, create from actual package offset api return type +impl ffi::PackageReadContextQueryCXX { + pub(crate) fn new( + offset_result: Result<Option<PackageReadContext>, AconfigStorageError>, + ) -> Self { + match offset_result { + Ok(offset_opt) => match offset_opt { + Some(offset) => Self { + query_success: true, + error_message: String::from(""), + package_exists: true, + package_id: offset.package_id, + boolean_start_index: offset.boolean_start_index, + }, + None => Self { + query_success: true, + error_message: String::from(""), + package_exists: false, + package_id: 0, + boolean_start_index: 0, + }, + }, + Err(errmsg) => Self { + query_success: false, + error_message: format!("{:?}", errmsg), + package_exists: false, + package_id: 0, + boolean_start_index: 0, + }, + } + } +} + +/// Implement the flag offset interlop return type, create from actual flag offset api return type +impl ffi::FlagReadContextQueryCXX { + pub(crate) fn new(offset_result: Result<Option<FlagReadContext>, AconfigStorageError>) -> Self { + match offset_result { + Ok(offset_opt) => match offset_opt { + Some(offset) => Self { + query_success: true, + error_message: String::from(""), + flag_exists: true, + flag_type: offset.flag_type as u16, + flag_index: offset.flag_index, + }, + None => Self { + query_success: true, + error_message: String::from(""), + flag_exists: false, + flag_type: 0u16, + flag_index: 0u16, + }, + }, + Err(errmsg) => Self { + query_success: false, + error_message: format!("{:?}", errmsg), + flag_exists: false, + flag_type: 0u16, + flag_index: 0u16, + }, + } + } +} + +/// Implement the flag value interlop return type, create from actual flag value api return type +impl ffi::BooleanFlagValueQueryCXX { + pub(crate) fn new(value_result: Result<bool, AconfigStorageError>) -> Self { + match value_result { + Ok(value) => { + Self { query_success: true, error_message: String::from(""), flag_value: value } + } + Err(errmsg) => Self { + query_success: false, + error_message: format!("{:?}", errmsg), + flag_value: false, + }, + } + } +} + +/// Implement the flag info interlop return type, create from actual flag info api return type +impl ffi::FlagAttributeQueryCXX { + pub(crate) fn new(info_result: Result<u8, AconfigStorageError>) -> Self { + match info_result { + Ok(info) => { + Self { query_success: true, error_message: String::from(""), flag_attribute: info } + } + Err(errmsg) => Self { + query_success: false, + error_message: format!("{:?}", errmsg), + flag_attribute: 0u8, + }, + } + } +} + +/// Implement the storage version number interlop return type, create from actual version number +/// api return type +impl ffi::VersionNumberQueryCXX { + pub(crate) fn new(version_result: Result<u32, AconfigStorageError>) -> Self { + match version_result { + Ok(version) => Self { + query_success: true, + error_message: String::from(""), + version_number: version, + }, + Err(errmsg) => Self { + query_success: false, + error_message: format!("{:?}", errmsg), + version_number: 0, + }, + } + } +} + +/// Get package read context cc interlop +pub fn get_package_read_context_cxx(file: &[u8], package: &str) -> ffi::PackageReadContextQueryCXX { + ffi::PackageReadContextQueryCXX::new(find_package_read_context(file, package)) +} + +/// Get flag read context cc interlop +pub fn get_flag_read_context_cxx( + file: &[u8], + package_id: u32, + flag: &str, +) -> ffi::FlagReadContextQueryCXX { + ffi::FlagReadContextQueryCXX::new(find_flag_read_context(file, package_id, flag)) +} + +/// Get boolean flag value cc interlop +pub fn get_boolean_flag_value_cxx(file: &[u8], offset: u32) -> ffi::BooleanFlagValueQueryCXX { + ffi::BooleanFlagValueQueryCXX::new(find_boolean_flag_value(file, offset)) +} + +/// Get flag attribute cc interlop +pub fn get_flag_attribute_cxx( + file: &[u8], + flag_type: u16, + flag_index: u32, +) -> ffi::FlagAttributeQueryCXX { + match FlagValueType::try_from(flag_type) { + Ok(value_type) => { + ffi::FlagAttributeQueryCXX::new(find_flag_attribute(file, value_type, flag_index)) + } + Err(errmsg) => ffi::FlagAttributeQueryCXX::new(Err(errmsg)), + } +} + +/// Get storage version number cc interlop +pub fn get_storage_file_version_cxx(file_path: &str) -> ffi::VersionNumberQueryCXX { + ffi::VersionNumberQueryCXX::new(get_storage_file_version(file_path)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mapped_file::get_mapped_file; + use crate::test_utils::copy_to_temp_file; + use aconfig_storage_file::protos::storage_record_pb::write_proto_to_temp_file; + use aconfig_storage_file::{FlagInfoBit, StoredFlagType}; + use tempfile::NamedTempFile; + + fn create_test_storage_files() -> [NamedTempFile; 5] { + let package_map = copy_to_temp_file("./tests/package.map").unwrap(); + let flag_map = copy_to_temp_file("./tests/flag.map").unwrap(); + let flag_val = copy_to_temp_file("./tests/flag.val").unwrap(); + let flag_info = copy_to_temp_file("./tests/flag.info").unwrap(); + + let text_proto = format!( + r#" +files {{ + version: 0 + container: "mockup" + package_map: "{}" + flag_map: "{}" + flag_val: "{}" + flag_info: "{}" + timestamp: 12345 +}} +"#, + package_map.path().display(), + flag_map.path().display(), + flag_val.path().display(), + flag_info.path().display() + ); + let pb_file = write_proto_to_temp_file(&text_proto).unwrap(); + [package_map, flag_map, flag_val, flag_info, pb_file] + } + + #[test] + // this test point locks down flag package read context query + fn test_package_context_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + let package_mapped_file = unsafe { + get_mapped_file(&pb_file_path, "mockup", StorageFileType::PackageMap).unwrap() + }; + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_1") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 0, boolean_start_index: 0 }; + assert_eq!(package_context, expected_package_context); + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_2") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 1, boolean_start_index: 3 }; + assert_eq!(package_context, expected_package_context); + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_4") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 2, boolean_start_index: 6 }; + assert_eq!(package_context, expected_package_context); + } + + #[test] + // this test point locks down flag read context query + fn test_flag_context_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + let flag_mapped_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagMap).unwrap() }; + + let baseline = vec![ + (0, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 1u16), + (0, "enabled_rw", StoredFlagType::ReadWriteBoolean, 2u16), + (2, "enabled_rw", StoredFlagType::ReadWriteBoolean, 1u16), + (1, "disabled_rw", StoredFlagType::ReadWriteBoolean, 0u16), + (1, "enabled_fixed_ro", StoredFlagType::FixedReadOnlyBoolean, 1u16), + (1, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 2u16), + (2, "enabled_fixed_ro", StoredFlagType::FixedReadOnlyBoolean, 0u16), + (0, "disabled_rw", StoredFlagType::ReadWriteBoolean, 0u16), + ]; + for (package_id, flag_name, flag_type, flag_index) in baseline.into_iter() { + let flag_context = + get_flag_read_context(&flag_mapped_file, package_id, flag_name).unwrap().unwrap(); + assert_eq!(flag_context.flag_type, flag_type); + assert_eq!(flag_context.flag_index, flag_index); + } + } + + #[test] + // this test point locks down flag value query + fn test_flag_value_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + let flag_value_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagVal).unwrap() }; + let baseline: Vec<bool> = vec![false, true, true, false, true, true, true, true]; + for (offset, expected_value) in baseline.into_iter().enumerate() { + let flag_value = get_boolean_flag_value(&flag_value_file, offset as u32).unwrap(); + assert_eq!(flag_value, expected_value); + } + } + + #[test] + // this test point locks donw flag info query + fn test_flag_info_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + let flag_info_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagInfo).unwrap() }; + let is_rw: Vec<bool> = vec![true, false, true, true, false, false, false, true]; + for (offset, expected_value) in is_rw.into_iter().enumerate() { + let attribute = + get_flag_attribute(&flag_info_file, FlagValueType::Boolean, offset as u32).unwrap(); + assert_eq!((attribute & FlagInfoBit::IsReadWrite as u8) != 0u8, expected_value); + assert!((attribute & FlagInfoBit::HasServerOverride as u8) == 0u8); + assert!((attribute & FlagInfoBit::HasLocalOverride as u8) == 0u8); + } + } + + #[test] + // this test point locks down flag storage file version number query api + fn test_storage_version_query() { + assert_eq!(get_storage_file_version("./tests/package.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/flag.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/flag.val").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/flag.info").unwrap(), 1); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs b/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs new file mode 100644 index 0000000000..378644317c --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::fs::File; +use std::io::{BufReader, Read}; + +use anyhow::anyhow; +use memmap2::Mmap; + +use crate::AconfigStorageError::{ + self, FileReadFail, MapFileFail, ProtobufParseFail, StorageFileNotFound, +}; +use crate::StorageFileType; +use aconfig_storage_file::protos::{ + storage_record_pb::try_from_binary_proto, ProtoStorageFileInfo, ProtoStorageFiles, +}; + +/// Find where storage files are stored for a particular container +pub fn find_container_storage_location( + location_pb_file: &str, + container: &str, +) -> Result<ProtoStorageFileInfo, AconfigStorageError> { + let file = File::open(location_pb_file).map_err(|errmsg| { + FileReadFail(anyhow!("Failed to open file {}: {}", location_pb_file, errmsg)) + })?; + let mut reader = BufReader::new(file); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).map_err(|errmsg| { + FileReadFail(anyhow!("Failed to read file {}: {}", location_pb_file, errmsg)) + })?; + let storage_locations: ProtoStorageFiles = try_from_binary_proto(&bytes).map_err(|errmsg| { + ProtobufParseFail(anyhow!( + "Failed to parse storage location pb file {}: {}", + location_pb_file, + errmsg + )) + })?; + for location_info in storage_locations.files.iter() { + if location_info.container() == container { + return Ok(location_info.clone()); + } + } + Err(StorageFileNotFound(anyhow!("Storage file does not exist for {}", container))) +} + +/// Get the read only memory mapping of a storage file +/// +/// # Safety +/// +/// The memory mapped file may have undefined behavior if there are writes to this +/// file after being mapped. Ensure no writes can happen to this file while this +/// mapping stays alive. +unsafe fn map_file(file_path: &str) -> Result<Mmap, AconfigStorageError> { + let file = File::open(file_path) + .map_err(|errmsg| FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)))?; + unsafe { + let mapped_file = Mmap::map(&file).map_err(|errmsg| { + MapFileFail(anyhow!("fail to map storage file {}: {}", file_path, errmsg)) + })?; + Ok(mapped_file) + } +} + +/// Get a mapped storage file given the container and file type +/// +/// # Safety +/// +/// The memory mapped file may have undefined behavior if there are writes to this +/// file after being mapped. Ensure no writes can happen to this file while this +/// mapping stays alive. +pub unsafe fn get_mapped_file( + location_pb_file: &str, + container: &str, + file_type: StorageFileType, +) -> Result<Mmap, AconfigStorageError> { + let files_location = find_container_storage_location(location_pb_file, container)?; + match file_type { + StorageFileType::PackageMap => unsafe { map_file(files_location.package_map()) }, + StorageFileType::FlagMap => unsafe { map_file(files_location.flag_map()) }, + StorageFileType::FlagVal => unsafe { map_file(files_location.flag_val()) }, + StorageFileType::FlagInfo => unsafe { map_file(files_location.flag_info()) }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::copy_to_temp_file; + use aconfig_storage_file::protos::storage_record_pb::write_proto_to_temp_file; + use tempfile::NamedTempFile; + + #[test] + fn test_find_storage_file_location() { + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "/system/etc/package.map" + flag_map: "/system/etc/flag.map" + flag_val: "/metadata/aconfig/system.val" + timestamp: 12345 +} +files { + version: 1 + container: "product" + package_map: "/product/etc/package.map" + flag_map: "/product/etc/flag.map" + flag_val: "/metadata/aconfig/product.val" + timestamp: 54321 +} +"#; + let file = write_proto_to_temp_file(&text_proto).unwrap(); + let file_full_path = file.path().display().to_string(); + let file_info = find_container_storage_location(&file_full_path, "system").unwrap(); + assert_eq!(file_info.version(), 0); + assert_eq!(file_info.container(), "system"); + assert_eq!(file_info.package_map(), "/system/etc/package.map"); + assert_eq!(file_info.flag_map(), "/system/etc/flag.map"); + assert_eq!(file_info.flag_val(), "/metadata/aconfig/system.val"); + assert_eq!(file_info.timestamp(), 12345); + + let file_info = find_container_storage_location(&file_full_path, "product").unwrap(); + assert_eq!(file_info.version(), 1); + assert_eq!(file_info.container(), "product"); + assert_eq!(file_info.package_map(), "/product/etc/package.map"); + assert_eq!(file_info.flag_map(), "/product/etc/flag.map"); + assert_eq!(file_info.flag_val(), "/metadata/aconfig/product.val"); + assert_eq!(file_info.timestamp(), 54321); + + let err = find_container_storage_location(&file_full_path, "vendor").unwrap_err(); + assert_eq!( + format!("{:?}", err), + "StorageFileNotFound(Storage file does not exist for vendor)" + ); + } + + fn map_and_verify(location_pb_file: &str, file_type: StorageFileType, actual_file: &str) { + let mut opened_file = File::open(actual_file).unwrap(); + let mut content = Vec::new(); + opened_file.read_to_end(&mut content).unwrap(); + + let mmaped_file = + unsafe { get_mapped_file(location_pb_file, "system", file_type).unwrap() }; + assert_eq!(mmaped_file[..], content[..]); + } + + fn create_test_storage_files() -> [NamedTempFile; 4] { + let package_map = copy_to_temp_file("./tests/package.map").unwrap(); + let flag_map = copy_to_temp_file("./tests/flag.map").unwrap(); + let flag_val = copy_to_temp_file("./tests/package.map").unwrap(); + + let text_proto = format!( + r#" +files {{ + version: 0 + container: "system" + package_map: "{}" + flag_map: "{}" + flag_val: "{}" + timestamp: 12345 +}} +"#, + package_map.path().display(), + flag_map.path().display(), + flag_val.path().display() + ); + let pb_file = write_proto_to_temp_file(&text_proto).unwrap(); + [package_map, flag_map, flag_val, pb_file] + } + + #[test] + fn test_mapped_file_contents() { + let [package_map, flag_map, flag_val, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + map_and_verify( + &pb_file_path, + StorageFileType::PackageMap, + &package_map.path().display().to_string(), + ); + map_and_verify( + &pb_file_path, + StorageFileType::FlagMap, + &flag_map.path().display().to_string(), + ); + map_and_verify( + &pb_file_path, + StorageFileType::FlagVal, + &flag_val.path().display().to_string(), + ); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs b/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs new file mode 100644 index 0000000000..2cb854b1b1 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! package table query module defines the package table file read from mapped bytes + +use crate::{AconfigStorageError, FILE_VERSION}; +use aconfig_storage_file::{ + package_table::PackageTableHeader, package_table::PackageTableNode, read_u32_from_bytes, +}; +use anyhow::anyhow; + +/// Package table query return +#[derive(PartialEq, Debug)] +pub struct PackageReadContext { + pub package_id: u32, + pub boolean_start_index: u32, +} + +/// Query package read context: package id and start index +pub fn find_package_read_context( + buf: &[u8], + package: &str, +) -> Result<Option<PackageReadContext>, AconfigStorageError> { + let interpreted_header = PackageTableHeader::from_bytes(buf)?; + if interpreted_header.version > FILE_VERSION { + return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( + "Cannot read storage file with a higher version of {} with lib version {}", + interpreted_header.version, + FILE_VERSION + ))); + } + + let num_buckets = (interpreted_header.node_offset - interpreted_header.bucket_offset) / 4; + let bucket_index = PackageTableNode::find_bucket_index(package, num_buckets); + + let mut pos = (interpreted_header.bucket_offset + 4 * bucket_index) as usize; + let mut package_node_offset = read_u32_from_bytes(buf, &mut pos)? as usize; + if package_node_offset < interpreted_header.node_offset as usize + || package_node_offset >= interpreted_header.file_size as usize + { + return Ok(None); + } + + loop { + let interpreted_node = PackageTableNode::from_bytes(&buf[package_node_offset..])?; + if interpreted_node.package_name == package { + return Ok(Some(PackageReadContext { + package_id: interpreted_node.package_id, + boolean_start_index: interpreted_node.boolean_start_index, + })); + } + match interpreted_node.next_offset { + Some(offset) => package_node_offset = offset as usize, + None => return Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_storage_file::test_utils::create_test_package_table; + + #[test] + // this test point locks down table query + fn test_package_query() { + let package_table = create_test_package_table().into_bytes(); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_1") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 0, boolean_start_index: 0 }; + assert_eq!(package_context, expected_package_context); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_2") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 1, boolean_start_index: 3 }; + assert_eq!(package_context, expected_package_context); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_4") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 2, boolean_start_index: 6 }; + assert_eq!(package_context, expected_package_context); + } + + #[test] + // this test point locks down table query of a non exist package + fn test_not_existed_package_query() { + // this will land at an empty bucket + let package_table = create_test_package_table().into_bytes(); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_3") + .unwrap(); + assert_eq!(package_context, None); + // this will land at the end of a linked list + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_5") + .unwrap(); + assert_eq!(package_context, None); + } + + #[test] + // this test point locks down query error when file has a higher version + fn test_higher_version_storage_file() { + let mut table = create_test_package_table(); + table.header.version = crate::FILE_VERSION + 1; + let package_table = table.into_bytes(); + let error = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_1") + .unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!( + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", + crate::FILE_VERSION + 1, + crate::FILE_VERSION + ) + ); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/src/test_utils.rs b/tools/aconfig/aconfig_storage_read_api/src/test_utils.rs new file mode 100644 index 0000000000..84f31aa710 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/src/test_utils.rs @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; +use std::fs; +use tempfile::NamedTempFile; + +/// Create temp file copy +pub(crate) fn copy_to_temp_file(source_file: &str) -> Result<NamedTempFile> { + let file = NamedTempFile::new()?; + fs::copy(source_file, file.path())?; + Ok(file) +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/Android.bp b/tools/aconfig/aconfig_storage_read_api/tests/Android.bp new file mode 100644 index 0000000000..6b05ca6fb1 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/Android.bp @@ -0,0 +1,45 @@ +rust_test { + name: "aconfig_storage_read_api.test.rust", + srcs: [ + "storage_read_api_test.rs" + ], + rustlibs: [ + "libanyhow", + "libaconfig_storage_file", + "libaconfig_storage_read_api", + "libprotobuf", + "libtempfile", + ], + data: [ + "package.map", + "flag.map", + "flag.val", + "flag.info", + ], + test_suites: ["general-tests"], +} + +cc_test { + name: "aconfig_storage_read_api.test.cpp", + srcs: [ + "storage_read_api_test.cpp", + ], + static_libs: [ + "libgmock", + "libaconfig_storage_protos_cc", + "libprotobuf-cpp-lite", + "libaconfig_storage_read_api_cc", + "libbase", + "liblog", + ], + data: [ + "package.map", + "flag.map", + "flag.val", + "flag.info", + ], + test_suites: [ + "device-tests", + "general-tests", + ], +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.info b/tools/aconfig/aconfig_storage_read_api/tests/flag.info Binary files differnew file mode 100644 index 0000000000..6223edf369 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/flag.info diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.map b/tools/aconfig/aconfig_storage_read_api/tests/flag.map Binary files differnew file mode 100644 index 0000000000..e868f53d7e --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/flag.map diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.val b/tools/aconfig/aconfig_storage_read_api/tests/flag.val Binary files differnew file mode 100644 index 0000000000..ed203d4d13 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/flag.val diff --git a/tools/aconfig/aconfig_storage_read_api/tests/package.map b/tools/aconfig/aconfig_storage_read_api/tests/package.map Binary files differnew file mode 100644 index 0000000000..6c46a0339c --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/package.map diff --git a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp new file mode 100644 index 0000000000..5393c49a7d --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> +#include <memory> +#include <cstdio> + +#include <sys/stat.h> +#include "aconfig_storage/aconfig_storage_read_api.hpp" +#include <gtest/gtest.h> +#include <protos/aconfig_storage_metadata.pb.h> +#include <android-base/file.h> +#include <android-base/result.h> + +using android::aconfig_storage_metadata::storage_files; +using namespace android::base; + +namespace api = aconfig_storage; +namespace private_api = aconfig_storage::private_internal_api; + +class AconfigStorageTest : public ::testing::Test { + protected: + Result<std::string> copy_to_temp_file(std::string const& source_file) { + auto temp_file = std::string(std::tmpnam(nullptr)); + auto content = std::string(); + if (!ReadFileToString(source_file, &content)) { + return Error() << "failed to read file: " << source_file; + } + if (!WriteStringToFile(content, temp_file)) { + return Error() << "failed to copy file: " << source_file; + } + return temp_file; + } + + Result<std::string> write_storage_location_pb_file(std::string const& package_map, + std::string const& flag_map, + std::string const& flag_val, + std::string const& flag_info) { + auto temp_file = std::tmpnam(nullptr); + auto proto = storage_files(); + auto* info = proto.add_files(); + info->set_version(0); + info->set_container("mockup"); + info->set_package_map(package_map); + info->set_flag_map(flag_map); + info->set_flag_val(flag_val); + info->set_flag_info(flag_info); + info->set_timestamp(12345); + + auto content = std::string(); + proto.SerializeToString(&content); + if (!WriteStringToFile(content, temp_file)) { + return Error() << "failed to write storage records pb file"; + } + return temp_file; + } + + void SetUp() override { + auto const test_dir = android::base::GetExecutableDirectory(); + package_map = *copy_to_temp_file(test_dir + "/package.map"); + flag_map = *copy_to_temp_file(test_dir + "/flag.map"); + flag_val = *copy_to_temp_file(test_dir + "/flag.val"); + flag_info = *copy_to_temp_file(test_dir + "/flag.info"); + storage_record_pb = *write_storage_location_pb_file( + package_map, flag_map, flag_val, flag_info); + } + + void TearDown() override { + std::remove(package_map.c_str()); + std::remove(flag_map.c_str()); + std::remove(flag_val.c_str()); + std::remove(flag_info.c_str()); + std::remove(storage_record_pb.c_str()); + } + + std::string package_map; + std::string flag_map; + std::string flag_val; + std::string flag_info; + std::string storage_record_pb; +}; + +/// Test to lock down storage file version query api +TEST_F(AconfigStorageTest, test_storage_version_query) { + auto version = api::get_storage_file_version(package_map); + ASSERT_TRUE(version.ok()); + ASSERT_EQ(*version, 1); + version = api::get_storage_file_version(flag_map); + ASSERT_TRUE(version.ok()); + ASSERT_EQ(*version, 1); + version = api::get_storage_file_version(flag_val); + ASSERT_TRUE(version.ok()); + ASSERT_EQ(*version, 1); + version = api::get_storage_file_version(flag_info); + ASSERT_TRUE(version.ok()); + ASSERT_EQ(*version, 1); +} + +/// Negative test to lock down the error when mapping none exist storage files +TEST_F(AconfigStorageTest, test_none_exist_storage_file_mapping) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "vendor", api::StorageFileType::package_map); + ASSERT_FALSE(mapped_file_result.ok()); + ASSERT_EQ(mapped_file_result.error().message(), + "Unable to find storage files for container vendor"); +} + +/// Test to lock down storage package context query api +TEST_F(AconfigStorageTest, test_package_context_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::package_map); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto context = api::get_package_read_context( + *mapped_file, "com.android.aconfig.storage.test_1"); + ASSERT_TRUE(context.ok()); + ASSERT_TRUE(context->package_exists); + ASSERT_EQ(context->package_id, 0); + ASSERT_EQ(context->boolean_start_index, 0); + + context = api::get_package_read_context( + *mapped_file, "com.android.aconfig.storage.test_2"); + ASSERT_TRUE(context.ok()); + ASSERT_TRUE(context->package_exists); + ASSERT_EQ(context->package_id, 1); + ASSERT_EQ(context->boolean_start_index, 3); + + context = api::get_package_read_context( + *mapped_file, "com.android.aconfig.storage.test_4"); + ASSERT_TRUE(context.ok()); + ASSERT_TRUE(context->package_exists); + ASSERT_EQ(context->package_id, 2); + ASSERT_EQ(context->boolean_start_index, 6); +} + +/// Test to lock down when querying none exist package +TEST_F(AconfigStorageTest, test_none_existent_package_context_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::package_map); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto context = api::get_package_read_context( + *mapped_file, "com.android.aconfig.storage.test_3"); + ASSERT_TRUE(context.ok()); + ASSERT_FALSE(context->package_exists); +} + +/// Test to lock down storage flag context query api +TEST_F(AconfigStorageTest, test_flag_context_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::flag_map); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto baseline = std::vector<std::tuple<int, std::string, api::StoredFlagType, int>>{ + {0, "enabled_ro", api::StoredFlagType::ReadOnlyBoolean, 1}, + {0, "enabled_rw", api::StoredFlagType::ReadWriteBoolean, 2}, + {2, "enabled_rw", api::StoredFlagType::ReadWriteBoolean, 1}, + {1, "disabled_rw", api::StoredFlagType::ReadWriteBoolean, 0}, + {1, "enabled_fixed_ro", api::StoredFlagType::FixedReadOnlyBoolean, 1}, + {1, "enabled_ro", api::StoredFlagType::ReadOnlyBoolean, 2}, + {2, "enabled_fixed_ro", api::StoredFlagType::FixedReadOnlyBoolean, 0}, + {0, "disabled_rw", api::StoredFlagType::ReadWriteBoolean, 0}, + }; + for (auto const&[package_id, flag_name, flag_type, flag_index] : baseline) { + auto context = api::get_flag_read_context(*mapped_file, package_id, flag_name); + ASSERT_TRUE(context.ok()); + ASSERT_TRUE(context->flag_exists); + ASSERT_EQ(context->flag_type, flag_type); + ASSERT_EQ(context->flag_index, flag_index); + } +} + +/// Test to lock down when querying none exist flag +TEST_F(AconfigStorageTest, test_none_existent_flag_context_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::flag_map); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto context = api::get_flag_read_context(*mapped_file, 0, "none_exist"); + ASSERT_TRUE(context.ok()); + ASSERT_FALSE(context->flag_exists); + + context = api::get_flag_read_context(*mapped_file, 3, "enabled_ro"); + ASSERT_TRUE(context.ok()); + ASSERT_FALSE(context->flag_exists); +} + +/// Test to lock down storage flag value query api +TEST_F(AconfigStorageTest, test_boolean_flag_value_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::flag_val); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto expected_value = std::vector<bool>{ + false, true, true, false, true, true, true, true}; + for (int index = 0; index < 8; ++index) { + auto value = api::get_boolean_flag_value(*mapped_file, index); + ASSERT_TRUE(value.ok()); + ASSERT_EQ(*value, expected_value[index]); + } +} + +/// Negative test to lock down the error when querying flag value out of range +TEST_F(AconfigStorageTest, test_invalid_boolean_flag_value_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::flag_val); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto value = api::get_boolean_flag_value(*mapped_file, 8); + ASSERT_FALSE(value.ok()); + ASSERT_EQ(value.error().message(), + std::string("InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)")); +} + +/// Test to lock down storage flag info query api +TEST_F(AconfigStorageTest, test_boolean_flag_info_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::flag_info); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto expected_value = std::vector<bool>{ + true, false, true, true, false, false, false, true}; + for (int index = 0; index < 8; ++index) { + auto attribute = api::get_flag_attribute(*mapped_file, api::FlagValueType::Boolean, index); + ASSERT_TRUE(attribute.ok()); + ASSERT_EQ(*attribute & static_cast<uint8_t>(api::FlagInfoBit::HasServerOverride), 0); + ASSERT_EQ((*attribute & static_cast<uint8_t>(api::FlagInfoBit::IsReadWrite)) != 0, + expected_value[index]); + ASSERT_EQ(*attribute & static_cast<uint8_t>(api::FlagInfoBit::HasLocalOverride), 0); + } +} + +/// Negative test to lock down the error when querying flag info out of range +TEST_F(AconfigStorageTest, test_invalid_boolean_flag_info_query) { + auto mapped_file_result = private_api::get_mapped_file_impl( + storage_record_pb, "mockup", api::StorageFileType::flag_info); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MappedStorageFile>(*mapped_file_result); + + auto attribute = api::get_flag_attribute(*mapped_file, api::FlagValueType::Boolean, 8); + ASSERT_FALSE(attribute.ok()); + ASSERT_EQ(attribute.error().message(), + std::string("InvalidStorageFileOffset(Flag info offset goes beyond the end of the file.)")); +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs new file mode 100644 index 0000000000..ecba573d82 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs @@ -0,0 +1,223 @@ +#[cfg(not(feature = "cargo"))] +mod aconfig_storage_rust_test { + use aconfig_storage_file::protos::storage_record_pb::write_proto_to_temp_file; + use aconfig_storage_file::{FlagInfoBit, FlagValueType, StorageFileType, StoredFlagType}; + use aconfig_storage_read_api::{ + get_boolean_flag_value, get_flag_attribute, get_flag_read_context, + get_package_read_context, get_storage_file_version, mapped_file::get_mapped_file, + PackageReadContext, + }; + use std::fs; + use tempfile::NamedTempFile; + + pub fn copy_to_temp_file(source_file: &str) -> NamedTempFile { + let file = NamedTempFile::new().unwrap(); + fs::copy(source_file, file.path()).unwrap(); + file + } + + fn create_test_storage_files() -> [NamedTempFile; 5] { + let package_map = copy_to_temp_file("./package.map"); + let flag_map = copy_to_temp_file("./flag.map"); + let flag_val = copy_to_temp_file("./flag.val"); + let flag_info = copy_to_temp_file("./flag.info"); + + let text_proto = format!( + r#" +files {{ + version: 0 + container: "mockup" + package_map: "{}" + flag_map: "{}" + flag_val: "{}" + flag_info: "{}" + timestamp: 12345 +}} +"#, + package_map.path().display(), + flag_map.path().display(), + flag_val.path().display(), + flag_info.path().display() + ); + let pb_file = write_proto_to_temp_file(&text_proto).unwrap(); + [package_map, flag_map, flag_val, flag_info, pb_file] + } + + #[test] + fn test_unavailable_stoarge() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let err = unsafe { + get_mapped_file(&pb_file_path, "vendor", StorageFileType::PackageMap).unwrap_err() + }; + assert_eq!( + format!("{:?}", err), + "StorageFileNotFound(Storage file does not exist for vendor)" + ); + } + + #[test] + fn test_package_context_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let package_mapped_file = unsafe { + get_mapped_file(&pb_file_path, "mockup", StorageFileType::PackageMap).unwrap() + }; + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_1") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 0, boolean_start_index: 0 }; + assert_eq!(package_context, expected_package_context); + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_2") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 1, boolean_start_index: 3 }; + assert_eq!(package_context, expected_package_context); + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_4") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { package_id: 2, boolean_start_index: 6 }; + assert_eq!(package_context, expected_package_context); + } + + #[test] + fn test_none_exist_package_context_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let package_mapped_file = unsafe { + get_mapped_file(&pb_file_path, "mockup", StorageFileType::PackageMap).unwrap() + }; + + let package_context_option = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_3") + .unwrap(); + assert_eq!(package_context_option, None); + } + + #[test] + fn test_flag_context_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let flag_mapped_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagMap).unwrap() }; + + let baseline = vec![ + (0, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 1u16), + (0, "enabled_rw", StoredFlagType::ReadWriteBoolean, 2u16), + (2, "enabled_rw", StoredFlagType::ReadWriteBoolean, 1u16), + (1, "disabled_rw", StoredFlagType::ReadWriteBoolean, 0u16), + (1, "enabled_fixed_ro", StoredFlagType::FixedReadOnlyBoolean, 1u16), + (1, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 2u16), + (2, "enabled_fixed_ro", StoredFlagType::FixedReadOnlyBoolean, 0u16), + (0, "disabled_rw", StoredFlagType::ReadWriteBoolean, 0u16), + ]; + for (package_id, flag_name, flag_type, flag_index) in baseline.into_iter() { + let flag_context = + get_flag_read_context(&flag_mapped_file, package_id, flag_name).unwrap().unwrap(); + assert_eq!(flag_context.flag_type, flag_type); + assert_eq!(flag_context.flag_index, flag_index); + } + } + + #[test] + fn test_none_exist_flag_context_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let flag_mapped_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagMap).unwrap() }; + let flag_context_option = + get_flag_read_context(&flag_mapped_file, 0, "none_exist").unwrap(); + assert_eq!(flag_context_option, None); + + let flag_context_option = + get_flag_read_context(&flag_mapped_file, 3, "enabled_ro").unwrap(); + assert_eq!(flag_context_option, None); + } + + #[test] + fn test_boolean_flag_value_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let flag_value_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagVal).unwrap() }; + let baseline: Vec<bool> = vec![false, true, true, false, true, true, true, true]; + for (offset, expected_value) in baseline.into_iter().enumerate() { + let flag_value = get_boolean_flag_value(&flag_value_file, offset as u32).unwrap(); + assert_eq!(flag_value, expected_value); + } + } + + #[test] + fn test_invalid_boolean_flag_value_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let flag_value_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagVal).unwrap() }; + let err = get_boolean_flag_value(&flag_value_file, 8u32).unwrap_err(); + assert_eq!( + format!("{:?}", err), + "InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)" + ); + } + + #[test] + fn test_flag_info_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let flag_info_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagInfo).unwrap() }; + let is_rw: Vec<bool> = vec![true, false, true, true, false, false, false, true]; + for (offset, expected_value) in is_rw.into_iter().enumerate() { + let attribute = + get_flag_attribute(&flag_info_file, FlagValueType::Boolean, offset as u32).unwrap(); + assert!((attribute & FlagInfoBit::HasServerOverride as u8) == 0u8); + assert_eq!((attribute & FlagInfoBit::IsReadWrite as u8) != 0u8, expected_value); + assert!((attribute & FlagInfoBit::HasLocalOverride as u8) == 0u8); + } + } + + #[test] + fn test_invalid_boolean_flag_info_query() { + let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files(); + let pb_file_path = pb_file.path().display().to_string(); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let flag_info_file = + unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagInfo).unwrap() }; + let err = get_flag_attribute(&flag_info_file, FlagValueType::Boolean, 8u32).unwrap_err(); + assert_eq!( + format!("{:?}", err), + "InvalidStorageFileOffset(Flag info offset goes beyond the end of the file.)" + ); + } + + #[test] + fn test_storage_version_query() { + assert_eq!(get_storage_file_version("./package.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./flag.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./flag.val").unwrap(), 1); + assert_eq!(get_storage_file_version("./flag.info").unwrap(), 1); + } +} diff --git a/tools/aconfig/aconfig_storage_write_api/Android.bp b/tools/aconfig/aconfig_storage_write_api/Android.bp new file mode 100644 index 0000000000..4dbdbbfb2f --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/Android.bp @@ -0,0 +1,84 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "aconfig_storage_write_api.defaults", + edition: "2021", + lints: "none", + srcs: ["src/lib.rs"], + rustlibs: [ + "libanyhow", + "libtempfile", + "libmemmap2", + "libcxx", + "libthiserror", + "libaconfig_storage_file", + "libaconfig_storage_read_api", + ], +} + +rust_library { + name: "libaconfig_storage_write_api", + crate_name: "aconfig_storage_write_api", + host_supported: true, + defaults: ["aconfig_storage_write_api.defaults"], +} + +rust_test_host { + name: "aconfig_storage_write_api.test", + test_suites: ["general-tests"], + defaults: ["aconfig_storage_write_api.defaults"], + data: [ + "tests/flag.val", + "tests/flag.info", + ], + rustlibs: [ + "libaconfig_storage_read_api", + ], +} + +// cxx source codegen from rust api +genrule { + name: "libcxx_aconfig_storage_write_api_bridge_code", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) > $(out)", + srcs: ["src/lib.rs"], + out: ["aconfig_storage/lib.rs.cc"], +} + +// cxx header codegen from rust api +genrule { + name: "libcxx_aconfig_storage_write_api_bridge_header", + tools: ["cxxbridge"], + cmd: "$(location cxxbridge) $(in) --header > $(out)", + srcs: ["src/lib.rs"], + out: ["aconfig_storage/lib.rs.h"], +} + +// a static cc lib based on generated code +rust_ffi_static { + name: "libaconfig_storage_write_api_cxx_bridge", + crate_name: "aconfig_storage_write_api_cxx_bridge", + host_supported: true, + defaults: ["aconfig_storage_write_api.defaults"], +} + +// flag write api cc interface +cc_library_static { + name: "libaconfig_storage_write_api_cc", + srcs: ["aconfig_storage_write_api.cpp"], + generated_headers: [ + "cxx-bridge-header", + "libcxx_aconfig_storage_write_api_bridge_header", + ], + generated_sources: ["libcxx_aconfig_storage_write_api_bridge_code"], + whole_static_libs: ["libaconfig_storage_write_api_cxx_bridge"], + export_include_dirs: ["include"], + static_libs: [ + "libaconfig_storage_read_api_cc", + "libaconfig_storage_protos_cc", + "libprotobuf-cpp-lite", + "libbase", + ], +} diff --git a/tools/aconfig/aconfig_storage_write_api/Cargo.toml b/tools/aconfig/aconfig_storage_write_api/Cargo.toml new file mode 100644 index 0000000000..eaa55f2531 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "aconfig_storage_write_api" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +cxx = "1.0" +memmap2 = "0.8.0" +tempfile = "3.9.0" +thiserror = "1.0.56" +protobuf = "3.2.0" +aconfig_storage_file = { path = "../aconfig_storage_file" } +aconfig_storage_read_api = { path = "../aconfig_storage_read_api" } + +[build-dependencies] +cxx-build = "1.0" diff --git a/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp b/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp new file mode 100644 index 0000000000..f529f7954c --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp @@ -0,0 +1,111 @@ + +#include <android-base/file.h> +#include <android-base/logging.h> +#include <protos/aconfig_storage_metadata.pb.h> + +#include <sys/mman.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "rust/cxx.h" +#include "aconfig_storage/lib.rs.h" +#include "aconfig_storage/aconfig_storage_write_api.hpp" + +using storage_records_pb = android::aconfig_storage_metadata::storage_files; +using storage_record_pb = android::aconfig_storage_metadata::storage_file_info; +using namespace android::base; + +namespace aconfig_storage { + +/// Map a storage file +Result<MutableMappedStorageFile*> map_mutable_storage_file(std::string const& file) { + struct stat file_stat; + if (stat(file.c_str(), &file_stat) < 0) { + return ErrnoError() << "stat failed"; + } + + if ((file_stat.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH)) == 0) { + return Error() << "cannot map nonwriteable file"; + } + + size_t file_size = file_stat.st_size; + + const int fd = open(file.c_str(), O_RDWR | O_NOFOLLOW | O_CLOEXEC); + if (fd == -1) { + return ErrnoError() << "failed to open " << file; + }; + + void* const map_result = + mmap(nullptr, file_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map_result == MAP_FAILED) { + return ErrnoError() << "mmap failed"; + } + + auto mapped_file = new MutableMappedStorageFile(); + mapped_file->file_ptr = map_result; + mapped_file->file_size = file_size; + + return mapped_file; +} + +/// Set boolean flag value +Result<void> set_boolean_flag_value( + const MutableMappedStorageFile& file, + uint32_t offset, + bool value) { + auto content = rust::Slice<uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto update_cxx = update_boolean_flag_value_cxx(content, offset, value); + if (!update_cxx.update_success) { + return Error() << std::string(update_cxx.error_message.c_str()); + } + return {}; +} + +/// Set if flag has server override +Result<void> set_flag_has_server_override( + const MutableMappedStorageFile& file, + FlagValueType value_type, + uint32_t offset, + bool value) { + auto content = rust::Slice<uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto update_cxx = update_flag_has_server_override_cxx( + content, static_cast<uint16_t>(value_type), offset, value); + if (!update_cxx.update_success) { + return Error() << std::string(update_cxx.error_message.c_str()); + } + return {}; +} + +/// Set if flag has local override +Result<void> set_flag_has_local_override( + const MutableMappedStorageFile& file, + FlagValueType value_type, + uint32_t offset, + bool value) { + auto content = rust::Slice<uint8_t>( + static_cast<uint8_t*>(file.file_ptr), file.file_size); + auto update_cxx = update_flag_has_local_override_cxx( + content, static_cast<uint16_t>(value_type), offset, value); + if (!update_cxx.update_success) { + return Error() << std::string(update_cxx.error_message.c_str()); + } + return {}; +} + +Result<void> create_flag_info( + std::string const& package_map, + std::string const& flag_map, + std::string const& flag_info_out) { + auto creation_cxx = create_flag_info_cxx( + rust::Str(package_map.c_str()), + rust::Str(flag_map.c_str()), + rust::Str(flag_info_out.c_str())); + if (creation_cxx.success) { + return {}; + } else { + return android::base::Error() << creation_cxx.error_message; + } +} +} // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_write_api/build.rs b/tools/aconfig/aconfig_storage_write_api/build.rs new file mode 100644 index 0000000000..7b1aa53b5f --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/build.rs @@ -0,0 +1,4 @@ +fn main() { + let _ = cxx_build::bridge("src/lib.rs"); + println!("cargo:rerun-if-changed=src/lib.rs"); +} diff --git a/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp b/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp new file mode 100644 index 0000000000..ff06cbc6de --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include <stdint.h> +#include <string> + +#include <android-base/result.h> +#include <aconfig_storage/aconfig_storage_read_api.hpp> + +using namespace android::base; + +namespace aconfig_storage { + +/// Mapped flag value file +struct MutableMappedStorageFile : MappedStorageFile {}; + +/// Map a storage file +Result<MutableMappedStorageFile*> map_mutable_storage_file( + std::string const& file); + +/// Set boolean flag value +Result<void> set_boolean_flag_value( + const MutableMappedStorageFile& file, + uint32_t offset, + bool value); + +/// Set if flag has server override +Result<void> set_flag_has_server_override( + const MutableMappedStorageFile& file, + FlagValueType value_type, + uint32_t offset, + bool value); + +/// Set if flag has local override +Result<void> set_flag_has_local_override( + const MutableMappedStorageFile& file, + FlagValueType value_type, + uint32_t offset, + bool value); + +/// Create flag info file based on package and flag map +/// \input package_map: package map file +/// \input flag_map: flag map file +/// \input flag_info_out: flag info file to be created +Result<void> create_flag_info( + std::string const& package_map, + std::string const& flag_map, + std::string const& flag_info_out); + +} // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs b/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs new file mode 100644 index 0000000000..6f03f128a5 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag info update module defines the flag info file write to mapped bytes + +use aconfig_storage_file::{ + read_u8_from_bytes, AconfigStorageError, FlagInfoBit, FlagInfoHeader, FlagValueType, + FILE_VERSION, +}; +use anyhow::anyhow; + +fn get_flag_info_offset( + buf: &mut [u8], + flag_type: FlagValueType, + flag_index: u32, +) -> Result<usize, AconfigStorageError> { + let interpreted_header = FlagInfoHeader::from_bytes(buf)?; + if interpreted_header.version > FILE_VERSION { + return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( + "Cannot write to storage file with a higher version of {} with lib version {}", + interpreted_header.version, + FILE_VERSION + ))); + } + + // get byte offset to the flag info + let head = match flag_type { + FlagValueType::Boolean => (interpreted_header.boolean_flag_offset + flag_index) as usize, + }; + + if head >= interpreted_header.file_size as usize { + return Err(AconfigStorageError::InvalidStorageFileOffset(anyhow!( + "Flag value offset goes beyond the end of the file." + ))); + } + + Ok(head) +} + +fn get_flag_attribute_and_offset( + buf: &mut [u8], + flag_type: FlagValueType, + flag_index: u32, +) -> Result<(u8, usize), AconfigStorageError> { + let head = get_flag_info_offset(buf, flag_type, flag_index)?; + let mut pos = head; + let attribute = read_u8_from_bytes(buf, &mut pos)?; + Ok((attribute, head)) +} + +/// Set if flag has server override +pub fn update_flag_has_server_override( + buf: &mut [u8], + flag_type: FlagValueType, + flag_index: u32, + value: bool, +) -> Result<(), AconfigStorageError> { + let (attribute, head) = get_flag_attribute_and_offset(buf, flag_type, flag_index)?; + let has_override = (attribute & (FlagInfoBit::HasServerOverride as u8)) != 0; + if has_override != value { + buf[head] = (attribute ^ FlagInfoBit::HasServerOverride as u8).to_le_bytes()[0]; + } + Ok(()) +} + +/// Set if flag has local override +pub fn update_flag_has_local_override( + buf: &mut [u8], + flag_type: FlagValueType, + flag_index: u32, + value: bool, +) -> Result<(), AconfigStorageError> { + let (attribute, head) = get_flag_attribute_and_offset(buf, flag_type, flag_index)?; + let has_override = (attribute & (FlagInfoBit::HasLocalOverride as u8)) != 0; + if has_override != value { + buf[head] = (attribute ^ FlagInfoBit::HasLocalOverride as u8).to_le_bytes()[0]; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_storage_file::test_utils::create_test_flag_info_list; + use aconfig_storage_read_api::flag_info_query::find_flag_attribute; + + #[test] + // this test point locks down has server override update + fn test_update_flag_has_server_override() { + let flag_info_list = create_test_flag_info_list(); + let mut buf = flag_info_list.into_bytes(); + for i in 0..flag_info_list.header.num_flags { + update_flag_has_server_override(&mut buf, FlagValueType::Boolean, i, true).unwrap(); + let attribute = find_flag_attribute(&buf, FlagValueType::Boolean, i).unwrap(); + assert!((attribute & (FlagInfoBit::HasServerOverride as u8)) != 0); + update_flag_has_server_override(&mut buf, FlagValueType::Boolean, i, false).unwrap(); + let attribute = find_flag_attribute(&buf, FlagValueType::Boolean, i).unwrap(); + assert!((attribute & (FlagInfoBit::HasServerOverride as u8)) == 0); + } + } + + #[test] + // this test point locks down has local override update + fn test_update_flag_has_local_override() { + let flag_info_list = create_test_flag_info_list(); + let mut buf = flag_info_list.into_bytes(); + for i in 0..flag_info_list.header.num_flags { + update_flag_has_local_override(&mut buf, FlagValueType::Boolean, i, true).unwrap(); + let attribute = find_flag_attribute(&buf, FlagValueType::Boolean, i).unwrap(); + assert!((attribute & (FlagInfoBit::HasLocalOverride as u8)) != 0); + update_flag_has_local_override(&mut buf, FlagValueType::Boolean, i, false).unwrap(); + let attribute = find_flag_attribute(&buf, FlagValueType::Boolean, i).unwrap(); + assert!((attribute & (FlagInfoBit::HasLocalOverride as u8)) == 0); + } + } +} diff --git a/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs b/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs new file mode 100644 index 0000000000..0938715714 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! flag value update module defines the flag value file write to mapped bytes + +use aconfig_storage_file::{AconfigStorageError, FlagValueHeader, FILE_VERSION}; +use anyhow::anyhow; + +/// Set flag value +pub fn update_boolean_flag_value( + buf: &mut [u8], + flag_index: u32, + flag_value: bool, +) -> Result<(), AconfigStorageError> { + let interpreted_header = FlagValueHeader::from_bytes(buf)?; + if interpreted_header.version > FILE_VERSION { + return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( + "Cannot write to storage file with a higher version of {} with lib version {}", + interpreted_header.version, + FILE_VERSION + ))); + } + + // get byte offset to the flag + let head = (interpreted_header.boolean_value_offset + flag_index) as usize; + if head >= interpreted_header.file_size as usize { + return Err(AconfigStorageError::InvalidStorageFileOffset(anyhow!( + "Flag value offset goes beyond the end of the file." + ))); + } + + buf[head] = u8::from(flag_value).to_le_bytes()[0]; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use aconfig_storage_file::test_utils::create_test_flag_value_list; + + #[test] + // this test point locks down flag value update + fn test_boolean_flag_value_update() { + let flag_value_list = create_test_flag_value_list(); + let value_offset = flag_value_list.header.boolean_value_offset; + let mut content = flag_value_list.into_bytes(); + let true_byte = u8::from(true).to_le_bytes()[0]; + let false_byte = u8::from(false).to_le_bytes()[0]; + + for i in 0..flag_value_list.header.num_flags { + let offset = (value_offset + i) as usize; + update_boolean_flag_value(&mut content, i, true).unwrap(); + assert_eq!(content[offset], true_byte); + update_boolean_flag_value(&mut content, i, false).unwrap(); + assert_eq!(content[offset], false_byte); + } + } + + #[test] + // this test point locks down update beyond the end of boolean section + fn test_boolean_out_of_range() { + let mut flag_value_list = create_test_flag_value_list().into_bytes(); + let error = update_boolean_flag_value(&mut flag_value_list[..], 8, true).unwrap_err(); + assert_eq!( + format!("{:?}", error), + "InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)" + ); + } + + #[test] + // this test point locks down query error when file has a higher version + fn test_higher_version_storage_file() { + let mut value_list = create_test_flag_value_list(); + value_list.header.version = FILE_VERSION + 1; + let mut flag_value = value_list.into_bytes(); + let error = update_boolean_flag_value(&mut flag_value[..], 4, true).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!( + "HigherStorageFileVersion(Cannot write to storage file with a higher version of {} with lib version {})", + FILE_VERSION + 1, + FILE_VERSION + ) + ); + } +} diff --git a/tools/aconfig/aconfig_storage_write_api/src/lib.rs b/tools/aconfig/aconfig_storage_write_api/src/lib.rs new file mode 100644 index 0000000000..aec28def68 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/src/lib.rs @@ -0,0 +1,457 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aconfig_storage_write_api` is a crate that defines write apis to update flag value +//! in storage file. It provides one api to interface with storage files. + +pub mod flag_info_update; +pub mod flag_value_update; +pub mod mapped_file; + +#[cfg(test)] +mod test_utils; + +use aconfig_storage_file::{ + AconfigStorageError, FlagInfoHeader, FlagInfoList, FlagInfoNode, FlagTable, FlagValueType, + PackageTable, StorageFileType, StoredFlagType, FILE_VERSION, +}; + +use anyhow::anyhow; +use memmap2::MmapMut; +use std::fs::File; +use std::io::{Read, Write}; + +/// Get read write mapped storage files. +/// +/// \input file_path: path to the storage file +/// +/// # Safety +/// +/// The memory mapped file may have undefined behavior if there are writes to this +/// file not thru this memory mapped file or there are concurrent writes to this +/// memory mapped file. Ensure all writes to the underlying file are thru this memory +/// mapped file and there are no concurrent writes. +pub unsafe fn map_mutable_storage_file(file_path: &str) -> Result<MmapMut, AconfigStorageError> { + crate::mapped_file::map_file(file_path) +} + +/// Set boolean flag value thru mapped file and flush the change to file +/// +/// \input mapped_file: the mapped flag value file +/// \input index: flag index +/// \input value: updated flag value +/// \return a result of () +/// +pub fn set_boolean_flag_value( + file: &mut MmapMut, + index: u32, + value: bool, +) -> Result<(), AconfigStorageError> { + crate::flag_value_update::update_boolean_flag_value(file, index, value)?; + file.flush().map_err(|errmsg| { + AconfigStorageError::MapFlushFail(anyhow!("fail to flush storage file: {}", errmsg)) + }) +} + +/// Set if flag is has server override thru mapped file and flush the change to file +/// +/// \input mapped_file: the mapped flag info file +/// \input index: flag index +/// \input value: updated flag has server override value +/// \return a result of () +/// +pub fn set_flag_has_server_override( + file: &mut MmapMut, + flag_type: FlagValueType, + index: u32, + value: bool, +) -> Result<(), AconfigStorageError> { + crate::flag_info_update::update_flag_has_server_override(file, flag_type, index, value)?; + file.flush().map_err(|errmsg| { + AconfigStorageError::MapFlushFail(anyhow!("fail to flush storage file: {}", errmsg)) + }) +} + +/// Set if flag has local override thru mapped file and flush the change to file +/// +/// \input mapped_file: the mapped flag info file +/// \input index: flag index +/// \input value: updated flag has local override value +/// \return a result of () +/// +pub fn set_flag_has_local_override( + file: &mut MmapMut, + flag_type: FlagValueType, + index: u32, + value: bool, +) -> Result<(), AconfigStorageError> { + crate::flag_info_update::update_flag_has_local_override(file, flag_type, index, value)?; + file.flush().map_err(|errmsg| { + AconfigStorageError::MapFlushFail(anyhow!("fail to flush storage file: {}", errmsg)) + }) +} + +/// Read in storage file as bytes +fn read_file_to_bytes(file_path: &str) -> Result<Vec<u8>, AconfigStorageError> { + let mut file = File::open(file_path).map_err(|errmsg| { + AconfigStorageError::FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)) + })?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).map_err(|errmsg| { + AconfigStorageError::FileReadFail(anyhow!( + "Failed to read bytes from file {}: {}", + file_path, + errmsg + )) + })?; + Ok(buffer) +} + +/// Create flag info file given package map file and flag map file +/// \input package_map: package map file +/// \input flag_map: flag map file +/// \output flag_info_out: created flag info file +pub fn create_flag_info( + package_map: &str, + flag_map: &str, + flag_info_out: &str, +) -> Result<(), AconfigStorageError> { + let package_table = PackageTable::from_bytes(&read_file_to_bytes(package_map)?)?; + let flag_table = FlagTable::from_bytes(&read_file_to_bytes(flag_map)?)?; + + if package_table.header.container != flag_table.header.container { + return Err(AconfigStorageError::FileCreationFail(anyhow!( + "container for package map {} and flag map {} does not match", + package_table.header.container, + flag_table.header.container, + ))); + } + + let mut package_start_index = vec![0; package_table.header.num_packages as usize]; + for node in package_table.nodes.iter() { + package_start_index[node.package_id as usize] = node.boolean_start_index; + } + + let mut is_flag_rw = vec![false; flag_table.header.num_flags as usize]; + for node in flag_table.nodes.iter() { + let flag_index = package_start_index[node.package_id as usize] + node.flag_index as u32; + is_flag_rw[flag_index as usize] = node.flag_type == StoredFlagType::ReadWriteBoolean; + } + + let mut list = FlagInfoList { + header: FlagInfoHeader { + version: FILE_VERSION, + container: flag_table.header.container, + file_type: StorageFileType::FlagInfo as u8, + file_size: 0, + num_flags: flag_table.header.num_flags, + boolean_flag_offset: 0, + }, + nodes: is_flag_rw.iter().map(|&rw| FlagInfoNode::create(rw)).collect(), + }; + + list.header.boolean_flag_offset = list.header.into_bytes().len() as u32; + list.header.file_size = list.into_bytes().len() as u32; + + let mut file = File::create(flag_info_out).map_err(|errmsg| { + AconfigStorageError::FileCreationFail(anyhow!( + "fail to create file {}: {}", + flag_info_out, + errmsg + )) + })?; + file.write_all(&list.into_bytes()).map_err(|errmsg| { + AconfigStorageError::FileCreationFail(anyhow!( + "fail to write to file {}: {}", + flag_info_out, + errmsg + )) + })?; + + Ok(()) +} + +// *************************************** // +// CC INTERLOP +// *************************************** // + +// Exported rust data structure and methods, c++ code will be generated +#[cxx::bridge] +mod ffi { + // Flag value update return for cc interlop + pub struct BooleanFlagValueUpdateCXX { + pub update_success: bool, + pub error_message: String, + } + + // Flag has server override update return for cc interlop + pub struct FlagHasServerOverrideUpdateCXX { + pub update_success: bool, + pub error_message: String, + } + + // Flag has local override update return for cc interlop + pub struct FlagHasLocalOverrideUpdateCXX { + pub update_success: bool, + pub error_message: String, + } + + // Flag info file creation return for cc interlop + pub struct FlagInfoCreationCXX { + pub success: bool, + pub error_message: String, + } + + // Rust export to c++ + extern "Rust" { + pub fn update_boolean_flag_value_cxx( + file: &mut [u8], + offset: u32, + value: bool, + ) -> BooleanFlagValueUpdateCXX; + + pub fn update_flag_has_server_override_cxx( + file: &mut [u8], + flag_type: u16, + offset: u32, + value: bool, + ) -> FlagHasServerOverrideUpdateCXX; + + pub fn update_flag_has_local_override_cxx( + file: &mut [u8], + flag_type: u16, + offset: u32, + value: bool, + ) -> FlagHasLocalOverrideUpdateCXX; + + pub fn create_flag_info_cxx( + package_map: &str, + flag_map: &str, + flag_info_out: &str, + ) -> FlagInfoCreationCXX; + } +} + +pub(crate) fn update_boolean_flag_value_cxx( + file: &mut [u8], + offset: u32, + value: bool, +) -> ffi::BooleanFlagValueUpdateCXX { + match crate::flag_value_update::update_boolean_flag_value(file, offset, value) { + Ok(()) => { + ffi::BooleanFlagValueUpdateCXX { update_success: true, error_message: String::from("") } + } + Err(errmsg) => ffi::BooleanFlagValueUpdateCXX { + update_success: false, + error_message: format!("{:?}", errmsg), + }, + } +} + +pub(crate) fn update_flag_has_server_override_cxx( + file: &mut [u8], + flag_type: u16, + offset: u32, + value: bool, +) -> ffi::FlagHasServerOverrideUpdateCXX { + match FlagValueType::try_from(flag_type) { + Ok(value_type) => { + match crate::flag_info_update::update_flag_has_server_override( + file, value_type, offset, value, + ) { + Ok(()) => ffi::FlagHasServerOverrideUpdateCXX { + update_success: true, + error_message: String::from(""), + }, + Err(errmsg) => ffi::FlagHasServerOverrideUpdateCXX { + update_success: false, + error_message: format!("{:?}", errmsg), + }, + } + } + Err(errmsg) => ffi::FlagHasServerOverrideUpdateCXX { + update_success: false, + error_message: format!("{:?}", errmsg), + }, + } +} + +pub(crate) fn update_flag_has_local_override_cxx( + file: &mut [u8], + flag_type: u16, + offset: u32, + value: bool, +) -> ffi::FlagHasLocalOverrideUpdateCXX { + match FlagValueType::try_from(flag_type) { + Ok(value_type) => { + match crate::flag_info_update::update_flag_has_local_override( + file, value_type, offset, value, + ) { + Ok(()) => ffi::FlagHasLocalOverrideUpdateCXX { + update_success: true, + error_message: String::from(""), + }, + Err(errmsg) => ffi::FlagHasLocalOverrideUpdateCXX { + update_success: false, + error_message: format!("{:?}", errmsg), + }, + } + } + Err(errmsg) => ffi::FlagHasLocalOverrideUpdateCXX { + update_success: false, + error_message: format!("{:?}", errmsg), + }, + } +} + +/// Create flag info file cc interlop +pub(crate) fn create_flag_info_cxx( + package_map: &str, + flag_map: &str, + flag_info_out: &str, +) -> ffi::FlagInfoCreationCXX { + match create_flag_info(package_map, flag_map, flag_info_out) { + Ok(()) => ffi::FlagInfoCreationCXX { success: true, error_message: String::from("") }, + Err(errmsg) => { + ffi::FlagInfoCreationCXX { success: false, error_message: format!("{:?}", errmsg) } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::copy_to_temp_file; + use aconfig_storage_file::test_utils::{ + create_test_flag_info_list, create_test_flag_table, create_test_package_table, + write_bytes_to_temp_file, + }; + use aconfig_storage_file::FlagInfoBit; + use aconfig_storage_read_api::flag_info_query::find_flag_attribute; + use aconfig_storage_read_api::flag_value_query::find_boolean_flag_value; + use std::fs::File; + use std::io::Read; + use tempfile::NamedTempFile; + + fn get_boolean_flag_value_at_offset(file: &str, offset: u32) -> bool { + let mut f = File::open(&file).unwrap(); + let mut bytes = Vec::new(); + f.read_to_end(&mut bytes).unwrap(); + find_boolean_flag_value(&bytes, offset).unwrap() + } + + #[test] + fn test_set_boolean_flag_value() { + let flag_value_file = copy_to_temp_file("./tests/flag.val", false).unwrap(); + let flag_value_path = flag_value_file.path().display().to_string(); + + // SAFETY: + // The safety here is guaranteed as only this single threaded test process will + // write to this file + unsafe { + let mut file = map_mutable_storage_file(&flag_value_path).unwrap(); + for i in 0..8 { + set_boolean_flag_value(&mut file, i, true).unwrap(); + let value = get_boolean_flag_value_at_offset(&flag_value_path, i); + assert_eq!(value, true); + + set_boolean_flag_value(&mut file, i, false).unwrap(); + let value = get_boolean_flag_value_at_offset(&flag_value_path, i); + assert_eq!(value, false); + } + } + } + + fn get_flag_attribute_at_offset(file: &str, value_type: FlagValueType, offset: u32) -> u8 { + let mut f = File::open(&file).unwrap(); + let mut bytes = Vec::new(); + f.read_to_end(&mut bytes).unwrap(); + find_flag_attribute(&bytes, value_type, offset).unwrap() + } + + #[test] + fn test_set_flag_has_server_override() { + let flag_info_file = copy_to_temp_file("./tests/flag.info", false).unwrap(); + let flag_info_path = flag_info_file.path().display().to_string(); + + // SAFETY: + // The safety here is guaranteed as only this single threaded test process will + // write to this file + unsafe { + let mut file = map_mutable_storage_file(&flag_info_path).unwrap(); + for i in 0..8 { + set_flag_has_server_override(&mut file, FlagValueType::Boolean, i, true).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasServerOverride as u8)) != 0); + set_flag_has_server_override(&mut file, FlagValueType::Boolean, i, false).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasServerOverride as u8)) == 0); + } + } + } + + #[test] + fn test_set_flag_has_local_override() { + let flag_info_file = copy_to_temp_file("./tests/flag.info", false).unwrap(); + let flag_info_path = flag_info_file.path().display().to_string(); + + // SAFETY: + // The safety here is guaranteed as only this single threaded test process will + // write to this file + unsafe { + let mut file = map_mutable_storage_file(&flag_info_path).unwrap(); + for i in 0..8 { + set_flag_has_local_override(&mut file, FlagValueType::Boolean, i, true).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasLocalOverride as u8)) != 0); + set_flag_has_local_override(&mut file, FlagValueType::Boolean, i, false).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasLocalOverride as u8)) == 0); + } + } + } + + fn create_empty_temp_file() -> Result<NamedTempFile, AconfigStorageError> { + let file = NamedTempFile::new().map_err(|_| { + AconfigStorageError::FileCreationFail(anyhow!("Failed to create temp file")) + })?; + Ok(file) + } + + #[test] + // this test point locks down the flag info creation + fn test_create_flag_info() { + let package_table = + write_bytes_to_temp_file(&create_test_package_table().into_bytes()).unwrap(); + let flag_table = write_bytes_to_temp_file(&create_test_flag_table().into_bytes()).unwrap(); + let flag_info = create_empty_temp_file().unwrap(); + + let package_table_path = package_table.path().display().to_string(); + let flag_table_path = flag_table.path().display().to_string(); + let flag_info_path = flag_info.path().display().to_string(); + + assert!(create_flag_info(&package_table_path, &flag_table_path, &flag_info_path).is_ok()); + + let flag_info = + FlagInfoList::from_bytes(&read_file_to_bytes(&flag_info_path).unwrap()).unwrap(); + let expected_flag_info = create_test_flag_info_list(); + assert_eq!(flag_info, expected_flag_info); + } +} diff --git a/tools/aconfig/aconfig_storage_write_api/src/mapped_file.rs b/tools/aconfig/aconfig_storage_write_api/src/mapped_file.rs new file mode 100644 index 0000000000..401d6b798a --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/src/mapped_file.rs @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::anyhow; +use memmap2::MmapMut; +use std::fs::{self, OpenOptions}; + +use aconfig_storage_file::AconfigStorageError::{self, FileReadFail, MapFileFail}; + +/// Get the mutable memory mapping of a storage file +/// +/// # Safety +/// +/// The memory mapped file may have undefined behavior if there are writes to this +/// file not thru this memory mapped file or there are concurrent writes to this +/// memory mapped file. Ensure all writes to the underlying file are thru this memory +/// mapped file and there are no concurrent writes. +pub(crate) unsafe fn map_file(file_path: &str) -> Result<MmapMut, AconfigStorageError> { + // make sure file has read write permission + let perms = fs::metadata(file_path).unwrap().permissions(); + if perms.readonly() { + return Err(MapFileFail(anyhow!("fail to map non read write storage file {}", file_path))); + } + + let file = + OpenOptions::new().read(true).write(true).open(file_path).map_err(|errmsg| { + FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)) + })?; + + unsafe { + let mapped_file = MmapMut::map_mut(&file).map_err(|errmsg| { + MapFileFail(anyhow!("fail to map storage file {}: {}", file_path, errmsg)) + })?; + Ok(mapped_file) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::copy_to_temp_file; + use std::io::Read; + + #[test] + fn test_mapped_file_contents() { + let mut rw_val_file = copy_to_temp_file("./tests/flag.val", false).unwrap(); + let mut rw_info_file = copy_to_temp_file("./tests/flag.info", false).unwrap(); + let flag_val = rw_val_file.path().display().to_string(); + let flag_info = rw_info_file.path().display().to_string(); + + let mut content = Vec::new(); + rw_val_file.read_to_end(&mut content).unwrap(); + + // SAFETY: + // The safety here is guaranteed here as no writes happens to this temp file + unsafe { + let mmaped_file = map_file(&flag_val).unwrap(); + assert_eq!(mmaped_file[..], content[..]); + } + + let mut content = Vec::new(); + rw_info_file.read_to_end(&mut content).unwrap(); + + // SAFETY: + // The safety here is guaranteed here as no writes happens to this temp file + unsafe { + let mmaped_file = map_file(&flag_info).unwrap(); + assert_eq!(mmaped_file[..], content[..]); + } + } + + #[test] + fn test_mapped_read_only_file() { + let ro_val_file = copy_to_temp_file("./tests/flag.val", true).unwrap(); + let flag_val = ro_val_file.path().display().to_string(); + + // SAFETY: + // The safety here is guaranteed here as no writes happens to this temp file + unsafe { + let error = map_file(&flag_val).unwrap_err(); + assert_eq!( + format!("{:?}", error), + format!("MapFileFail(fail to map non read write storage file {})", flag_val) + ); + } + } +} diff --git a/tools/aconfig/aconfig_storage_write_api/src/test_utils.rs b/tools/aconfig/aconfig_storage_write_api/src/test_utils.rs new file mode 100644 index 0000000000..06e2e221c8 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/src/test_utils.rs @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; +use std::fs; +use tempfile::NamedTempFile; + +/// Create temp file copy +pub(crate) fn copy_to_temp_file(source_file: &str, read_only: bool) -> Result<NamedTempFile> { + let file = NamedTempFile::new()?; + fs::copy(source_file, file.path())?; + if read_only { + let file_name = file.path().display().to_string(); + let mut perms = fs::metadata(file_name).unwrap().permissions(); + perms.set_readonly(true); + fs::set_permissions(file.path(), perms.clone()).unwrap(); + } + Ok(file) +} diff --git a/tools/aconfig/aconfig_storage_write_api/tests/Android.bp b/tools/aconfig/aconfig_storage_write_api/tests/Android.bp new file mode 100644 index 0000000000..85568e063b --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/tests/Android.bp @@ -0,0 +1,44 @@ +rust_test { + name: "aconfig_storage_write_api.test.rust", + srcs: [ + "storage_write_api_test.rs", + ], + rustlibs: [ + "libanyhow", + "libaconfig_storage_file", + "libaconfig_storage_read_api", + "libaconfig_storage_write_api", + "libprotobuf", + "libtempfile", + ], + data: [ + "flag.val", + "flag.info", + ], + test_suites: ["general-tests"], +} + +cc_test { + name: "aconfig_storage_write_api.test.cpp", + srcs: [ + "storage_write_api_test.cpp", + ], + static_libs: [ + "libgmock", + "libaconfig_storage_protos_cc", + "libprotobuf-cpp-lite", + "libaconfig_storage_read_api_cc", + "libaconfig_storage_write_api_cc", + "libbase", + "liblog", + ], + data: [ + "flag.val", + "flag.info", + ], + test_suites: [ + "device-tests", + "general-tests", + ], + ldflags: ["-Wl,--allow-multiple-definition"], +} diff --git a/tools/aconfig/aconfig_storage_write_api/tests/flag.info b/tools/aconfig/aconfig_storage_write_api/tests/flag.info Binary files differnew file mode 100644 index 0000000000..6223edf369 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/tests/flag.info diff --git a/tools/aconfig/aconfig_storage_write_api/tests/flag.val b/tools/aconfig/aconfig_storage_write_api/tests/flag.val Binary files differnew file mode 100644 index 0000000000..ed203d4d13 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/tests/flag.val diff --git a/tools/aconfig/aconfig_storage_write_api/tests/storage_write_api_test.cpp b/tools/aconfig/aconfig_storage_write_api/tests/storage_write_api_test.cpp new file mode 100644 index 0000000000..54373798c9 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/tests/storage_write_api_test.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> +#include <cstdio> + +#include <sys/stat.h> +#include "aconfig_storage/aconfig_storage_read_api.hpp" +#include "aconfig_storage/aconfig_storage_write_api.hpp" +#include <gtest/gtest.h> +#include <protos/aconfig_storage_metadata.pb.h> +#include <android-base/file.h> +#include <android-base/result.h> + +using android::aconfig_storage_metadata::storage_files; +using namespace android::base; + +namespace api = aconfig_storage; +namespace private_api = aconfig_storage::private_internal_api; + +class AconfigStorageTest : public ::testing::Test { + protected: + Result<std::string> copy_to_rw_temp_file(std::string const& source_file) { + auto temp_file = std::string(std::tmpnam(nullptr)); + auto content = std::string(); + if (!ReadFileToString(source_file, &content)) { + return Error() << "failed to read file: " << source_file; + } + if (!WriteStringToFile(content, temp_file)) { + return Error() << "failed to copy file: " << source_file; + } + if (chmod(temp_file.c_str(), + S_IRUSR | S_IRGRP | S_IROTH | S_IWUSR | S_IWGRP | S_IWOTH) == -1) { + return Error() << "failed to chmod"; + } + return temp_file; + } + + void SetUp() override { + auto const test_dir = android::base::GetExecutableDirectory(); + flag_val = *copy_to_rw_temp_file(test_dir + "/flag.val"); + flag_info = *copy_to_rw_temp_file(test_dir + "/flag.info"); + } + + void TearDown() override { + std::remove(flag_val.c_str()); + std::remove(flag_info.c_str()); + } + + std::string flag_val; + std::string flag_info; +}; + +/// Negative test to lock down the error when mapping a non writeable storage file +TEST_F(AconfigStorageTest, test_non_writable_storage_file_mapping) { + ASSERT_TRUE(chmod(flag_val.c_str(), S_IRUSR | S_IRGRP | S_IROTH) != -1); + auto mapped_file_result = api::map_mutable_storage_file(flag_val); + ASSERT_FALSE(mapped_file_result.ok()); + auto it = mapped_file_result.error().message().find("cannot map nonwriteable file"); + ASSERT_TRUE(it != std::string::npos) << mapped_file_result.error().message(); +} + +/// Test to lock down storage flag value update api +TEST_F(AconfigStorageTest, test_boolean_flag_value_update) { + auto mapped_file_result = api::map_mutable_storage_file(flag_val); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MutableMappedStorageFile>(*mapped_file_result); + + for (int offset = 0; offset < 8; ++offset) { + auto update_result = api::set_boolean_flag_value(*mapped_file, offset, true); + ASSERT_TRUE(update_result.ok()); + auto value = api::get_boolean_flag_value(*mapped_file, offset); + ASSERT_TRUE(value.ok()); + ASSERT_TRUE(*value); + } +} + +/// Negative test to lock down the error when querying flag value out of range +TEST_F(AconfigStorageTest, test_invalid_boolean_flag_value_update) { + auto mapped_file_result = api::map_mutable_storage_file(flag_val); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MutableMappedStorageFile>(*mapped_file_result); + auto update_result = api::set_boolean_flag_value(*mapped_file, 8, true); + ASSERT_FALSE(update_result.ok()); + ASSERT_EQ(update_result.error().message(), + std::string("InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)")); +} + +/// Test to lock down storage flag has server override update api +TEST_F(AconfigStorageTest, test_flag_has_server_override_update) { + auto mapped_file_result = api::map_mutable_storage_file(flag_info); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MutableMappedStorageFile>(*mapped_file_result); + + for (int offset = 0; offset < 8; ++offset) { + auto update_result = api::set_flag_has_server_override( + *mapped_file, api::FlagValueType::Boolean, offset, true); + ASSERT_TRUE(update_result.ok()) << update_result.error(); + auto attribute = api::get_flag_attribute( + *mapped_file, api::FlagValueType::Boolean, offset); + ASSERT_TRUE(attribute.ok()); + ASSERT_TRUE(*attribute & api::FlagInfoBit::HasServerOverride); + + update_result = api::set_flag_has_server_override( + *mapped_file, api::FlagValueType::Boolean, offset, false); + ASSERT_TRUE(update_result.ok()); + attribute = api::get_flag_attribute( + *mapped_file, api::FlagValueType::Boolean, offset); + ASSERT_TRUE(attribute.ok()); + ASSERT_FALSE(*attribute & api::FlagInfoBit::HasServerOverride); + } +} + +/// Test to lock down storage flag has local override update api +TEST_F(AconfigStorageTest, test_flag_has_local_override_update) { + auto mapped_file_result = api::map_mutable_storage_file(flag_info); + ASSERT_TRUE(mapped_file_result.ok()); + auto mapped_file = std::unique_ptr<api::MutableMappedStorageFile>(*mapped_file_result); + + for (int offset = 0; offset < 8; ++offset) { + auto update_result = api::set_flag_has_local_override( + *mapped_file, api::FlagValueType::Boolean, offset, true); + ASSERT_TRUE(update_result.ok()); + auto attribute = api::get_flag_attribute( + *mapped_file, api::FlagValueType::Boolean, offset); + ASSERT_TRUE(attribute.ok()); + ASSERT_TRUE(*attribute & api::FlagInfoBit::HasLocalOverride); + + update_result = api::set_flag_has_local_override( + *mapped_file, api::FlagValueType::Boolean, offset, false); + ASSERT_TRUE(update_result.ok()); + attribute = api::get_flag_attribute( + *mapped_file, api::FlagValueType::Boolean, offset); + ASSERT_TRUE(attribute.ok()); + ASSERT_FALSE(*attribute & api::FlagInfoBit::HasLocalOverride); + } +} diff --git a/tools/aconfig/aconfig_storage_write_api/tests/storage_write_api_test.rs b/tools/aconfig/aconfig_storage_write_api/tests/storage_write_api_test.rs new file mode 100644 index 0000000000..367569def4 --- /dev/null +++ b/tools/aconfig/aconfig_storage_write_api/tests/storage_write_api_test.rs @@ -0,0 +1,102 @@ +#[cfg(not(feature = "cargo"))] +mod aconfig_storage_write_api_test { + use aconfig_storage_file::{FlagInfoBit, FlagValueType}; + use aconfig_storage_read_api::flag_info_query::find_flag_attribute; + use aconfig_storage_read_api::flag_value_query::find_boolean_flag_value; + use aconfig_storage_write_api::{ + map_mutable_storage_file, set_boolean_flag_value, set_flag_has_local_override, + set_flag_has_server_override, + }; + + use std::fs::{self, File}; + use std::io::Read; + use tempfile::NamedTempFile; + + /// Create temp file copy + fn copy_to_temp_rw_file(source_file: &str) -> NamedTempFile { + let file = NamedTempFile::new().unwrap(); + fs::copy(source_file, file.path()).unwrap(); + file + } + + /// Get boolean flag value from offset + fn get_boolean_flag_value_at_offset(file: &str, offset: u32) -> bool { + let mut f = File::open(file).unwrap(); + let mut bytes = Vec::new(); + f.read_to_end(&mut bytes).unwrap(); + find_boolean_flag_value(&bytes, offset).unwrap() + } + + /// Get flag attribute at offset + fn get_flag_attribute_at_offset(file: &str, value_type: FlagValueType, offset: u32) -> u8 { + let mut f = File::open(file).unwrap(); + let mut bytes = Vec::new(); + f.read_to_end(&mut bytes).unwrap(); + find_flag_attribute(&bytes, value_type, offset).unwrap() + } + + #[test] + /// Test to lock down flag value update api + fn test_boolean_flag_value_update() { + let flag_value_file = copy_to_temp_rw_file("./flag.val"); + let flag_value_path = flag_value_file.path().display().to_string(); + + // SAFETY: + // The safety here is ensured as only this single threaded test process will + // write to this file + let mut file = unsafe { map_mutable_storage_file(&flag_value_path).unwrap() }; + for i in 0..8 { + set_boolean_flag_value(&mut file, i, true).unwrap(); + let value = get_boolean_flag_value_at_offset(&flag_value_path, i); + assert!(value); + + set_boolean_flag_value(&mut file, i, false).unwrap(); + let value = get_boolean_flag_value_at_offset(&flag_value_path, i); + assert!(!value); + } + } + + #[test] + /// Test to lock down flag has server override update api + fn test_set_flag_has_server_override() { + let flag_info_file = copy_to_temp_rw_file("./flag.info"); + let flag_info_path = flag_info_file.path().display().to_string(); + + // SAFETY: + // The safety here is ensured as only this single threaded test process will + // write to this file + let mut file = unsafe { map_mutable_storage_file(&flag_info_path).unwrap() }; + for i in 0..8 { + set_flag_has_server_override(&mut file, FlagValueType::Boolean, i, true).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasServerOverride as u8)) != 0); + set_flag_has_server_override(&mut file, FlagValueType::Boolean, i, false).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasServerOverride as u8)) == 0); + } + } + + #[test] + /// Test to lock down flag has local override update api + fn test_set_flag_has_local_override() { + let flag_info_file = copy_to_temp_rw_file("./flag.info"); + let flag_info_path = flag_info_file.path().display().to_string(); + + // SAFETY: + // The safety here is ensured as only this single threaded test process will + // write to this file + let mut file = unsafe { map_mutable_storage_file(&flag_info_path).unwrap() }; + for i in 0..8 { + set_flag_has_local_override(&mut file, FlagValueType::Boolean, i, true).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasLocalOverride as u8)) != 0); + set_flag_has_local_override(&mut file, FlagValueType::Boolean, i, false).unwrap(); + let attribute = + get_flag_attribute_at_offset(&flag_info_path, FlagValueType::Boolean, i); + assert!((attribute & (FlagInfoBit::HasLocalOverride as u8)) == 0); + } + } +} diff --git a/tools/aconfig/aflags/Android.bp b/tools/aconfig/aflags/Android.bp new file mode 100644 index 0000000000..2a023792b6 --- /dev/null +++ b/tools/aconfig/aflags/Android.bp @@ -0,0 +1,33 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "aflags.defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + srcs: ["src/main.rs"], + rustlibs: [ + "libaconfig_device_paths", + "libaconfig_protos", + "libaconfig_storage_read_api", + "libaconfig_storage_file", + "libanyhow", + "libclap", + "libnix", + "libprotobuf", + "libregex", + ], +} + +rust_binary { + name: "aflags", + defaults: ["aflags.defaults"], +} + +rust_test_host { + name: "aflags.test", + defaults: ["aflags.defaults"], + test_suites: ["general-tests"], +} diff --git a/tools/aconfig/aflags/Cargo.toml b/tools/aconfig/aflags/Cargo.toml new file mode 100644 index 0000000000..eeae295316 --- /dev/null +++ b/tools/aconfig/aflags/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "aflags" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.69" +paste = "1.0.11" +protobuf = "3.2.0" +regex = "1.10.3" +aconfig_protos = { path = "../aconfig_protos" } +nix = { version = "0.28.0", features = ["user"] } +aconfig_storage_file = { version = "0.1.0", path = "../aconfig_storage_file" } +aconfig_storage_read_api = { version = "0.1.0", path = "../aconfig_storage_read_api" } +clap = {version = "4.5.2" } +aconfig_device_paths = { version = "0.1.0", path = "../aconfig_device_paths" } diff --git a/tools/aconfig/aflags/src/aconfig_storage_source.rs b/tools/aconfig/aflags/src/aconfig_storage_source.rs new file mode 100644 index 0000000000..c21c5424bb --- /dev/null +++ b/tools/aconfig/aflags/src/aconfig_storage_source.rs @@ -0,0 +1,56 @@ +use crate::{Flag, FlagPermission, FlagSource, FlagValue, ValuePickedFrom}; +use anyhow::{anyhow, Result}; + +use std::fs::File; +use std::io::Read; + +pub struct AconfigStorageSource {} + +use aconfig_storage_file::protos::ProtoStorageFiles; + +static STORAGE_INFO_FILE_PATH: &str = "/metadata/aconfig/persistent_storage_file_records.pb"; + +impl FlagSource for AconfigStorageSource { + fn list_flags() -> Result<Vec<Flag>> { + let mut result = Vec::new(); + + let mut file = File::open(STORAGE_INFO_FILE_PATH)?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + let storage_file_info: ProtoStorageFiles = protobuf::Message::parse_from_bytes(&bytes)?; + + for file_info in storage_file_info.files { + let package_map = + file_info.package_map.ok_or(anyhow!("storage file is missing package map"))?; + let flag_map = file_info.flag_map.ok_or(anyhow!("storage file is missing flag map"))?; + let flag_val = file_info.flag_val.ok_or(anyhow!("storage file is missing flag val"))?; + let container = + file_info.container.ok_or(anyhow!("storage file is missing container"))?; + + for listed_flag in + aconfig_storage_file::list_flags(&package_map, &flag_map, &flag_val)? + { + result.push(Flag { + name: listed_flag.flag_name, + package: listed_flag.package_name, + value: FlagValue::try_from(listed_flag.flag_value.as_str())?, + container: container.to_string(), + + // TODO(b/324436145): delete namespace field once DeviceConfig isn't in CLI. + namespace: "-".to_string(), + + // TODO(b/324436145): Populate with real values once API is available. + staged_value: None, + permission: FlagPermission::ReadOnly, + value_picked_from: ValuePickedFrom::Default, + }); + } + } + + Ok(result) + } + + fn override_flag(_namespace: &str, _qualified_name: &str, _value: &str) -> Result<()> { + todo!() + } +} diff --git a/tools/aconfig/aflags/src/device_config_source.rs b/tools/aconfig/aflags/src/device_config_source.rs new file mode 100644 index 0000000000..cf6ab28e8b --- /dev/null +++ b/tools/aconfig/aflags/src/device_config_source.rs @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::load_protos; +use crate::{Flag, FlagSource, FlagValue, ValuePickedFrom}; + +use anyhow::{anyhow, bail, Result}; +use regex::Regex; +use std::collections::HashMap; +use std::process::Command; +use std::str; + +pub struct DeviceConfigSource {} + +fn parse_device_config(raw: &str) -> Result<HashMap<String, FlagValue>> { + let mut flags = HashMap::new(); + let regex = Regex::new(r"(?m)^([[[:alnum:]]_]+/[[[:alnum:]]_\.]+)=(true|false)$")?; + for capture in regex.captures_iter(raw) { + let key = + capture.get(1).ok_or(anyhow!("invalid device_config output"))?.as_str().to_string(); + let value = FlagValue::try_from( + capture.get(2).ok_or(anyhow!("invalid device_config output"))?.as_str(), + )?; + flags.insert(key, value); + } + Ok(flags) +} + +fn read_device_config_output(command: &[&str]) -> Result<String> { + let output = Command::new("/system/bin/device_config").args(command).output()?; + if !output.status.success() { + let reason = match output.status.code() { + Some(code) => { + let output = str::from_utf8(&output.stdout)?; + if !output.is_empty() { + format!("exit code {code}, output was {output}") + } else { + format!("exit code {code}") + } + } + None => "terminated by signal".to_string(), + }; + bail!("failed to access flag storage: {}", reason); + } + Ok(str::from_utf8(&output.stdout)?.to_string()) +} + +fn read_device_config_flags() -> Result<HashMap<String, FlagValue>> { + let list_output = read_device_config_output(&["list"])?; + parse_device_config(&list_output) +} + +/// Parse the list of newline-separated staged flags. +/// +/// The output is a newline-sepaarated list of entries which follow this format: +/// `namespace*flagname=value` +/// +/// The resulting map maps from `namespace/flagname` to `value`, if a staged flag exists for +/// `namespace/flagname`. +fn parse_staged_flags(raw: &str) -> Result<HashMap<String, FlagValue>> { + let mut flags = HashMap::new(); + for line in raw.split('\n') { + match (line.find('*'), line.find('=')) { + (Some(star_index), Some(equal_index)) => { + let namespace = &line[..star_index]; + let flag = &line[star_index + 1..equal_index]; + if let Ok(value) = FlagValue::try_from(&line[equal_index + 1..]) { + flags.insert(namespace.to_owned() + "/" + flag, value); + } + } + _ => continue, + }; + } + Ok(flags) +} + +fn read_staged_flags() -> Result<HashMap<String, FlagValue>> { + let staged_flags_output = read_device_config_output(&["list", "staged"])?; + parse_staged_flags(&staged_flags_output) +} + +fn reconcile( + pb_flags: &[Flag], + dc_flags: HashMap<String, FlagValue>, + staged_flags: HashMap<String, FlagValue>, +) -> Vec<Flag> { + pb_flags + .iter() + .map(|f| { + let server_override = dc_flags.get(&format!("{}/{}", f.namespace, f.qualified_name())); + let (value_picked_from, selected_value) = match server_override { + Some(value) if *value != f.value => (ValuePickedFrom::Server, *value), + _ => (ValuePickedFrom::Default, f.value), + }; + Flag { value_picked_from, value: selected_value, ..f.clone() } + }) + .map(|f| { + let staged_value = staged_flags + .get(&format!("{}/{}", f.namespace, f.qualified_name())) + .map(|value| if *value != f.value { Some(*value) } else { None }) + .unwrap_or(None); + Flag { staged_value, ..f } + }) + .collect() +} + +impl FlagSource for DeviceConfigSource { + fn list_flags() -> Result<Vec<Flag>> { + let pb_flags = load_protos::load()?; + let dc_flags = read_device_config_flags()?; + let staged_flags = read_staged_flags()?; + + let flags = reconcile(&pb_flags, dc_flags, staged_flags); + Ok(flags) + } + + fn override_flag(namespace: &str, qualified_name: &str, value: &str) -> Result<()> { + read_device_config_output(&["put", namespace, qualified_name, value]).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_device_config() { + let input = r#" +namespace_one/com.foo.bar.flag_one=true +namespace_one/com.foo.bar.flag_two=false +random_noise; +namespace_two/android.flag_one=true +namespace_two/android.flag_two=nonsense +"#; + let expected = HashMap::from([ + ("namespace_one/com.foo.bar.flag_one".to_string(), FlagValue::Enabled), + ("namespace_one/com.foo.bar.flag_two".to_string(), FlagValue::Disabled), + ("namespace_two/android.flag_one".to_string(), FlagValue::Enabled), + ]); + let actual = parse_device_config(input).unwrap(); + assert_eq!(expected, actual); + } +} diff --git a/tools/aconfig/aflags/src/load_protos.rs b/tools/aconfig/aflags/src/load_protos.rs new file mode 100644 index 0000000000..90d8599145 --- /dev/null +++ b/tools/aconfig/aflags/src/load_protos.rs @@ -0,0 +1,62 @@ +use crate::{Flag, FlagPermission, FlagValue, ValuePickedFrom}; +use aconfig_protos::ProtoFlagPermission as ProtoPermission; +use aconfig_protos::ProtoFlagState as ProtoState; +use aconfig_protos::ProtoParsedFlag; +use aconfig_protos::ProtoParsedFlags; +use anyhow::Result; +use std::fs; +use std::path::Path; + +// TODO(b/329875578): use container field directly instead of inferring. +fn infer_container(path: &Path) -> String { + let path_str = path.to_string_lossy(); + path_str + .strip_prefix("/apex/") + .or_else(|| path_str.strip_prefix('/')) + .unwrap_or(&path_str) + .strip_suffix("/etc/aconfig_flags.pb") + .unwrap_or(&path_str) + .to_string() +} + +fn convert_parsed_flag(path: &Path, flag: &ProtoParsedFlag) -> Flag { + let namespace = flag.namespace().to_string(); + let package = flag.package().to_string(); + let name = flag.name().to_string(); + + let value = match flag.state() { + ProtoState::ENABLED => FlagValue::Enabled, + ProtoState::DISABLED => FlagValue::Disabled, + }; + + let permission = match flag.permission() { + ProtoPermission::READ_ONLY => FlagPermission::ReadOnly, + ProtoPermission::READ_WRITE => FlagPermission::ReadWrite, + }; + + Flag { + namespace, + package, + name, + container: infer_container(path), + value, + staged_value: None, + permission, + value_picked_from: ValuePickedFrom::Default, + } +} + +pub(crate) fn load() -> Result<Vec<Flag>> { + let mut result = Vec::new(); + + let paths = aconfig_device_paths::parsed_flags_proto_paths()?; + for path in paths { + let bytes = fs::read(path.clone())?; + let parsed_flags: ProtoParsedFlags = protobuf::Message::parse_from_bytes(&bytes)?; + for flag in parsed_flags.parsed_flag { + // TODO(b/334954748): enforce one-container-per-flag invariant. + result.push(convert_parsed_flag(&path, &flag)); + } + } + Ok(result) +} diff --git a/tools/aconfig/aflags/src/main.rs b/tools/aconfig/aflags/src/main.rs new file mode 100644 index 0000000000..4ce0d35ba1 --- /dev/null +++ b/tools/aconfig/aflags/src/main.rs @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `aflags` is a device binary to read and write aconfig flags. + +use anyhow::{anyhow, ensure, Result}; +use clap::Parser; + +mod device_config_source; +use device_config_source::DeviceConfigSource; + +mod aconfig_storage_source; +use aconfig_storage_source::AconfigStorageSource; + +mod load_protos; + +#[derive(Clone, PartialEq, Debug)] +enum FlagPermission { + ReadOnly, + ReadWrite, +} + +impl ToString for FlagPermission { + fn to_string(&self) -> String { + match &self { + Self::ReadOnly => "read-only".into(), + Self::ReadWrite => "read-write".into(), + } + } +} + +#[derive(Clone, Debug)] +enum ValuePickedFrom { + Default, + Server, +} + +impl ToString for ValuePickedFrom { + fn to_string(&self) -> String { + match &self { + Self::Default => "default".into(), + Self::Server => "server".into(), + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum FlagValue { + Enabled, + Disabled, +} + +impl TryFrom<&str> for FlagValue { + type Error = anyhow::Error; + + fn try_from(value: &str) -> std::result::Result<Self, Self::Error> { + match value { + "true" | "enabled" => Ok(Self::Enabled), + "false" | "disabled" => Ok(Self::Disabled), + _ => Err(anyhow!("cannot convert string '{}' to FlagValue", value)), + } + } +} + +impl ToString for FlagValue { + fn to_string(&self) -> String { + match &self { + Self::Enabled => "enabled".into(), + Self::Disabled => "disabled".into(), + } + } +} + +#[derive(Clone, Debug)] +struct Flag { + namespace: String, + name: String, + package: String, + container: String, + value: FlagValue, + staged_value: Option<FlagValue>, + permission: FlagPermission, + value_picked_from: ValuePickedFrom, +} + +impl Flag { + fn qualified_name(&self) -> String { + format!("{}.{}", self.package, self.name) + } + + fn display_staged_value(&self) -> String { + match self.staged_value { + Some(v) => format!("(->{})", v.to_string()), + None => "-".to_string(), + } + } +} + +trait FlagSource { + fn list_flags() -> Result<Vec<Flag>>; + fn override_flag(namespace: &str, qualified_name: &str, value: &str) -> Result<()>; +} + +enum FlagSourceType { + DeviceConfig, + AconfigStorage, +} + +const ABOUT_TEXT: &str = "Tool for reading and writing flags. + +Rows in the table from the `list` command follow this format: + + package flag_name value provenance permission container + + * `package`: package set for this flag in its .aconfig definition. + * `flag_name`: flag name, also set in definition. + * `value`: the value read from the flag. + * `staged_value`: the value on next boot: + + `-`: same as current value + + `(->enabled) flipped to enabled on boot. + + `(->disabled) flipped to disabled on boot. + * `provenance`: one of: + + `default`: the flag value comes from its build-time default. + + `server`: the flag value comes from a server override. + * `permission`: read-write or read-only. + * `container`: the container for the flag, configured in its definition. +"; + +#[derive(Parser, Debug)] +#[clap(long_about=ABOUT_TEXT)] +struct Cli { + #[clap(subcommand)] + command: Command, +} + +#[derive(Parser, Debug)] +enum Command { + /// List all aconfig flags on this device. + List { + /// Read from the new flag storage. + #[clap(long)] + use_new_storage: bool, + }, + + /// Enable an aconfig flag on this device, on the next boot. + Enable { + /// <package>.<flag_name> + qualified_name: String, + }, + + /// Disable an aconfig flag on this device, on the next boot. + Disable { + /// <package>.<flag_name> + qualified_name: String, + }, +} + +struct PaddingInfo { + longest_flag_col: usize, + longest_val_col: usize, + longest_staged_val_col: usize, + longest_value_picked_from_col: usize, + longest_permission_col: usize, +} + +fn format_flag_row(flag: &Flag, info: &PaddingInfo) -> String { + let full_name = flag.qualified_name(); + let p0 = info.longest_flag_col + 1; + + let val = flag.value.to_string(); + let p1 = info.longest_val_col + 1; + + let staged_val = flag.display_staged_value(); + let p2 = info.longest_staged_val_col + 1; + + let value_picked_from = flag.value_picked_from.to_string(); + let p3 = info.longest_value_picked_from_col + 1; + + let perm = flag.permission.to_string(); + let p4 = info.longest_permission_col + 1; + + let container = &flag.container; + + format!( + "{full_name:p0$}{val:p1$}{staged_val:p2$}{value_picked_from:p3$}{perm:p4$}{container}\n" + ) +} + +fn set_flag(qualified_name: &str, value: &str) -> Result<()> { + ensure!(nix::unistd::Uid::current().is_root(), "must be root to mutate flags"); + + let flags_binding = DeviceConfigSource::list_flags()?; + let flag = flags_binding.iter().find(|f| f.qualified_name() == qualified_name).ok_or( + anyhow!("no aconfig flag '{qualified_name}'. Does the flag have an .aconfig definition?"), + )?; + + ensure!(flag.permission == FlagPermission::ReadWrite, + format!("could not write flag '{qualified_name}', it is read-only for the current release configuration.")); + + DeviceConfigSource::override_flag(&flag.namespace, qualified_name, value)?; + + Ok(()) +} + +fn list(source_type: FlagSourceType) -> Result<String> { + let flags = match source_type { + FlagSourceType::DeviceConfig => DeviceConfigSource::list_flags()?, + FlagSourceType::AconfigStorage => AconfigStorageSource::list_flags()?, + }; + let padding_info = PaddingInfo { + longest_flag_col: flags.iter().map(|f| f.qualified_name().len()).max().unwrap_or(0), + longest_val_col: flags.iter().map(|f| f.value.to_string().len()).max().unwrap_or(0), + longest_staged_val_col: flags + .iter() + .map(|f| f.display_staged_value().len()) + .max() + .unwrap_or(0), + longest_value_picked_from_col: flags + .iter() + .map(|f| f.value_picked_from.to_string().len()) + .max() + .unwrap_or(0), + longest_permission_col: flags + .iter() + .map(|f| f.permission.to_string().len()) + .max() + .unwrap_or(0), + }; + + let mut result = String::from(""); + for flag in flags { + let row = format_flag_row(&flag, &padding_info); + result.push_str(&row); + } + Ok(result) +} + +fn main() { + let cli = Cli::parse(); + let output = match cli.command { + Command::List { use_new_storage: true } => list(FlagSourceType::AconfigStorage).map(Some), + Command::List { use_new_storage: false } => list(FlagSourceType::DeviceConfig).map(Some), + Command::Enable { qualified_name } => set_flag(&qualified_name, "true").map(|_| None), + Command::Disable { qualified_name } => set_flag(&qualified_name, "false").map(|_| None), + }; + match output { + Ok(Some(text)) => println!("{text}"), + Ok(None) => (), + Err(message) => println!("Error: {message}"), + } +} diff --git a/tools/aconfig/fake_device_config/Android.bp b/tools/aconfig/fake_device_config/Android.bp new file mode 100644 index 0000000000..4566bf985a --- /dev/null +++ b/tools/aconfig/fake_device_config/Android.bp @@ -0,0 +1,22 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +java_library { + name: "fake_device_config", + srcs: ["src/**/*.java"], + sdk_version: "none", + system_modules: "core-all-system-modules", + host_supported: true, +} + diff --git a/tools/aconfig/fake_device_config/src/android/provider/DeviceConfig.java b/tools/aconfig/fake_device_config/src/android/provider/DeviceConfig.java new file mode 100644 index 0000000000..dbb07ac983 --- /dev/null +++ b/tools/aconfig/fake_device_config/src/android/provider/DeviceConfig.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.provider; + +/* + * This class allows generated aconfig code to compile independently of the framework. + */ +public class DeviceConfig { + private DeviceConfig() { + } + + public static boolean getBoolean(String ns, String name, boolean def) { + return false; + } + + public static Properties getProperties(String namespace, String... names) { + return new Properties(); + } + + public static class Properties { + public boolean getBoolean(String name, boolean def) { + return false; + } + } +} diff --git a/tools/aconfig/overrideflags/overrideflags.py b/tools/aconfig/overrideflags/overrideflags.py new file mode 100644 index 0000000000..e355c21cc7 --- /dev/null +++ b/tools/aconfig/overrideflags/overrideflags.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Create Aconfig value building rules. + +This script will help to create Aconfig flag value building rules. It will +parse necessary information in the value file to create the building rules, but +it will not validate the value file. The validation will defer to the building +system. +""" + +import argparse +import pathlib +import re +import sys + + +_VALUE_LIST_TEMPLATE: str = """ +ACONFIG_VALUES_LIST_LOCAL = [{}] +""" + +_ACONFIG_VALUES_TEMPLATE: str = """ +aconfig_values {{ + name: "{}", + package: "{}", + srcs: [ + "{}", + ] +}} +""" + +_ACONFIG_VALUES_NAME_SUFFIX: str = "aconfig-local-override-{}" + +_PACKAGE_REGEX = re.compile(r"^package\:\s*\"([\w\d\.]+)\"") +_ANDROID_BP_FILE_NAME = r"Android.bp" + + +def _parse_packages(file: pathlib.Path) -> set[str]: + packages = set() + with open(file) as f: + for line in f: + line = line.strip() + package_match = _PACKAGE_REGEX.match(line) + if package_match is None: + continue + package_name = package_match.group(1) + packages.add(package_name) + + return packages + + +def _create_android_bp(packages: set[str], file_name: str) -> str: + android_bp = "" + value_list = ",\n ".join( + map(f'"{_ACONFIG_VALUES_NAME_SUFFIX}"'.format, packages) + ) + if value_list: + value_list = "\n " + value_list + "\n" + android_bp += _VALUE_LIST_TEMPLATE.format(value_list) + "\n" + + for package in packages: + android_bp += _ACONFIG_VALUES_TEMPLATE.format( + _ACONFIG_VALUES_NAME_SUFFIX.format(package), package, file_name + ) + android_bp += "\n" + + return android_bp + + +def _write_android_bp(new_android_bp: str, out: pathlib.Path) -> None: + if not out.is_dir(): + out.mkdir(parents=True, exist_ok=True) + + output = out.joinpath(_ANDROID_BP_FILE_NAME) + with open(output, "r+", encoding="utf8") as file: + lines = [] + for line in file: + line = line.rstrip("\n") + if line.startswith("ACONFIG_VALUES_LIST_LOCAL"): + break + lines.append(line) + # Overwrite the file with the updated contents. + file.seek(0) + file.truncate() + file.write("\n".join(lines)) + file.write(new_android_bp) + + +def main(args): + """Program entry point.""" + args_parser = argparse.ArgumentParser() + args_parser.add_argument( + "--overrides", + required=True, + help="The path to override file.", + ) + args_parser.add_argument( + "--out", + required=True, + help="The path to output directory.", + ) + + args = args_parser.parse_args(args) + file = pathlib.Path(args.overrides) + out = pathlib.Path(args.out) + if not file.is_file(): + raise FileNotFoundError(f"File '{file}' is not found") + + packages = _parse_packages(file) + new_android_bp = _create_android_bp(packages, file.name) + _write_android_bp(new_android_bp, out) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/aconfig/printflags/Android.bp b/tools/aconfig/printflags/Android.bp new file mode 100644 index 0000000000..d50a77d072 --- /dev/null +++ b/tools/aconfig/printflags/Android.bp @@ -0,0 +1,28 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "printflags.defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + srcs: ["src/main.rs"], + rustlibs: [ + "libaconfig_protos", + "libanyhow", + "libprotobuf", + "libregex", + ], +} + +rust_binary { + name: "printflags", + defaults: ["printflags.defaults"], +} + +rust_test_host { + name: "printflags.test", + defaults: ["printflags.defaults"], + test_suites: ["general-tests"], +} diff --git a/tools/aconfig/printflags/Cargo.toml b/tools/aconfig/printflags/Cargo.toml new file mode 100644 index 0000000000..7313f5d044 --- /dev/null +++ b/tools/aconfig/printflags/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "printflags" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +paste = "1.0.11" +protobuf = "3.2.0" +regex = "1.10.3" +aconfig_protos = { path = "../aconfig_protos" } diff --git a/tools/aconfig/printflags/src/main.rs b/tools/aconfig/printflags/src/main.rs new file mode 100644 index 0000000000..7838b51e62 --- /dev/null +++ b/tools/aconfig/printflags/src/main.rs @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `printflags` is a device binary to print feature flags. + +use aconfig_protos::ProtoFlagState as State; +use aconfig_protos::ProtoParsedFlags; +use anyhow::{bail, Context, Result}; +use regex::Regex; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::process::Command; +use std::{fs, str}; + +fn parse_device_config(raw: &str) -> HashMap<String, String> { + let mut flags = HashMap::new(); + let regex = Regex::new(r"(?m)^([[[:alnum:]]_]+/[[[:alnum:]]_\.]+)=(true|false)$").unwrap(); + for capture in regex.captures_iter(raw) { + let key = capture.get(1).unwrap().as_str().to_string(); + let value = match capture.get(2).unwrap().as_str() { + "true" => format!("{:?} (device_config)", State::ENABLED), + "false" => format!("{:?} (device_config)", State::DISABLED), + _ => panic!(), + }; + flags.insert(key, value); + } + flags +} + +fn xxd(bytes: &[u8]) -> String { + let n = 8.min(bytes.len()); + let mut v = Vec::with_capacity(n); + for byte in bytes.iter().take(n) { + v.push(format!("{:02x}", byte)); + } + let trailer = match bytes.len() { + 0..=8 => "", + _ => " ..", + }; + format!("[{}{}]", v.join(" "), trailer) +} + +fn main() -> Result<()> { + // read device_config + let output = Command::new("/system/bin/device_config").arg("list").output()?; + if !output.status.success() { + let reason = match output.status.code() { + Some(code) => format!("exit code {}", code), + None => "terminated by signal".to_string(), + }; + bail!("failed to execute device_config: {}", reason); + } + let dc_stdout = str::from_utf8(&output.stdout)?; + let device_config_flags = parse_device_config(dc_stdout); + + // read aconfig_flags.pb files + let apex_pattern = Regex::new(r"^/apex/[^@]+\.[^@]+$").unwrap(); + let mut mount_points = vec![ + "system".to_string(), + "system_ext".to_string(), + "product".to_string(), + "vendor".to_string(), + ]; + for apex in fs::read_dir("/apex")? { + let path_name = apex?.path().display().to_string(); + if let Some(canonical_path) = apex_pattern.captures(&path_name) { + mount_points.push(canonical_path.get(0).unwrap().as_str().to_owned()); + } + } + + let mut flags: BTreeMap<String, Vec<String>> = BTreeMap::new(); + for mount_point in mount_points { + let path = format!("/{}/etc/aconfig_flags.pb", mount_point); + let Ok(bytes) = fs::read(&path) else { + eprintln!("warning: failed to read {}", path); + continue; + }; + let parsed_flags: ProtoParsedFlags = protobuf::Message::parse_from_bytes(&bytes) + .with_context(|| { + format!("failed to parse {} ({}, {} byte(s))", path, xxd(&bytes), bytes.len()) + })?; + for flag in parsed_flags.parsed_flag { + let key = format!("{}/{}.{}", flag.namespace(), flag.package(), flag.name()); + let value = format!("{:?} + {:?} ({})", flag.permission(), flag.state(), mount_point); + flags.entry(key).or_default().push(value); + } + } + + // print flags + for (key, mut value) in flags { + if let Some(dc_value) = device_config_flags.get(&key) { + value.push(dc_value.to_string()); + } + println!("{}: {}", key, value.join(", ")); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_device_config() { + let input = r#" +namespace_one/com.foo.bar.flag_one=true +namespace_one/com.foo.bar.flag_two=false +random_noise; +namespace_two/android.flag_one=true +namespace_two/android.flag_two=nonsense +"#; + let expected = HashMap::from([ + ( + "namespace_one/com.foo.bar.flag_one".to_string(), + "ENABLED (device_config)".to_string(), + ), + ( + "namespace_one/com.foo.bar.flag_two".to_string(), + "DISABLED (device_config)".to_string(), + ), + ("namespace_two/android.flag_one".to_string(), "ENABLED (device_config)".to_string()), + ]); + let actual = parse_device_config(input); + assert_eq!(expected, actual); + } + + #[test] + fn test_xxd() { + let input = [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9]; + assert_eq!("[]", &xxd(&input[0..0])); + assert_eq!("[00]", &xxd(&input[0..1])); + assert_eq!("[00 01]", &xxd(&input[0..2])); + assert_eq!("[00 01 02 03 04 05 06]", &xxd(&input[0..7])); + assert_eq!("[00 01 02 03 04 05 06 07]", &xxd(&input[0..8])); + assert_eq!("[00 01 02 03 04 05 06 07 ..]", &xxd(&input[0..9])); + assert_eq!("[00 01 02 03 04 05 06 07 ..]", &xxd(&input)); + } +} diff --git a/tools/aconfig/rustfmt.toml b/tools/aconfig/rustfmt.toml new file mode 120000 index 0000000000..291e99b28f --- /dev/null +++ b/tools/aconfig/rustfmt.toml @@ -0,0 +1 @@ +../../../soong/scripts/rustfmt.toml
\ No newline at end of file diff --git a/tools/auto_gen_test_config.py b/tools/auto_gen_test_config.py index ce6416000b..8ee599a1ec 100755 --- a/tools/auto_gen_test_config.py +++ b/tools/auto_gen_test_config.py @@ -17,6 +17,8 @@ """A tool to generate TradeFed test config file. """ +import argparse +import re import os import shutil import sys @@ -42,42 +44,85 @@ def main(argv): Returns: 0 if no error, otherwise 1. """ - if len(argv) != 4 and len(argv) != 6: - sys.stderr.write( - 'Invalid arguments. The script requires 4 arguments for file paths: ' - 'target_config android_manifest empty_config ' - 'instrumentation_test_config_template ' - 'and 2 optional arguments for extra configs: ' - '--extra-configs \'EXTRA_CONFIGS\'.\n') - return 1 - - target_config = argv[0] - android_manifest = argv[1] - empty_config = argv[2] - instrumentation_test_config_template = argv[3] - extra_configs = '\n'.join(argv[5].split('\\n')) if len(argv) == 6 else '' - - manifest = parse(android_manifest) - instrumentation_elements = manifest.getElementsByTagName('instrumentation') - manifest_elements = manifest.getElementsByTagName('manifest') - if len(instrumentation_elements) != 1 or len(manifest_elements) != 1: - # Failed to locate instrumentation or manifest element in AndroidManifest. - # file. Empty test config file will be created. - shutil.copyfile(empty_config, target_config) - return 0 + + parser = argparse.ArgumentParser() + parser.add_argument( + "target_config", + help="Path to the generated output config.") + parser.add_argument( + "android_manifest", + help="Path to AndroidManifest.xml or output of 'aapt2 dump xmltree' with .xmltree extension.") + parser.add_argument( + "empty_config", + help="Path to the empty config template.") + parser.add_argument( + "instrumentation_test_config_template", + help="Path to the instrumentation test config template.") + parser.add_argument("--extra-configs", default="") + args = parser.parse_args(argv) + + target_config = args.target_config + android_manifest = args.android_manifest + empty_config = args.empty_config + instrumentation_test_config_template = args.instrumentation_test_config_template + extra_configs = '\n'.join(args.extra_configs.split('\\n')) module = os.path.splitext(os.path.basename(target_config))[0] - instrumentation = instrumentation_elements[0] - manifest = manifest_elements[0] - if ATTRIBUTE_LABEL in instrumentation.attributes: - label = instrumentation.attributes[ATTRIBUTE_LABEL].value - else: + + # If the AndroidManifest.xml is not available, but the APK is, this tool also + # accepts the output of `aapt2 dump xmltree <apk> AndroidManifest.xml` written + # into a file. This is a custom structured aapt2 output - not raw XML! + if android_manifest.endswith(".xmltree"): label = module - runner = instrumentation.attributes[ATTRIBUTE_RUNNER].value - package = manifest.attributes[ATTRIBUTE_PACKAGE].value + with open(android_manifest, encoding="utf-8") as manifest: + # e.g. A: package="android.test.example.helloworld" (Raw: "android.test.example.helloworld") + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + pattern = re.compile(r"\(Raw:\s\"(.*)\"\)$") + curr_element = None + for line in manifest: + curr_line = line.strip() + if curr_line.startswith("E:"): + # e.g. "E: instrumentation (line=9)" + # ^^^^^^^^^^^^^^^ + curr_element = curr_line.split(" ")[1] + if curr_element == "instrumentation": + if ATTRIBUTE_RUNNER in curr_line: + runner = re.findall(pattern, curr_line)[0] + if ATTRIBUTE_LABEL in curr_line: + label = re.findall(pattern, curr_line)[0] + if curr_element == "manifest": + if ATTRIBUTE_PACKAGE in curr_line: + package = re.findall(pattern, curr_line)[0] + + if not (runner and label and package): + # Failed to locate instrumentation or manifest element in AndroidManifest. + # file. Empty test config file will be created. + shutil.copyfile(empty_config, target_config) + return 0 + + else: + # If the AndroidManifest.xml file is directly available, read it as an XML file. + manifest = parse(android_manifest) + instrumentation_elements = manifest.getElementsByTagName('instrumentation') + manifest_elements = manifest.getElementsByTagName('manifest') + if len(instrumentation_elements) != 1 or len(manifest_elements) != 1: + # Failed to locate instrumentation or manifest element in AndroidManifest. + # file. Empty test config file will be created. + shutil.copyfile(empty_config, target_config) + return 0 + + instrumentation = instrumentation_elements[0] + manifest = manifest_elements[0] + if ATTRIBUTE_LABEL in instrumentation.attributes: + label = instrumentation.attributes[ATTRIBUTE_LABEL].value + else: + label = module + runner = instrumentation.attributes[ATTRIBUTE_RUNNER].value + package = manifest.attributes[ATTRIBUTE_PACKAGE].value + test_type = ('InstrumentationTest' - if runner.endswith('.InstrumentationTestRunner') - else 'AndroidJUnitTest') + if runner.endswith('.InstrumentationTestRunner') + else 'AndroidJUnitTest') with open(instrumentation_test_config_template) as template: config = template.read() diff --git a/tools/auto_gen_test_config_test.py b/tools/auto_gen_test_config_test.py index 51a8583f2e..b7ef0b0b2e 100644 --- a/tools/auto_gen_test_config_test.py +++ b/tools/auto_gen_test_config_test.py @@ -30,6 +30,24 @@ MANIFEST_INVALID = """<?xml version="1.0" encoding="utf-8"?> </manifest> """ +XMLTREE_JUNIT_TEST = """N: android=http://schemas.android.com/apk/res/android (line=2) + E: manifest (line=2) + A: package="com.android.my.tests.x" (Raw: "com.android.my.tests.x") + E: instrumentation (line=9) + A: http://schemas.android.com/apk/res/android:label(0x01010001)="TestModule" (Raw: "TestModule") + A: http://schemas.android.com/apk/res/android:name(0x01010003)="androidx.test.runner.AndroidJUnitRunner" (Raw: "androidx.test.runner.AndroidJUnitRunner") + A: http://schemas.android.com/apk/res/android:targetPackage(0x01010021)="com.android.my.tests" (Raw: "com.android.my.tests") +""" + +XMLTREE_INSTRUMENTATION_TEST = """N: android=http://schemas.android.com/apk/res/android (line=2) + E: manifest (line=2) + A: package="com.android.my.tests.x" (Raw: "com.android.my.tests.x") + E: instrumentation (line=9) + A: http://schemas.android.com/apk/res/android:label(0x01010001)="TestModule" (Raw: "TestModule") + A: http://schemas.android.com/apk/res/android:name(0x01010003)="android.test.InstrumentationTestRunner" (Raw: "android.test.InstrumentationTestRunner") + A: http://schemas.android.com/apk/res/android:targetPackage(0x01010021)="com.android.my.tests" (Raw: "com.android.my.tests") +""" + MANIFEST_JUNIT_TEST = """<?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.android.my.tests.x"> @@ -45,12 +63,12 @@ MANIFEST_INSTRUMENTATION_TEST = """<?xml version="1.0" encoding="utf-8"?> <instrumentation android:name="android.test.InstrumentationTestRunner" android:targetPackage="com.android.my.tests" - android:label="My Tests" /> + android:label="TestModule" /> </manifest> """ EXPECTED_JUNIT_TEST_CONFIG = """<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2017 The Android Open Source Project +<!-- Copyright (C) 2023 The Android Open Source Project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -66,7 +84,11 @@ EXPECTED_JUNIT_TEST_CONFIG = """<?xml version="1.0" encoding="utf-8"?> --> <!-- This test config file is auto-generated. --> <configuration description="Runs TestModule."> + <option name="test-suite-tag" value="apct" /> + <option name="test-suite-tag" value="apct-instrumentation" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller"> + <option name="cleanup-apks" value="true" /> <option name="test-file-name" value="TestModule.apk" /> </target_preparer> @@ -78,7 +100,7 @@ EXPECTED_JUNIT_TEST_CONFIG = """<?xml version="1.0" encoding="utf-8"?> """ EXPECTED_INSTRUMENTATION_TEST_CONFIG = """<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2017 The Android Open Source Project +<!-- Copyright (C) 2023 The Android Open Source Project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -93,8 +115,12 @@ EXPECTED_INSTRUMENTATION_TEST_CONFIG = """<?xml version="1.0" encoding="utf-8"?> limitations under the License. --> <!-- This test config file is auto-generated. --> -<configuration description="Runs My Tests."> +<configuration description="Runs TestModule."> + <option name="test-suite-tag" value="apct" /> + <option name="test-suite-tag" value="apct-instrumentation" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller"> + <option name="cleanup-apks" value="true" /> <option name="test-file-name" value="TestModule.apk" /> </target_preparer> @@ -105,11 +131,58 @@ EXPECTED_INSTRUMENTATION_TEST_CONFIG = """<?xml version="1.0" encoding="utf-8"?> </configuration> """ -TOOLS_DIR = os.path.dirname(os.path.dirname(__file__)) -EMPTY_TEST_CONFIG = os.path.join( - TOOLS_DIR, '..', 'core', 'empty_test_config.xml') -INSTRUMENTATION_TEST_CONFIG_TEMPLATE = os.path.join( - TOOLS_DIR, '..', 'core', 'instrumentation_test_config_template.xml') +EMPTY_TEST_CONFIG_CONTENT = """<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2017 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- No AndroidTest.xml was provided and the manifest does not include + instrumentation, hence this apk is not instrumentable. +--> +<configuration description="Empty Configuration" /> +""" + +INSTRUMENTATION_TEST_CONFIG_TEMPLATE_CONTENT = """<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- This test config file is auto-generated. --> +<configuration description="Runs {LABEL}."> + <option name="test-suite-tag" value="apct" /> + <option name="test-suite-tag" value="apct-instrumentation" /> +{EXTRA_CONFIGS} + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller"> + <option name="cleanup-apks" value="true" /> + <option name="test-file-name" value="{MODULE}.apk" /> + </target_preparer> + + <test class="com.android.tradefed.testtype.{TEST_TYPE}" > + <option name="package" value="{PACKAGE}" /> + <option name="runner" value="{RUNNER}" /> + </test> +</configuration> +""" class AutoGenTestConfigUnittests(unittest.TestCase): @@ -120,6 +193,16 @@ class AutoGenTestConfigUnittests(unittest.TestCase): self.test_dir = tempfile.mkdtemp() self.config_file = os.path.join(self.test_dir, TEST_MODULE + '.config') self.manifest_file = os.path.join(self.test_dir, 'AndroidManifest.xml') + self.xmltree_file = os.path.join(self.test_dir, TEST_MODULE + '.xmltree') + self.empty_test_config_file = os.path.join(self.test_dir, 'empty.config') + self.instrumentation_test_config_template_file = os.path.join( + self.test_dir, 'instrumentation.config') + + with open(self.empty_test_config_file, 'w') as f: + f.write(EMPTY_TEST_CONFIG_CONTENT) + + with open(self.instrumentation_test_config_template_file, 'w') as f: + f.write(INSTRUMENTATION_TEST_CONFIG_TEMPLATE_CONTENT) def tearDown(self): """Cleanup the test directory.""" @@ -133,11 +216,11 @@ class AutoGenTestConfigUnittests(unittest.TestCase): argv = [self.config_file, self.manifest_file, - EMPTY_TEST_CONFIG, - INSTRUMENTATION_TEST_CONFIG_TEMPLATE] + self.empty_test_config_file, + self.instrumentation_test_config_template_file] auto_gen_test_config.main(argv) with open(self.config_file) as config_file: - with open(EMPTY_TEST_CONFIG) as empty_config: + with open(self.empty_test_config_file) as empty_config: self.assertEqual(config_file.read(), empty_config.read()) def testCreateJUnitTestConfig(self): @@ -148,8 +231,8 @@ class AutoGenTestConfigUnittests(unittest.TestCase): argv = [self.config_file, self.manifest_file, - EMPTY_TEST_CONFIG, - INSTRUMENTATION_TEST_CONFIG_TEMPLATE] + self.empty_test_config_file, + self.instrumentation_test_config_template_file] auto_gen_test_config.main(argv) with open(self.config_file) as config_file: self.assertEqual(config_file.read(), EXPECTED_JUNIT_TEST_CONFIG) @@ -162,8 +245,37 @@ class AutoGenTestConfigUnittests(unittest.TestCase): argv = [self.config_file, self.manifest_file, - EMPTY_TEST_CONFIG, - INSTRUMENTATION_TEST_CONFIG_TEMPLATE] + self.empty_test_config_file, + self.instrumentation_test_config_template_file] + auto_gen_test_config.main(argv) + with open(self.config_file) as config_file: + self.assertEqual( + config_file.read(), EXPECTED_INSTRUMENTATION_TEST_CONFIG) + + def testCreateJUnitTestConfigWithXMLTree(self): + """Test creating test config for AndroidJUnitTest. + """ + with open(self.xmltree_file, 'w') as f: + f.write(XMLTREE_JUNIT_TEST) + + argv = [self.config_file, + self.xmltree_file, + self.empty_test_config_file, + self.instrumentation_test_config_template_file] + auto_gen_test_config.main(argv) + with open(self.config_file) as config_file: + self.assertEqual(config_file.read(), EXPECTED_JUNIT_TEST_CONFIG) + + def testCreateInstrumentationTestConfigWithXMLTree(self): + """Test creating test config for InstrumentationTest. + """ + with open(self.xmltree_file, 'w') as f: + f.write(XMLTREE_INSTRUMENTATION_TEST) + + argv = [self.config_file, + self.xmltree_file, + self.empty_test_config_file, + self.instrumentation_test_config_template_file] auto_gen_test_config.main(argv) with open(self.config_file) as config_file: self.assertEqual( diff --git a/tools/buildinfo.sh b/tools/buildinfo.sh deleted file mode 100755 index c2e36dfeee..0000000000 --- a/tools/buildinfo.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -echo "# begin build properties" -echo "# autogenerated by buildinfo.sh" - -# The ro.build.id will be set dynamically by init, by appending the unique vbmeta digest. -if [ "$BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT" = "true" ] ; then - echo "ro.build.legacy.id=$BUILD_ID" -else - echo "ro.build.id=$BUILD_ID" -fi -echo "ro.build.display.id=$BUILD_DISPLAY_ID" -echo "ro.build.version.incremental=$BUILD_NUMBER" -echo "ro.build.version.sdk=$PLATFORM_SDK_VERSION" -echo "ro.build.version.preview_sdk=$PLATFORM_PREVIEW_SDK_VERSION" -echo "ro.build.version.preview_sdk_fingerprint=$PLATFORM_PREVIEW_SDK_FINGERPRINT" -echo "ro.build.version.codename=$PLATFORM_VERSION_CODENAME" -echo "ro.build.version.all_codenames=$PLATFORM_VERSION_ALL_CODENAMES" -echo "ro.build.version.known_codenames=$PLATFORM_VERSION_KNOWN_CODENAMES" -echo "ro.build.version.release=$PLATFORM_VERSION_LAST_STABLE" -echo "ro.build.version.release_or_codename=$PLATFORM_VERSION" -echo "ro.build.version.release_or_preview_display=$PLATFORM_DISPLAY_VERSION" -echo "ro.build.version.security_patch=$PLATFORM_SECURITY_PATCH" -echo "ro.build.version.base_os=$PLATFORM_BASE_OS" -echo "ro.build.version.min_supported_target_sdk=$PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION" -echo "ro.build.date=`$DATE`" -echo "ro.build.date.utc=`$DATE +%s`" -echo "ro.build.type=$TARGET_BUILD_TYPE" -echo "ro.build.user=$BUILD_USERNAME" -echo "ro.build.host=$BUILD_HOSTNAME" -echo "ro.build.tags=$BUILD_VERSION_TAGS" -echo "ro.build.flavor=$TARGET_BUILD_FLAVOR" - -# These values are deprecated, use "ro.product.cpu.abilist" -# instead (see below). -echo "# ro.product.cpu.abi and ro.product.cpu.abi2 are obsolete," -echo "# use ro.product.cpu.abilist instead." -echo "ro.product.cpu.abi=$TARGET_CPU_ABI" -if [ -n "$TARGET_CPU_ABI2" ] ; then - echo "ro.product.cpu.abi2=$TARGET_CPU_ABI2" -fi - -if [ -n "$PRODUCT_DEFAULT_LOCALE" ] ; then - echo "ro.product.locale=$PRODUCT_DEFAULT_LOCALE" -fi -echo "ro.wifi.channels=$PRODUCT_DEFAULT_WIFI_CHANNELS" - -echo "# ro.build.product is obsolete; use ro.product.device" -echo "ro.build.product=$TARGET_DEVICE" - -echo "# Do not try to parse description or thumbprint" -echo "ro.build.description=$PRIVATE_BUILD_DESC" -if [ -n "$BUILD_THUMBPRINT" ] ; then - echo "ro.build.thumbprint=$BUILD_THUMBPRINT" -fi - -echo "# end build properties" diff --git a/tools/characteristics_rro_generator.py b/tools/characteristics_rro_generator.py new file mode 100644 index 0000000000..cf873eebcd --- /dev/null +++ b/tools/characteristics_rro_generator.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +import sys + +if __name__ == '__main__': + if len(sys.argv) != 3: + sys.exit(f"usage: {sys_argv[0]} target_package_name output\n") + with open(sys.argv[2], "w") as f: + f.write(f'''<?xml version="1.0" encoding="utf-8"?> + <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="{sys.argv[1]}.auto_generated_characteristics_rro"> + <application android:hasCode="false" /> + <overlay android:targetPackage="{sys.argv[1]}" + android:isStatic="true" + android:priority="0" /> +</manifest> +''') diff --git a/tools/check-flagged-apis/Android.bp b/tools/check-flagged-apis/Android.bp new file mode 100644 index 0000000000..43c9c8e975 --- /dev/null +++ b/tools/check-flagged-apis/Android.bp @@ -0,0 +1,51 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_team: "trendy_team_updatable_sdk_apis", + default_applicable_licenses: ["Android-Apache-2.0"], +} + +java_defaults { + name: "check-flagged-apis-defaults", + srcs: [ + "src/com/android/checkflaggedapis/Main.kt", + ], + static_libs: [ + "libaconfig_java_proto_lite", + "metalava-signature-reader", + "metalava-tools-common-m2-deps", + ], +} + +java_binary_host { + name: "check-flagged-apis", + defaults: [ + "check-flagged-apis-defaults", + ], + main_class: "com.android.checkflaggedapis.Main", +} + +java_test_host { + name: "check-flagged-apis-test", + defaults: [ + "check-flagged-apis-defaults", + ], + srcs: [ + "src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt", + ], + static_libs: [ + "junit", + ], +} diff --git a/tools/check-flagged-apis/OWNERS b/tools/check-flagged-apis/OWNERS new file mode 100644 index 0000000000..289e21e4b6 --- /dev/null +++ b/tools/check-flagged-apis/OWNERS @@ -0,0 +1,4 @@ +amhk@google.com +gurpreetgs@google.com +michaelwr@google.com +paulduffin@google.com diff --git a/tools/check-flagged-apis/check-flagged-apis.sh b/tools/check-flagged-apis/check-flagged-apis.sh new file mode 100755 index 0000000000..d9934a11fe --- /dev/null +++ b/tools/check-flagged-apis/check-flagged-apis.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run check-flagged-apis for public APIs and the three @SystemApi flavours +# Usage: lunch <your-target> && source <this script> + +source $(cd $(dirname $BASH_SOURCE) &> /dev/null && pwd)/../../shell_utils.sh +require_top + +PUBLIC_XML_VERSIONS=out/target/common/obj/PACKAGING/api_versions_public_generated-api-versions.xml +SYSTEM_XML_VERSIONS=out/target/common/obj/PACKAGING/api_versions_system_generated-api-versions.xml +SYSTEM_SERVER_XML_VERSONS=out/target/common/obj/PACKAGING/api_versions_system_server_complete_generated-api-versions.xml +MODULE_LIB_XML_VERSIONS=out/target/common/obj/PACKAGING/api_versions_module_lib_complete_generated-api-versions.xml + +function m() { + $(gettop)/build/soong/soong_ui.bash --build-mode --all-modules --dir="$(pwd)" "$@" +} + +function build() { + m \ + check-flagged-apis \ + all_aconfig_declarations \ + frameworks-base-api-current.txt \ + frameworks-base-api-system-current.txt \ + frameworks-base-api-system-server-current.txt \ + frameworks-base-api-module-lib-current.txt \ + $PUBLIC_XML_VERSIONS \ + $SYSTEM_XML_VERSIONS \ + $SYSTEM_SERVER_XML_VERSONS \ + $MODULE_LIB_XML_VERSIONS +} + +function aninja() { + local T="$(gettop)" + (\cd "${T}" && prebuilts/build-tools/linux-x86/bin/ninja -f out/combined-${TARGET_PRODUCT}.ninja "$@") +} + +function path_to_api_signature_file { + aninja -t query device_"$1"_all_targets | grep -A1 -e input: | tail -n1 +} + +function run() { + local errors=0 + + echo "# current" + check-flagged-apis \ + --api-signature $(path_to_api_signature_file "frameworks-base-api-current.txt") \ + --flag-values $(gettop)/out/soong/.intermediates/all_aconfig_declarations.pb \ + --api-versions $PUBLIC_XML_VERSIONS + (( errors += $? )) + + echo + echo "# system-current" + check-flagged-apis \ + --api-signature $(path_to_api_signature_file "frameworks-base-api-system-current.txt") \ + --flag-values $(gettop)/out/soong/.intermediates/all_aconfig_declarations.pb \ + --api-versions $SYSTEM_XML_VERSIONS + (( errors += $? )) + + echo + echo "# system-server-current" + check-flagged-apis \ + --api-signature $(path_to_api_signature_file "frameworks-base-api-system-server-current.txt") \ + --flag-values $(gettop)/out/soong/.intermediates/all_aconfig_declarations.pb \ + --api-versions $SYSTEM_SERVER_XML_VERSONS + (( errors += $? )) + + echo + echo "# module-lib" + check-flagged-apis \ + --api-signature $(path_to_api_signature_file "frameworks-base-api-module-lib-current.txt") \ + --flag-values $(gettop)/out/soong/.intermediates/all_aconfig_declarations.pb \ + --api-versions $MODULE_LIB_XML_VERSIONS + (( errors += $? )) + + return $errors +} + +if [[ "$1" != "--skip-build" ]]; then + build && run +else + run +fi diff --git a/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt b/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt new file mode 100644 index 0000000000..cd859448b0 --- /dev/null +++ b/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.checkflaggedapis + +import android.aconfig.Aconfig +import android.aconfig.Aconfig.flag_state.DISABLED +import android.aconfig.Aconfig.flag_state.ENABLED +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.InputStream +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +private val API_SIGNATURE = + """ + // Signature format: 2.0 + package android { + @FlaggedApi("android.flag.foo") public final class Clazz { + ctor @FlaggedApi("android.flag.foo") public Clazz(); + field @FlaggedApi("android.flag.foo") public static final int FOO = 1; // 0x1 + method @FlaggedApi("android.flag.foo") public int getErrorCode(); + method @FlaggedApi("android.flag.foo") public boolean setData(int, int[][], @NonNull android.util.Utility<T, U>); + method @FlaggedApi("android.flag.foo") public boolean setVariableData(int, android.util.Atom...); + method @FlaggedApi("android.flag.foo") public boolean innerClassArg(android.Clazz.Builder); + } + @FlaggedApi("android.flag.bar") public static class Clazz.Builder { + } + } +""" + .trim() + +private val API_VERSIONS = + """ + <?xml version="1.0" encoding="utf-8"?> + <api version="3"> + <class name="android/Clazz" since="1"> + <extends name="java/lang/Object"/> + <method name="<init>()V"/> + <field name="FOO"/> + <method name="getErrorCode()I"/> + <method name="setData(I[[ILandroid/util/Utility;)Z"/> + <method name="setVariableData(I[Landroid/util/Atom;)Z"/> + <method name="innerClassArg(Landroid/Clazz${"$"}Builder;)"/> + </class> + <class name="android/Clazz${"$"}Builder" since="2"> + <extends name="java/lang/Object"/> + </class> + </api> +""" + .trim() + +private fun generateFlagsProto( + fooState: Aconfig.flag_state, + barState: Aconfig.flag_state +): InputStream { + val fooFlag = + Aconfig.parsed_flag + .newBuilder() + .setPackage("android.flag") + .setName("foo") + .setState(fooState) + .setPermission(Aconfig.flag_permission.READ_ONLY) + .build() + val barFlag = + Aconfig.parsed_flag + .newBuilder() + .setPackage("android.flag") + .setName("bar") + .setState(barState) + .setPermission(Aconfig.flag_permission.READ_ONLY) + .build() + val flags = + Aconfig.parsed_flags.newBuilder().addParsedFlag(fooFlag).addParsedFlag(barFlag).build() + val binaryProto = ByteArrayOutputStream() + flags.writeTo(binaryProto) + return ByteArrayInputStream(binaryProto.toByteArray()) +} + +@RunWith(JUnit4::class) +class CheckFlaggedApisTest { + @Test + fun testParseApiSignature() { + val expected = + setOf( + Pair( + Symbol.createClass("android/Clazz", "java/lang/Object", setOf()), + Flag("android.flag.foo")), + Pair(Symbol.createMethod("android/Clazz", "Clazz()"), Flag("android.flag.foo")), + Pair(Symbol.createField("android/Clazz", "FOO"), Flag("android.flag.foo")), + Pair(Symbol.createMethod("android/Clazz", "getErrorCode()"), Flag("android.flag.foo")), + Pair( + Symbol.createMethod("android/Clazz", "setData(I[[ILandroid/util/Utility;)"), + Flag("android.flag.foo")), + Pair( + Symbol.createMethod("android/Clazz", "setVariableData(I[Landroid/util/Atom;)"), + Flag("android.flag.foo")), + Pair( + Symbol.createMethod("android/Clazz", "innerClassArg(Landroid/Clazz/Builder;)"), + Flag("android.flag.foo")), + Pair( + Symbol.createClass("android/Clazz/Builder", "java/lang/Object", setOf()), + Flag("android.flag.bar")), + ) + val actual = parseApiSignature("in-memory", API_SIGNATURE.byteInputStream()) + assertEquals(expected, actual) + } + + @Test + fun testParseFlagValues() { + val expected: Map<Flag, Boolean> = + mapOf(Flag("android.flag.foo") to true, Flag("android.flag.bar") to true) + val actual = parseFlagValues(generateFlagsProto(ENABLED, ENABLED)) + assertEquals(expected, actual) + } + + @Test + fun testParseApiVersions() { + val expected: Set<Symbol> = + setOf( + Symbol.createClass("android/Clazz", "java/lang/Object", setOf()), + Symbol.createMethod("android/Clazz", "Clazz()"), + Symbol.createField("android/Clazz", "FOO"), + Symbol.createMethod("android/Clazz", "getErrorCode()"), + Symbol.createMethod("android/Clazz", "setData(I[[ILandroid/util/Utility;)"), + Symbol.createMethod("android/Clazz", "setVariableData(I[Landroid/util/Atom;)"), + Symbol.createMethod("android/Clazz", "innerClassArg(Landroid/Clazz/Builder;)"), + Symbol.createClass("android/Clazz/Builder", "java/lang/Object", setOf()), + ) + val actual = parseApiVersions(API_VERSIONS.byteInputStream()) + assertEquals(expected, actual) + } + + @Test + fun testParseApiVersionsNestedClasses() { + val apiVersions = + """ + <?xml version="1.0" encoding="utf-8"?> + <api version="3"> + <class name="android/Clazz${'$'}Foo${'$'}Bar" since="1"> + <extends name="java/lang/Object"/> + <method name="<init>()V"/> + </class> + </api> + """ + .trim() + val expected: Set<Symbol> = + setOf( + Symbol.createClass("android/Clazz/Foo/Bar", "java/lang/Object", setOf()), + Symbol.createMethod("android/Clazz/Foo/Bar", "Bar()"), + ) + val actual = parseApiVersions(apiVersions.byteInputStream()) + assertEquals(expected, actual) + } + + @Test + fun testFindErrorsNoErrors() { + val expected = setOf<ApiError>() + val actual = + findErrors( + parseApiSignature("in-memory", API_SIGNATURE.byteInputStream()), + parseFlagValues(generateFlagsProto(ENABLED, ENABLED)), + parseApiVersions(API_VERSIONS.byteInputStream())) + assertEquals(expected, actual) + } + + @Test + fun testFindErrorsVerifyImplements() { + val apiSignature = + """ + // Signature format: 2.0 + package android { + @FlaggedApi("android.flag.foo") public final class Clazz implements android.Interface { + method @FlaggedApi("android.flag.foo") public boolean foo(); + method @FlaggedApi("android.flag.foo") public boolean bar(); + } + public interface Interface { + method public boolean bar(); + } + } + """ + .trim() + + val apiVersions = + """ + <?xml version="1.0" encoding="utf-8"?> + <api version="3"> + <class name="android/Clazz" since="1"> + <extends name="java/lang/Object"/> + <implements name="android/Interface"/> + <method name="foo()Z"/> + </class> + <class name="android/Interface" since="1"> + <method name="bar()Z"/> + </class> + </api> + """ + .trim() + + val expected = setOf<ApiError>() + val actual = + findErrors( + parseApiSignature("in-memory", apiSignature.byteInputStream()), + parseFlagValues(generateFlagsProto(ENABLED, ENABLED)), + parseApiVersions(apiVersions.byteInputStream())) + assertEquals(expected, actual) + } + + @Test + fun testFindErrorsVerifySuperclass() { + val apiSignature = + """ + // Signature format: 2.0 + package android { + @FlaggedApi("android.flag.foo") public final class C extends android.B { + method @FlaggedApi("android.flag.foo") public boolean c(); + method @FlaggedApi("android.flag.foo") public boolean b(); + method @FlaggedApi("android.flag.foo") public boolean a(); + } + public final class B extends android.A { + method public boolean b(); + } + public final class A { + method public boolean a(); + } + } + """ + .trim() + + val apiVersions = + """ + <?xml version="1.0" encoding="utf-8"?> + <api version="3"> + <class name="android/C" since="1"> + <extends name="android/B"/> + <method name="c()Z"/> + </class> + <class name="android/B" since="1"> + <extends name="android/A"/> + <method name="b()Z"/> + </class> + <class name="android/A" since="1"> + <method name="a()Z"/> + </class> + </api> + """ + .trim() + + val expected = setOf<ApiError>() + val actual = + findErrors( + parseApiSignature("in-memory", apiSignature.byteInputStream()), + parseFlagValues(generateFlagsProto(ENABLED, ENABLED)), + parseApiVersions(apiVersions.byteInputStream())) + assertEquals(expected, actual) + } + + @Test + fun testFindErrorsDisabledFlaggedApiIsPresent() { + val expected = + setOf<ApiError>( + DisabledFlaggedApiIsPresentError( + Symbol.createClass("android/Clazz", "java/lang/Object", setOf()), + Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createMethod("android/Clazz", "Clazz()"), Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createField("android/Clazz", "FOO"), Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createMethod("android/Clazz", "getErrorCode()"), Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createMethod("android/Clazz", "setData(I[[ILandroid/util/Utility;)"), + Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createMethod("android/Clazz", "setVariableData(I[Landroid/util/Atom;)"), + Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createMethod("android/Clazz", "innerClassArg(Landroid/Clazz/Builder;)"), + Flag("android.flag.foo")), + DisabledFlaggedApiIsPresentError( + Symbol.createClass("android/Clazz/Builder", "java/lang/Object", setOf()), + Flag("android.flag.bar")), + ) + val actual = + findErrors( + parseApiSignature("in-memory", API_SIGNATURE.byteInputStream()), + parseFlagValues(generateFlagsProto(DISABLED, DISABLED)), + parseApiVersions(API_VERSIONS.byteInputStream())) + assertEquals(expected, actual) + } +} diff --git a/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt b/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt new file mode 100644 index 0000000000..b514048bd6 --- /dev/null +++ b/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt @@ -0,0 +1,407 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:JvmName("Main") + +package com.android.checkflaggedapis + +import android.aconfig.Aconfig +import com.android.tools.metalava.model.BaseItemVisitor +import com.android.tools.metalava.model.ClassItem +import com.android.tools.metalava.model.FieldItem +import com.android.tools.metalava.model.Item +import com.android.tools.metalava.model.MethodItem +import com.android.tools.metalava.model.text.ApiFile +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.parameters.options.help +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.path +import java.io.InputStream +import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Node + +/** + * Class representing the fully qualified name of a class, method or field. + * + * This tool reads a multitude of input formats all of which represents the fully qualified path to + * a Java symbol slightly differently. To keep things consistent, all parsed APIs are converted to + * Symbols. + * + * Symbols are encoded using the format similar to the one described in section 4.3.2 of the JVM + * spec [1], that is, "package.class.inner-class.method(int, int[], android.util.Clazz)" is + * represented as + * <pre> + * package.class.inner-class.method(II[Landroid/util/Clazz;) + * <pre> + * + * Where possible, the format has been simplified (to make translation of the + * various input formats easier): for instance, only / is used as delimiter (# + * and $ are never used). + * + * 1. https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html#jvms-4.3.2 + */ +internal sealed class Symbol { + companion object { + private val FORBIDDEN_CHARS = listOf('#', '$', '.') + + fun createClass(clazz: String, superclass: String?, interfaces: Set<String>): Symbol { + return ClassSymbol( + toInternalFormat(clazz), + superclass?.let { toInternalFormat(it) }, + interfaces.map { toInternalFormat(it) }.toSet()) + } + + fun createField(clazz: String, field: String): Symbol { + require(!field.contains("(") && !field.contains(")")) + return MemberSymbol(toInternalFormat(clazz), toInternalFormat(field)) + } + + fun createMethod(clazz: String, method: String): Symbol { + return MemberSymbol(toInternalFormat(clazz), toInternalFormat(method)) + } + + protected fun toInternalFormat(name: String): String { + var internalName = name + for (ch in FORBIDDEN_CHARS) { + internalName = internalName.replace(ch, '/') + } + return internalName + } + } + + abstract fun toPrettyString(): String +} + +internal data class ClassSymbol( + val clazz: String, + val superclass: String?, + val interfaces: Set<String> +) : Symbol() { + override fun toPrettyString(): String = "$clazz" +} + +internal data class MemberSymbol(val clazz: String, val member: String) : Symbol() { + override fun toPrettyString(): String = "$clazz/$member" +} + +/** + * Class representing the fully qualified name of an aconfig flag. + * + * This includes both the flag's package and name, separated by a dot, e.g.: + * <pre> + * com.android.aconfig.test.disabled_ro + * <pre> + */ +@JvmInline +internal value class Flag(val name: String) { + override fun toString(): String = name.toString() +} + +internal sealed class ApiError { + abstract val symbol: Symbol + abstract val flag: Flag +} + +internal data class EnabledFlaggedApiNotPresentError( + override val symbol: Symbol, + override val flag: Flag +) : ApiError() { + override fun toString(): String { + return "error: enabled @FlaggedApi not present in built artifact: symbol=${symbol.toPrettyString()} flag=$flag" + } +} + +internal data class DisabledFlaggedApiIsPresentError( + override val symbol: Symbol, + override val flag: Flag +) : ApiError() { + override fun toString(): String { + return "error: disabled @FlaggedApi is present in built artifact: symbol=${symbol.toPrettyString()} flag=$flag" + } +} + +internal data class UnknownFlagError(override val symbol: Symbol, override val flag: Flag) : + ApiError() { + override fun toString(): String { + return "error: unknown flag: symbol=${symbol.toPrettyString()} flag=$flag" + } +} + +class CheckCommand : + CliktCommand( + help = + """ +Check that all flagged APIs are used in the correct way. + +This tool reads the API signature file and checks that all flagged APIs are used in the correct way. + +The tool will exit with a non-zero exit code if any flagged APIs are found to be used in the incorrect way. +""") { + private val apiSignaturePath by + option("--api-signature") + .help( + """ + Path to API signature file. + Usually named *current.txt. + Tip: `m frameworks-base-api-current.txt` will generate a file that includes all platform and mainline APIs. + """) + .path(mustExist = true, canBeDir = false, mustBeReadable = true) + .required() + private val flagValuesPath by + option("--flag-values") + .help( + """ + Path to aconfig parsed_flags binary proto file. + Tip: `m all_aconfig_declarations` will generate a file that includes all information about all flags. + """) + .path(mustExist = true, canBeDir = false, mustBeReadable = true) + .required() + private val apiVersionsPath by + option("--api-versions") + .help( + """ + Path to API versions XML file. + Usually named xml-versions.xml. + Tip: `m sdk dist` will generate a file that includes all platform and mainline APIs. + """) + .path(mustExist = true, canBeDir = false, mustBeReadable = true) + .required() + + override fun run() { + val flaggedSymbols = + apiSignaturePath.toFile().inputStream().use { + parseApiSignature(apiSignaturePath.toString(), it) + } + val flags = flagValuesPath.toFile().inputStream().use { parseFlagValues(it) } + val exportedSymbols = apiVersionsPath.toFile().inputStream().use { parseApiVersions(it) } + val errors = findErrors(flaggedSymbols, flags, exportedSymbols) + for (e in errors) { + println(e) + } + throw ProgramResult(errors.size) + } +} + +internal fun parseApiSignature(path: String, input: InputStream): Set<Pair<Symbol, Flag>> { + val output = mutableSetOf<Pair<Symbol, Flag>>() + val visitor = + object : BaseItemVisitor() { + override fun visitClass(cls: ClassItem) { + getFlagOrNull(cls)?.let { flag -> + val symbol = + Symbol.createClass( + cls.baselineElementId(), + cls.superClass()?.baselineElementId(), + cls.allInterfaces() + .map { it.baselineElementId() } + .filter { it != cls.baselineElementId() } + .toSet()) + output.add(Pair(symbol, flag)) + } + } + + override fun visitField(field: FieldItem) { + getFlagOrNull(field)?.let { flag -> + val symbol = + Symbol.createField(field.containingClass().baselineElementId(), field.name()) + output.add(Pair(symbol, flag)) + } + } + + override fun visitMethod(method: MethodItem) { + getFlagOrNull(method)?.let { flag -> + val methodName = buildString { + append(method.name()) + append("(") + method.parameters().joinTo(this, separator = "") { it.type().internalName() } + append(")") + } + val symbol = Symbol.createMethod(method.containingClass().qualifiedName(), methodName) + output.add(Pair(symbol, flag)) + } + } + + private fun getFlagOrNull(item: Item): Flag? { + return item.modifiers + .findAnnotation("android.annotation.FlaggedApi") + ?.findAttribute("value") + ?.value + ?.let { Flag(it.value() as String) } + } + } + val codebase = ApiFile.parseApi(path, input) + codebase.accept(visitor) + return output +} + +internal fun parseFlagValues(input: InputStream): Map<Flag, Boolean> { + val parsedFlags = Aconfig.parsed_flags.parseFrom(input).getParsedFlagList() + return parsedFlags.associateBy( + { Flag("${it.getPackage()}.${it.getName()}") }, + { it.getState() == Aconfig.flag_state.ENABLED }) +} + +internal fun parseApiVersions(input: InputStream): Set<Symbol> { + fun Node.getAttribute(name: String): String? = getAttributes()?.getNamedItem(name)?.getNodeValue() + + val output = mutableSetOf<Symbol>() + val factory = DocumentBuilderFactory.newInstance() + val parser = factory.newDocumentBuilder() + val document = parser.parse(input) + + val classes = document.getElementsByTagName("class") + // ktfmt doesn't understand the `..<` range syntax; explicitly call .rangeUntil instead + for (i in 0.rangeUntil(classes.getLength())) { + val cls = classes.item(i) + val className = + requireNotNull(cls.getAttribute("name")) { + "Bad XML: <class> element without name attribute" + } + var superclass: String? = null + val interfaces = mutableSetOf<String>() + val children = cls.getChildNodes() + for (j in 0.rangeUntil(children.getLength())) { + val child = children.item(j) + when (child.getNodeName()) { + "extends" -> { + superclass = + requireNotNull(child.getAttribute("name")) { + "Bad XML: <extends> element without name attribute" + } + } + "implements" -> { + val interfaceName = + requireNotNull(child.getAttribute("name")) { + "Bad XML: <implements> element without name attribute" + } + interfaces.add(interfaceName) + } + } + } + output.add(Symbol.createClass(className, superclass, interfaces)) + } + + val fields = document.getElementsByTagName("field") + // ktfmt doesn't understand the `..<` range syntax; explicitly call .rangeUntil instead + for (i in 0.rangeUntil(fields.getLength())) { + val field = fields.item(i) + val fieldName = + requireNotNull(field.getAttribute("name")) { + "Bad XML: <field> element without name attribute" + } + val className = + requireNotNull(field.getParentNode()?.getAttribute("name")) { + "Bad XML: top level <field> element" + } + output.add(Symbol.createField(className, fieldName)) + } + + val methods = document.getElementsByTagName("method") + // ktfmt doesn't understand the `..<` range syntax; explicitly call .rangeUntil instead + for (i in 0.rangeUntil(methods.getLength())) { + val method = methods.item(i) + val methodSignature = + requireNotNull(method.getAttribute("name")) { + "Bad XML: <method> element without name attribute" + } + val methodSignatureParts = methodSignature.split(Regex("\\(|\\)")) + if (methodSignatureParts.size != 3) { + throw Exception("Bad XML: method signature '$methodSignature'") + } + var (methodName, methodArgs, _) = methodSignatureParts + val packageAndClassName = + requireNotNull(method.getParentNode()?.getAttribute("name")) { + "Bad XML: top level <method> element, or <class> element missing name attribute" + } + .replace("$", "/") + if (methodName == "<init>") { + methodName = packageAndClassName.split("/").last() + } + output.add(Symbol.createMethod(packageAndClassName, "$methodName($methodArgs)")) + } + + return output +} + +/** + * Find errors in the given data. + * + * @param flaggedSymbolsInSource the set of symbols that are flagged in the source code + * @param flags the set of flags and their values + * @param symbolsInOutput the set of symbols that are present in the output + * @return the set of errors found + */ +internal fun findErrors( + flaggedSymbolsInSource: Set<Pair<Symbol, Flag>>, + flags: Map<Flag, Boolean>, + symbolsInOutput: Set<Symbol> +): Set<ApiError> { + fun Set<Symbol>.containsSymbol(symbol: Symbol): Boolean { + // trivial case: the symbol is explicitly listed in api-versions.xml + if (contains(symbol)) { + return true + } + + // non-trivial case: the symbol could be part of the surrounding class' + // super class or interfaces + val (className, memberName) = + when (symbol) { + is ClassSymbol -> return false + is MemberSymbol -> { + Pair(symbol.clazz, symbol.member) + } + } + val clazz = find { it is ClassSymbol && it.clazz == className } as? ClassSymbol? + if (clazz == null) { + return false + } + + for (interfaceName in clazz.interfaces) { + // createMethod is the same as createField, except it allows parenthesis + val interfaceSymbol = Symbol.createMethod(interfaceName, memberName) + if (contains(interfaceSymbol)) { + return true + } + } + + if (clazz.superclass != null) { + val superclassSymbol = Symbol.createMethod(clazz.superclass, memberName) + return containsSymbol(superclassSymbol) + } + + return false + } + val errors = mutableSetOf<ApiError>() + for ((symbol, flag) in flaggedSymbolsInSource) { + try { + if (flags.getValue(flag)) { + if (!symbolsInOutput.containsSymbol(symbol)) { + errors.add(EnabledFlaggedApiNotPresentError(symbol, flag)) + } + } else { + if (symbolsInOutput.containsSymbol(symbol)) { + errors.add(DisabledFlaggedApiIsPresentError(symbol, flag)) + } + } + } catch (e: NoSuchElementException) { + errors.add(UnknownFlagError(symbol, flag)) + } + } + return errors +} + +fun main(args: Array<String>) = CheckCommand().main(args) diff --git a/tools/checkowners.py b/tools/checkowners.py deleted file mode 100755 index f037321059..0000000000 --- a/tools/checkowners.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/python - -"""Parse and check syntax errors of a given OWNERS file.""" - -import argparse -import re -import sys -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse - -parser = argparse.ArgumentParser(description='Check OWNERS file syntax') -parser.add_argument('-v', '--verbose', dest='verbose', - action='store_true', default=False, - help='Verbose output to debug') -parser.add_argument('-c', '--check_address', dest='check_address', - action='store_true', default=False, - help='Check email addresses') -parser.add_argument(dest='owners', metavar='OWNERS', nargs='+', - help='Path to OWNERS file') -args = parser.parse_args() - -gerrit_server = 'https://android-review.googlesource.com' -checked_addresses = {} - - -def echo(msg): - if args.verbose: - print(msg) - - -def find_address(address): - if address not in checked_addresses: - request = (gerrit_server + '/accounts/?n=1&q=email:' - + urllib.parse.quote(address)) - echo('Checking email address: ' + address) - result = urllib.request.urlopen(request).read() - checked_addresses[address] = result.find('"_account_id":') >= 0 - if checked_addresses[address]: - echo('Found email address: ' + address) - return checked_addresses[address] - - -def check_address(fname, num, address): - if find_address(address): - return 0 - print('%s:%d: ERROR: unknown email address: %s' % (fname, num, address)) - return 1 - - -def main(): - # One regular expression to check all valid lines. - noparent = 'set +noparent' - email = '([^@ ]+@[^ @]+|\\*)' - emails = '(%s( *, *%s)*)' % (email, email) - file_directive = 'file: *([^ :]+ *: *)?[^ ]+' - directive = '(%s|%s|%s)' % (emails, noparent, file_directive) - glob = '[a-zA-Z0-9_\\.\\-\\*\\?]+' - globs = '(%s( *, *%s)*)' % (glob, glob) - perfile = 'per-file +' + globs + ' *= *' + directive - include = 'include +([^ :]+ *: *)?[^ ]+' - pats = '(|%s|%s|%s|%s|%s)$' % (noparent, email, perfile, include, file_directive) - patterns = re.compile(pats) - address_pattern = re.compile('([^@ ]+@[^ @]+)') - perfile_pattern = re.compile('per-file +.*=(.*)') - - error = 0 - for fname in args.owners: - echo('Checking file: ' + fname) - num = 0 - for line in open(fname, 'r'): - num += 1 - stripped_line = re.sub('#.*$', '', line).strip() - if not patterns.match(stripped_line): - error += 1 - print('%s:%d: ERROR: unknown line [%s]' % (fname, num, line.strip())) - elif args.check_address: - if perfile_pattern.match(stripped_line): - for addr in perfile_pattern.match(stripped_line).group(1).split(','): - a = addr.strip() - if a and a != '*': - error += check_address(fname, num, addr.strip()) - elif address_pattern.match(stripped_line): - error += check_address(fname, num, stripped_line) - sys.exit(error) - -if __name__ == '__main__': - main() diff --git a/tools/compliance/Android.bp b/tools/compliance/Android.bp index 8e13f2f661..ef5c760cfc 100644 --- a/tools/compliance/Android.bp +++ b/tools/compliance/Android.bp @@ -138,6 +138,11 @@ blueprint_go_binary { "compliance-module", "blueprint-deptools", "soong-response", + "spdx-tools-spdxv2_2", + "spdx-tools-builder2v2", + "spdx-tools-spdxcommon", + "spdx-tools-spdx-json", + "spdx-tools-spdxlib", ], testSrcs: ["cmd/sbom/sbom_test.go"], } diff --git a/tools/compliance/cmd/rtrace/rtrace.go b/tools/compliance/cmd/rtrace/rtrace.go index 667cdcea87..3e7e69bd02 100644 --- a/tools/compliance/cmd/rtrace/rtrace.go +++ b/tools/compliance/cmd/rtrace/rtrace.go @@ -93,17 +93,17 @@ func main() { flags.Usage = func() { fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...} -Outputs a space-separated Target ActsOn Origin Condition tuple for each -resolution in the graph. When -dot flag given, outputs nodes and edges -in graphviz directed graph format. +Calculates the source-sharing requirements in reverse starting at the +-rtrace projects or metadata files that inherited source-sharing and +working back to the targets where the source-sharing requirmements +originate. -If one or more '-c condition' conditions are given, outputs the -resolution for the union of the conditions. Otherwise, outputs the -resolution for all conditions. +Outputs a space-separated pair where the first field is an originating +target with one or more restricted conditions and where the second +field is a colon-separated list of the restricted conditions. -In plain text mode, when '-label_conditions' is requested, the Target -and Origin have colon-separated license conditions appended: -i.e. target:condition1:condition2 etc. +Outputs a count of the originating targets, and if the count is zero, +outputs a warning to check the -rtrace projects and/or filenames. Options: `, filepath.Base(os.Args[0])) diff --git a/tools/compliance/cmd/sbom/sbom.go b/tools/compliance/cmd/sbom/sbom.go index 0f8a876f24..a53741ffb2 100644 --- a/tools/compliance/cmd/sbom/sbom.go +++ b/tools/compliance/cmd/sbom/sbom.go @@ -16,6 +16,8 @@ package main import ( "bytes" + "crypto/sha1" + "encoding/hex" "flag" "fmt" "io" @@ -31,6 +33,12 @@ import ( "android/soong/tools/compliance/projectmetadata" "github.com/google/blueprint/deptools" + + "github.com/spdx/tools-golang/builder/builder2v2" + spdx_json "github.com/spdx/tools-golang/json" + "github.com/spdx/tools-golang/spdx/common" + spdx "github.com/spdx/tools-golang/spdx/v2_2" + "github.com/spdx/tools-golang/spdxlib" ) var ( @@ -38,6 +46,8 @@ var ( failNoLicenses = fmt.Errorf("No licenses found") ) +const NOASSERTION = "NOASSERTION" + type context struct { stdout io.Writer stderr io.Writer @@ -45,6 +55,7 @@ type context struct { product string stripPrefix []string creationTime creationTimeGetter + buildid string } func (ctx context) strip(installPath string) string { @@ -114,6 +125,7 @@ Options: depsFile := flags.String("d", "", "Where to write the deps file") product := flags.String("product", "", "The name of the product for which the notice is generated.") stripPrefix := newMultiString(flags, "strip_prefix", "Prefix to remove from paths. i.e. path to root (multiple allowed)") + buildid := flags.String("build_id", "", "Uniquely identifies the build. (default timestamp)") flags.Parse(expandedArgs) @@ -152,9 +164,10 @@ Options: ofile = obuf } - ctx := &context{ofile, os.Stderr, compliance.FS, *product, *stripPrefix, actualTime} + ctx := &context{ofile, os.Stderr, compliance.FS, *product, *stripPrefix, actualTime, *buildid} + + spdxDoc, deps, err := sbomGenerator(ctx, flags.Args()...) - deps, err := sbomGenerator(ctx, flags.Args()...) if err != nil { if err == failNoneRequested { flags.Usage() @@ -163,6 +176,12 @@ Options: os.Exit(1) } + // writing the spdx Doc created + if err := spdx_json.Save2_2(spdxDoc, ofile); err != nil { + fmt.Fprintf(os.Stderr, "failed to write document to %v: %v", *outputFile, err) + os.Exit(1) + } + if *outputFile != "-" { err := os.WriteFile(*outputFile, obuf.Bytes(), 0666) if err != nil { @@ -181,11 +200,12 @@ Options: os.Exit(0) } -type creationTimeGetter func() time.Time +type creationTimeGetter func() string // actualTime returns current time in UTC -func actualTime() time.Time { - return time.Now().UTC() +func actualTime() string { + t := time.Now().UTC() + return t.UTC().Format("2006-01-02T15:04:05Z") } // replaceSlashes replaces "/" by "-" for the library path to be used for packages & files SPDXID @@ -193,6 +213,23 @@ func replaceSlashes(x string) string { return strings.ReplaceAll(x, "/", "-") } +// stripDocName removes the outdir prefix and meta_lic suffix from a target Name +func stripDocName(name string) string { + // remove outdir prefix + if strings.HasPrefix(name, "out/") { + name = name[4:] + } + + // remove suffix + if strings.HasSuffix(name, ".meta_lic") { + name = name[:len(name)-9] + } else if strings.HasSuffix(name, "/meta_lic") { + name = name[:len(name)-9] + "/" + } + + return name +} + // getPackageName returns a package name of a target Node func getPackageName(_ *context, tn *compliance.TargetNode) string { return replaceSlashes(tn.Name()) @@ -210,25 +247,24 @@ func getDocumentName(ctx *context, tn *compliance.TargetNode, pm *projectmetadat return replaceSlashes(tn.ModuleName()) } - // TO DO: Replace tn.Name() with pm.Name() + parts of the target name - return replaceSlashes(tn.Name()) + return stripDocName(replaceSlashes(tn.Name())) } // getDownloadUrl returns the download URL if available (GIT, SVN, etc..), // or NOASSERTION if not available, none determined or ambiguous func getDownloadUrl(_ *context, pm *projectmetadata.ProjectMetadata) string { if pm == nil { - return "NOASSERTION" + return NOASSERTION } urlsByTypeName := pm.UrlsByTypeName() if urlsByTypeName == nil { - return "NOASSERTION" + return NOASSERTION } url := urlsByTypeName.DownloadUrl() if url == "" { - return "NOASSERTION" + return NOASSERTION } return url } @@ -238,7 +274,7 @@ func getProjectMetadata(_ *context, pmix *projectmetadata.Index, tn *compliance.TargetNode) (*projectmetadata.ProjectMetadata, error) { pms, err := pmix.MetadataForProjects(tn.Projects()...) if err != nil { - return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn, err) + return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn.Name(), err) } if len(pms) == 0 { return nil, nil @@ -274,7 +310,7 @@ func getProjectMetadata(_ *context, pmix *projectmetadata.Index, // inputFiles returns the complete list of files read func inputFiles(lg *compliance.LicenseGraph, pmix *projectmetadata.Index, licenseTexts []string) []string { projectMeta := pmix.AllMetadataFiles() - targets := lg.TargetNames() + targets := lg.TargetNames() files := make([]string, 0, len(licenseTexts)+len(targets)+len(projectMeta)) files = append(files, licenseTexts...) files = append(files, targets...) @@ -282,6 +318,26 @@ func inputFiles(lg *compliance.LicenseGraph, pmix *projectmetadata.Index, licens return files } +// generateSPDXNamespace generates a unique SPDX Document Namespace using a SHA1 checksum +func generateSPDXNamespace(buildid string, created string, files ...string) string { + + seed := strings.Join(files, "") + + if buildid == "" { + seed += created + } else { + seed += buildid + } + + // Compute a SHA1 checksum of the seed. + hash := sha1.Sum([]byte(seed)) + uuid := hex.EncodeToString(hash[:]) + + namespace := fmt.Sprintf("SPDXRef-DOCUMENT-%s", uuid) + + return namespace +} + // sbomGenerator implements the spdx bom utility // SBOM is part of the new government regulation issued to improve national cyber security @@ -289,10 +345,10 @@ func inputFiles(lg *compliance.LicenseGraph, pmix *projectmetadata.Index, licens // sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/) // sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide) -func sbomGenerator(ctx *context, files ...string) ([]string, error) { +func sbomGenerator(ctx *context, files ...string) (*spdx.Document, []string, error) { // Must be at least one root file. if len(files) < 1 { - return nil, failNoneRequested + return nil, nil, failNoneRequested } pmix := projectmetadata.NewIndex(ctx.rootFS) @@ -300,9 +356,24 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) { lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files) if err != nil { - return nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) + return nil, nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) } + // creating the packages section + pkgs := []*spdx.Package{} + + // creating the relationship section + relationships := []*spdx.Relationship{} + + // creating the license section + otherLicenses := []*spdx.OtherLicense{} + + // spdx document name + var docName string + + // main package name + var mainPkgName string + // implementing the licenses references for the packages licenses := make(map[string]string) concludedLicenses := func(licenseTexts []string) string { @@ -325,7 +396,6 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) { } isMainPackage := true - var mainPackage string visitedNodes := make(map[*compliance.TargetNode]struct{}) // performing a Breadth-first top down walk of licensegraph and building package information @@ -341,45 +411,51 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) { } if isMainPackage { - mainPackage = getDocumentName(ctx, tn, pm) - fmt.Fprintf(ctx.stdout, "SPDXVersion: SPDX-2.2\n") - fmt.Fprintf(ctx.stdout, "DataLicense: CC0-1.0\n") - fmt.Fprintf(ctx.stdout, "DocumentName: %s\n", mainPackage) - fmt.Fprintf(ctx.stdout, "SPDXID: SPDXRef-DOCUMENT\n") - fmt.Fprintf(ctx.stdout, "DocumentNamespace: Android\n") - fmt.Fprintf(ctx.stdout, "Creator: Organization: Google LLC\n") - fmt.Fprintf(ctx.stdout, "Created: %s\n", ctx.creationTime().Format("2006-01-02T15:04:05Z")) + docName = getDocumentName(ctx, tn, pm) + mainPkgName = replaceSlashes(getPackageName(ctx, tn)) isMainPackage = false } - relationships := make([]string, 0, 1) - defer func() { - if r := recover(); r != nil { - panic(r) - } - for _, relationship := range relationships { - fmt.Fprintln(ctx.stdout, relationship) - } - }() if len(path) == 0 { - relationships = append(relationships, - fmt.Sprintf("Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-Package-%s", - getPackageName(ctx, tn))) + // Add the describe relationship for the main package + rln := &spdx.Relationship{ + RefA: common.MakeDocElementID("" /* this document */, "DOCUMENT"), + RefB: common.MakeDocElementID("", mainPkgName), + Relationship: "DESCRIBES", + } + relationships = append(relationships, rln) + } else { // Check parent and identify annotation parent := path[len(path)-1] targetEdge := parent.Edge() if targetEdge.IsRuntimeDependency() { // Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship - relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s RUNTIME_DEPENDENCY_OF SPDXRef-Package-%s", getPackageName(ctx, tn), getPackageName(ctx, targetEdge.Target()))) + rln := &spdx.Relationship{ + RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), + RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), + Relationship: "RUNTIME_DEPENDENCY_OF", + } + relationships = append(relationships, rln) } else if targetEdge.IsDerivation() { // Adding the derivation annotation as a CONTAINS relationship - relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s CONTAINS SPDXRef-Package-%s", getPackageName(ctx, targetEdge.Target()), getPackageName(ctx, tn))) + rln := &spdx.Relationship{ + RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), + RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), + Relationship: "CONTAINS", + } + relationships = append(relationships, rln) } else if targetEdge.IsBuildTool() { // Adding the toolchain annotation as a BUILD_TOOL_OF relationship - relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s BUILD_TOOL_OF SPDXRef-Package-%s", getPackageName(ctx, tn), getPackageName(ctx, targetEdge.Target()))) + rln := &spdx.Relationship{ + RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), + RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), + Relationship: "BUILD_TOOL_OF", + } + relationships = append(relationships, rln) + } else { panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations())) } @@ -390,18 +466,27 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) { } visitedNodes[tn] = struct{}{} pkgName := getPackageName(ctx, tn) - fmt.Fprintf(ctx.stdout, "##### Package: %s\n", strings.Replace(pkgName, "-", "/", -2)) - fmt.Fprintf(ctx.stdout, "PackageName: %s\n", pkgName) + + // Making an spdx package and adding it to pkgs + pkg := &spdx.Package{ + PackageName: replaceSlashes(pkgName), + PackageDownloadLocation: getDownloadUrl(ctx, pm), + PackageSPDXIdentifier: common.ElementID(replaceSlashes(pkgName)), + PackageLicenseConcluded: concludedLicenses(tn.LicenseTexts()), + } + if pm != nil && pm.Version() != "" { - fmt.Fprintf(ctx.stdout, "PackageVersion: %s\n", pm.Version()) + pkg.PackageVersion = pm.Version() + } else { + pkg.PackageVersion = NOASSERTION } - fmt.Fprintf(ctx.stdout, "SPDXID: SPDXRef-Package-%s\n", pkgName) - fmt.Fprintf(ctx.stdout, "PackageDownloadLocation: %s\n", getDownloadUrl(ctx, pm)) - fmt.Fprintf(ctx.stdout, "PackageLicenseConcluded: %s\n", concludedLicenses(tn.LicenseTexts())) + + pkgs = append(pkgs, pkg) + return true }) - fmt.Fprintf(ctx.stdout, "##### Non-standard license:\n") + // Adding Non-standard licenses licenseTexts := make([]string, 0, len(licenses)) @@ -412,23 +497,51 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) { sort.Strings(licenseTexts) for _, licenseText := range licenseTexts { - fmt.Fprintf(ctx.stdout, "LicenseID: %s\n", licenses[licenseText]) // open the file f, err := ctx.rootFS.Open(filepath.Clean(licenseText)) if err != nil { - return nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err) + return nil, nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err) } // read the file text, err := io.ReadAll(f) if err != nil { - return nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err) + return nil, nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err) } - // adding the extracted license text - fmt.Fprintf(ctx.stdout, "ExtractedText: <text>%v</text>\n", string(text)) + // Making an spdx License and adding it to otherLicenses + otherLicenses = append(otherLicenses, &spdx.OtherLicense{ + LicenseName: strings.Replace(licenses[licenseText], "LicenseRef-", "", -1), + LicenseIdentifier: string(licenses[licenseText]), + ExtractedText: string(text), + }) } deps := inputFiles(lg, pmix, licenseTexts) sort.Strings(deps) - return deps, nil + + // Making the SPDX doc + ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil) + if err != nil { + return nil, nil, fmt.Errorf("Unable to build creation info section for SPDX doc: %v\n", err) + } + + ci.Created = ctx.creationTime() + + doc := &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: docName, + DocumentNamespace: generateSPDXNamespace(ctx.buildid, ci.Created, files...), + CreationInfo: ci, + Packages: pkgs, + Relationships: relationships, + OtherLicenses: otherLicenses, + } + + if err := spdxlib.ValidateDocument2_2(doc); err != nil { + return nil, nil, fmt.Errorf("Unable to validate the SPDX doc: %v\n", err) + } + + return doc, deps, nil } diff --git a/tools/compliance/cmd/sbom/sbom_test.go b/tools/compliance/cmd/sbom/sbom_test.go index 6df74e2dcc..13ba66db99 100644 --- a/tools/compliance/cmd/sbom/sbom_test.go +++ b/tools/compliance/cmd/sbom/sbom_test.go @@ -15,37 +15,20 @@ package main import ( - "bufio" "bytes" + "encoding/json" "fmt" "os" "reflect" - "regexp" "strings" "testing" "time" "android/soong/tools/compliance" -) -var ( - spdxVersionTag = regexp.MustCompile(`^\s*SPDXVersion: SPDX-2.2\s*$`) - spdxDataLicenseTag = regexp.MustCompile(`^\s*DataLicense: CC0-1.0\s*$`) - spdxDocumentNameTag = regexp.MustCompile(`^\s*DocumentName:\s*Android*\s*$`) - spdxIDTag = regexp.MustCompile(`^\s*SPDXID:\s*SPDXRef-DOCUMENT\s*$`) - spdxDocumentNameSpaceTag = regexp.MustCompile(`^\s*DocumentNamespace:\s*Android\s*$`) - spdxCreatorOrganizationTag = regexp.MustCompile(`^\s*Creator:\s*Organization:\s*Google LLC\s*$`) - spdxCreatedTimeTag = regexp.MustCompile(`^\s*Created: 1970-01-01T00:00:00Z\s*$`) - spdxPackageTag = regexp.MustCompile(`^\s*#####\s*Package:\s*(.*)\s*$`) - spdxPackageNameTag = regexp.MustCompile(`^\s*PackageName:\s*(.*)\s*$`) - spdxPkgIDTag = regexp.MustCompile(`^\s*SPDXID:\s*SPDXRef-Package-(.*)\s*$`) - spdxPkgDownloadLocationTag = regexp.MustCompile(`^\s*PackageDownloadLocation:\s*NOASSERTION\s*$`) - spdxPkgLicenseDeclaredTag = regexp.MustCompile(`^\s*PackageLicenseConcluded:\s*LicenseRef-(.*)\s*$`) - spdxRelationshipTag = regexp.MustCompile(`^\s*Relationship:\s*SPDXRef-(.*)\s*(DESCRIBES|CONTAINS|BUILD_TOOL_OF|RUNTIME_DEPENDENCY_OF)\s*SPDXRef-Package-(.*)\s*$`) - spdxLicenseTag = regexp.MustCompile(`^\s*##### Non-standard license:\s*$`) - spdxLicenseIDTag = regexp.MustCompile(`^\s*LicenseID: LicenseRef-(.*)\s*$`) - spdxExtractedTextTag = regexp.MustCompile(`^\s*ExtractedText:\s*<text>(.*)\s*$`) - spdxExtractedClosingTextTag = regexp.MustCompile(`^\s*</text>\s*$`) + "github.com/spdx/tools-golang/builder/builder2v2" + "github.com/spdx/tools-golang/spdx/common" + spdx "github.com/spdx/tools-golang/spdx/v2_2" ) func TestMain(m *testing.M) { @@ -65,69 +48,125 @@ func Test(t *testing.T) { outDir string roots []string stripPrefix string - expectedOut []matcher + expectedOut *spdx.Document expectedDeps []string }{ { condition: "firstparty", name: "apex", roots: []string{"highest.apex.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/firstparty/highest.apex.meta_lic"}, - packageName{"testdata/firstparty/highest.apex.meta_lic"}, - spdxPkgID{"testdata/firstparty/highest.apex.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata-firstparty-highest.apex.meta_lic", "DESCRIBES"}, - packageTag{"testdata/firstparty/bin/bin1.meta_lic"}, - packageName{"testdata/firstparty/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/firstparty/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/highest.apex.meta_lic ", "testdata/firstparty/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/bin/bin2.meta_lic"}, - packageName{"testdata/firstparty/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/firstparty/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/highest.apex.meta_lic ", "testdata-firstparty-bin-bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/liba.so.meta_lic"}, - packageName{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/highest.apex.meta_lic ", "testdata/firstparty/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/libb.so.meta_lic"}, - packageName{"testdata/firstparty/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/highest.apex.meta_lic ", "testdata/firstparty/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/firstparty/bin/bin1.meta_lic ", "testdata/firstparty/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/libc.a.meta_lic"}, - packageName{"testdata/firstparty/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata-firstparty-bin-bin1.meta_lic ", "testdata/firstparty/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/firstparty/lib/libb.so.meta_lic ", "testdata/firstparty/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/firstparty/lib/libd.so.meta_lic"}, - packageName{"testdata/firstparty/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/lib/libd.so.meta_lic ", "testdata/firstparty/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-firstparty-highest.apex", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/firstparty/highest.apex.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-firstparty-highest.apex.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-highest.apex.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-firstparty-highest.apex.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -144,42 +183,72 @@ func Test(t *testing.T) { condition: "firstparty", name: "application", roots: []string{"application.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/firstparty/application.meta_lic"}, - packageName{"testdata/firstparty/application.meta_lic"}, - spdxPkgID{"testdata/firstparty/application.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/firstparty/application.meta_lic", "DESCRIBES"}, - packageTag{"testdata/firstparty/bin/bin3.meta_lic"}, - packageName{"testdata/firstparty/bin/bin3.meta_lic"}, - spdxPkgID{"testdata/firstparty/bin/bin3.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/bin/bin3.meta_lic ", "testdata-firstparty-application.meta_lic", "BUILD_TOOL_OF"}, - packageTag{"testdata/firstparty/lib/liba.so.meta_lic"}, - packageName{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/application.meta_lic ", "testdata/firstparty/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/libb.so.meta_lic"}, - packageName{"testdata/firstparty/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/lib/libb.so.meta_lic ", "testdata-firstparty-application.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-firstparty-application", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/firstparty/application.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-firstparty-application.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-application.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-bin-bin3.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-bin-bin3.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-firstparty-application.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin3.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-application.meta_lic"), + Relationship: "BUILD_TOOL_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-application.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-application.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -193,62 +262,118 @@ func Test(t *testing.T) { condition: "firstparty", name: "container", roots: []string{"container.zip.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/firstparty/container.zip.meta_lic"}, - packageName{"testdata/firstparty/container.zip.meta_lic"}, - spdxPkgID{"testdata/firstparty/container.zip.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/firstparty/container.zip.meta_lic", "DESCRIBES"}, - packageTag{"testdata/firstparty/bin/bin1.meta_lic"}, - packageName{"testdata/firstparty/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/firstparty/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/container.zip.meta_lic ", "testdata/firstparty/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/bin/bin2.meta_lic"}, - packageName{"testdata/firstparty/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/firstparty/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/container.zip.meta_lic ", "testdata/firstparty/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/liba.so.meta_lic"}, - packageName{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/container.zip.meta_lic ", "testdata/firstparty/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/libb.so.meta_lic"}, - packageName{"testdata/firstparty/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/container.zip.meta_lic ", "testdata/firstparty/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/firstparty/bin/bin1.meta_lic ", "testdata/firstparty/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/libc.a.meta_lic"}, - packageName{"testdata/firstparty/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/bin/bin1.meta_lic ", "testdata/firstparty/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/firstparty/lib/libb.so.meta_lic ", "testdata/firstparty/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/firstparty/lib/libd.so.meta_lic"}, - packageName{"testdata/firstparty/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/lib/libd.so.meta_lic ", "testdata/firstparty/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-firstparty-container.zip", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/firstparty/container.zip.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-firstparty-container.zip.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-container.zip.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-firstparty-container.zip.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -265,36 +390,60 @@ func Test(t *testing.T) { condition: "firstparty", name: "binary", roots: []string{"bin/bin1.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/firstparty/bin/bin1.meta_lic"}, - packageName{"testdata/firstparty/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/firstparty/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/firstparty/bin/bin1.meta_lic", "DESCRIBES"}, - packageTag{"testdata/firstparty/lib/liba.so.meta_lic"}, - packageName{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/bin/bin1.meta_lic ", "testdata/firstparty/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/firstparty/lib/libc.a.meta_lic"}, - packageName{"testdata/firstparty/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/firstparty/bin/bin1.meta_lic ", "testdata/firstparty/lib/libc.a.meta_lic", "CONTAINS"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-firstparty-bin-bin1", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/firstparty/bin/bin1.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-firstparty-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-firstparty-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-firstparty-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -307,24 +456,36 @@ func Test(t *testing.T) { condition: "firstparty", name: "library", roots: []string{"lib/libd.so.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/firstparty/lib/libd.so.meta_lic"}, - packageName{"testdata/firstparty/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/firstparty/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/firstparty/lib/libd.so.meta_lic", "DESCRIBES"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-firstparty-lib-libd.so", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/firstparty/lib/libd.so.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-firstparty-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-firstparty-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-firstparty-lib-libd.so.meta_lic"), + Relationship: "DESCRIBES", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -335,65 +496,123 @@ func Test(t *testing.T) { condition: "notice", name: "apex", roots: []string{"highest.apex.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/notice/highest.apex.meta_lic"}, - packageName{"testdata/notice/highest.apex.meta_lic"}, - spdxPkgID{"testdata/notice/highest.apex.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/notice/highest.apex.meta_lic", "DESCRIBES"}, - packageTag{"testdata/notice/bin/bin1.meta_lic"}, - packageName{"testdata/notice/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/notice/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/notice/highest.apex.meta_lic ", "testdata/notice/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/bin/bin2.meta_lic"}, - packageName{"testdata/notice/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/notice/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/notice/highest.apex.meta_lic ", "testdata/notice/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/liba.so.meta_lic"}, - packageName{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/highest.apex.meta_lic ", "testdata/notice/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/libb.so.meta_lic"}, - packageName{"testdata/notice/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/notice/highest.apex.meta_lic ", "testdata/notice/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/notice/bin/bin1.meta_lic ", "testdata/notice/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/libc.a.meta_lic"}, - packageName{"testdata/notice/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/bin/bin1.meta_lic ", "testdata/notice/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/notice/lib/libb.so.meta_lic ", "testdata/notice/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/notice/lib/libd.so.meta_lic"}, - packageName{"testdata/notice/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/lib/libd.so.meta_lic ", "testdata/notice/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-notice-highest.apex", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/notice/highest.apex.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-notice-highest.apex.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-highest.apex.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-notice-highest.apex.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -411,65 +630,123 @@ func Test(t *testing.T) { condition: "notice", name: "container", roots: []string{"container.zip.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/notice/container.zip.meta_lic"}, - packageName{"testdata/notice/container.zip.meta_lic"}, - spdxPkgID{"testdata/notice/container.zip.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/notice/container.zip.meta_lic", "DESCRIBES"}, - packageTag{"testdata/notice/bin/bin1.meta_lic"}, - packageName{"testdata/notice/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/notice/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/notice/container.zip.meta_lic ", "testdata/notice/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/bin/bin2.meta_lic"}, - packageName{"testdata/notice/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/notice/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/notice/container.zip.meta_lic ", "testdata/notice/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/liba.so.meta_lic"}, - packageName{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/container.zip.meta_lic ", "testdata/notice/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/libb.so.meta_lic"}, - packageName{"testdata/notice/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/notice/container.zip.meta_lic ", "testdata/notice/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/notice/bin/bin1.meta_lic ", "testdata/notice/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/libc.a.meta_lic"}, - packageName{"testdata/notice/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/bin/bin1.meta_lic ", "testdata/notice/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/notice/lib/libb.so.meta_lic ", "testdata/notice/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/notice/lib/libd.so.meta_lic"}, - packageName{"testdata/notice/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/lib/libd.so.meta_lic ", "testdata/notice/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-notice-container.zip", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/notice/container.zip.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-notice-container.zip.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-container.zip.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-notice-container.zip.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -487,45 +764,77 @@ func Test(t *testing.T) { condition: "notice", name: "application", roots: []string{"application.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/notice/application.meta_lic"}, - packageName{"testdata/notice/application.meta_lic"}, - spdxPkgID{"testdata/notice/application.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata-notice-application.meta_lic", "DESCRIBES"}, - packageTag{"testdata/notice/bin/bin3.meta_lic"}, - packageName{"testdata/notice/bin/bin3.meta_lic"}, - spdxPkgID{"testdata/notice/bin/bin3.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata-notice-bin-bin3.meta_lic ", "testdata/notice/application.meta_lic", "BUILD_TOOL_OF"}, - packageTag{"testdata/notice/lib/liba.so.meta_lic"}, - packageName{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/application.meta_lic ", "testdata-notice-lib-liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/libb.so.meta_lic"}, - packageName{"testdata/notice/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata-notice-lib-libb.so.meta_lic ", "testdata/notice/application.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-notice-application", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/notice/application.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-notice-application.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-application.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-bin-bin3.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-bin-bin3.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-notice-application.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin3.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-application.meta_lic"), + Relationship: "BUILD_TOOL_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-application.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-application.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -540,39 +849,65 @@ func Test(t *testing.T) { condition: "notice", name: "binary", roots: []string{"bin/bin1.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/notice/bin/bin1.meta_lic"}, - packageName{"testdata/notice/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/notice/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/notice/bin/bin1.meta_lic", "DESCRIBES"}, - packageTag{"testdata/notice/lib/liba.so.meta_lic"}, - packageName{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/bin/bin1.meta_lic ", "testdata/notice/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/notice/lib/libc.a.meta_lic"}, - packageName{"testdata/notice/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/notice/bin/bin1.meta_lic ", "testdata/notice/lib/libc.a.meta_lic", "CONTAINS"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-notice-bin-bin1", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/notice/bin/bin1.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-notice-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-notice-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-notice-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-notice-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -586,24 +921,36 @@ func Test(t *testing.T) { condition: "notice", name: "library", roots: []string{"lib/libd.so.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/notice/lib/libd.so.meta_lic"}, - packageName{"testdata/notice/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/notice/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/notice/lib/libd.so.meta_lic", "DESCRIBES"}, - spdxLicense{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-notice-lib-libd.so", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/notice/lib/libd.so.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-notice-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-notice-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-notice-lib-libd.so.meta_lic"), + Relationship: "DESCRIBES", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/notice/NOTICE_LICENSE", @@ -614,68 +961,128 @@ func Test(t *testing.T) { condition: "reciprocal", name: "apex", roots: []string{"highest.apex.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/reciprocal/highest.apex.meta_lic"}, - packageName{"testdata/reciprocal/highest.apex.meta_lic"}, - spdxPkgID{"testdata/reciprocal/highest.apex.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/reciprocal/highest.apex.meta_lic", "DESCRIBES"}, - packageTag{"testdata/reciprocal/bin/bin1.meta_lic"}, - packageName{"testdata/reciprocal/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/reciprocal/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/highest.apex.meta_lic ", "testdata-reciprocal-bin-bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/bin/bin2.meta_lic"}, - packageName{"testdata/reciprocal/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/reciprocal/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/highest.apex.meta_lic ", "testdata-reciprocal-bin-bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/liba.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/highest.apex.meta_lic ", "testdata/reciprocal/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/libb.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/highest.apex.meta_lic ", "testdata/reciprocal/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/reciprocal/bin/bin1.meta_lic ", "testdata/reciprocal/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/libc.a.meta_lic"}, - packageName{"testdata/reciprocal/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/bin/bin1.meta_lic ", "testdata/reciprocal/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/reciprocal/lib/libb.so.meta_lic ", "testdata/reciprocal/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/reciprocal/lib/libd.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/lib/libd.so.meta_lic ", "testdata/reciprocal/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-reciprocal-highest.apex", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/reciprocal/highest.apex.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-reciprocal-highest.apex.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-highest.apex.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-reciprocal-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-reciprocal-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-highest.apex.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + ExtractedText: "$$$Reciprocal License$$$\n", + LicenseName: "testdata-reciprocal-RECIPROCAL_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -692,130 +1099,84 @@ func Test(t *testing.T) { }, { condition: "reciprocal", - name: "container", - roots: []string{"container.zip.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/reciprocal/container.zip.meta_lic"}, - packageName{"testdata/reciprocal/container.zip.meta_lic"}, - spdxPkgID{"testdata/reciprocal/container.zip.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/reciprocal/container.zip.meta_lic", "DESCRIBES"}, - packageTag{"testdata/reciprocal/bin/bin1.meta_lic"}, - packageName{"testdata/reciprocal/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/reciprocal/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/container.zip.meta_lic ", "testdata-reciprocal-bin-bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/bin/bin2.meta_lic"}, - packageName{"testdata/reciprocal/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/reciprocal/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/container.zip.meta_lic ", "testdata-reciprocal-bin-bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/liba.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/container.zip.meta_lic ", "testdata/reciprocal/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/libb.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/container.zip.meta_lic ", "testdata/reciprocal/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/reciprocal/bin/bin1.meta_lic ", "testdata/reciprocal/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/libc.a.meta_lic"}, - packageName{"testdata/reciprocal/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/bin/bin1.meta_lic ", "testdata/reciprocal/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/reciprocal/lib/libb.so.meta_lic ", "testdata/reciprocal/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/reciprocal/lib/libd.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/lib/libd.so.meta_lic ", "testdata/reciprocal/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, - }, - expectedDeps: []string{ - "testdata/firstparty/FIRST_PARTY_LICENSE", - "testdata/notice/NOTICE_LICENSE", - "testdata/reciprocal/RECIPROCAL_LICENSE", - "testdata/reciprocal/bin/bin1.meta_lic", - "testdata/reciprocal/bin/bin2.meta_lic", - "testdata/reciprocal/container.zip.meta_lic", - "testdata/reciprocal/lib/liba.so.meta_lic", - "testdata/reciprocal/lib/libb.so.meta_lic", - "testdata/reciprocal/lib/libc.a.meta_lic", - "testdata/reciprocal/lib/libd.so.meta_lic", - }, - }, - { - condition: "reciprocal", name: "application", roots: []string{"application.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/reciprocal/application.meta_lic"}, - packageName{"testdata/reciprocal/application.meta_lic"}, - spdxPkgID{"testdata/reciprocal/application.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/reciprocal/application.meta_lic", "DESCRIBES"}, - packageTag{"testdata/reciprocal/bin/bin3.meta_lic"}, - packageName{"testdata/reciprocal/bin/bin3.meta_lic"}, - spdxPkgID{"testdata/reciprocal/bin/bin3.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata-reciprocal-bin-bin3.meta_lic ", "testdata/reciprocal/application.meta_lic", "BUILD_TOOL_OF"}, - packageTag{"testdata/reciprocal/lib/liba.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/application.meta_lic ", "testdata/reciprocal/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/libb.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/lib/libb.so.meta_lic ", "testdata/reciprocal/application.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-reciprocal-application", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/reciprocal/application.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-reciprocal-application.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-application.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-reciprocal-bin-bin3.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-bin-bin3.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-application.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-bin-bin3.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-application.meta_lic"), + Relationship: "BUILD_TOOL_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-application.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-application.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + ExtractedText: "$$$Reciprocal License$$$\n", + LicenseName: "testdata-reciprocal-RECIPROCAL_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -831,39 +1192,65 @@ func Test(t *testing.T) { condition: "reciprocal", name: "binary", roots: []string{"bin/bin1.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/reciprocal/bin/bin1.meta_lic"}, - packageName{"testdata/reciprocal/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/reciprocal/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/reciprocal/bin/bin1.meta_lic", "DESCRIBES"}, - packageTag{"testdata/reciprocal/lib/liba.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/bin/bin1.meta_lic ", "testdata/reciprocal/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/reciprocal/lib/libc.a.meta_lic"}, - packageName{"testdata/reciprocal/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/reciprocal/bin/bin1.meta_lic ", "testdata/reciprocal/lib/libc.a.meta_lic", "CONTAINS"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-reciprocal-bin-bin1", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/reciprocal/bin/bin1.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-reciprocal-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + PackageName: "testdata-reciprocal-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-bin-bin1.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-reciprocal-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + ExtractedText: "$$$Reciprocal License$$$\n", + LicenseName: "testdata-reciprocal-RECIPROCAL_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -877,24 +1264,36 @@ func Test(t *testing.T) { condition: "reciprocal", name: "library", roots: []string{"lib/libd.so.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/reciprocal/lib/libd.so.meta_lic"}, - packageName{"testdata/reciprocal/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/reciprocal/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/reciprocal/lib/libd.so.meta_lic", "DESCRIBES"}, - spdxLicense{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-reciprocal-lib-libd.so", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/reciprocal/lib/libd.so.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-reciprocal-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-reciprocal-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-reciprocal-lib-libd.so.meta_lic"), + Relationship: "DESCRIBES", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/notice/NOTICE_LICENSE", @@ -902,75 +1301,136 @@ func Test(t *testing.T) { }, }, { - condition: "restricted", - name: "apex", - roots: []string{"highest.apex.meta_lic"}, - stripPrefix: "out/target/product/fictional/system/apex/", - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/restricted/highest.apex.meta_lic"}, - packageName{"testdata/restricted/highest.apex.meta_lic"}, - spdxPkgID{"testdata/restricted/highest.apex.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/restricted/highest.apex.meta_lic", "DESCRIBES"}, - packageTag{"testdata/restricted/bin/bin1.meta_lic"}, - packageName{"testdata/restricted/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/restricted/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/highest.apex.meta_lic ", "testdata/restricted/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/bin/bin2.meta_lic"}, - packageName{"testdata/restricted/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/restricted/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/highest.apex.meta_lic ", "testdata/restricted/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/liba.so.meta_lic"}, - packageName{"testdata/restricted/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/highest.apex.meta_lic ", "testdata/restricted/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/libb.so.meta_lic"}, - packageName{"testdata/restricted/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/highest.apex.meta_lic ", "testdata/restricted/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/restricted/bin/bin1.meta_lic ", "testdata/restricted/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/libc.a.meta_lic"}, - packageName{"testdata/restricted/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/bin/bin1.meta_lic ", "testdata/restricted/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/restricted/lib/libb.so.meta_lic ", "testdata/restricted/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/restricted/lib/libd.so.meta_lic"}, - packageName{"testdata/restricted/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/lib/libd.so.meta_lic ", "testdata/restricted/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxExtractedText{"###Restricted License###"}, - spdxExtractedClosingText{}, + condition: "restricted", + name: "apex", + roots: []string{"highest.apex.meta_lic"}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-restricted-highest.apex", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/restricted/highest.apex.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-restricted-highest.apex.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-highest.apex.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-restricted-highest.apex.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + ExtractedText: "$$$Reciprocal License$$$\n", + LicenseName: "testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + ExtractedText: "###Restricted License###\n", + LicenseName: "testdata-restricted-RESTRICTED_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -987,75 +1447,136 @@ func Test(t *testing.T) { }, }, { - condition: "restricted", - name: "container", - roots: []string{"container.zip.meta_lic"}, - stripPrefix: "out/target/product/fictional/system/apex/", - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/restricted/container.zip.meta_lic"}, - packageName{"testdata/restricted/container.zip.meta_lic"}, - spdxPkgID{"testdata/restricted/container.zip.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/restricted/container.zip.meta_lic", "DESCRIBES"}, - packageTag{"testdata/restricted/bin/bin1.meta_lic"}, - packageName{"testdata/restricted/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/restricted/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/container.zip.meta_lic ", "testdata/restricted/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/bin/bin2.meta_lic"}, - packageName{"testdata/restricted/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/restricted/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/container.zip.meta_lic ", "testdata/restricted/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/liba.so.meta_lic"}, - packageName{"testdata/restricted/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/container.zip.meta_lic ", "testdata/restricted/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/libb.so.meta_lic"}, - packageName{"testdata/restricted/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/container.zip.meta_lic ", "testdata/restricted/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/restricted/bin/bin1.meta_lic ", "testdata/restricted/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/libc.a.meta_lic"}, - packageName{"testdata/restricted/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/bin/bin1.meta_lic ", "testdata/restricted/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/restricted/lib/libb.so.meta_lic ", "testdata/restricted/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/restricted/lib/libd.so.meta_lic"}, - packageName{"testdata/restricted/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/lib/libd.so.meta_lic ", "testdata/restricted/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxExtractedText{"###Restricted License###"}, - spdxExtractedClosingText{}, + condition: "restricted", + name: "container", + roots: []string{"container.zip.meta_lic"}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-restricted-container.zip", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/restricted/container.zip.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-restricted-container.zip.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-container.zip.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-restricted-container.zip.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + ExtractedText: "$$$Reciprocal License$$$\n", + LicenseName: "testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + ExtractedText: "###Restricted License###\n", + LicenseName: "testdata-restricted-RESTRICTED_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -1075,42 +1596,70 @@ func Test(t *testing.T) { condition: "restricted", name: "binary", roots: []string{"bin/bin1.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/restricted/bin/bin1.meta_lic"}, - packageName{"testdata/restricted/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/restricted/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/restricted/bin/bin1.meta_lic", "DESCRIBES"}, - packageTag{"testdata/restricted/lib/liba.so.meta_lic"}, - packageName{"testdata/restricted/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/bin/bin1.meta_lic ", "testdata/restricted/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/restricted/lib/libc.a.meta_lic"}, - packageName{"testdata/restricted/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxRelationship{"Package-testdata/restricted/bin/bin1.meta_lic ", "testdata/restricted/lib/libc.a.meta_lic", "CONTAINS"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-reciprocal-RECIPROCAL_LICENSE"}, - spdxExtractedText{"$$$Reciprocal License$$$"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxExtractedText{"###Restricted License###"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-restricted-bin-bin1", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/restricted/bin/bin1.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-restricted-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-restricted-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-restricted-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-reciprocal-RECIPROCAL_LICENSE", + ExtractedText: "$$$Reciprocal License$$$\n", + LicenseName: "testdata-reciprocal-RECIPROCAL_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + ExtractedText: "###Restricted License###\n", + LicenseName: "testdata-restricted-RESTRICTED_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -1125,24 +1674,36 @@ func Test(t *testing.T) { condition: "restricted", name: "library", roots: []string{"lib/libd.so.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/restricted/lib/libd.so.meta_lic"}, - packageName{"testdata/restricted/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/restricted/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/restricted/lib/libd.so.meta_lic", "DESCRIBES"}, - spdxLicense{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-restricted-lib-libd.so", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/restricted/lib/libd.so.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-restricted-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-restricted-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-restricted-lib-libd.so.meta_lic"), + Relationship: "DESCRIBES", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/notice/NOTICE_LICENSE", @@ -1153,71 +1714,133 @@ func Test(t *testing.T) { condition: "proprietary", name: "apex", roots: []string{"highest.apex.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/proprietary/highest.apex.meta_lic"}, - packageName{"testdata/proprietary/highest.apex.meta_lic"}, - spdxPkgID{"testdata/proprietary/highest.apex.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/proprietary/highest.apex.meta_lic", "DESCRIBES"}, - packageTag{"testdata/proprietary/bin/bin1.meta_lic"}, - packageName{"testdata/proprietary/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/proprietary/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/highest.apex.meta_lic ", "testdata/proprietary/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/bin/bin2.meta_lic"}, - packageName{"testdata/proprietary/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/proprietary/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/highest.apex.meta_lic ", "testdata/proprietary/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/liba.so.meta_lic"}, - packageName{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/highest.apex.meta_lic ", "testdata/proprietary/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/libb.so.meta_lic"}, - packageName{"testdata/proprietary/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/highest.apex.meta_lic ", "testdata/proprietary/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin1.meta_lic ", "testdata/proprietary/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/libc.a.meta_lic"}, - packageName{"testdata/proprietary/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin1.meta_lic ", "testdata/proprietary/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata-proprietary-lib-libb.so.meta_lic ", "testdata/proprietary/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/proprietary/lib/libd.so.meta_lic"}, - packageName{"testdata/proprietary/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata-proprietary-lib-libd.so.meta_lic ", "testdata/proprietary/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxExtractedText{"@@@Proprietary License@@@"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxExtractedText{"###Restricted License###"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-proprietary-highest.apex", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/proprietary/highest.apex.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-proprietary-highest.apex.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-highest.apex.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-proprietary-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-proprietary-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-proprietary-highest.apex.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-highest.apex.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + ExtractedText: "@@@Proprietary License@@@\n", + LicenseName: "testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + ExtractedText: "###Restricted License###\n", + LicenseName: "testdata-restricted-RESTRICTED_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -1237,71 +1860,133 @@ func Test(t *testing.T) { condition: "proprietary", name: "container", roots: []string{"container.zip.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/proprietary/container.zip.meta_lic"}, - packageName{"testdata/proprietary/container.zip.meta_lic"}, - spdxPkgID{"testdata/proprietary/container.zip.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/proprietary/container.zip.meta_lic", "DESCRIBES"}, - packageTag{"testdata/proprietary/bin/bin1.meta_lic"}, - packageName{"testdata/proprietary/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/proprietary/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/container.zip.meta_lic ", "testdata/proprietary/bin/bin1.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/bin/bin2.meta_lic"}, - packageName{"testdata/proprietary/bin/bin2.meta_lic"}, - spdxPkgID{"testdata/proprietary/bin/bin2.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/container.zip.meta_lic ", "testdata/proprietary/bin/bin2.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/liba.so.meta_lic"}, - packageName{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/container.zip.meta_lic ", "testdata/proprietary/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/libb.so.meta_lic"}, - packageName{"testdata/proprietary/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/container.zip.meta_lic ", "testdata/proprietary/lib/libb.so.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin1.meta_lic ", "testdata/proprietary/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/libc.a.meta_lic"}, - packageName{"testdata/proprietary/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin1.meta_lic ", "testdata/proprietary/lib/libc.a.meta_lic", "CONTAINS"}, - spdxRelationship{"Package-testdata-proprietary-lib-libb.so.meta_lic ", "testdata/proprietary/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - packageTag{"testdata/proprietary/lib/libd.so.meta_lic"}, - packageName{"testdata/proprietary/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"Package-testdata-proprietary-lib-libd.so.meta_lic ", "testdata/proprietary/bin/bin2.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxExtractedText{"@@@Proprietary License@@@"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxExtractedText{"###Restricted License###"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-proprietary-container.zip", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/proprietary/container.zip.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-proprietary-container.zip.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-container.zip.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-proprietary-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-proprietary-bin-bin2.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-bin-bin2.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-proprietary-container.zip.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin2.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-container.zip.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-libb.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-lib-libd.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin2.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + ExtractedText: "@@@Proprietary License@@@\n", + LicenseName: "testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + ExtractedText: "###Restricted License###\n", + LicenseName: "testdata-restricted-RESTRICTED_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -1321,48 +2006,82 @@ func Test(t *testing.T) { condition: "proprietary", name: "application", roots: []string{"application.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/proprietary/application.meta_lic"}, - packageName{"testdata/proprietary/application.meta_lic"}, - spdxPkgID{"testdata/proprietary/application.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/proprietary/application.meta_lic", "DESCRIBES"}, - packageTag{"testdata/proprietary/bin/bin3.meta_lic"}, - packageName{"testdata/proprietary/bin/bin3.meta_lic"}, - spdxPkgID{"testdata/proprietary/bin/bin3.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin3.meta_lic ", "testdata/proprietary/application.meta_lic", "BUILD_TOOL_OF"}, - packageTag{"testdata/proprietary/lib/liba.so.meta_lic"}, - packageName{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/application.meta_lic ", "testdata/proprietary/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/libb.so.meta_lic"}, - packageName{"testdata/proprietary/lib/libb.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libb.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/lib/libb.so.meta_lic ", "testdata/proprietary/application.meta_lic", "RUNTIME_DEPENDENCY_OF"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxExtractedText{"@@@Proprietary License@@@"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-restricted-RESTRICTED_LICENSE"}, - spdxExtractedText{"###Restricted License###"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-proprietary-application", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/proprietary/application.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-proprietary-application.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-application.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-proprietary-bin-bin3.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-bin-bin3.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libb.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libb.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-proprietary-application.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin3.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-application.meta_lic"), + Relationship: "BUILD_TOOL_OF", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-application.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-lib-libb.so.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-application.meta_lic"), + Relationship: "RUNTIME_DEPENDENCY_OF", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + ExtractedText: "@@@Proprietary License@@@\n", + LicenseName: "testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-restricted-RESTRICTED_LICENSE", + ExtractedText: "###Restricted License###\n", + LicenseName: "testdata-restricted-RESTRICTED_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -1378,39 +2097,65 @@ func Test(t *testing.T) { condition: "proprietary", name: "binary", roots: []string{"bin/bin1.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/proprietary/bin/bin1.meta_lic"}, - packageName{"testdata/proprietary/bin/bin1.meta_lic"}, - spdxPkgID{"testdata/proprietary/bin/bin1.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/proprietary/bin/bin1.meta_lic", "DESCRIBES"}, - packageTag{"testdata/proprietary/lib/liba.so.meta_lic"}, - packageName{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/liba.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin1.meta_lic ", "testdata/proprietary/lib/liba.so.meta_lic", "CONTAINS"}, - packageTag{"testdata/proprietary/lib/libc.a.meta_lic"}, - packageName{"testdata/proprietary/lib/libc.a.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libc.a.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxRelationship{"Package-testdata/proprietary/bin/bin1.meta_lic ", "testdata/proprietary/lib/libc.a.meta_lic", "CONTAINS"}, - spdxLicense{}, - spdxLicenseID{"testdata-firstparty-FIRST_PARTY_LICENSE"}, - spdxExtractedText{"&&&First Party License&&&"}, - spdxExtractedClosingText{}, - spdxLicenseID{"testdata-proprietary-PROPRIETARY_LICENSE"}, - spdxExtractedText{"@@@Proprietary License@@@"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-proprietary-bin-bin1", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/proprietary/bin/bin1.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-proprietary-bin-bin1.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-bin-bin1.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-liba.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-liba.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + { + PackageName: "testdata-proprietary-lib-libc.a.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libc.a.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + Relationship: "DESCRIBES", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-liba.so.meta_lic"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "testdata-proprietary-bin-bin1.meta_lic"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-libc.a.meta_lic"), + Relationship: "CONTAINS", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-firstparty-FIRST_PARTY_LICENSE", + ExtractedText: "&&&First Party License&&&\n", + LicenseName: "testdata-firstparty-FIRST_PARTY_LICENSE", + }, + { + LicenseIdentifier: "LicenseRef-testdata-proprietary-PROPRIETARY_LICENSE", + ExtractedText: "@@@Proprietary License@@@\n", + LicenseName: "testdata-proprietary-PROPRIETARY_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", @@ -1424,24 +2169,36 @@ func Test(t *testing.T) { condition: "proprietary", name: "library", roots: []string{"lib/libd.so.meta_lic"}, - expectedOut: []matcher{ - spdxVersion{}, - spdxDataLicense{}, - spdxDocumentName{"Android"}, - spdxID{}, - spdxDocumentNameSpace{}, - spdxCreatorOrganization{}, - spdxCreatedTime{}, - packageTag{"testdata/proprietary/lib/libd.so.meta_lic"}, - packageName{"testdata/proprietary/lib/libd.so.meta_lic"}, - spdxPkgID{"testdata/proprietary/lib/libd.so.meta_lic"}, - spdxPkgDownloadLocation{"NOASSERTION"}, - spdxPkgLicenseDeclared{"testdata-notice-NOTICE_LICENSE"}, - spdxRelationship{"DOCUMENT ", "testdata/proprietary/lib/libd.so.meta_lic", "DESCRIBES"}, - spdxLicense{}, - spdxLicenseID{"testdata-notice-NOTICE_LICENSE"}, - spdxExtractedText{"%%%Notice License%%%"}, - spdxExtractedClosingText{}, + expectedOut: &spdx.Document{ + SPDXVersion: "SPDX-2.2", + DataLicense: "CC0-1.0", + SPDXIdentifier: "DOCUMENT", + DocumentName: "testdata-proprietary-lib-libd.so", + DocumentNamespace: generateSPDXNamespace("", "1970-01-01T00:00:00Z", "testdata/proprietary/lib/libd.so.meta_lic"), + CreationInfo: getCreationInfo(t), + Packages: []*spdx.Package{ + { + PackageName: "testdata-proprietary-lib-libd.so.meta_lic", + PackageVersion: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + PackageSPDXIdentifier: common.ElementID("testdata-proprietary-lib-libd.so.meta_lic"), + PackageLicenseConcluded: "LicenseRef-testdata-notice-NOTICE_LICENSE", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: common.MakeDocElementID("", "DOCUMENT"), + RefB: common.MakeDocElementID("", "testdata-proprietary-lib-libd.so.meta_lic"), + Relationship: "DESCRIBES", + }, + }, + OtherLicenses: []*spdx.OtherLicense{ + { + LicenseIdentifier: "LicenseRef-testdata-notice-NOTICE_LICENSE", + ExtractedText: "%%%Notice License%%%\n", + LicenseName: "testdata-notice-NOTICE_LICENSE", + }, + }, }, expectedDeps: []string{ "testdata/notice/NOTICE_LICENSE", @@ -1459,9 +2216,9 @@ func Test(t *testing.T) { rootFiles = append(rootFiles, "testdata/"+tt.condition+"/"+r) } - ctx := context{stdout, stderr, compliance.GetFS(tt.outDir), "Android", []string{tt.stripPrefix}, fakeTime} + ctx := context{stdout, stderr, compliance.GetFS(tt.outDir), "", []string{tt.stripPrefix}, fakeTime, ""} - deps, err := sbomGenerator(&ctx, rootFiles...) + spdxDoc, deps, err := sbomGenerator(&ctx, rootFiles...) if err != nil { t.Fatalf("sbom: error = %v, stderr = %v", err, stderr) return @@ -1470,28 +2227,30 @@ func Test(t *testing.T) { t.Errorf("sbom: gotStderr = %v, want none", stderr) } - t.Logf("got stdout: %s", stdout.String()) - - t.Logf("want stdout: %s", matcherList(tt.expectedOut).String()) - - out := bufio.NewScanner(stdout) - lineno := 0 - for out.Scan() { - line := out.Text() - if strings.TrimLeft(line, " ") == "" { - continue - } - if len(tt.expectedOut) <= lineno { - t.Errorf("sbom: unexpected output at line %d: got %q, want nothing (wanted %d lines)", lineno+1, line, len(tt.expectedOut)) - } else if !tt.expectedOut[lineno].isMatch(line) { - t.Errorf("sbom: unexpected output at line %d: got %q, want %q", lineno+1, line, tt.expectedOut[lineno]) - } - lineno++ + if err := validate(spdxDoc); err != nil { + t.Fatalf("sbom: document fails to validate: %v", err) } - for ; lineno < len(tt.expectedOut); lineno++ { - t.Errorf("bom: missing output line %d: ended early, want %q", lineno+1, tt.expectedOut[lineno]) + + gotData, err := json.Marshal(spdxDoc) + if err != nil { + t.Fatalf("sbom: failed to marshal spdx doc: %v", err) + return } + t.Logf("Got SPDX Doc: %s", string(gotData)) + + expectedData, err := json.Marshal(tt.expectedOut) + if err != nil { + t.Fatalf("sbom: failed to marshal spdx doc: %v", err) + return + } + + t.Logf("Want SPDX Doc: %s", string(expectedData)) + + // compare the spdx Docs + compareSpdxDocs(t, spdxDoc, tt.expectedOut) + + // compare deps t.Logf("got deps: %q", deps) t.Logf("want deps: %q", tt.expectedDeps) @@ -1504,242 +2263,296 @@ func Test(t *testing.T) { } } -type matcher interface { - isMatch(line string) bool - String() string -} +func TestGenerateSPDXNamespace(t *testing.T) { -type packageTag struct { - name string -} + buildID1 := "example-1" + buildID2 := "example-2" + files1 := "file1" + timestamp1 := "2022-05-01" + timestamp2 := "2022-05-02" + files2 := "file2" -func (m packageTag) isMatch(line string) bool { - groups := spdxPackageTag.FindStringSubmatch(line) - if len(groups) != 2 { - return false + // Test case 1: different timestamps, same files + nsh1 := generateSPDXNamespace("", timestamp1, files1) + nsh2 := generateSPDXNamespace("", timestamp2, files1) + + if nsh1 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", "", timestamp1, files1) } - return groups[1] == m.name -} -func (m packageTag) String() string { - return "##### Package: " + m.name -} + if nsh2 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", "", timestamp2, files1) + } -type packageName struct { - name string -} + if nsh1 == nsh2 { + t.Errorf("generateSPDXNamespace(%s, %s, %s) and generateSPDXNamespace(%s, %s, %s): expected different namespace hashes, but got the same", "", timestamp1, files1, "", timestamp2, files1) + } -func (m packageName) isMatch(line string) bool { - groups := spdxPackageNameTag.FindStringSubmatch(line) - if len(groups) != 2 { - return false + // Test case 2: different build ids, same timestamps and files + nsh1 = generateSPDXNamespace(buildID1, timestamp1, files1) + nsh2 = generateSPDXNamespace(buildID2, timestamp1, files1) + + if nsh1 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", buildID1, timestamp1, files1) } - return groups[1] == replaceSlashes(m.name) -} -func (m packageName) String() string { - return "PackageName: " + replaceSlashes(m.name) -} + if nsh2 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", buildID2, timestamp1, files1) + } -type spdxID struct {} + if nsh1 == nsh2 { + t.Errorf("generateSPDXNamespace(%s, %s, %s) and generateSPDXNamespace(%s, %s, %s): expected different namespace hashes, but got the same", buildID1, timestamp1, files1, buildID2, timestamp1, files1) + } -func (m spdxID) isMatch(line string) bool { - return spdxIDTag.MatchString(line) -} + // Test case 3: same build ids and files, different timestamps + nsh1 = generateSPDXNamespace(buildID1, timestamp1, files1) + nsh2 = generateSPDXNamespace(buildID1, timestamp2, files1) -func (m spdxID) String() string { - return "SPDXID: SPDXRef-DOCUMENT" -} + if nsh1 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", buildID1, timestamp1, files1) + } -type spdxPkgID struct { - name string -} + if nsh2 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", buildID1, timestamp2, files1) + } -func (m spdxPkgID) isMatch(line string) bool { - groups := spdxPkgIDTag.FindStringSubmatch(line) - if len(groups) != 2 { - return false + if nsh1 != nsh2 { + t.Errorf("generateSPDXNamespace(%s, %s, %s) and generateSPDXNamespace(%s, %s, %s): expected same namespace hashes, but got different: %s and %s", buildID1, timestamp1, files1, buildID2, timestamp1, files1, nsh1, nsh2) } - return groups[1] == replaceSlashes(m.name) -} -func (m spdxPkgID) String() string { - return "SPDXID: SPDXRef-Package-" + replaceSlashes(m.name) -} + // Test case 4: same build ids and timestamps, different files + nsh1 = generateSPDXNamespace(buildID1, timestamp1, files1) + nsh2 = generateSPDXNamespace(buildID1, timestamp1, files2) -type spdxVersion struct{} + if nsh1 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", buildID1, timestamp1, files1) + } -func (m spdxVersion) isMatch(line string) bool { - return spdxVersionTag.MatchString(line) -} + if nsh2 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", buildID1, timestamp1, files2) + } -func (m spdxVersion) String() string { - return "SPDXVersion: SPDX-2.2" -} + if nsh1 == nsh2 { + t.Errorf("generateSPDXNamespace(%s, %s, %s) and generateSPDXNamespace(%s, %s, %s): expected different namespace hashes, but got the same", buildID1, timestamp1, files1, buildID1, timestamp1, files2) + } -type spdxDataLicense struct{} + // Test case 5: empty build ids, same timestamps and different files + nsh1 = generateSPDXNamespace("", timestamp1, files1) + nsh2 = generateSPDXNamespace("", timestamp1, files2) -func (m spdxDataLicense) isMatch(line string) bool { - return spdxDataLicenseTag.MatchString(line) -} + if nsh1 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", "", timestamp1, files1) + } -func (m spdxDataLicense) String() string { - return "DataLicense: CC0-1.0" -} + if nsh2 == "" { + t.Errorf("generateSPDXNamespace(%s, %s, %s): expected non-empty string, but got empty string", "", timestamp1, files2) + } -type spdxDocumentName struct { - name string + if nsh1 == nsh2 { + t.Errorf("generateSPDXNamespace(%s, %s, %s) and generateSPDXNamespace(%s, %s, %s): expected different namespace hashes, but got the same", "", timestamp1, files1, "", timestamp1, files2) + } } -func (m spdxDocumentName) isMatch(line string) bool { - return spdxDocumentNameTag.MatchString(line) +func getCreationInfo(t *testing.T) *spdx.CreationInfo { + ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil) + if err != nil { + t.Errorf("Unable to get creation info: %v", err) + return nil + } + return ci } -func (m spdxDocumentName) String() string { - return "DocumentName: " + m.name -} +// validate returns an error if the Document is found to be invalid +func validate(doc *spdx.Document) error { + if doc.SPDXVersion == "" { + return fmt.Errorf("SPDXVersion: got nothing, want spdx version") + } + if doc.DataLicense == "" { + return fmt.Errorf("DataLicense: got nothing, want Data License") + } + if doc.SPDXIdentifier == "" { + return fmt.Errorf("SPDXIdentifier: got nothing, want SPDX Identifier") + } + if doc.DocumentName == "" { + return fmt.Errorf("DocumentName: got nothing, want Document Name") + } + if c := fmt.Sprintf("%v", doc.CreationInfo.Creators[1].Creator); c != "Google LLC" { + return fmt.Errorf("Creator: got %v, want 'Google LLC'", c) + } + _, err := time.Parse(time.RFC3339, doc.CreationInfo.Created) + if err != nil { + return fmt.Errorf("Invalid time spec: %q: got error %q, want no error", doc.CreationInfo.Created, err) + } -type spdxDocumentNameSpace struct { - name string + for _, license := range doc.OtherLicenses { + if license.ExtractedText == "" { + return fmt.Errorf("License file: %q: got nothing, want license text", license.LicenseName) + } + } + return nil } -func (m spdxDocumentNameSpace) isMatch(line string) bool { - return spdxDocumentNameSpaceTag.MatchString(line) -} +// compareSpdxDocs deep-compares two spdx docs by going through the info section, packages, relationships and licenses +func compareSpdxDocs(t *testing.T, actual, expected *spdx.Document) { -func (m spdxDocumentNameSpace) String() string { - return "DocumentNameSpace: Android" -} + if actual == nil || expected == nil { + t.Errorf("SBOM: SPDX Doc is nil! Got %v: Expected %v", actual, expected) + } -type spdxCreatorOrganization struct{} + if actual.DocumentName != expected.DocumentName { + t.Errorf("sbom: unexpected SPDX Document Name got %q, want %q", actual.DocumentName, expected.DocumentName) + } -func (m spdxCreatorOrganization) isMatch(line string) bool { - return spdxCreatorOrganizationTag.MatchString(line) -} + if actual.SPDXVersion != expected.SPDXVersion { + t.Errorf("sbom: unexpected SPDX Version got %s, want %s", actual.SPDXVersion, expected.SPDXVersion) + } -func (m spdxCreatorOrganization) String() string { - return "Creator: Organization: Google LLC" -} + if actual.DataLicense != expected.DataLicense { + t.Errorf("sbom: unexpected SPDX DataLicense got %s, want %s", actual.DataLicense, expected.DataLicense) + } -func fakeTime() time.Time { - return time.UnixMicro(0).UTC() -} + if actual.SPDXIdentifier != expected.SPDXIdentifier { + t.Errorf("sbom: unexpected SPDX Identified got %s, want %s", actual.SPDXIdentifier, expected.SPDXIdentifier) + } -type spdxCreatedTime struct{} + if actual.DocumentNamespace != expected.DocumentNamespace { + t.Errorf("sbom: unexpected SPDX Document Namespace got %s, want %s", actual.DocumentNamespace, expected.DocumentNamespace) + } -func (m spdxCreatedTime) isMatch(line string) bool { - return spdxCreatedTimeTag.MatchString(line) -} + // compare creation info + compareSpdxCreationInfo(t, actual.CreationInfo, expected.CreationInfo) -func (m spdxCreatedTime) String() string { - return "Created: 1970-01-01T00:00:00Z" -} + // compare packages + if len(actual.Packages) != len(expected.Packages) { + t.Errorf("SBOM: Number of Packages is different! Got %d: Expected %d", len(actual.Packages), len(expected.Packages)) + } -type spdxPkgDownloadLocation struct { - name string -} + for i, pkg := range actual.Packages { + if !compareSpdxPackages(t, i, pkg, expected.Packages[i]) { + break + } + } -func (m spdxPkgDownloadLocation) isMatch(line string) bool { - return spdxPkgDownloadLocationTag.MatchString(line) -} + // compare licenses + if len(actual.OtherLicenses) != len(expected.OtherLicenses) { + t.Errorf("SBOM: Number of Licenses in actual is different! Got %d: Expected %d", len(actual.OtherLicenses), len(expected.OtherLicenses)) + } + for i, license := range actual.OtherLicenses { + if !compareLicenses(t, i, license, expected.OtherLicenses[i]) { + break + } + } -func (m spdxPkgDownloadLocation) String() string { - return "PackageDownloadLocation: " + m.name + //compare Relationships + if len(actual.Relationships) != len(expected.Relationships) { + t.Errorf("SBOM: Number of Licenses in actual is different! Got %d: Expected %d", len(actual.Relationships), len(expected.Relationships)) + } + for i, rl := range actual.Relationships { + if !compareRelationShips(t, i, rl, expected.Relationships[i]) { + break + } + } } -type spdxPkgLicenseDeclared struct { - name string -} +func compareSpdxCreationInfo(t *testing.T, actual, expected *spdx.CreationInfo) { + if actual == nil || expected == nil { + t.Errorf("SBOM: Creation info is nil! Got %q: Expected %q", actual, expected) + } -func (m spdxPkgLicenseDeclared) isMatch(line string) bool { - groups := spdxPkgLicenseDeclaredTag.FindStringSubmatch(line) - if len(groups) != 2 { - return false + if actual.LicenseListVersion != expected.LicenseListVersion { + t.Errorf("SBOM: Creation info license version Error! Got %s: Expected %s", actual.LicenseListVersion, expected.LicenseListVersion) } - return groups[1] == replaceSlashes(m.name) -} -func (m spdxPkgLicenseDeclared) String() string { - return "PackageLicenseConcluded: LicenseRef-" + m.name -} + if len(actual.Creators) != len(expected.Creators) { + t.Errorf("SBOM: Creation info creators Error! Got %d: Expected %d", len(actual.Creators), len(expected.Creators)) + } -type spdxRelationship struct { - pkg1 string - pkg2 string - relation string + for i, info := range actual.Creators { + if info != expected.Creators[i] { + t.Errorf("SBOM: Creation info creators Error! Got %q: Expected %q", info, expected.Creators[i]) + } + } } -func (m spdxRelationship) isMatch(line string) bool { - groups := spdxRelationshipTag.FindStringSubmatch(line) - if len(groups) != 4 { +func compareSpdxPackages(t *testing.T, i int, actual, expected *spdx.Package) bool { + if actual == nil || expected == nil { + t.Errorf("SBOM: Packages are nil at index %d! Got %v: Expected %v", i, actual, expected) + return false + } + if actual.PackageName != expected.PackageName { + t.Errorf("SBOM: Package name Error at index %d! Got %s: Expected %s", i, actual.PackageName, expected.PackageName) return false } - return groups[1] == replaceSlashes(m.pkg1) && groups[2] == m.relation && groups[3] == replaceSlashes(m.pkg2) -} - -func (m spdxRelationship) String() string { - return "Relationship: SPDXRef-" + replaceSlashes(m.pkg1) + " " + m.relation + " SPDXRef-Package-" + replaceSlashes(m.pkg2) -} -type spdxLicense struct{} + if actual.PackageVersion != expected.PackageVersion { + t.Errorf("SBOM: Package version Error at index %d! Got %s: Expected %s", i, actual.PackageVersion, expected.PackageVersion) + return false + } -func (m spdxLicense) isMatch(line string) bool { - return spdxLicenseTag.MatchString(line) -} + if actual.PackageSPDXIdentifier != expected.PackageSPDXIdentifier { + t.Errorf("SBOM: Package identifier Error at index %d! Got %s: Expected %s", i, actual.PackageSPDXIdentifier, expected.PackageSPDXIdentifier) + return false + } -func (m spdxLicense) String() string { - return "##### Non-standard license:" -} + if actual.PackageDownloadLocation != expected.PackageDownloadLocation { + t.Errorf("SBOM: Package download location Error at index %d! Got %s: Expected %s", i, actual.PackageDownloadLocation, expected.PackageDownloadLocation) + return false + } -type spdxLicenseID struct { - name string + if actual.PackageLicenseConcluded != expected.PackageLicenseConcluded { + t.Errorf("SBOM: Package license concluded Error at index %d! Got %s: Expected %s", i, actual.PackageLicenseConcluded, expected.PackageLicenseConcluded) + return false + } + return true } -func (m spdxLicenseID) isMatch(line string) bool { - groups := spdxLicenseIDTag.FindStringSubmatch(line) - if len(groups) != 2 { +func compareRelationShips(t *testing.T, i int, actual, expected *spdx.Relationship) bool { + if actual == nil || expected == nil { + t.Errorf("SBOM: Relationships is nil at index %d! Got %v: Expected %v", i, actual, expected) return false } - return groups[1] == replaceSlashes(m.name) -} -func (m spdxLicenseID) String() string { - return "LicenseID: LicenseRef-" + m.name -} + if actual.RefA != expected.RefA { + t.Errorf("SBOM: Relationship RefA Error at index %d! Got %s: Expected %s", i, actual.RefA, expected.RefA) + return false + } -type spdxExtractedText struct { - name string -} + if actual.RefB != expected.RefB { + t.Errorf("SBOM: Relationship RefB Error at index %d! Got %s: Expected %s", i, actual.RefB, expected.RefB) + return false + } -func (m spdxExtractedText) isMatch(line string) bool { - groups := spdxExtractedTextTag.FindStringSubmatch(line) - if len(groups) != 2 { + if actual.Relationship != expected.Relationship { + t.Errorf("SBOM: Relationship type Error at index %d! Got %s: Expected %s", i, actual.Relationship, expected.Relationship) return false } - return groups[1] == replaceSlashes(m.name) + return true } -func (m spdxExtractedText) String() string { - return "ExtractedText: <text>" + m.name -} +func compareLicenses(t *testing.T, i int, actual, expected *spdx.OtherLicense) bool { + if actual == nil || expected == nil { + t.Errorf("SBOM: Licenses is nil at index %d! Got %v: Expected %v", i, actual, expected) + return false + } -type spdxExtractedClosingText struct{} + if actual.LicenseName != expected.LicenseName { + t.Errorf("SBOM: License Name Error at index %d! Got %s: Expected %s", i, actual.LicenseName, expected.LicenseName) + return false + } -func (m spdxExtractedClosingText) isMatch(line string) bool { - return spdxExtractedClosingTextTag.MatchString(line) -} + if actual.LicenseIdentifier != expected.LicenseIdentifier { + t.Errorf("SBOM: License Identifier Error at index %d! Got %s: Expected %s", i, actual.LicenseIdentifier, expected.LicenseIdentifier) + return false + } -func (m spdxExtractedClosingText) String() string { - return "</text>" + if actual.ExtractedText != expected.ExtractedText { + t.Errorf("SBOM: License Extracted Text Error at index %d! Got: %q want: %q", i, actual.ExtractedText, expected.ExtractedText) + return false + } + return true } -type matcherList []matcher - -func (l matcherList) String() string { - var sb strings.Builder - for _, m := range l { - s := m.String() - fmt.Fprintf(&sb, "%s\n", s) - } - return sb.String() +func fakeTime() string { + t := time.UnixMicro(0) + return t.UTC().Format("2006-01-02T15:04:05Z") } diff --git a/tools/compliance/go.mod b/tools/compliance/go.mod index 088915a33f..bd040774b1 100644 --- a/tools/compliance/go.mod +++ b/tools/compliance/go.mod @@ -7,8 +7,11 @@ replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobu require ( android/soong v0.0.0 github.com/google/blueprint v0.0.0 + github.com/spdx/tools-golang v0.0.0 ) +replace github.com/spdx/tools-golang v0.0.0 => ../../../../external/spdx-tools + require golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect replace android/soong v0.0.0 => ../../../soong @@ -23,4 +26,4 @@ replace github.com/google/go-cmp v0.5.5 => ../../../../external/go-cmp // Indirect dep from go-cmp exclude golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 -go 1.18 +go 1.21 diff --git a/tools/compliance/projectmetadata/projectmetadata.go b/tools/compliance/projectmetadata/projectmetadata.go index b137a120dc..30a63258d6 100644 --- a/tools/compliance/projectmetadata/projectmetadata.go +++ b/tools/compliance/projectmetadata/projectmetadata.go @@ -63,12 +63,12 @@ func (pm *ProjectMetadata) Project() string { return pm.project } -// ProjectName returns the name of the project. +// Name returns the name of the project. func (pm *ProjectMetadata) Name() string { return pm.proto.GetName() } -// ProjectVersion returns the version of the project if available. +// Version returns the version of the project if available. func (pm *ProjectMetadata) Version() string { tp := pm.proto.GetThirdParty() if tp != nil { diff --git a/tools/finalization/OWNERS b/tools/finalization/OWNERS index 518b60db7e..b00b774b72 100644 --- a/tools/finalization/OWNERS +++ b/tools/finalization/OWNERS @@ -1,5 +1,7 @@ include platform/build/soong:/OWNERS -smoreland@google.com -alexbuy@google.com +amhk@google.com +gurpreetgs@google.com +michaelwr@google.com patb@google.com +smoreland@google.com zyy@google.com diff --git a/tools/finalization/README.md b/tools/finalization/README.md new file mode 100644 index 0000000000..cc97d1f4a5 --- /dev/null +++ b/tools/finalization/README.md @@ -0,0 +1,20 @@ +# Finalization tools +This folder contains automation and CI scripts for [finalizing](https://go/android-finalization) Android before release. + +## Automation: +1. [Environment setup](./environment.sh). Set values for varios finalization constants. +2. [Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh). Prepare the branch for SDK release. SDK contains Android Java APIs and other stable APIs. Commonly referred as a 1st step. +3. [Finalize Android](./finalize-sdk-rel.sh). Mark branch as "REL", i.e. prepares for Android release. Any signed build containing these changes will be considered an official Android Release. Referred as a 2nd finalization step. +4. [Finalize SDK and submit](./step-1.sh). Do [Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh) step, create CLs, organize them into topic and send to Gerrit. + a. [Update SDK and submit](./update-step-1.sh). Same as above, but updates the existings CLs. +5. [Finalize Android and submit](./step-2.sh). Do [Finalize Android](./finalize-sdk-rel.sh) step, create CLs, organize them into topic and send to Gerrit. + a. [Update Android and submit](./update-step-2.sh). Same as above, but updates the existings CLs. + +## CI: +Performed in build targets in Finalization branches. +1. [Finalization Step 1, git_main-fina-1-release](https://android-build.corp.google.com/build_explorer/branch/git_main-fina-1-release). Test [1st step/Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh). +3. [Finalization Step 2, git_main-fina-2-release](https://android-build.corp.google.com/build_explorer/branch/git_main-fina-2-release). Test [1st step/Finalize SDK](./finalize-aidl-vndk-sdk-resources.sh) and [2nd step/Finalize Android](./finalize-sdk-rel.sh). Use [local finalization](./localonly-steps.sh) to build and copy presubmits. +5. [Local finalization steps](./localonly-steps.sh) are done only during local testing or in the CI lab. Normally these steps use artifacts from other builds. + +## Utility: +[Full cleanup](./cleanup.sh). Remove all local changes and switch each project into head-less state. This is the best state to sync/rebase/finalize the branch. diff --git a/tools/finalization/build-step-1-and-2.sh b/tools/finalization/build-step-1-and-2.sh index eaaf0cd3a4..84e2782659 100755 --- a/tools/finalization/build-step-1-and-2.sh +++ b/tools/finalization/build-step-1-and-2.sh @@ -6,17 +6,18 @@ function finalize_main_step12() { local top="$(dirname "$0")"/../../../.. source $top/build/make/tools/finalization/environment.sh - # default target to modify tree and build SDK - local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" - - # SDK codename -> int - source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh - - # Platform/Mainline SDKs build and move to prebuilts - source $top/build/make/tools/finalization/localonly-finalize-mainline-sdk.sh - - # REL - source $top/build/make/tools/finalization/finalize-sdk-rel.sh + if [ "$FINAL_STATE" = "unfinalized" ] ; then + # SDK codename -> int + source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh + fi; + + if [ "$FINAL_STATE" = "unfinalized" ] || [ "$FINAL_STATE" = "sdk" ] ; then + # ADB, Platform/Mainline SDKs build and move to prebuilts + source $top/build/make/tools/finalization/localonly-steps.sh + + # REL + source $top/build/make/tools/finalization/finalize-sdk-rel.sh + fi; } finalize_main_step12 diff --git a/tools/finalization/build-step-1.sh b/tools/finalization/build-step-1.sh index edf497ed03..3d5eadbbc6 100755 --- a/tools/finalization/build-step-1.sh +++ b/tools/finalization/build-step-1.sh @@ -6,11 +6,10 @@ function finalize_main_step1() { local top="$(dirname "$0")"/../../../.. source $top/build/make/tools/finalization/environment.sh - # default target to modify tree and build SDK - local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" - - # Build finalization artifacts. - source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh + if [ "$FINAL_STATE" = "unfinalized" ] ; then + # Build finalization artifacts. + source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh + fi; } finalize_main_step1 diff --git a/tools/finalization/environment.sh b/tools/finalization/environment.sh index 983e19dc4a..d9c42c8bf9 100755 --- a/tools/finalization/environment.sh +++ b/tools/finalization/environment.sh @@ -2,14 +2,25 @@ set -ex -export FINAL_BUG_ID='0' +export FINAL_BUG_ID='0' # CI only -export FINAL_PLATFORM_CODENAME='UpsideDownCake' +export FINAL_PLATFORM_CODENAME='VanillaIceCream' export CURRENT_PLATFORM_CODENAME='VanillaIceCream' -export FINAL_PLATFORM_CODENAME_JAVA='UPSIDE_DOWN_CAKE' -export FINAL_PLATFORM_SDK_VERSION='34' -export FINAL_PLATFORM_VERSION='14' +export FINAL_PLATFORM_CODENAME_JAVA='VANILLA_ICE_CREAM' +export FINAL_PLATFORM_VERSION='15' -export FINAL_BUILD_PREFIX='UP1A' +# Set arbitrary large values for CI. +# SDK_VERSION needs to be <61 (lint/libs/lint-api/src/main/java/com/android/tools/lint/detector/api/ApiConstraint.kt) +# There are multiple places where we rely on next SDK version to be previous + 1, e.g. RESOURCES_SDK_INT. +# We might or might not fix this in future, but for now let's keep it +1. +export FINAL_PLATFORM_SDK_VERSION='35' +# Feel free to randomize once in a while to detect buggy version detection code. +export FINAL_MAINLINE_EXTENSION='58' -export FINAL_MAINLINE_EXTENSION='6'
\ No newline at end of file +# Options: +# 'unfinalized' - branch is in development state, +# 'sdk' - SDK/API is finalized +# 'rel' - branch is finalized, switched to REL +export FINAL_STATE='unfinalized' + +export BUILD_FROM_SOURCE_STUB=true
\ No newline at end of file diff --git a/tools/finalization/finalize-aidl-vndk-sdk-resources.sh b/tools/finalization/finalize-aidl-vndk-sdk-resources.sh index e23e585311..75379ff661 100755 --- a/tools/finalization/finalize-aidl-vndk-sdk-resources.sh +++ b/tools/finalization/finalize-aidl-vndk-sdk-resources.sh @@ -4,7 +4,15 @@ set -ex function apply_droidstubs_hack() { if ! grep -q 'STOPSHIP: RESTORE THIS LOGIC WHEN DECLARING "REL" BUILD' "$top/build/soong/java/droidstubs.go" ; then - git -C "$top/build/soong" apply --allow-empty ../../build/make/tools/finalization/build_soong_java_droidstubs.go.apply_hack.diff + local build_soong_git_root="$(readlink -f $top/build/soong)" + patch --strip=1 --no-backup-if-mismatch --directory="$build_soong_git_root" --input=../../build/make/tools/finalization/build_soong_java_droidstubs.go.apply_hack.diff + fi +} + +function apply_resources_sdk_int_fix() { + if ! grep -q 'public static final int RESOURCES_SDK_INT = SDK_INT;' "$top/frameworks/base/core/java/android/os/Build.java" ; then + local base_git_root="$(readlink -f $top/frameworks/base)" + patch --strip=1 --no-backup-if-mismatch --directory="$base_git_root" --input=../../build/make/tools/finalization/frameworks_base.apply_resource_sdk_int.diff fi } @@ -61,6 +69,33 @@ inline bool IsAtLeast${FINAL_PLATFORM_CODENAME:0:1}() { return android_get_devic rm "$tmpfile" } +function bumpSdkExtensionsVersion() { + local SDKEXT="packages/modules/SdkExtensions/" + + # This used to call bump_sdk.sh utility. + # However due to TS, we have to build the gen_sdk with a correct set of settings. + + # "$top/packages/modules/SdkExtensions/gen_sdk/bump_sdk.sh" ${FINAL_MAINLINE_EXTENSION} + # Leave the last commit as a set of modified files. + # The code to create a finalization topic will pick it up later. + # git -C ${SDKEXT} reset HEAD~1 + + local sdk="${FINAL_MAINLINE_EXTENSION}" + local modules_arg= + + TARGET_PRODUCT=aosp_arm64 \ + TARGET_RELEASE=fina_1 \ + TARGET_BUILD_VARIANT=userdebug \ + DIST_DIR=out/dist \ + $top/build/soong/soong_ui.bash --make-mode --soong-only gen_sdk + + ANDROID_BUILD_TOP="$top" out/soong/host/linux-x86/bin/gen_sdk \ + --database ${SDKEXT}/gen_sdk/extensions_db.textpb \ + --action new_sdk \ + --sdk "$sdk" \ + $modules_arg +} + function finalize_aidl_vndk_sdk_resources() { local top="$(dirname "$0")"/../../../.. source $top/build/make/tools/finalization/environment.sh @@ -68,9 +103,6 @@ function finalize_aidl_vndk_sdk_resources() { local SDK_CODENAME="public static final int $FINAL_PLATFORM_CODENAME_JAVA = CUR_DEVELOPMENT;" local SDK_VERSION="public static final int $FINAL_PLATFORM_CODENAME_JAVA = $FINAL_PLATFORM_SDK_VERSION;" - # default target to modify tree and build SDK - local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" - # The full process can be found at (INTERNAL) go/android-sdk-finalization. # apply droidstubs hack to prevent tools from incrementing an API version @@ -79,37 +111,33 @@ function finalize_aidl_vndk_sdk_resources() { # bionic/NDK finalize_bionic_ndk - # VNDK definitions for new SDK version - cp "$top/development/vndk/tools/definition-tool/datasets/vndk-lib-extra-list-current.txt" \ - "$top/development/vndk/tools/definition-tool/datasets/vndk-lib-extra-list-$FINAL_PLATFORM_SDK_VERSION.txt" - - AIDL_TRANSITIVE_FREEZE=true $m aidl-freeze-api create_reference_dumps + # pre-finalization build target (trunk) + local aidl_m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_RELEASE=trunk TARGET_BUILD_VARIANT=userdebug DIST_DIR=out/dist" + AIDL_TRANSITIVE_FREEZE=true $aidl_m aidl-freeze-api - # Generate ABI dumps - ANDROID_BUILD_TOP="$top" \ - out/host/linux-x86/bin/create_reference_dumps \ - -p aosp_arm64 --build-variant user - - echo "NOTE: THIS INTENTIONALLY MAY FAIL AND REPAIR ITSELF (until 'DONE')" - # Update new versions of files. See update-vndk-list.sh (which requires envsetup.sh) - $m check-vndk-list || \ - { cp $top/out/soong/vndk/vndk.libraries.txt $top/build/make/target/product/gsi/current.txt; } - echo "DONE: THIS INTENTIONALLY MAY FAIL AND REPAIR ITSELF" + # TODO(b/309880485) + # Add back create_reference_dumps and $top/build/make/target/product/gsi/current.txt # Finalize SDK # frameworks/libs/modules-utils finalize_modules_utils + # development/sdk + local platform_source="$top/development/sdk/platform_source.prop_template" + sed -i -e 's/Pkg\.Revision.*/Pkg\.Revision=1/g' $platform_source + local build_tools_source="$top/development/sdk/build_tools_source.prop_template" + sed -i -e 's/Pkg\.Revision.*/Pkg\.Revision=${PLATFORM_SDK_VERSION}.0.0/g' $build_tools_source + # build/make - local version_defaults="$top/build/make/core/version_defaults.mk" - sed -i -e "s/PLATFORM_SDK_VERSION := .*/PLATFORM_SDK_VERSION := ${FINAL_PLATFORM_SDK_VERSION}/g" $version_defaults - sed -i -e "s/PLATFORM_VERSION_LAST_STABLE := .*/PLATFORM_VERSION_LAST_STABLE := ${FINAL_PLATFORM_VERSION}/g" $version_defaults sed -i -e "s/sepolicy_major_vers := .*/sepolicy_major_vers := ${FINAL_PLATFORM_SDK_VERSION}/g" "$top/build/make/core/config.mk" cp "$top/build/make/target/product/gsi/current.txt" "$top/build/make/target/product/gsi/$FINAL_PLATFORM_SDK_VERSION.txt" # build/soong - sed -i -e "/:.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\\t\t\t\"${FINAL_PLATFORM_CODENAME}\": ${FINAL_PLATFORM_SDK_VERSION}," "$top/build/soong/android/api_levels.go" + local codename_version="\"${FINAL_PLATFORM_CODENAME}\": ${FINAL_PLATFORM_SDK_VERSION}" + if ! grep -q "$codename_version" "$top/build/soong/android/api_levels.go" ; then + sed -i -e "/:.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\\t\t$codename_version," "$top/build/soong/android/api_levels.go" + fi # cts echo ${FINAL_PLATFORM_VERSION} > "$top/cts/tests/tests/os/assets/platform_releases.txt" @@ -122,6 +150,13 @@ function finalize_aidl_vndk_sdk_resources() { local version_codes="$top/platform_testing/libraries/compatibility-common-util/src/com/android/compatibility/common/util/VersionCodes.java" sed -i -e "/=.*$((${FINAL_PLATFORM_SDK_VERSION}-1));/a \\ ${SDK_VERSION}" $version_codes + # tools/platform-compat + local class2nonsdklist="$top/tools/platform-compat/java/com/android/class2nonsdklist/Class2NonSdkList.java" + if ! grep -q "\.*map.put($((${FINAL_PLATFORM_SDK_VERSION}))" $class2nonsdklist ; then + local sdk_version="map.put(${FINAL_PLATFORM_SDK_VERSION}, FLAG_UNSUPPORTED);" + sed -i -e "/.*map.put($((${FINAL_PLATFORM_SDK_VERSION}-1))/a \\ ${sdk_version}" $class2nonsdklist + fi + # Finalize resources "$top/frameworks/base/tools/aapt2/tools/finalize_res.py" \ "$top/frameworks/base/core/res/res/values/public-staging.xml" \ @@ -129,17 +164,19 @@ function finalize_aidl_vndk_sdk_resources() { # frameworks/base sed -i "s%$SDK_CODENAME%$SDK_VERSION%g" "$top/frameworks/base/core/java/android/os/Build.java" + apply_resources_sdk_int_fix sed -i -e "/=.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\ SDK_${FINAL_PLATFORM_CODENAME_JAVA} = ${FINAL_PLATFORM_SDK_VERSION}," "$top/frameworks/base/tools/aapt/SdkConstants.h" sed -i -e "/=.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\ SDK_${FINAL_PLATFORM_CODENAME_JAVA} = ${FINAL_PLATFORM_SDK_VERSION}," "$top/frameworks/base/tools/aapt2/SdkConstants.h" # Bump Mainline SDK extension version. - "$top/packages/modules/SdkExtensions/gen_sdk/bump_sdk.sh" ${FINAL_MAINLINE_EXTENSION} - local version_defaults="$top/build/make/core/version_defaults.mk" - sed -i -e "s/PLATFORM_SDK_EXTENSION_VERSION := .*/PLATFORM_SDK_EXTENSION_VERSION := ${FINAL_MAINLINE_EXTENSION}/g" $version_defaults + bumpSdkExtensionsVersion + + # target to build SDK + local sdk_m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug DIST_DIR=out/dist" # Force update current.txt - $m clobber - $m update-api + $sdk_m clobber + $sdk_m update-api } finalize_aidl_vndk_sdk_resources diff --git a/tools/finalization/finalize-sdk-rel.sh b/tools/finalization/finalize-sdk-rel.sh index 56f3bc3c87..245305bfad 100755 --- a/tools/finalization/finalize-sdk-rel.sh +++ b/tools/finalization/finalize-sdk-rel.sh @@ -4,13 +4,19 @@ set -ex function revert_droidstubs_hack() { if grep -q 'STOPSHIP: RESTORE THIS LOGIC WHEN DECLARING "REL" BUILD' "$top/build/soong/java/droidstubs.go" ; then - git -C "$top/build/soong" apply --allow-empty ../../build/make/tools/finalization/build_soong_java_droidstubs.go.revert_hack.diff + patch --strip=1 --no-backup-if-mismatch --directory="$top/build/soong" --input=../../build/make/tools/finalization/build_soong_java_droidstubs.go.revert_hack.diff + fi +} + +function revert_resources_sdk_int_fix() { + if grep -q 'public static final int RESOURCES_SDK_INT = SDK_INT;' "$top/frameworks/base/core/java/android/os/Build.java" ; then + patch --strip=1 --no-backup-if-mismatch --directory="$top/frameworks/base" --input=../../build/make/tools/finalization/frameworks_base.revert_resource_sdk_int.diff fi } function apply_prerelease_sdk_hack() { if ! grep -q 'STOPSHIP: hack for the pre-release SDK' "$top/frameworks/base/core/java/android/content/pm/parsing/FrameworkParsingPackageUtils.java" ; then - git -C "$top/frameworks/base" apply --allow-empty ../../build/make/tools/finalization/frameworks_base.apply_hack.diff + patch --strip=1 --no-backup-if-mismatch --directory="$top/frameworks/base" --input=../../build/make/tools/finalization/frameworks_base.apply_hack.diff fi } @@ -18,21 +24,14 @@ function finalize_sdk_rel() { local top="$(dirname "$0")"/../../../.. source $top/build/make/tools/finalization/environment.sh - # default target to modify tree and build SDK - local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" - # revert droidstubs hack now we are switching to REL revert_droidstubs_hack # let the apps built with pre-release SDK parse apply_prerelease_sdk_hack - # adb keys - $m adb - LOGNAME=android-eng HOSTNAME=google.com "$top/out/host/linux-x86/bin/adb" keygen "$top/vendor/google/security/adb/${FINAL_PLATFORM_VERSION}.adb_key" - - # build/make/core/version_defaults.mk - sed -i -e "s/PLATFORM_VERSION_CODENAME.${FINAL_BUILD_PREFIX} := .*/PLATFORM_VERSION_CODENAME.${FINAL_BUILD_PREFIX} := REL/g" "$top/build/make/core/version_defaults.mk" + # in REL mode, resources would correctly set the resources_sdk_int, no fix required + revert_resources_sdk_int_fix # cts echo "$FINAL_PLATFORM_VERSION" > "$top/cts/tests/tests/os/assets/platform_versions.txt" @@ -42,18 +41,21 @@ function finalize_sdk_rel() { git -C "$top/cts" mv hostsidetests/theme/assets/${FINAL_PLATFORM_CODENAME} hostsidetests/theme/assets/${FINAL_PLATFORM_SDK_VERSION} # system/sepolicy - mkdir -p "$top/system/sepolicy/prebuilts/api/${FINAL_PLATFORM_SDK_VERSION}.0/" - cp -r "$top/system/sepolicy/public/" "$top/system/sepolicy/prebuilts/api/${FINAL_PLATFORM_SDK_VERSION}.0/" - cp -r "$top/system/sepolicy/private/" "$top/system/sepolicy/prebuilts/api/${FINAL_PLATFORM_SDK_VERSION}.0/" - - # prebuilts/abi-dumps/ndk - mv "$top/prebuilts/abi-dumps/ndk/current" "$top/prebuilts/abi-dumps/ndk/$FINAL_PLATFORM_SDK_VERSION" - - # prebuilts/abi-dumps/vndk - mv "$top/prebuilts/abi-dumps/vndk/$CURRENT_PLATFORM_CODENAME" "$top/prebuilts/abi-dumps/vndk/$FINAL_PLATFORM_SDK_VERSION" + system/sepolicy/tools/finalize-sdk-rel.sh "$top" "$FINAL_PLATFORM_SDK_VERSION" # prebuilts/abi-dumps/platform - mv "$top/prebuilts/abi-dumps/platform/current" "$top/prebuilts/abi-dumps/platform/$FINAL_PLATFORM_SDK_VERSION" + mkdir -p "$top/prebuilts/abi-dumps/platform/$FINAL_PLATFORM_SDK_VERSION" + cp -r "$top/prebuilts/abi-dumps/platform/current/64/" "$top/prebuilts/abi-dumps/platform/$FINAL_PLATFORM_SDK_VERSION/" + + # TODO(b/309880485) + # uncomment and update + # prebuilts/abi-dumps/ndk + #mkdir -p "$top/prebuilts/abi-dumps/ndk/$FINAL_PLATFORM_SDK_VERSION" + #cp -r "$top/prebuilts/abi-dumps/ndk/current/64/" "$top/prebuilts/abi-dumps/ndk/$FINAL_PLATFORM_SDK_VERSION/" + #if [ "$FINAL_STATE" != "sdk" ] || [ "$FINAL_PLATFORM_CODENAME" == "$CURRENT_PLATFORM_CODENAME" ] ; then + # prebuilts/abi-dumps/vndk + #mv "$top/prebuilts/abi-dumps/vndk/$CURRENT_PLATFORM_CODENAME" "$top/prebuilts/abi-dumps/vndk/$FINAL_PLATFORM_SDK_VERSION" + #fi; } finalize_sdk_rel diff --git a/tools/finalization/frameworks_base.apply_resource_sdk_int.diff b/tools/finalization/frameworks_base.apply_resource_sdk_int.diff new file mode 100644 index 0000000000..f0576d0851 --- /dev/null +++ b/tools/finalization/frameworks_base.apply_resource_sdk_int.diff @@ -0,0 +1,24 @@ +From cdb47fc90b8d6860ec1dc5efada1f9ccd471618b Mon Sep 17 00:00:00 2001 +From: Alex Buynytskyy <alexbuy@google.com> +Date: Tue, 11 Apr 2023 22:12:44 +0000 +Subject: [PATCH] Don't force +1 for resource resolution. + +Bug: 277674088 +Fixes: 277674088 +Test: boots, no crashes +Change-Id: I17e743a0f1cf6f98fddd40c358dea5a8b9cc7723 +--- + +diff --git a/core/java/android/os/Build.java b/core/java/android/os/Build.java +index eb47170..4d3e92b 100755 +--- a/core/java/android/os/Build.java ++++ b/core/java/android/os/Build.java +@@ -493,7 +493,7 @@ + * @hide + */ + @TestApi +- public static final int RESOURCES_SDK_INT = SDK_INT + ACTIVE_CODENAMES.length; ++ public static final int RESOURCES_SDK_INT = SDK_INT; + + /** + * The current lowest supported value of app target SDK. Applications targeting diff --git a/tools/finalization/frameworks_base.revert_resource_sdk_int.diff b/tools/finalization/frameworks_base.revert_resource_sdk_int.diff new file mode 100644 index 0000000000..2ade499e20 --- /dev/null +++ b/tools/finalization/frameworks_base.revert_resource_sdk_int.diff @@ -0,0 +1,27 @@ +From c7e460bb19071d867cd7ca04282ce42694f4f358 Mon Sep 17 00:00:00 2001 +From: Alex Buynytskyy <alexbuy@google.com> +Date: Wed, 12 Apr 2023 01:06:26 +0000 +Subject: [PATCH] Revert "Don't force +1 for resource resolution." + +It's not required for master. + +This reverts commit f1cb683988f81579a76ddbf9993848a4a06dd28c. + +Bug: 277674088 +Test: boots, no crashes +Change-Id: Ia1692548f26496fdc6f1e4f0557213c7996d6823 +--- + +diff --git a/core/java/android/os/Build.java b/core/java/android/os/Build.java +index 4d3e92b..eb47170 100755 +--- a/core/java/android/os/Build.java ++++ b/core/java/android/os/Build.java +@@ -493,7 +493,7 @@ + * @hide + */ + @TestApi +- public static final int RESOURCES_SDK_INT = SDK_INT; ++ public static final int RESOURCES_SDK_INT = SDK_INT + ACTIVE_CODENAMES.length; + + /** + * The current lowest supported value of app target SDK. Applications targeting diff --git a/tools/finalization/localonly-finalize-mainline-sdk.sh b/tools/finalization/localonly-finalize-mainline-sdk.sh deleted file mode 100755 index 104b6acb04..0000000000 --- a/tools/finalization/localonly-finalize-mainline-sdk.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ex - -function finalize_locally_mainline_sdk() { - local top="$(dirname "$0")"/../../../.. - source $top/build/make/tools/finalization/environment.sh - - # Build Platform SDKs. - $top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=sdk TARGET_BUILD_VARIANT=userdebug sdk dist sdk_repo - - # Build Modules SDKs. - TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true "$top/vendor/google/build/mainline_modules_sdks.sh" - - # Update prebuilts. - # "$top/prebuilts/build-tools/path/linux-x86/python3" "$top/packages/modules/common/tools/finalize_sdk.py" -l -b 0 -f ${FINAL_MAINLINE_EXTENSION} -r '' 0 - "$top/prebuilts/build-tools/path/linux-x86/python3" "$top/prebuilts/sdk/update_prebuilts.py" --local_mode -f ${FINAL_PLATFORM_SDK_VERSION} -e ${FINAL_MAINLINE_EXTENSION} --bug 1 1 -} - -finalize_locally_mainline_sdk - diff --git a/tools/finalization/localonly-steps.sh b/tools/finalization/localonly-steps.sh new file mode 100755 index 0000000000..bebd563bea --- /dev/null +++ b/tools/finalization/localonly-steps.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +function finalize_locally() { + local top="$(dirname "$0")"/../../../.. + source $top/build/make/tools/finalization/environment.sh + + # default target to modify tree and build SDK + local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug DIST_DIR=out/dist" + + # adb keys + $m adb + LOGNAME=android-eng HOSTNAME=google.com "$top/out/host/linux-x86/bin/adb" keygen "$top/vendor/google/security/adb/${FINAL_PLATFORM_VERSION}.adb_key" + + # Build Platform SDKs. + $top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=sdk TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug sdk dist sdk_repo DIST_DIR=out/dist + + # Build Modules SDKs. + TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true DIST_DIR=out/dist "$top/vendor/google/build/mainline_modules_sdks.sh" --build-release=latest + + # Update prebuilts. + "$top/prebuilts/build-tools/path/linux-x86/python3" -W ignore::DeprecationWarning "$top/prebuilts/sdk/update_prebuilts.py" --local_mode -f ${FINAL_PLATFORM_SDK_VERSION} -e ${FINAL_MAINLINE_EXTENSION} --bug 1 1 +} + +finalize_locally diff --git a/tools/finalization/step-1.sh b/tools/finalization/step-1.sh index cf21e4511f..0dd4b3afc3 100755 --- a/tools/finalization/step-1.sh +++ b/tools/finalization/step-1.sh @@ -9,7 +9,7 @@ function commit_step_1_changes() { if [[ $(git status --short) ]]; then repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization" ; git add -A . ; - git commit -m "$FINAL_PLATFORM_CODENAME is now $FINAL_PLATFORM_SDK_VERSION" \ + git commit -m "$FINAL_PLATFORM_CODENAME is now $FINAL_PLATFORM_SDK_VERSION and extension version $FINAL_MAINLINE_EXTENSION" \ -m "Ignore-AOSP-First: $FINAL_PLATFORM_CODENAME Finalization Bug: $FINAL_BUG_ID Test: build"; diff --git a/tools/finalization/update-step-1.sh b/tools/finalization/update-step-1.sh new file mode 100755 index 0000000000..b469988f2b --- /dev/null +++ b/tools/finalization/update-step-1.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Script to perform a 1st step of Android Finalization: API/SDK finalization, update CLs and upload to Gerrit. + +# WIP, does not work yet +exit 10 + +set -ex + +function update_step_1_changes() { + set +e + repo forall -c '\ + if [[ $(git status --short) ]]; then + git stash -u ; + repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization" ; + git stash pop ; + git add -A . ; + git commit --amend --no-edit ; + repo upload --cbr --no-verify -o nokeycheck -t -y . ; + fi' +} + +function update_step_1_main() { + local top="$(dirname "$0")"/../../../.. + source $top/build/make/tools/finalization/environment.sh + + + local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" + + # vndk etc finalization + source $top/build/make/tools/finalization/finalize-aidl-vndk-sdk-resources.sh + + # update existing CLs and upload to gerrit + update_step_1_changes + + # build to confirm everything is OK + AIDL_FROZEN_REL=true $m +} + +update_step_1_main diff --git a/tools/finalization/update-step-2.sh b/tools/finalization/update-step-2.sh new file mode 100755 index 0000000000..d2b8592c9d --- /dev/null +++ b/tools/finalization/update-step-2.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Script to perform a 2nd step of Android Finalization: REL finalization, create CLs and upload to Gerrit. + +# WIP, does not work yet +exit 10 + +set -ex + +function update_step_2_changes() { + set +e + repo forall -c '\ + if [[ $(git status --short) ]]; then + git stash -u ; + repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization-Rel" ; + git stash pop ; + git add -A . ; + git commit --amend --no-edit ; + repo upload --cbr --no-verify -o nokeycheck -t -y . ; + fi' +} + +function update_step_2_main() { + local top="$(dirname "$0")"/../../../.. + source $top/build/make/tools/finalization/environment.sh + + local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug" + + # prebuilts etc + source $top/build/make/tools/finalization/finalize-sdk-rel.sh + + # move all changes to finalization branch/topic and upload to gerrit + update_step_2_changes + + # build to confirm everything is OK + AIDL_FROZEN_REL=true $m +} + +update_step_2_main diff --git a/tools/find_static_candidates.py b/tools/find_static_candidates.py new file mode 100644 index 0000000000..2e50627cab --- /dev/null +++ b/tools/find_static_candidates.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +"""Tool to find static libraries that maybe should be shared libraries and shared libraries that maybe should be static libraries. + +This tool only looks at the module-info.json for the current target. + +Example of "class" types for each of the modules in module-info.json + "EXECUTABLES": 2307, + "ETC": 9094, + "NATIVE_TESTS": 10461, + "APPS": 2885, + "JAVA_LIBRARIES": 5205, + "EXECUTABLES/JAVA_LIBRARIES": 119, + "FAKE": 553, + "SHARED_LIBRARIES/STATIC_LIBRARIES": 7591, + "STATIC_LIBRARIES": 11535, + "SHARED_LIBRARIES": 10852, + "HEADER_LIBRARIES": 1897, + "DYLIB_LIBRARIES": 1262, + "RLIB_LIBRARIES": 3413, + "ROBOLECTRIC": 39, + "PACKAGING": 5, + "PROC_MACRO_LIBRARIES": 36, + "RENDERSCRIPT_BITCODE": 17, + "DYLIB_LIBRARIES/RLIB_LIBRARIES": 8, + "ETC/FAKE": 1 + +None of the "SHARED_LIBRARIES/STATIC_LIBRARIES" are double counted in the +modules with one class +RLIB/ + +All of these classes have shared_libs and/or static_libs + "EXECUTABLES", + "SHARED_LIBRARIES", + "STATIC_LIBRARIES", + "SHARED_LIBRARIES/STATIC_LIBRARIES", # cc_library + "HEADER_LIBRARIES", + "NATIVE_TESTS", # test modules + "DYLIB_LIBRARIES", # rust + "RLIB_LIBRARIES", # rust + "ETC", # rust_bindgen +""" + +from collections import defaultdict + +import json, os, argparse + +ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT") +# If a shared library is used less than MAX_SHARED_INCLUSIONS times in a target, +# then it will likely save memory by changing it to a static library +# This move will also use less storage +MAX_SHARED_INCLUSIONS = 2 +# If a static library is used more than MAX_STATIC_INCLUSIONS times in a target, +# then it will likely save memory by changing it to a shared library +# This move will also likely use less storage +MIN_STATIC_INCLUSIONS = 3 + + +def parse_args(): + parser = argparse.ArgumentParser( + description=( + "Parse module-info.jso and display information about static and" + " shared library dependencies." + ) + ) + parser.add_argument( + "--module", dest="module", help="Print the info for the module." + ) + parser.add_argument( + "--shared", + dest="print_shared", + action=argparse.BooleanOptionalAction, + help=( + "Print the list of libraries that are shared_libs for fewer than {}" + " modules.".format(MAX_SHARED_INCLUSIONS) + ), + ) + parser.add_argument( + "--static", + dest="print_static", + action=argparse.BooleanOptionalAction, + help=( + "Print the list of libraries that are static_libs for more than {}" + " modules.".format(MIN_STATIC_INCLUSIONS) + ), + ) + parser.add_argument( + "--recursive", + dest="recursive", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Gather all dependencies of EXECUTABLES recursvily before calculating" + " the stats. This eliminates duplicates from multiple libraries" + " including the same dependencies in a single binary." + ), + ) + parser.add_argument( + "--both", + dest="both", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "Print a list of libraries that are including libraries as both" + " static and shared" + ), + ) + return parser.parse_args() + + +class TransitiveHelper: + + def __init__(self): + # keep a list of already expanded libraries so we don't end up in a cycle + self.visited = defaultdict(lambda: defaultdict(set)) + + # module is an object from the module-info dictionary + # module_info is the dictionary from module-info.json + # modify the module's shared_libs and static_libs with all of the transient + # dependencies required from all of the explicit dependencies + def flattenDeps(self, module, module_info): + libs_snapshot = dict(shared_libs = set(module.get("shared_libs",{})), static_libs = set(module.get("static_libs",{}))) + + for lib_class in ["shared_libs", "static_libs"]: + for lib in libs_snapshot[lib_class]: + if not lib or lib not in module_info or lib_class not in module: + continue + if lib in self.visited: + module[lib_class].update(self.visited[lib][lib_class]) + else: + res = self.flattenDeps(module_info[lib], module_info) + module[lib_class].update(res.get(lib_class, {})) + self.visited[lib][lib_class].update(res.get(lib_class, {})) + + return module + +def main(): + module_info = json.load(open(ANDROID_PRODUCT_OUT + "/module-info.json")) + + args = parse_args() + + if args.module: + if args.module not in module_info: + print("Module {} does not exist".format(args.module)) + exit(1) + + # turn all of the static_libs and shared_libs lists into sets to make them + # easier to update + for _, module in module_info.items(): + module["shared_libs"] = set(module.get("shared_libs", {})) + module["static_libs"] = set(module.get("static_libs", {})) + + includedStatically = defaultdict(set) + includedSharedly = defaultdict(set) + includedBothly = defaultdict(set) + transitive = TransitiveHelper() + for name, module in module_info.items(): + if args.recursive: + # in this recursive mode we only want to see what is included by the executables + if "EXECUTABLES" not in module["class"]: + continue + module = transitive.flattenDeps(module, module_info) + # filter out fuzzers by their dependency on clang + if "static_libs" in module: + if "libclang_rt.fuzzer" in module["static_libs"]: + continue + else: + if "NATIVE_TESTS" in module["class"]: + # We don't care about how tests are including libraries + continue + + # count all of the shared and static libs included in this module + if "shared_libs" in module: + for lib in module["shared_libs"]: + includedSharedly[lib].add(name) + if "static_libs" in module: + for lib in module["static_libs"]: + includedStatically[lib].add(name) + + if "shared_libs" in module and "static_libs" in module: + intersection = set(module["shared_libs"]).intersection( + module["static_libs"] + ) + if intersection: + includedBothly[name] = intersection + + if args.print_shared: + print( + "Shared libraries that are included by fewer than {} modules on a" + " device:".format(MAX_SHARED_INCLUSIONS) + ) + for name, libs in includedSharedly.items(): + if len(libs) < MAX_SHARED_INCLUSIONS: + print("{}: {} included by: {}".format(name, len(libs), libs)) + + if args.print_static: + print( + "Libraries that are included statically by more than {} modules on a" + " device:".format(MIN_STATIC_INCLUSIONS) + ) + for name, libs in includedStatically.items(): + if len(libs) > MIN_STATIC_INCLUSIONS: + print("{}: {} included by: {}".format(name, len(libs), libs)) + + if args.both: + allIncludedBothly = set() + for name, libs in includedBothly.items(): + allIncludedBothly.update(libs) + + print( + "List of libraries used both statically and shared in the same" + " processes:\n {}\n\n".format("\n".join(sorted(allIncludedBothly))) + ) + print( + "List of libraries used both statically and shared in any processes:\n {}".format("\n".join(sorted(includedStatically.keys() & includedSharedly.keys())))) + + if args.module: + print(json.dumps(module_info[args.module], default=list, indent=2)) + print( + "{} is included in shared_libs {} times by these modules: {}".format( + args.module, len(includedSharedly[args.module]), + includedSharedly[args.module] + ) + ) + print( + "{} is included in static_libs {} times by these modules: {}".format( + args.module, len(includedStatically[args.module]), + includedStatically[args.module] + ) + ) + print("Shared libs included by this module that are used in fewer than {} processes:\n{}".format( + MAX_SHARED_INCLUSIONS, [x for x in module_info[args.module]["shared_libs"] if len(includedSharedly[x]) < MAX_SHARED_INCLUSIONS])) + + + +if __name__ == "__main__": + main() diff --git a/tools/fs_config/Android.bp b/tools/fs_config/Android.bp index 55fdca43a1..6aa528963d 100644 --- a/tools/fs_config/Android.bp +++ b/tools/fs_config/Android.bp @@ -35,7 +35,6 @@ cc_binary_host { srcs: ["fs_config.c"], shared_libs: [ "libcutils", - "libselinux", ], cflags: ["-Werror"], } @@ -259,3 +258,173 @@ prebuilt_etc { system_ext_specific: true, src: ":group_gen_system_ext", } + +fs_config_cmd = "$(location fs_config_generator) fsconfig " + + "--aid-header $(location :android_filesystem_config_header) " + + "--capability-header $(location :linux_capability_header) " + + "--out_file $(out) " +fs_config_cmd_dirs = fs_config_cmd + "--dirs " +fs_config_cmd_files = fs_config_cmd + "--files " + +genrule_defaults { + name: "fs_config_defaults", + tools: ["fs_config_generator"], + srcs: [ + ":android_filesystem_config_header", + ":linux_capability_header", + ":target_fs_config_gen", + ], + out: ["out"], +} + +genrule { + name: "fs_config_dirs_system_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition system " + + "--all-partitions vendor,oem,odm,vendor_dlkm,odm_dlkm,system_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_system", + filename: "fs_config_dirs", + src: ":fs_config_dirs_system_gen", +} + +genrule { + name: "fs_config_files_system_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition system " + + "--all-partitions vendor,oem,odm,vendor_dlkm,odm_dlkm,system_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_system", + filename: "fs_config_files", + src: ":fs_config_files_system_gen", +} + +genrule { + name: "fs_config_dirs_system_ext_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition system_ext " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_system_ext", + filename: "fs_config_dirs", + src: ":fs_config_dirs_system_ext_gen", + system_ext_specific: true, +} + +genrule { + name: "fs_config_files_system_ext_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition system_ext " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_system_ext", + filename: "fs_config_files", + src: ":fs_config_files_system_ext_gen", + system_ext_specific: true, +} + +genrule { + name: "fs_config_dirs_product_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition product " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_product", + filename: "fs_config_dirs", + src: ":fs_config_dirs_product_gen", + product_specific: true, +} + +genrule { + name: "fs_config_files_product_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition product " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_product", + filename: "fs_config_files", + src: ":fs_config_files_product_gen", + product_specific: true, +} + +genrule { + name: "fs_config_dirs_vendor_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition vendor " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_vendor", + filename: "fs_config_dirs", + src: ":fs_config_dirs_vendor_gen", + vendor: true, +} + +genrule { + name: "fs_config_files_vendor_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition vendor " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_vendor", + filename: "fs_config_files", + src: ":fs_config_files_vendor_gen", + vendor: true, +} + +genrule { + name: "fs_config_dirs_odm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition odm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_odm", + filename: "fs_config_dirs", + src: ":fs_config_dirs_odm_gen", + device_specific: true, +} + +genrule { + name: "fs_config_files_odm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition odm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_odm", + filename: "fs_config_files", + src: ":fs_config_files_odm_gen", + device_specific: true, +} + +// TODO(jiyong): add fs_config for oem, system_dlkm, vendor_dlkm, odm_dlkm partitions diff --git a/tools/fs_config/Android.mk b/tools/fs_config/Android.mk index c36c3aa6a2..e4c362630f 100644 --- a/tools/fs_config/Android.mk +++ b/tools/fs_config/Android.mk @@ -24,23 +24,8 @@ ifneq ($(wildcard $(TARGET_DEVICE_DIR)/android_filesystem_config.h),) $(error Using $(TARGET_DEVICE_DIR)/android_filesystem_config.h is deprecated, please use TARGET_FS_CONFIG_GEN instead) endif -system_android_filesystem_config := system/core/libcutils/include/private/android_filesystem_config.h -system_capability_header := bionic/libc/kernel/uapi/linux/capability.h - -# Use snapshots if exist -vendor_android_filesystem_config := $(strip \ - $(if $(filter-out current,$(BOARD_VNDK_VERSION)), \ - $(SOONG_VENDOR_$(BOARD_VNDK_VERSION)_SNAPSHOT_DIR)/include/$(system_android_filesystem_config))) -ifeq (,$(wildcard $(vendor_android_filesystem_config))) -vendor_android_filesystem_config := $(system_android_filesystem_config) -endif - -vendor_capability_header := $(strip \ - $(if $(filter-out current,$(BOARD_VNDK_VERSION)), \ - $(SOONG_VENDOR_$(BOARD_VNDK_VERSION)_SNAPSHOT_DIR)/include/$(system_capability_header))) -ifeq (,$(wildcard $(vendor_capability_header))) -vendor_capability_header := $(system_capability_header) -endif +android_filesystem_config := system/core/libcutils/include/private/android_filesystem_config.h +capability_header := bionic/libc/kernel/uapi/linux/capability.h # List of supported vendor, oem, odm, vendor_dlkm, odm_dlkm, and system_dlkm Partitions fs_config_generate_extra_partition_list := $(strip \ @@ -85,58 +70,6 @@ LOCAL_REQUIRED_MODULES := \ include $(BUILD_PHONY_PACKAGE) ################################## -# Generate the system_ext/etc/fs_config_dirs binary file for the target if the -# system_ext partition is generated. Add fs_config_dirs or fs_config_dirs_system_ext -# to PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_system_ext -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(if $(BOARD_USES_SYSTEM_EXTIMAGE)$(BOARD_SYSTEM_EXTIMAGE_FILE_SYSTEM_TYPE),_fs_config_dirs_system_ext) -include $(BUILD_PHONY_PACKAGE) - -################################## -# Generate the system_ext/etc/fs_config_files binary file for the target if the -# system_ext partition is generated. Add fs_config_files or fs_config_files_system_ext -# to PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_system_ext -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(if $(BOARD_USES_SYSTEM_EXTIMAGE)$(BOARD_SYSTEM_EXTIMAGE_FILE_SYSTEM_TYPE),_fs_config_files_system_ext) -include $(BUILD_PHONY_PACKAGE) - -################################## -# Generate the product/etc/fs_config_dirs binary file for the target if the -# product partition is generated. Add fs_config_dirs or fs_config_dirs_product -# to PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_product -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(if $(BOARD_USES_PRODUCTIMAGE)$(BOARD_PRODUCTIMAGE_FILE_SYSTEM_TYPE),_fs_config_dirs_product) -include $(BUILD_PHONY_PACKAGE) - -################################## -# Generate the product/etc/fs_config_files binary file for the target if the -# product partition is generated. Add fs_config_files or fs_config_files_product -# to PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_product -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(if $(BOARD_USES_PRODUCTIMAGE)$(BOARD_PRODUCTIMAGE_FILE_SYSTEM_TYPE),_fs_config_files_product) -include $(BUILD_PHONY_PACKAGE) - -################################## # Generate the <p>/etc/fs_config_dirs binary files for all enabled partitions # excluding /system, /system_ext and /product. Add fs_config_dirs_nonsystem to # PRODUCT_PACKAGES in the device make file to enable. @@ -146,7 +79,7 @@ LOCAL_MODULE := fs_config_dirs_nonsystem LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(foreach t,$(fs_config_generate_extra_partition_list),_fs_config_dirs_$(t)) +LOCAL_REQUIRED_MODULES := $(foreach t,$(fs_config_generate_extra_partition_list),fs_config_dirs_$(t)) include $(BUILD_PHONY_PACKAGE) ################################## @@ -159,122 +92,9 @@ LOCAL_MODULE := fs_config_files_nonsystem LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(foreach t,$(fs_config_generate_extra_partition_list),_fs_config_files_$(t)) +LOCAL_REQUIRED_MODULES := $(foreach t,$(fs_config_generate_extra_partition_list),fs_config_files_$(t)) include $(BUILD_PHONY_PACKAGE) -################################## -# Generate the system/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_system to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_system -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_PARTITION_LIST := $(fs_config_generate_extra_partition_list) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition system \ - --all-partitions "$(subst $(space),$(comma),$(PRIVATE_PARTITION_LIST))" \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the system/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_system to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_system -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_PARTITION_LIST := $(fs_config_generate_extra_partition_list) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition system \ - --all-partitions "$(subst $(space),$(comma),$(PRIVATE_PARTITION_LIST))" \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -ifneq ($(filter vendor,$(fs_config_generate_extra_partition_list)),) -################################## -# Generate the vendor/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_dirs_vendor -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition vendor \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the vendor/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_files_vendor -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition vendor \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -endif - ifneq ($(filter oem,$(fs_config_generate_extra_partition_list)),) ################################## # Generate the oem/etc/fs_config_dirs binary file for the target @@ -282,7 +102,7 @@ ifneq ($(filter oem,$(fs_config_generate_extra_partition_list)),) # in the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_dirs_oem +LOCAL_MODULE := fs_config_dirs_oem LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -290,10 +110,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_OEM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -309,7 +129,7 @@ $(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_G # in the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_files_oem +LOCAL_MODULE := fs_config_files_oem LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -317,10 +137,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_OEM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -332,63 +152,6 @@ $(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_G endif -ifneq ($(filter odm,$(fs_config_generate_extra_partition_list)),) -################################## -# Generate the odm/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_dirs_odm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_ODM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition odm \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the odm/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_files_odm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_ODM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition odm \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -endif - ifneq ($(filter vendor_dlkm,$(fs_config_generate_extra_partition_list)),) ################################## # Generate the vendor_dlkm/etc/fs_config_dirs binary file for the target @@ -396,7 +159,7 @@ ifneq ($(filter vendor_dlkm,$(fs_config_generate_extra_partition_list)),) # the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_dirs_vendor_dlkm +LOCAL_MODULE := fs_config_dirs_vendor_dlkm LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -404,10 +167,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -423,7 +186,7 @@ $(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_G # the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_files_vendor_dlkm +LOCAL_MODULE := fs_config_files_vendor_dlkm LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -431,10 +194,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -453,7 +216,7 @@ ifneq ($(filter odm_dlkm,$(fs_config_generate_extra_partition_list)),) # in the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_dirs_odm_dlkm +LOCAL_MODULE := fs_config_dirs_odm_dlkm LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -461,10 +224,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_ODM_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -480,7 +243,7 @@ $(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_G # in the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_files_odm_dlkm +LOCAL_MODULE := fs_config_files_odm_dlkm LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -488,10 +251,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_ODM_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -510,7 +273,7 @@ ifneq ($(filter system_dlkm,$(fs_config_generate_extra_partition_list)),) # in the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_dirs_system_dlkm +LOCAL_MODULE := fs_config_dirs_system_dlkm LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -518,10 +281,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -537,7 +300,7 @@ $(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_G # in the device make file to enable include $(CLEAR_VARS) -LOCAL_MODULE := _fs_config_files_system_dlkm +LOCAL_MODULE := fs_config_files_system_dlkm LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 LOCAL_LICENSE_CONDITIONS := notice LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE @@ -545,10 +308,10 @@ LOCAL_MODULE_CLASS := ETC LOCAL_INSTALLED_MODULE_STEM := fs_config_files LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_DLKM)/etc include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(vendor_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) +$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) $(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(vendor_android_filesystem_config) $(vendor_capability_header) +$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) @mkdir -p $(dir $@) $< fsconfig \ --aid-header $(PRIVATE_ANDROID_FS_HDR) \ @@ -560,118 +323,6 @@ $(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_G endif -ifneq ($(BOARD_USES_PRODUCTIMAGE)$(BOARD_PRODUCTIMAGE_FILE_SYSTEM_TYPE),) -################################## -# Generate the product/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_product to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_dirs_product -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition product \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the product/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_product to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_files_product -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition product \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) -endif - -ifneq ($(BOARD_USES_SYSTEM_EXTIMAGE)$(BOARD_SYSTEM_EXTIMAGE_FILE_SYSTEM_TYPE),) -################################## -# Generate the system_ext/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_system_ext to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_dirs_system_ext -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_EXT)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition system_ext \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the system_ext/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_system_ext to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := _fs_config_files_system_ext -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_EXT)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(system_android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(system_capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(system_android_filesystem_config) $(system_capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition system_ext \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) -endif - -system_android_filesystem_config := -system_capability_header := +android_filesystem_config := +capability_header := fs_config_generate_extra_partition_list := diff --git a/tools/fs_config/fs_config.c b/tools/fs_config/fs_config.c index 2a75add096..80bd3c1591 100644 --- a/tools/fs_config/fs_config.c +++ b/tools/fs_config/fs_config.c @@ -22,9 +22,6 @@ #include <string.h> #include <inttypes.h> -#include <selinux/selinux.h> -#include <selinux/label.h> - #include "private/android_filesystem_config.h" #include "private/fs_config.h" @@ -35,8 +32,8 @@ // // After the first 4 columns, optional key=value pairs are emitted // for each file. Currently, the following keys are supported: -// * -S: selabel=[selinux_label] -// * -C: capabilities=[hex capabilities value] +// +// -C: capabilities=[hex capabilities value] // // Example input: // @@ -48,45 +45,24 @@ // system/etc/dbus.conf 1002 1002 440 // data/app 1000 1000 771 // -// or if, for example, -S is used: -// -// system/etc/dbus.conf 1002 1002 440 selabel=u:object_r:system_file:s0 -// data/app 1000 1000 771 selabel=u:object_r:apk_data_file:s0 -// // Note that the output will omit the trailing slash from // directories. -static struct selabel_handle* get_sehnd(const char* context_file) { - struct selinux_opt seopts[] = { { SELABEL_OPT_PATH, context_file } }; - struct selabel_handle* sehnd = selabel_open(SELABEL_CTX_FILE, seopts, 1); - - if (!sehnd) { - perror("error running selabel_open"); - exit(EXIT_FAILURE); - } - return sehnd; -} - static void usage() { - fprintf(stderr, "Usage: fs_config [-D product_out_path] [-S context_file] [-R root] [-C]\n"); + fprintf(stderr, "Usage: fs_config [-D product_out_path] [-R root] [-C]\n"); } int main(int argc, char** argv) { char buffer[1024]; - const char* context_file = NULL; const char* product_out_path = NULL; char* root_path = NULL; - struct selabel_handle* sehnd = NULL; int print_capabilities = 0; int opt; - while((opt = getopt(argc, argv, "CS:R:D:")) != -1) { + while((opt = getopt(argc, argv, "CR:D:")) != -1) { switch(opt) { case 'C': print_capabilities = 1; break; - case 'S': - context_file = optarg; - break; case 'R': root_path = optarg; break; @@ -99,10 +75,6 @@ int main(int argc, char** argv) { } } - if (context_file != NULL) { - sehnd = get_sehnd(context_file); - } - if (root_path != NULL) { size_t root_len = strlen(root_path); /* Trim any trailing slashes from the root path. */ @@ -141,33 +113,6 @@ int main(int argc, char** argv) { } printf("%s %d %d %o", buffer, uid, gid, mode); - if (sehnd != NULL) { - size_t buffer_strlen = strnlen(buffer, sizeof(buffer)); - if (buffer_strlen >= sizeof(buffer)) { - fprintf(stderr, "non null terminated buffer, aborting\n"); - exit(EXIT_FAILURE); - } - size_t full_name_size = buffer_strlen + 2; - char* full_name = (char*) malloc(full_name_size); - if (full_name == NULL) { - perror("malloc"); - exit(EXIT_FAILURE); - } - - full_name[0] = '/'; - strncpy(full_name + 1, buffer, full_name_size - 1); - full_name[full_name_size - 1] = '\0'; - - char* secontext; - if (selabel_lookup(sehnd, &secontext, full_name, ( mode | (is_dir ? S_IFDIR : S_IFREG)))) { - secontext = strdup("u:object_r:unlabeled:s0"); - } - - printf(" selabel=%s", secontext); - free(full_name); - freecon(secontext); - } - if (print_capabilities) { printf(" capabilities=0x%" PRIx64, capabilities); } diff --git a/tools/fs_get_stats/Android.bp b/tools/fs_get_stats/Android.bp deleted file mode 100644 index 0697999090..0000000000 --- a/tools/fs_get_stats/Android.bp +++ /dev/null @@ -1,14 +0,0 @@ -package { - // See: http://go/android-license-faq - default_applicable_licenses: ["Android-Apache-2.0"], -} - -cc_binary_host { - name: "fs_get_stats", - srcs: ["fs_get_stats.c"], - cflags: ["-Wall", "-Werror"], - shared_libs: [ - "libcutils", - "liblog", - ], -} diff --git a/tools/fs_get_stats/fs_get_stats.c b/tools/fs_get_stats/fs_get_stats.c deleted file mode 100644 index 64ef0e2312..0000000000 --- a/tools/fs_get_stats/fs_get_stats.c +++ /dev/null @@ -1,67 +0,0 @@ -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <unistd.h> - -#include <private/android_filesystem_config.h> -#include <private/fs_config.h> - -#define DO_DEBUG 1 - -#define ERROR(fmt,args...) \ - do { \ - fprintf(stderr, "%s:%d: ERROR: " fmt, \ - __FILE__, __LINE__, ##args); \ - } while (0) - -#if DO_DEBUG -#define DEBUG(fmt,args...) \ - do { fprintf(stderr, "DEBUG: " fmt, ##args); } while(0) -#else -#define DEBUG(x...) do {} while(0) -#endif - -void -print_help(void) -{ - fprintf(stderr, "fs_get_stats: retrieve the target file stats " - "for the specified file\n"); - fprintf(stderr, "usage: fs_get_stats cur_perms is_dir filename targetout\n"); - fprintf(stderr, "\tcur_perms - The current permissions of " - "the file\n"); - fprintf(stderr, "\tis_dir - Is filename is a dir, 1. Otherwise, 0.\n"); - fprintf(stderr, "\tfilename - The filename to lookup\n"); - fprintf(stderr, "\ttargetout - The target out path to query device specific FS configs\n"); - fprintf(stderr, "\n"); -} - -int -main(int argc, const char *argv[]) -{ - char *endptr; - char is_dir = 0; - unsigned perms = 0; - unsigned uid = (unsigned)-1; - unsigned gid = (unsigned)-1; - - if (argc < 5) { - ERROR("Invalid arguments\n"); - print_help(); - exit(-1); - } - - perms = (unsigned)strtoul(argv[1], &endptr, 0); - if (!endptr || (endptr == argv[1]) || (*endptr != '\0')) { - ERROR("current permissions must be a number. Got '%s'.\n", argv[1]); - exit(-1); - } - - if (!strcmp(argv[2], "1")) - is_dir = 1; - - uint64_t capabilities; - fs_config(argv[3], is_dir, argv[4], &uid, &gid, &perms, &capabilities); - fprintf(stdout, "%d %d 0%o\n", uid, gid, perms); - - return 0; -} diff --git a/tools/generate_gts_shared_report.py b/tools/generate_gts_shared_report.py index 11c9364189..3067ae172c 100644 --- a/tools/generate_gts_shared_report.py +++ b/tools/generate_gts_shared_report.py @@ -18,14 +18,12 @@ Checks and generates a report for gts modules that should be open-sourced. Usage: generate_gts_open_source_report.py - --gtsv-metalic [gts-verifier meta_lic] --gts-test-metalic [android-gts meta_lic] --checkshare [COMPLIANCE_CHECKSHARE] --gts-test-dir [directory of android-gts] --output [output file] Output example: - GTS-Verifier: PASS/FAIL GTS-Modules: PASS/FAIL GtsIncrementalInstallTestCases_BackgroundProcess GtsUnsignedNetworkStackTestCases @@ -39,9 +37,6 @@ def _get_args(): """Parses input arguments.""" parser = argparse.ArgumentParser() parser.add_argument( - '--gtsv-metalic', required=True, - help='license meta_lic file path of gts-verifier.zip') - parser.add_argument( '--gts-test-metalic', required=True, help='license meta_lic file path of android-gts.zip') parser.add_argument( @@ -55,23 +50,6 @@ def _get_args(): help='file path of the output report') return parser.parse_args() -def _check_gtsv(checkshare: str, gtsv_metalic: str) -> str: - """Checks gts-verifier license. - - Args: - checkshare: path of the COMPLIANCE_CHECKSHARE tool - gtsv_metalic: license meta_lic file path of gts-verifier.zip - - Returns: - PASS when gts-verifier.zip doesn't need to be shared, and FAIL - when gts-verifier.zip need to be shared. - """ - cmd = f'{checkshare} {gtsv_metalic}' - proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc.communicate() - return 'PASS' if proc.returncode == 0 else 'FAIL' - def _check_gts_test(checkshare: str, gts_test_metalic: str, gts_test_dir: str) -> tuple[str, set[str]]: """Checks android-gts license. @@ -109,15 +87,12 @@ def _check_gts_test(checkshare: str, gts_test_metalic: str, def main(argv): args = _get_args() - gtsv_metalic = args.gtsv_metalic gts_test_metalic = args.gts_test_metalic output_file = args.output checkshare = args.checkshare gts_test_dir = args.gts_test_dir with open(output_file, 'w') as file: - result = _check_gtsv(checkshare, gtsv_metalic) - file.write(f'GTS-Verifier: {result}\n') result, open_source_modules = _check_gts_test( checkshare, gts_test_metalic, gts_test_dir) file.write(f'GTS-Modules: {result}\n') @@ -125,4 +100,4 @@ def main(argv): file.write(f'\t{open_source_module}\n') if __name__ == "__main__": - main(sys.argv)
\ No newline at end of file + main(sys.argv) diff --git a/tools/ide_query/cc_analyzer/Android.bp b/tools/ide_query/cc_analyzer/Android.bp new file mode 100644 index 0000000000..3cbbb0587f --- /dev/null +++ b/tools/ide_query/cc_analyzer/Android.bp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +cc_defaults { + name: "ide_query_cc_analyzer_defaults", + compile_multilib: "64", + defaults: [ + "llvm-build-host-tools-defaults", + ], + cflags: [ + // LLVM Sources do have unused parameters :( + "-Wno-unused-parameter", + ], + target: { + host: { + cppflags: [ + "-fno-rtti", + ], + }, + }, +} + +cc_library_host_static { + name: "builtin_headers", + srcs: ["builtin_headers.cc"], + generated_headers: ["clang_builtin_headers_resources"], + defaults: ["ide_query_cc_analyzer_defaults"], +} + +cc_library_host_static { + name: "include_scanner", + srcs: ["include_scanner.cc"], + shared_libs: ["libclang-cpp_host"], + static_libs: ["builtin_headers"], + defaults: ["ide_query_cc_analyzer_defaults"], +} + +cc_library_host_static { + name: "analyzer", + srcs: ["analyzer.cc"], + shared_libs: ["libclang-cpp_host"], + static_libs: [ + "include_scanner", + "ide_query_proto", + ], + defaults: ["ide_query_cc_analyzer_defaults"], +} + +cc_binary_host { + name: "ide_query_cc_analyzer", + defaults: ["ide_query_cc_analyzer_defaults"], + srcs: ["main.cc"], + shared_libs: [ + "libclang-cpp_host", + "libprotobuf-cpp-full", + ], + static_libs: [ + "ide_query_proto", + "builtin_headers", + "include_scanner", + "analyzer", + ], +} diff --git a/tools/ide_query/cc_analyzer/analyzer.cc b/tools/ide_query/cc_analyzer/analyzer.cc new file mode 100644 index 0000000000..bb7ca0b5bc --- /dev/null +++ b/tools/ide_query/cc_analyzer/analyzer.cc @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "analyzer.h" + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "clang/Tooling/CompilationDatabase.h" +#include "clang/Tooling/JSONCompilationDatabase.h" +#include "ide_query.pb.h" +#include "include_scanner.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace tools::ide_query::cc_analyzer { +namespace { +llvm::Expected<std::unique_ptr<clang::tooling::CompilationDatabase>> LoadCompDB( + llvm::StringRef comp_db_path) { + std::string err; + std::unique_ptr<clang::tooling::CompilationDatabase> db = + clang::tooling::JSONCompilationDatabase::loadFromFile( + comp_db_path, err, clang::tooling::JSONCommandLineSyntax::AutoDetect); + if (!db) { + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Failed to load CDB: " + err); + } + // Provide some heuristic support for missing files. + return inferMissingCompileCommands(std::move(db)); +} +} // namespace + +::ide_query::DepsResponse GetDeps(::ide_query::RepoState state) { + ::ide_query::DepsResponse results; + auto db = LoadCompDB(state.comp_db_path()); + if (!db) { + results.mutable_status()->set_code(::ide_query::Status::FAILURE); + results.mutable_status()->set_message(llvm::toString(db.takeError())); + return results; + } + for (llvm::StringRef active_file : state.active_file_path()) { + auto& result = *results.add_deps(); + + llvm::SmallString<256> abs_file(state.repo_dir()); + llvm::sys::path::append(abs_file, active_file); + auto cmds = db->get()->getCompileCommands(active_file); + if (cmds.empty()) { + result.mutable_status()->set_code(::ide_query::Status::FAILURE); + result.mutable_status()->set_message( + llvm::Twine("Can't find compile flags for file: ", abs_file).str()); + continue; + } + result.set_source_file(active_file.str()); + llvm::StringRef file = cmds[0].Filename; + if (llvm::StringRef actual_file(cmds[0].Heuristic); + actual_file.consume_front("inferred from ")) { + file = actual_file; + } + // TODO: Query ninja graph to figure out a minimal set of targets to build. + result.add_build_target(file.str() + "^"); + } + return results; +} + +::ide_query::IdeAnalysis GetBuildInputs(::ide_query::RepoState state) { + auto db = LoadCompDB(state.comp_db_path()); + ::ide_query::IdeAnalysis results; + if (!db) { + results.mutable_status()->set_code(::ide_query::Status::FAILURE); + results.mutable_status()->set_message(llvm::toString(db.takeError())); + return results; + } + std::string repo_dir = state.repo_dir(); + if (!repo_dir.empty() && repo_dir.back() == '/') repo_dir.pop_back(); + + llvm::SmallString<256> genfile_root_abs(repo_dir); + llvm::sys::path::append(genfile_root_abs, state.out_dir()); + if (genfile_root_abs.empty() || genfile_root_abs.back() != '/') { + genfile_root_abs.push_back('/'); + } + + results.set_build_artifact_root(state.out_dir()); + for (llvm::StringRef active_file : state.active_file_path()) { + auto& result = *results.add_sources(); + result.set_path(active_file.str()); + + llvm::SmallString<256> abs_file(repo_dir); + llvm::sys::path::append(abs_file, active_file); + auto cmds = db->get()->getCompileCommands(abs_file); + if (cmds.empty()) { + result.mutable_status()->set_code(::ide_query::Status::FAILURE); + result.mutable_status()->set_message( + llvm::Twine("Can't find compile flags for file: ", abs_file).str()); + continue; + } + const auto& cmd = cmds.front(); + llvm::StringRef working_dir = cmd.Directory; + if (!working_dir.consume_front(repo_dir)) { + result.mutable_status()->set_code(::ide_query::Status::FAILURE); + result.mutable_status()->set_message("Command working dir " + + working_dir.str() + + " outside repository " + repo_dir); + continue; + } + working_dir = working_dir.ltrim('/'); + result.set_working_dir(working_dir.str()); + for (auto& arg : cmd.CommandLine) result.add_compiler_arguments(arg); + + auto includes = + ScanIncludes(cmds.front(), llvm::vfs::createPhysicalFileSystem()); + if (!includes) { + result.mutable_status()->set_code(::ide_query::Status::FAILURE); + result.mutable_status()->set_message( + llvm::toString(includes.takeError())); + continue; + } + + for (auto& [req_input, contents] : *includes) { + llvm::StringRef req_input_ref(req_input); + // We're only interested in generated files. + if (!req_input_ref.consume_front(genfile_root_abs)) continue; + auto& genfile = *result.add_generated(); + genfile.set_path(req_input_ref.str()); + genfile.set_contents(std::move(contents)); + } + } + return results; +} +} // namespace tools::ide_query::cc_analyzer diff --git a/tools/ide_query/cc_analyzer/analyzer.h b/tools/ide_query/cc_analyzer/analyzer.h new file mode 100644 index 0000000000..3133795217 --- /dev/null +++ b/tools/ide_query/cc_analyzer/analyzer.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _TOOLS_IDE_QUERY_CC_ANALYZER_ANALYZER_H_ +#define _TOOLS_IDE_QUERY_CC_ANALYZER_ANALYZER_H_ + +#include "ide_query.pb.h" + +namespace tools::ide_query::cc_analyzer { + +// Scans the build graph and returns target names from the build graph to +// generate all the dependencies for the active files. +::ide_query::DepsResponse GetDeps(::ide_query::RepoState state); + +// Scans the sources and returns all the source files required for analyzing the +// active files. +::ide_query::IdeAnalysis GetBuildInputs(::ide_query::RepoState state); + +} // namespace tools::ide_query::cc_analyzer + +#endif diff --git a/tools/ide_query/cc_analyzer/builtin_headers.cc b/tools/ide_query/cc_analyzer/builtin_headers.cc new file mode 100644 index 0000000000..6d44ce7d27 --- /dev/null +++ b/tools/ide_query/cc_analyzer/builtin_headers.cc @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "builtin_headers.h" + +#include <cstddef> + +#include "clang_builtin_headers_resources.inc" + +const struct FileToc *builtin_headers_create() { return kPackedFiles; } +size_t builtin_headers_size() { + return sizeof(kPackedFiles) / sizeof(FileToc) - 1; +} diff --git a/tools/ide_query/cc_analyzer/builtin_headers.h b/tools/ide_query/cc_analyzer/builtin_headers.h new file mode 100644 index 0000000000..eda722fe1c --- /dev/null +++ b/tools/ide_query/cc_analyzer/builtin_headers.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _TOOLS_IDE_QUERY_CC_ANALYZER_BUILTIN_HEADERS_H_ +#define _TOOLS_IDE_QUERY_CC_ANALYZER_BUILTIN_HEADERS_H_ + +#include <cstddef> + +struct FileToc { + const char *name; + const char *data; +}; + +const struct FileToc *builtin_headers_create(); +size_t builtin_headers_size(); + +#endif diff --git a/tools/ide_query/cc_analyzer/include_scanner.cc b/tools/ide_query/cc_analyzer/include_scanner.cc new file mode 100644 index 0000000000..8916a3edd6 --- /dev/null +++ b/tools/ide_query/cc_analyzer/include_scanner.cc @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "include_scanner.h" + +#include <memory> +#include <string> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "builtin_headers.h" +#include "clang/Basic/FileEntry.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Module.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Lex/PPCallbacks.h" +#include "clang/Tooling/ArgumentsAdjusters.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace tools::ide_query::cc_analyzer { +namespace { +std::string CleanPath(llvm::StringRef path) { + // both ./ and ../ has `./` in them. + if (!path.contains("./")) return path.str(); + llvm::SmallString<256> clean_path(path); + llvm::sys::path::remove_dots(clean_path, /*remove_dot_dot=*/true); + return clean_path.str().str(); +} + +// Returns the absolute path to file_name, treating it as relative to cwd if it +// isn't already absolute. +std::string GetAbsolutePath(llvm::StringRef cwd, llvm::StringRef file_name) { + if (llvm::sys::path::is_absolute(file_name)) return CleanPath(file_name); + llvm::SmallString<256> abs_path(cwd); + llvm::sys::path::append(abs_path, file_name); + llvm::sys::path::remove_dots(abs_path, /*remove_dot_dot=*/true); + return abs_path.str().str(); +} + +class IncludeRecordingPP : public clang::PPCallbacks { + public: + explicit IncludeRecordingPP( + std::unordered_map<std::string, std::string> &abs_paths, std::string cwd, + const clang::SourceManager &sm) + : abs_paths_(abs_paths), cwd_(std::move(cwd)), sm_(sm) {} + + void LexedFileChanged(clang::FileID FID, LexedFileChangeReason Reason, + clang::SrcMgr::CharacteristicKind FileType, + clang::FileID PrevFID, + clang::SourceLocation Loc) override { + auto file_entry = sm_.getFileEntryRefForID(FID); + if (!file_entry) return; + auto abs_path = GetAbsolutePath(cwd_, file_entry->getName()); + auto [it, inserted] = abs_paths_.try_emplace(abs_path); + if (inserted) it->second = sm_.getBufferData(FID); + } + + std::unordered_map<std::string, std::string> &abs_paths_; + const std::string cwd_; + const clang::SourceManager &sm_; +}; + +class IncludeScanningAction final : public clang::PreprocessOnlyAction { + public: + explicit IncludeScanningAction( + std::unordered_map<std::string, std::string> &abs_paths) + : abs_paths_(abs_paths) {} + bool BeginSourceFileAction(clang::CompilerInstance &ci) override { + std::string cwd; + auto cwd_or_err = ci.getVirtualFileSystem().getCurrentWorkingDirectory(); + if (!cwd_or_err || cwd_or_err.get().empty()) return false; + cwd = cwd_or_err.get(); + ci.getPreprocessor().addPPCallbacks(std::make_unique<IncludeRecordingPP>( + abs_paths_, std::move(cwd), ci.getSourceManager())); + return true; + } + + private: + std::unordered_map<std::string, std::string> &abs_paths_; +}; + +llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayBuiltinHeaders( + std::vector<std::string> &argv, + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> base) { + static constexpr llvm::StringLiteral kResourceDir = "/resources"; + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> overlay( + new llvm::vfs::OverlayFileSystem(std::move(base))); + llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> builtin_headers( + new llvm::vfs::InMemoryFileSystem); + + llvm::SmallString<256> file_path; + for (const auto &builtin_header : + llvm::ArrayRef(builtin_headers_create(), builtin_headers_size())) { + file_path.clear(); + llvm::sys::path::append(file_path, kResourceDir, "include", + builtin_header.name); + builtin_headers->addFile( + file_path, + /*ModificationTime=*/0, + llvm::MemoryBuffer::getMemBuffer(builtin_header.data)); + } + overlay->pushOverlay(std::move(builtin_headers)); + argv.insert(llvm::find(argv, "--"), + llvm::Twine("-resource-dir=", kResourceDir).str()); + return overlay; +} + +} // namespace + +llvm::Expected<std::vector<std::pair<std::string, std::string>>> ScanIncludes( + const clang::tooling::CompileCommand &cmd, + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs) { + if (fs->setCurrentWorkingDirectory(cmd.Directory)) { + return llvm::createStringError( + llvm::inconvertibleErrorCode(), + "Failed to set working directory to: " + cmd.Directory); + } + + auto main_file = fs->getBufferForFile(cmd.Filename); + if (!main_file) { + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Main file doesn't exist: " + cmd.Filename); + } + std::unordered_map<std::string, std::string> abs_paths; + abs_paths.try_emplace(GetAbsolutePath(cmd.Directory, cmd.Filename), + main_file.get()->getBuffer().str()); + + std::vector<std::string> argv = cmd.CommandLine; + fs = OverlayBuiltinHeaders(argv, std::move(fs)); + + llvm::IntrusiveRefCntPtr<clang::FileManager> files( + new clang::FileManager(/*FileSystemOpts=*/{}, std::move(fs))); + clang::tooling::ToolInvocation tool( + argv, std::make_unique<IncludeScanningAction>(abs_paths), files.get()); + if (!tool.run()) { + return llvm::createStringError( + llvm::inconvertibleErrorCode(), + "Failed to scan includes for: " + cmd.Filename); + } + + std::vector<std::pair<std::string, std::string>> result; + result.reserve(abs_paths.size()); + for (auto &entry : abs_paths) { + result.emplace_back(entry.first, std::move(entry.second)); + } + return result; +} +} // namespace tools::ide_query::cc_analyzer diff --git a/tools/ide_query/cc_analyzer/include_scanner.h b/tools/ide_query/cc_analyzer/include_scanner.h new file mode 100644 index 0000000000..e814e72cb1 --- /dev/null +++ b/tools/ide_query/cc_analyzer/include_scanner.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _TOOLS_IDE_QUERY_CC_ANALYZER_INCLUDE_SCANNER_H_ +#define _TOOLS_IDE_QUERY_CC_ANALYZER_INCLUDE_SCANNER_H_ + +#include <string> +#include <utility> +#include <vector> + +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace tools::ide_query::cc_analyzer { + +// Returns absolute paths and contents for all the includes necessary for +// compiling source file in command. +llvm::Expected<std::vector<std::pair<std::string, std::string>>> ScanIncludes( + const clang::tooling::CompileCommand &cmd, + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs); + +} // namespace tools::ide_query::cc_analyzer + +#endif diff --git a/tools/ide_query/cc_analyzer/main.cc b/tools/ide_query/cc_analyzer/main.cc new file mode 100644 index 0000000000..8e00c6396a --- /dev/null +++ b/tools/ide_query/cc_analyzer/main.cc @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Driver for c++ extractor. Operates in two modes: +// - DEPS, scans build graph for active files and reports targets that need to +// be build for analyzing that file. +// - INPUTS, scans the source code for active files and returns all the sources +// required for analyzing that file. +// +// Uses stdin/stdout to take in requests and provide responses. +#include <unistd.h> + +#include <memory> +#include <utility> + +#include "analyzer.h" +#include "google/protobuf/message.h" +#include "ide_query.pb.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" + +namespace { +enum class OpMode { + DEPS = 0, + INPUTS = 1, +}; +llvm::cl::opt<OpMode> mode{ + "mode", + llvm::cl::values(clEnumValN(OpMode::DEPS, "deps", + "Figure out targets that need to be build"), + clEnumValN(OpMode::INPUTS, "inputs", + "Figure out generated files used")), + llvm::cl::desc("Print the list of headers to insert and remove"), +}; + +ide_query::IdeAnalysis ReturnError(llvm::StringRef message) { + ide_query::IdeAnalysis result; + result.mutable_status()->set_code(ide_query::Status::FAILURE); + result.mutable_status()->set_message(message.str()); + return result; +} + +} // namespace + +int main(int argc, char* argv[]) { + llvm::InitializeAllTargetInfos(); + llvm::cl::ParseCommandLineOptions(argc, argv); + + ide_query::RepoState state; + if (!state.ParseFromFileDescriptor(STDIN_FILENO)) { + llvm::errs() << "Failed to parse input!\n"; + return 1; + } + + std::unique_ptr<google::protobuf::Message> result; + switch (mode) { + case OpMode::DEPS: { + result = std::make_unique<ide_query::DepsResponse>( + tools::ide_query::cc_analyzer::GetDeps(std::move(state))); + break; + } + case OpMode::INPUTS: { + result = std::make_unique<ide_query::IdeAnalysis>( + tools::ide_query::cc_analyzer::GetBuildInputs(std::move(state))); + break; + } + default: + llvm::errs() << "Unknown operation mode!\n"; + return 1; + } + if (!result->SerializeToFileDescriptor(STDOUT_FILENO)) { + llvm::errs() << "Failed to serialize result!\n"; + return 1; + } + + return 0; +} diff --git a/tools/ide_query/go.mod b/tools/ide_query/go.mod new file mode 100644 index 0000000000..f9d727f3ab --- /dev/null +++ b/tools/ide_query/go.mod @@ -0,0 +1,7 @@ +module ide_query + +go 1.21 + +require ( + google.golang.org/protobuf v0.0.0 +) diff --git a/tools/ide_query/go.work b/tools/ide_query/go.work new file mode 100644 index 0000000000..851f352af6 --- /dev/null +++ b/tools/ide_query/go.work @@ -0,0 +1,9 @@ +go 1.21 + +use ( + . +) + +replace ( + google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf +)
\ No newline at end of file diff --git a/tools/ide_query/go.work.sum b/tools/ide_query/go.work.sum new file mode 100644 index 0000000000..cf42b48eb0 --- /dev/null +++ b/tools/ide_query/go.work.sum @@ -0,0 +1,5 @@ +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go new file mode 100644 index 0000000000..de84fbe3e4 --- /dev/null +++ b/tools/ide_query/ide_query.go @@ -0,0 +1,392 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Binary ide_query generates and analyzes build artifacts. +// The produced result can be consumed by IDEs to provide language features. +package main + +import ( + "bytes" + "container/list" + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "os/exec" + "path" + "slices" + "strings" + + "google.golang.org/protobuf/proto" + pb "ide_query/ide_query_proto" +) + +// Env contains information about the current environment. +type Env struct { + LunchTarget LunchTarget + RepoDir string + OutDir string + ClangToolsRoot string + + CcFiles []string + JavaFiles []string +} + +// LunchTarget is a parsed Android lunch target. +// Input format: <product_name>-<release_type>-<build_variant> +type LunchTarget struct { + Product string + Release string + Variant string +} + +var _ flag.Value = (*LunchTarget)(nil) + +// // Get implements flag.Value. +// func (l *LunchTarget) Get() any { +// return l +// } + +// Set implements flag.Value. +func (l *LunchTarget) Set(s string) error { + parts := strings.Split(s, "-") + if len(parts) != 3 { + return fmt.Errorf("invalid lunch target: %q, must have form <product_name>-<release_type>-<build_variant>", s) + } + *l = LunchTarget{ + Product: parts[0], + Release: parts[1], + Variant: parts[2], + } + return nil +} + +// String implements flag.Value. +func (l *LunchTarget) String() string { + return fmt.Sprintf("%s-%s-%s", l.Product, l.Release, l.Variant) +} + +func main() { + var env Env + env.OutDir = os.Getenv("OUT_DIR") + env.RepoDir = os.Getenv("ANDROID_BUILD_TOP") + env.ClangToolsRoot = os.Getenv("PREBUILTS_CLANG_TOOLS_ROOT") + flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query") + flag.Parse() + files := flag.Args() + if len(files) == 0 { + fmt.Println("No files provided.") + os.Exit(1) + return + } + + for _, f := range files { + switch { + case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"): + env.JavaFiles = append(env.JavaFiles, f) + case strings.HasSuffix(f, ".cc") || strings.HasSuffix(f, ".cpp") || strings.HasSuffix(f, ".h"): + env.CcFiles = append(env.CcFiles, f) + default: + log.Printf("File %q is supported - will be skipped.", f) + } + } + + ctx := context.Background() + // TODO(michaelmerg): Figure out if module_bp_java_deps.json and compile_commands.json is outdated. + runMake(ctx, env, "nothing") + + javaModules, javaFileToModuleMap, err := loadJavaModules(&env) + if err != nil { + log.Printf("Failed to load java modules: %v", err) + } + toMake := getJavaTargets(javaFileToModuleMap) + + ccTargets, status := getCCTargets(ctx, &env) + if status != nil && status.Code != pb.Status_OK { + log.Fatalf("Failed to query cc targets: %v", *status.Message) + } + toMake = append(toMake, ccTargets...) + fmt.Fprintf(os.Stderr, "Running make for modules: %v\n", strings.Join(toMake, ", ")) + if err := runMake(ctx, env, toMake...); err != nil { + log.Printf("Building deps failed: %v", err) + } + + res := getJavaInputs(&env, javaModules, javaFileToModuleMap) + ccAnalysis := getCCInputs(ctx, &env) + proto.Merge(res, ccAnalysis) + + res.BuildArtifactRoot = env.OutDir + data, err := proto.Marshal(res) + if err != nil { + log.Fatalf("Failed to marshal result proto: %v", err) + } + + _, err = os.Stdout.Write(data) + if err != nil { + log.Fatalf("Failed to write result proto: %v", err) + } + + for _, s := range res.Sources { + fmt.Fprintf(os.Stderr, "%s: %v (Deps: %d, Generated: %d)\n", s.GetPath(), s.GetStatus(), len(s.GetDeps()), len(s.GetGenerated())) + } +} + +func repoState(env *Env) *pb.RepoState { + const compDbPath = "soong/development/ide/compdb/compile_commands.json" + return &pb.RepoState{ + RepoDir: env.RepoDir, + ActiveFilePath: env.CcFiles, + OutDir: env.OutDir, + CompDbPath: path.Join(env.OutDir, compDbPath), + } +} + +func runCCanalyzer(ctx context.Context, env *Env, mode string, in []byte) ([]byte, error) { + ccAnalyzerPath := path.Join(env.ClangToolsRoot, "bin/ide_query_cc_analyzer") + outBuffer := new(bytes.Buffer) + + inBuffer := new(bytes.Buffer) + inBuffer.Write(in) + + cmd := exec.CommandContext(ctx, ccAnalyzerPath, "--mode="+mode) + cmd.Dir = env.RepoDir + + cmd.Stdin = inBuffer + cmd.Stdout = outBuffer + cmd.Stderr = os.Stderr + + err := cmd.Run() + + return outBuffer.Bytes(), err +} + +// Execute cc_analyzer and get all the targets that needs to be build for analyzing files. +func getCCTargets(ctx context.Context, env *Env) ([]string, *pb.Status) { + state := repoState(env) + bytes, err := proto.Marshal(state) + if err != nil { + log.Fatalln("Failed to serialize state:", err) + } + + resp := new(pb.DepsResponse) + result, err := runCCanalyzer(ctx, env, "deps", bytes) + if marshal_err := proto.Unmarshal(result, resp); marshal_err != nil { + return nil, &pb.Status{ + Code: pb.Status_FAILURE, + Message: proto.String("Malformed response from cc_analyzer: " + marshal_err.Error()), + } + } + + var targets []string + if resp.Status != nil && resp.Status.Code != pb.Status_OK { + return targets, resp.Status + } + for _, deps := range resp.Deps { + targets = append(targets, deps.BuildTarget...) + } + + status := &pb.Status{Code: pb.Status_OK} + if err != nil { + status = &pb.Status{ + Code: pb.Status_FAILURE, + Message: proto.String(err.Error()), + } + } + return targets, status +} + +func getCCInputs(ctx context.Context, env *Env) *pb.IdeAnalysis { + state := repoState(env) + bytes, err := proto.Marshal(state) + if err != nil { + log.Fatalln("Failed to serialize state:", err) + } + + resp := new(pb.IdeAnalysis) + result, err := runCCanalyzer(ctx, env, "inputs", bytes) + if marshal_err := proto.Unmarshal(result, resp); marshal_err != nil { + resp.Status = &pb.Status{ + Code: pb.Status_FAILURE, + Message: proto.String("Malformed response from cc_analyzer: " + marshal_err.Error()), + } + return resp + } + + if err != nil && (resp.Status == nil || resp.Status.Code == pb.Status_OK) { + resp.Status = &pb.Status{ + Code: pb.Status_FAILURE, + Message: proto.String(err.Error()), + } + } + return resp +} + +func getJavaTargets(javaFileToModuleMap map[string]*javaModule) []string { + var targets []string + for _, m := range javaFileToModuleMap { + targets = append(targets, m.Name) + } + return targets +} + +func getJavaInputs(env *Env, javaModules map[string]*javaModule, javaFileToModuleMap map[string]*javaModule) *pb.IdeAnalysis { + var sources []*pb.SourceFile + type depsAndGenerated struct { + Deps []string + Generated []*pb.GeneratedFile + } + moduleToDeps := make(map[string]*depsAndGenerated) + for _, f := range env.JavaFiles { + file := &pb.SourceFile{ + Path: f, + } + sources = append(sources, file) + + m := javaFileToModuleMap[f] + if m == nil { + file.Status = &pb.Status{ + Code: pb.Status_FAILURE, + Message: proto.String("File not found in any module."), + } + continue + } + + file.Status = &pb.Status{Code: pb.Status_OK} + if moduleToDeps[m.Name] != nil { + file.Generated = moduleToDeps[m.Name].Generated + file.Deps = moduleToDeps[m.Name].Deps + continue + } + + deps := transitiveDeps(m, javaModules) + var generated []*pb.GeneratedFile + outPrefix := env.OutDir + "/" + for _, d := range deps { + if relPath, ok := strings.CutPrefix(d, outPrefix); ok { + contents, err := os.ReadFile(d) + if err != nil { + fmt.Printf("Generated file %q not found - will be skipped.\n", d) + continue + } + + generated = append(generated, &pb.GeneratedFile{ + Path: relPath, + Contents: contents, + }) + } + } + moduleToDeps[m.Name] = &depsAndGenerated{deps, generated} + file.Generated = generated + file.Deps = deps + } + return &pb.IdeAnalysis{ + Sources: sources, + } +} + +// runMake runs Soong build for the given modules. +func runMake(ctx context.Context, env Env, modules ...string) error { + args := []string{ + "--make-mode", + "ANDROID_BUILD_ENVIRONMENT_CONFIG=googler-cog", + "SOONG_GEN_COMPDB=1", + "TARGET_PRODUCT=" + env.LunchTarget.Product, + "TARGET_RELEASE=" + env.LunchTarget.Release, + "TARGET_BUILD_VARIANT=" + env.LunchTarget.Variant, + "-k", + } + args = append(args, modules...) + cmd := exec.CommandContext(ctx, "build/soong/soong_ui.bash", args...) + cmd.Dir = env.RepoDir + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + return cmd.Run() +} + +type javaModule struct { + Name string + Path []string `json:"path,omitempty"` + Deps []string `json:"dependencies,omitempty"` + Srcs []string `json:"srcs,omitempty"` + Jars []string `json:"jars,omitempty"` + SrcJars []string `json:"srcjars,omitempty"` +} + +func loadJavaModules(env *Env) (map[string]*javaModule, map[string]*javaModule, error) { + javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json") + data, err := os.ReadFile(javaDepsPath) + if err != nil { + return nil, nil, err + } + + var moduleMapping map[string]*javaModule // module name -> module + if err = json.Unmarshal(data, &moduleMapping); err != nil { + return nil, nil, err + } + + javaModules := make(map[string]*javaModule) + javaFileToModuleMap := make(map[string]*javaModule) + for name, module := range moduleMapping { + if strings.HasSuffix(name, "-jarjar") || strings.HasSuffix(name, ".impl") { + continue + } + module.Name = name + javaModules[name] = module + for _, src := range module.Srcs { + if !slices.Contains(env.JavaFiles, src) { + // We are only interested in active files. + continue + } + if javaFileToModuleMap[src] != nil { + // TODO(michaelmerg): Handle the case where a file is covered by multiple modules. + log.Printf("File %q found in module %q but is already covered by module %q", src, module.Name, javaFileToModuleMap[src].Name) + continue + } + javaFileToModuleMap[src] = module + } + } + return javaModules, javaFileToModuleMap, nil +} + +func transitiveDeps(m *javaModule, modules map[string]*javaModule) []string { + var ret []string + q := list.New() + q.PushBack(m.Name) + seen := make(map[string]bool) // module names -> true + for q.Len() > 0 { + name := q.Remove(q.Front()).(string) + mod := modules[name] + if mod == nil { + continue + } + + ret = append(ret, mod.Srcs...) + ret = append(ret, mod.SrcJars...) + ret = append(ret, mod.Jars...) + for _, d := range mod.Deps { + if seen[d] { + continue + } + seen[d] = true + q.PushBack(d) + } + } + slices.Sort(ret) + ret = slices.Compact(ret) + return ret +} diff --git a/tools/ide_query/ide_query.sh b/tools/ide_query/ide_query.sh new file mode 100755 index 0000000000..2df48d0129 --- /dev/null +++ b/tools/ide_query/ide_query.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e + +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd $(dirname $BASH_SOURCE) +source $(pwd)/../../shell_utils.sh +require_top + +# Ensure cogsetup (out/ will be symlink outside the repo) +. ${TOP}/build/make/cogsetup.sh + +case $(uname -s) in + Linux) + export PREBUILTS_CLANG_TOOLS_ROOT="${TOP}/prebuilts/clang-tools/linux-x86/" + PREBUILTS_GO_ROOT="${TOP}/prebuilts/go/linux-x86/" + ;; + *) + echo "Only supported for linux hosts" >&2 + exit 1 + ;; +esac + +export ANDROID_BUILD_TOP=$TOP +export OUT_DIR=${OUT_DIR} +exec "${PREBUILTS_GO_ROOT}/bin/go" "run" "ide_query" "$@" diff --git a/tools/ide_query/ide_query_proto/Android.bp b/tools/ide_query/ide_query_proto/Android.bp new file mode 100644 index 0000000000..70f15cd4a0 --- /dev/null +++ b/tools/ide_query/ide_query_proto/Android.bp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +cc_library_host_static { + name: "ide_query_proto", + srcs: [ + "ide_query.proto", + ], + proto: { + export_proto_headers: true, + type: "full", + canonical_path_from_root: false, + }, + compile_multilib: "64", + shared_libs: ["libprotobuf-cpp-full"], +} diff --git a/tools/ide_query/ide_query_proto/ide_query.pb.go b/tools/ide_query/ide_query_proto/ide_query.pb.go new file mode 100644 index 0000000000..f3a016d36b --- /dev/null +++ b/tools/ide_query/ide_query_proto/ide_query.pb.go @@ -0,0 +1,786 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0-devel +// protoc v3.21.12 +// source: ide_query.proto + +package ide_query_proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Status_Code int32 + +const ( + Status_OK Status_Code = 0 + Status_FAILURE Status_Code = 1 +) + +// Enum value maps for Status_Code. +var ( + Status_Code_name = map[int32]string{ + 0: "OK", + 1: "FAILURE", + } + Status_Code_value = map[string]int32{ + "OK": 0, + "FAILURE": 1, + } +) + +func (x Status_Code) Enum() *Status_Code { + p := new(Status_Code) + *p = x + return p +} + +func (x Status_Code) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Status_Code) Descriptor() protoreflect.EnumDescriptor { + return file_ide_query_proto_enumTypes[0].Descriptor() +} + +func (Status_Code) Type() protoreflect.EnumType { + return &file_ide_query_proto_enumTypes[0] +} + +func (x Status_Code) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Status_Code.Descriptor instead. +func (Status_Code) EnumDescriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{0, 0} +} + +// Indicates the success/failure for analysis. +type Status struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code Status_Code `protobuf:"varint,1,opt,name=code,proto3,enum=ide_query.Status_Code" json:"code,omitempty"` + // Details about the status, might be displayed to user. + Message *string `protobuf:"bytes,2,opt,name=message,proto3,oneof" json:"message,omitempty"` +} + +func (x *Status) Reset() { + *x = Status{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Status) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Status) ProtoMessage() {} + +func (x *Status) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Status.ProtoReflect.Descriptor instead. +func (*Status) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{0} +} + +func (x *Status) GetCode() Status_Code { + if x != nil { + return x.Code + } + return Status_OK +} + +func (x *Status) GetMessage() string { + if x != nil && x.Message != nil { + return *x.Message + } + return "" +} + +// Represents an Android checkout on user's workstation. +type RepoState struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Absolute path for the checkout in the workstation. + // e.g. /home/user/work/android/ + RepoDir string `protobuf:"bytes,1,opt,name=repo_dir,json=repoDir,proto3" json:"repo_dir,omitempty"` + // Relative to repo_dir. + ActiveFilePath []string `protobuf:"bytes,2,rep,name=active_file_path,json=activeFilePath,proto3" json:"active_file_path,omitempty"` + // Repository relative path to output directory in workstation. + OutDir string `protobuf:"bytes,3,opt,name=out_dir,json=outDir,proto3" json:"out_dir,omitempty"` + // Repository relative path to compile_commands.json in workstation. + CompDbPath string `protobuf:"bytes,4,opt,name=comp_db_path,json=compDbPath,proto3" json:"comp_db_path,omitempty"` +} + +func (x *RepoState) Reset() { + *x = RepoState{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RepoState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RepoState) ProtoMessage() {} + +func (x *RepoState) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RepoState.ProtoReflect.Descriptor instead. +func (*RepoState) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{1} +} + +func (x *RepoState) GetRepoDir() string { + if x != nil { + return x.RepoDir + } + return "" +} + +func (x *RepoState) GetActiveFilePath() []string { + if x != nil { + return x.ActiveFilePath + } + return nil +} + +func (x *RepoState) GetOutDir() string { + if x != nil { + return x.OutDir + } + return "" +} + +func (x *RepoState) GetCompDbPath() string { + if x != nil { + return x.CompDbPath + } + return "" +} + +// Provides all the targets that are pre-requisities for running language +// services on active_file_paths. +type DepsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Deps []*DepsResponse_Deps `protobuf:"bytes,1,rep,name=deps,proto3" json:"deps,omitempty"` + Status *Status `protobuf:"bytes,2,opt,name=status,proto3,oneof" json:"status,omitempty"` +} + +func (x *DepsResponse) Reset() { + *x = DepsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DepsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DepsResponse) ProtoMessage() {} + +func (x *DepsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DepsResponse.ProtoReflect.Descriptor instead. +func (*DepsResponse) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{2} +} + +func (x *DepsResponse) GetDeps() []*DepsResponse_Deps { + if x != nil { + return x.Deps + } + return nil +} + +func (x *DepsResponse) GetStatus() *Status { + if x != nil { + return x.Status + } + return nil +} + +// Returns all the information necessary for providing language services for the +// active files. +type GeneratedFile struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Path to the file relative to IdeAnalysis.build_artifact_root. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // The text of the generated file, if not provided contents will be read + // from the path above in user's workstation. + Contents []byte `protobuf:"bytes,2,opt,name=contents,proto3,oneof" json:"contents,omitempty"` +} + +func (x *GeneratedFile) Reset() { + *x = GeneratedFile{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GeneratedFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GeneratedFile) ProtoMessage() {} + +func (x *GeneratedFile) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GeneratedFile.ProtoReflect.Descriptor instead. +func (*GeneratedFile) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{3} +} + +func (x *GeneratedFile) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *GeneratedFile) GetContents() []byte { + if x != nil { + return x.Contents + } + return nil +} + +type SourceFile struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Path to the source file relative to repository root. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // Working directory used by the build system. All the relative + // paths in compiler_arguments should be relative to this path. + // Relative to repository root. + WorkingDir string `protobuf:"bytes,2,opt,name=working_dir,json=workingDir,proto3" json:"working_dir,omitempty"` + // Compiler arguments to compile the source file. If multiple variants + // of the module being compiled are possible, the query script will choose + // one. + CompilerArguments []string `protobuf:"bytes,3,rep,name=compiler_arguments,json=compilerArguments,proto3" json:"compiler_arguments,omitempty"` + // Any generated files that are used in compiling the file. + Generated []*GeneratedFile `protobuf:"bytes,4,rep,name=generated,proto3" json:"generated,omitempty"` + // Paths to all of the sources, like build files, code generators, + // proto files etc. that were used during analysis. Used to figure + // out when a set of build artifacts are stale and the query tool + // must be re-run. + // Relative to repository root. + Deps []string `protobuf:"bytes,5,rep,name=deps,proto3" json:"deps,omitempty"` + // Represents analysis status for this particular file. e.g. not part + // of the build graph. + Status *Status `protobuf:"bytes,6,opt,name=status,proto3,oneof" json:"status,omitempty"` +} + +func (x *SourceFile) Reset() { + *x = SourceFile{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SourceFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceFile) ProtoMessage() {} + +func (x *SourceFile) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SourceFile.ProtoReflect.Descriptor instead. +func (*SourceFile) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{4} +} + +func (x *SourceFile) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *SourceFile) GetWorkingDir() string { + if x != nil { + return x.WorkingDir + } + return "" +} + +func (x *SourceFile) GetCompilerArguments() []string { + if x != nil { + return x.CompilerArguments + } + return nil +} + +func (x *SourceFile) GetGenerated() []*GeneratedFile { + if x != nil { + return x.Generated + } + return nil +} + +func (x *SourceFile) GetDeps() []string { + if x != nil { + return x.Deps + } + return nil +} + +func (x *SourceFile) GetStatus() *Status { + if x != nil { + return x.Status + } + return nil +} + +type IdeAnalysis struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Path relative to repository root, containing all the artifacts + // generated by the build system. GeneratedFile.path are always + // relative to this directory. + BuildArtifactRoot string `protobuf:"bytes,1,opt,name=build_artifact_root,json=buildArtifactRoot,proto3" json:"build_artifact_root,omitempty"` + Sources []*SourceFile `protobuf:"bytes,2,rep,name=sources,proto3" json:"sources,omitempty"` + // Status representing overall analysis. + // Should fail only when no analysis can be performed. + Status *Status `protobuf:"bytes,3,opt,name=status,proto3,oneof" json:"status,omitempty"` +} + +func (x *IdeAnalysis) Reset() { + *x = IdeAnalysis{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IdeAnalysis) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IdeAnalysis) ProtoMessage() {} + +func (x *IdeAnalysis) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IdeAnalysis.ProtoReflect.Descriptor instead. +func (*IdeAnalysis) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{5} +} + +func (x *IdeAnalysis) GetBuildArtifactRoot() string { + if x != nil { + return x.BuildArtifactRoot + } + return "" +} + +func (x *IdeAnalysis) GetSources() []*SourceFile { + if x != nil { + return x.Sources + } + return nil +} + +func (x *IdeAnalysis) GetStatus() *Status { + if x != nil { + return x.Status + } + return nil +} + +// Build dependencies of a source file for providing language services. +type DepsResponse_Deps struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Relative to repo_dir. + SourceFile string `protobuf:"bytes,1,opt,name=source_file,json=sourceFile,proto3" json:"source_file,omitempty"` + // Build target to execute for generating dep. + BuildTarget []string `protobuf:"bytes,2,rep,name=build_target,json=buildTarget,proto3" json:"build_target,omitempty"` + Status *Status `protobuf:"bytes,3,opt,name=status,proto3,oneof" json:"status,omitempty"` +} + +func (x *DepsResponse_Deps) Reset() { + *x = DepsResponse_Deps{} + if protoimpl.UnsafeEnabled { + mi := &file_ide_query_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DepsResponse_Deps) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DepsResponse_Deps) ProtoMessage() {} + +func (x *DepsResponse_Deps) ProtoReflect() protoreflect.Message { + mi := &file_ide_query_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DepsResponse_Deps.ProtoReflect.Descriptor instead. +func (*DepsResponse_Deps) Descriptor() ([]byte, []int) { + return file_ide_query_proto_rawDescGZIP(), []int{2, 0} +} + +func (x *DepsResponse_Deps) GetSourceFile() string { + if x != nil { + return x.SourceFile + } + return "" +} + +func (x *DepsResponse_Deps) GetBuildTarget() []string { + if x != nil { + return x.BuildTarget + } + return nil +} + +func (x *DepsResponse_Deps) GetStatus() *Status { + if x != nil { + return x.Status + } + return nil +} + +var File_ide_query_proto protoreflect.FileDescriptor + +var file_ide_query_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x09, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x7c, 0x0a, 0x06, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, + 0x64, 0x65, 0x12, 0x1d, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x88, 0x01, + 0x01, 0x22, 0x1b, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x4b, 0x10, + 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x42, 0x0a, + 0x0a, 0x08, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x52, + 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x65, 0x70, 0x6f, + 0x5f, 0x64, 0x69, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x70, 0x6f, + 0x44, 0x69, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x66, 0x69, + 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x17, 0x0a, + 0x07, 0x6f, 0x75, 0x74, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x6f, 0x75, 0x74, 0x44, 0x69, 0x72, 0x12, 0x20, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x5f, 0x64, + 0x62, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, + 0x6d, 0x70, 0x44, 0x62, 0x50, 0x61, 0x74, 0x68, 0x22, 0x83, 0x02, 0x0a, 0x0c, 0x44, 0x65, 0x70, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x65, 0x70, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x2e, 0x44, 0x65, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x44, 0x65, 0x70, 0x73, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73, 0x12, 0x2e, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x69, 0x64, + 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, 0x1a, 0x85, 0x01, 0x0a, 0x04, + 0x44, 0x65, 0x70, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2e, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x69, 0x64, 0x65, 0x5f, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x51, + 0x0a, 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x22, 0xf7, 0x01, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x5f, + 0x64, 0x69, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x69, + 0x6e, 0x67, 0x44, 0x69, 0x72, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x72, 0x5f, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x72, 0x41, 0x72, 0x67, 0x75, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x36, 0x0a, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x64, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, + 0x65, 0x52, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x65, 0x70, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73, + 0x12, 0x2e, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x11, 0x2e, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, + 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0xa9, 0x01, 0x0a, 0x0b, + 0x49, 0x64, 0x65, 0x41, 0x6e, 0x61, 0x6c, 0x79, 0x73, 0x69, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x72, 0x6f, + 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, + 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x52, 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x69, + 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, + 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x1b, 0x5a, 0x19, 0x69, 0x64, 0x65, 0x5f, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x2f, 0x69, 0x64, 0x65, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_ide_query_proto_rawDescOnce sync.Once + file_ide_query_proto_rawDescData = file_ide_query_proto_rawDesc +) + +func file_ide_query_proto_rawDescGZIP() []byte { + file_ide_query_proto_rawDescOnce.Do(func() { + file_ide_query_proto_rawDescData = protoimpl.X.CompressGZIP(file_ide_query_proto_rawDescData) + }) + return file_ide_query_proto_rawDescData +} + +var file_ide_query_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_ide_query_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_ide_query_proto_goTypes = []interface{}{ + (Status_Code)(0), // 0: ide_query.Status.Code + (*Status)(nil), // 1: ide_query.Status + (*RepoState)(nil), // 2: ide_query.RepoState + (*DepsResponse)(nil), // 3: ide_query.DepsResponse + (*GeneratedFile)(nil), // 4: ide_query.GeneratedFile + (*SourceFile)(nil), // 5: ide_query.SourceFile + (*IdeAnalysis)(nil), // 6: ide_query.IdeAnalysis + (*DepsResponse_Deps)(nil), // 7: ide_query.DepsResponse.Deps +} +var file_ide_query_proto_depIdxs = []int32{ + 0, // 0: ide_query.Status.code:type_name -> ide_query.Status.Code + 7, // 1: ide_query.DepsResponse.deps:type_name -> ide_query.DepsResponse.Deps + 1, // 2: ide_query.DepsResponse.status:type_name -> ide_query.Status + 4, // 3: ide_query.SourceFile.generated:type_name -> ide_query.GeneratedFile + 1, // 4: ide_query.SourceFile.status:type_name -> ide_query.Status + 5, // 5: ide_query.IdeAnalysis.sources:type_name -> ide_query.SourceFile + 1, // 6: ide_query.IdeAnalysis.status:type_name -> ide_query.Status + 1, // 7: ide_query.DepsResponse.Deps.status:type_name -> ide_query.Status + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_ide_query_proto_init() } +func file_ide_query_proto_init() { + if File_ide_query_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_ide_query_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ide_query_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RepoState); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ide_query_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DepsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ide_query_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GeneratedFile); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ide_query_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SourceFile); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ide_query_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IdeAnalysis); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ide_query_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DepsResponse_Deps); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_ide_query_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_ide_query_proto_msgTypes[2].OneofWrappers = []interface{}{} + file_ide_query_proto_msgTypes[3].OneofWrappers = []interface{}{} + file_ide_query_proto_msgTypes[4].OneofWrappers = []interface{}{} + file_ide_query_proto_msgTypes[5].OneofWrappers = []interface{}{} + file_ide_query_proto_msgTypes[6].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_ide_query_proto_rawDesc, + NumEnums: 1, + NumMessages: 7, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_ide_query_proto_goTypes, + DependencyIndexes: file_ide_query_proto_depIdxs, + EnumInfos: file_ide_query_proto_enumTypes, + MessageInfos: file_ide_query_proto_msgTypes, + }.Build() + File_ide_query_proto = out.File + file_ide_query_proto_rawDesc = nil + file_ide_query_proto_goTypes = nil + file_ide_query_proto_depIdxs = nil +} diff --git a/tools/ide_query/ide_query_proto/ide_query.proto b/tools/ide_query/ide_query_proto/ide_query.proto new file mode 100644 index 0000000000..3d7a8e760a --- /dev/null +++ b/tools/ide_query/ide_query_proto/ide_query.proto @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; + +package ide_query; +option go_package = "ide_query/ide_query_proto"; + +// Indicates the success/failure for analysis. +message Status { + enum Code { + OK = 0; + FAILURE = 1; + } + Code code = 1; + // Details about the status, might be displayed to user. + optional string message = 2; +} + +// Represents an Android checkout on user's workstation. +message RepoState { + // Absolute path for the checkout in the workstation. + // e.g. /home/user/work/android/ + string repo_dir = 1; + // Relative to repo_dir. + repeated string active_file_path = 2; + // Repository relative path to output directory in workstation. + string out_dir = 3; + // Repository relative path to compile_commands.json in workstation. + string comp_db_path = 4; +} + +// Provides all the targets that are pre-requisities for running language +// services on active_file_paths. +message DepsResponse { + // Build dependencies of a source file for providing language services. + message Deps { + // Relative to repo_dir. + string source_file = 1; + // Build target to execute for generating dep. + repeated string build_target = 2; + optional Status status = 3; + } + repeated Deps deps = 1; + optional Status status = 2; +} + +// Returns all the information necessary for providing language services for the +// active files. +message GeneratedFile { + // Path to the file relative to IdeAnalysis.build_artifact_root. + string path = 1; + + // The text of the generated file, if not provided contents will be read + // from the path above in user's workstation. + optional bytes contents = 2; +} + +message SourceFile { + // Path to the source file relative to repository root. + string path = 1; + + // Working directory used by the build system. All the relative + // paths in compiler_arguments should be relative to this path. + // Relative to repository root. + string working_dir = 2; + + // Compiler arguments to compile the source file. If multiple variants + // of the module being compiled are possible, the query script will choose + // one. + repeated string compiler_arguments = 3; + + // Any generated files that are used in compiling the file. + repeated GeneratedFile generated = 4; + + // Paths to all of the sources, like build files, code generators, + // proto files etc. that were used during analysis. Used to figure + // out when a set of build artifacts are stale and the query tool + // must be re-run. + // Relative to repository root. + repeated string deps = 5; + + // Represents analysis status for this particular file. e.g. not part + // of the build graph. + optional Status status = 6; +} + +message IdeAnalysis { + // Path relative to repository root, containing all the artifacts + // generated by the build system. GeneratedFile.path are always + // relative to this directory. + string build_artifact_root = 1; + + repeated SourceFile sources = 2; + + // Status representing overall analysis. + // Should fail only when no analysis can be performed. + optional Status status = 3; +} diff --git a/tools/ide_query/ide_query_proto/regen.sh b/tools/ide_query/ide_query_proto/regen.sh new file mode 100755 index 0000000000..eec4f3731b --- /dev/null +++ b/tools/ide_query/ide_query_proto/regen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aprotoc --go_out=paths=source_relative:. ide_query.proto diff --git a/tools/list_files.py b/tools/list_files.py new file mode 100644 index 0000000000..4f666aa5fe --- /dev/null +++ b/tools/list_files.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List +from glob import glob +from pathlib import Path +from os.path import join, relpath +from itertools import chain +import argparse + +class FileLister: + def __init__(self, args) -> None: + self.out_file = args.out_file + + self.folder_dir = args.dir + self.extensions = [e if e.startswith(".") else "." + e for e in args.extensions] + self.root = args.root + self.files_list : List[str] = list() + self.classes = args.classes + + def get_files(self) -> None: + """Get all files directory in the input directory including the files in the subdirectories + + Recursively finds all files in the input directory. + Set file_list as a list of file directory strings, + which do not include directories but only files. + List is sorted in alphabetical order of the file directories. + + Args: + dir: Directory to get the files. String. + + Raises: + FileNotFoundError: An error occurred accessing the non-existing directory + """ + + if not dir_exists(self.folder_dir): + raise FileNotFoundError(f"Directory {self.folder_dir} does not exist") + + if self.folder_dir[:-2] != "**": + self.folder_dir = join(self.folder_dir, "**") + + self.files_list = list() + for file in sorted(glob(self.folder_dir, recursive=True)): + if Path(file).is_file(): + if self.root: + file = join(self.root, relpath(file, self.folder_dir[:-2])) + self.files_list.append(file) + + + def list(self) -> None: + self.get_files() + self.files_list = [f for f in self.files_list if not self.extensions or Path(f).suffix in self.extensions] + + # If files_list is as below: + # A/B/C.java + # A/B/D.java + # A/B/E.txt + # --classes flag converts files_list in the following format: + # A/B/C.class + # A/B/C$*.class + # A/B/D.class + # A/B/D$*.class + # Additional `$*`-suffixed line is appended after each line + # to take multiple top level classes in a single java file into account. + # Note that non-java files in files_list are filtered out. + if self.classes: + self.files_list = list(chain.from_iterable([ + (class_files := str(Path(ff).with_suffix(".class")), + class_files.replace(".class", "$*.class")) + for ff in self.files_list if ff.endswith(".java") + ])) + + self.write() + + def write(self) -> None: + if self.out_file == "": + pprint(self.files_list) + else: + write_lines(self.out_file, self.files_list) + +### +# Helper functions +### +def pprint(l: List[str]) -> None: + for line in l: + print(line) + +def dir_exists(dir: str) -> bool: + return Path(dir).exists() + +def write_lines(out_file: str, lines: List[str]) -> None: + with open(out_file, "w+") as f: + f.writelines(line + '\n' for line in lines) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('dir', action='store', type=str, + help="directory to list all subdirectory files") + parser.add_argument('--out', dest='out_file', + action='store', default="", type=str, + help="optional directory to write subdirectory files. If not set, will print to console") + parser.add_argument('--root', dest='root', + action='store', default="", type=str, + help="optional directory to replace the root directories of output.") + parser.add_argument('--extensions', nargs='*', default=list(), dest='extensions', + help="Extensions to include in the output. If not set, all files are included") + parser.add_argument('--classes', dest='classes', action=argparse.BooleanOptionalAction, + help="Optional flag. If passed, outputs a list of pattern of class files \ + that will be produced by compiling java files in the input dir. \ + Non-java files in the input directory will be ignored.") + + args = parser.parse_args() + + file_lister = FileLister(args) + file_lister.list() diff --git a/tools/lunchable b/tools/lunchable new file mode 100755 index 0000000000..fce2c2719d --- /dev/null +++ b/tools/lunchable @@ -0,0 +1,72 @@ +#!/bin/bash + +# TODO: Currently only checks trunk_staging. Should check trunk_staging first, +# then use the product-specfic releases. Only applies to -c though. + +function Help() { +cat <<@EOF@ +Usage: lunchable [options] + +Lists products that have no functioning lunch combo. + +options: +-c prints all failing lunch combos for all targets; +-w why? Prints the error message after each failed lunch combo. Only + works with -c + +@EOF@ +} + +complete=0 +why=0 +while getopts "cwh" option; do + case $option in + c) + complete=1;; + w) + why=1;; + h) + Help + exit;; + esac +done + +# Getting all named products can fail if we haven't lunched anything +source $(pwd)/build/envsetup.sh &> /dev/null +all_named_products=( $(get_build_var all_named_products 2> /dev/null) ) +if [[ $? -ne 0 ]]; then + echo "get_build_var all_named_products failed. Lunch something first?" >&2 + exit 1 +fi +total_products=${#all_named_products[@]} +current_product=0 + +for product in "${all_named_products[@]}"; do + (( current_product += 1 )) + single_pass=0 + printf " Checking ${current_product}/${total_products} \r" >&2 + for release in trunk_staging; do + for variant in eng user userdebug; do + lunchcombo="${product}-${release}-${variant}" + lunch_error="$(lunch $lunchcombo 2>&1 > /dev/null)" + if [[ $? -ne 0 ]]; then + # Lunch failed + if [[ $complete -eq 1 ]]; then + echo -e "${product} : ${lunchcombo}" + if [[ $why -eq 1 ]]; then + echo -e "$(sed 's/^/ /g' <<<$lunch_error)" + fi + fi + elif [[ $complete -ne 1 ]]; then + single_pass=1 + break # skip variant + fi + done + if [[ $single_pass -eq 1 ]]; then + break # skip release + fi + done + if [[ $complete -eq 0 ]] && [[ $single_pass -eq 0 ]]; then + echo "${product}" + fi +done diff --git a/tools/metadata/Android.bp b/tools/metadata/Android.bp new file mode 100644 index 0000000000..77d106d705 --- /dev/null +++ b/tools/metadata/Android.bp @@ -0,0 +1,16 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "metadata", + deps: [ + "soong-testing-test_spec_proto", + "soong-testing-code_metadata_proto", + "soong-testing-code_metadata_internal_proto", + "golang-protobuf-proto", + ], + srcs: [ + "generator.go", + ] +}
\ No newline at end of file diff --git a/tools/metadata/OWNERS b/tools/metadata/OWNERS new file mode 100644 index 0000000000..03bcdf1c40 --- /dev/null +++ b/tools/metadata/OWNERS @@ -0,0 +1,4 @@ +dariofreni@google.com +joeo@google.com +ronish@google.com +caditya@google.com diff --git a/tools/metadata/generator.go b/tools/metadata/generator.go new file mode 100644 index 0000000000..b7668be44f --- /dev/null +++ b/tools/metadata/generator.go @@ -0,0 +1,328 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "sort" + "strings" + "sync" + + "android/soong/testing/code_metadata_internal_proto" + "android/soong/testing/code_metadata_proto" + "android/soong/testing/test_spec_proto" + "google.golang.org/protobuf/proto" +) + +type keyToLocksMap struct { + locks sync.Map +} + +func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex { + mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{}) + return mutex.(*sync.Mutex) +} + +// Define a struct to hold the combination of team ID and multi-ownership flag for validation +type sourceFileAttributes struct { + TeamID string + MultiOwnership bool + Path string +} + +func getSortedKeys(syncMap *sync.Map) []string { + var allKeys []string + syncMap.Range( + func(key, _ interface{}) bool { + allKeys = append(allKeys, key.(string)) + return true + }, + ) + + sort.Strings(allKeys) + return allKeys +} + +// writeProtoToFile marshals a protobuf message and writes it to a file +func writeProtoToFile(outputFile string, message proto.Message) { + data, err := proto.Marshal(message) + if err != nil { + log.Fatal(err) + } + file, err := os.Create(outputFile) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + _, err = file.Write(data) + if err != nil { + log.Fatal(err) + } +} + +func readFileToString(filePath string) string { + file, err := os.Open(filePath) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + log.Fatal(err) + } + return string(data) +} + +func writeEmptyOutputProto(outputFile string, metadataRule string) { + file, err := os.Create(outputFile) + if err != nil { + log.Fatal(err) + } + var message proto.Message + if metadataRule == "test_spec" { + message = &test_spec_proto.TestSpec{} + } else if metadataRule == "code_metadata" { + message = &code_metadata_proto.CodeMetadata{} + } + data, err := proto.Marshal(message) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + _, err = file.Write([]byte(data)) + if err != nil { + log.Fatal(err) + } +} + +func processTestSpecProtobuf( + filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap, + errCh chan error, wg *sync.WaitGroup, +) { + defer wg.Done() + + fileContent := strings.TrimRight(readFileToString(filePath), "\n") + testData := test_spec_proto.TestSpec{} + err := proto.Unmarshal([]byte(fileContent), &testData) + if err != nil { + errCh <- err + return + } + + ownershipMetadata := testData.GetOwnershipMetadataList() + for _, metadata := range ownershipMetadata { + key := metadata.GetTargetName() + lock := keyLocks.GetLockForKey(key) + lock.Lock() + + value, loaded := ownershipMetadataMap.LoadOrStore( + key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata}, + ) + if loaded { + existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata) + isDuplicate := false + for _, existing := range existingMetadata { + if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() { + errCh <- fmt.Errorf( + "Conflicting trendy team IDs found for %s at:\n%s with teamId"+ + ": %s,\n%s with teamId: %s", + key, + metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(), + existing.GetTrendyTeamId(), + ) + + lock.Unlock() + return + } + if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() { + isDuplicate = true + break + } + } + if !isDuplicate { + existingMetadata = append(existingMetadata, metadata) + ownershipMetadataMap.Store(key, existingMetadata) + } + } + + lock.Unlock() + } +} + +// processCodeMetadataProtobuf processes CodeMetadata protobuf files +func processCodeMetadataProtobuf( + filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap, + errCh chan error, wg *sync.WaitGroup, +) { + defer wg.Done() + + fileContent := strings.TrimRight(readFileToString(filePath), "\n") + internalCodeData := code_metadata_internal_proto.CodeMetadataInternal{} + err := proto.Unmarshal([]byte(fileContent), &internalCodeData) + if err != nil { + errCh <- err + return + } + + // Process each TargetOwnership entry + for _, internalMetadata := range internalCodeData.GetTargetOwnershipList() { + key := internalMetadata.GetTargetName() + lock := keyLocks.GetLockForKey(key) + lock.Lock() + + for _, srcFile := range internalMetadata.GetSourceFiles() { + srcFileKey := srcFile + srcFileLock := keyLocks.GetLockForKey(srcFileKey) + srcFileLock.Lock() + attributes := sourceFileAttributes{ + TeamID: internalMetadata.GetTrendyTeamId(), + MultiOwnership: internalMetadata.GetMultiOwnership(), + Path: internalMetadata.GetPath(), + } + + existingAttributes, exists := sourceFileMetadataMap.Load(srcFileKey) + if exists { + existing := existingAttributes.(sourceFileAttributes) + if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership || !existing.MultiOwnership) { + errCh <- fmt.Errorf( + "Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+ + " If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+ + "Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.", + srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path, + ) + srcFileLock.Unlock() + lock.Unlock() + return + } + } else { + // Store the metadata if no conflict + sourceFileMetadataMap.Store(srcFileKey, attributes) + } + srcFileLock.Unlock() + } + + value, loaded := ownershipMetadataMap.LoadOrStore( + key, []*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership{internalMetadata}, + ) + if loaded { + existingMetadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership) + isDuplicate := false + for _, existing := range existingMetadata { + if internalMetadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && internalMetadata.GetPath() == existing.GetPath() { + isDuplicate = true + break + } + } + if !isDuplicate { + existingMetadata = append(existingMetadata, internalMetadata) + ownershipMetadataMap.Store(key, existingMetadata) + } + } + + lock.Unlock() + } +} + +func main() { + inputFile := flag.String("inputFile", "", "Input file path") + outputFile := flag.String("outputFile", "", "Output file path") + rule := flag.String( + "rule", "", "Metadata rule (Hint: test_spec or code_metadata)", + ) + flag.Parse() + + if *inputFile == "" || *outputFile == "" || *rule == "" { + fmt.Println("Usage: metadata -rule <rule> -inputFile <input file path> -outputFile <output file path>") + os.Exit(1) + } + + inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n") + filePaths := strings.Split(inputFileData, " ") + if len(filePaths) == 1 && filePaths[0] == "" { + writeEmptyOutputProto(*outputFile, *rule) + return + } + ownershipMetadataMap := &sync.Map{} + keyLocks := &keyToLocksMap{} + errCh := make(chan error, len(filePaths)) + var wg sync.WaitGroup + + switch *rule { + case "test_spec": + for _, filePath := range filePaths { + wg.Add(1) + go processTestSpecProtobuf( + filePath, ownershipMetadataMap, keyLocks, errCh, &wg, + ) + } + + wg.Wait() + close(errCh) + + for err := range errCh { + log.Fatal(err) + } + + allKeys := getSortedKeys(ownershipMetadataMap) + var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata + + for _, key := range allKeys { + value, _ := ownershipMetadataMap.Load(key) + metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata) + allMetadata = append(allMetadata, metadataList...) + } + + testSpec := &test_spec_proto.TestSpec{ + OwnershipMetadataList: allMetadata, + } + writeProtoToFile(*outputFile, testSpec) + break + case "code_metadata": + sourceFileMetadataMap := &sync.Map{} + for _, filePath := range filePaths { + wg.Add(1) + go processCodeMetadataProtobuf( + filePath, ownershipMetadataMap, sourceFileMetadataMap, keyLocks, errCh, &wg, + ) + } + + wg.Wait() + close(errCh) + + for err := range errCh { + log.Fatal(err) + } + + sortedKeys := getSortedKeys(ownershipMetadataMap) + allMetadata := make([]*code_metadata_proto.CodeMetadata_TargetOwnership, 0) + for _, key := range sortedKeys { + value, _ := ownershipMetadataMap.Load(key) + metadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership) + for _, m := range metadata { + targetName := m.GetTargetName() + path := m.GetPath() + trendyTeamId := m.GetTrendyTeamId() + + allMetadata = append(allMetadata, &code_metadata_proto.CodeMetadata_TargetOwnership{ + TargetName: &targetName, + Path: &path, + TrendyTeamId: &trendyTeamId, + SourceFiles: m.GetSourceFiles(), + }) + } + } + + finalMetadata := &code_metadata_proto.CodeMetadata{ + TargetOwnershipList: allMetadata, + } + writeProtoToFile(*outputFile, finalMetadata) + break + default: + log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule) + } +} diff --git a/tools/metadata/go.mod b/tools/metadata/go.mod new file mode 100644 index 0000000000..e9d04b16f6 --- /dev/null +++ b/tools/metadata/go.mod @@ -0,0 +1,7 @@ +module android/soong/tools/metadata + +require google.golang.org/protobuf v0.0.0 + +replace google.golang.org/protobuf v0.0.0 => ../../../external/golang-protobuf + +go 1.18
\ No newline at end of file diff --git a/tools/metadata/go.work b/tools/metadata/go.work new file mode 100644 index 0000000000..f2cdf8ec98 --- /dev/null +++ b/tools/metadata/go.work @@ -0,0 +1,11 @@ +go 1.18 + +use ( + . + ../../../../external/golang-protobuf + ../../../soong/testing/test_spec_proto + ../../../soong/testing/code_metadata_proto + ../../../soong/testing/code_metadata_proto_internal +) + +replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf diff --git a/tools/metadata/testdata/emptyInputFile.txt b/tools/metadata/testdata/emptyInputFile.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/tools/metadata/testdata/emptyInputFile.txt @@ -0,0 +1 @@ + diff --git a/tools/metadata/testdata/expectedCodeMetadataOutput.txt b/tools/metadata/testdata/expectedCodeMetadataOutput.txt new file mode 100644 index 0000000000..755cf40a30 --- /dev/null +++ b/tools/metadata/testdata/expectedCodeMetadataOutput.txt @@ -0,0 +1,7 @@ + + +bar +Android.bp12346"b.java + +foo +Android.bp12345"a.java
\ No newline at end of file diff --git a/tools/metadata/testdata/expectedOutputFile.txt b/tools/metadata/testdata/expectedOutputFile.txt new file mode 100644 index 0000000000..b0d382f279 --- /dev/null +++ b/tools/metadata/testdata/expectedOutputFile.txt @@ -0,0 +1,22 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/metadata/testdata/file1.txt b/tools/metadata/testdata/file1.txt new file mode 100644 index 0000000000..81beed00ab --- /dev/null +++ b/tools/metadata/testdata/file1.txt @@ -0,0 +1,13 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345 diff --git a/tools/metadata/testdata/file2.txt b/tools/metadata/testdata/file2.txt new file mode 100644 index 0000000000..32a753fef5 --- /dev/null +++ b/tools/metadata/testdata/file2.txt @@ -0,0 +1,25 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 diff --git a/tools/metadata/testdata/file3.txt b/tools/metadata/testdata/file3.txt new file mode 100644 index 0000000000..81beed00ab --- /dev/null +++ b/tools/metadata/testdata/file3.txt @@ -0,0 +1,13 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345 diff --git a/tools/metadata/testdata/file4.txt b/tools/metadata/testdata/file4.txt new file mode 100644 index 0000000000..6a7590021d --- /dev/null +++ b/tools/metadata/testdata/file4.txt @@ -0,0 +1,25 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-one +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 diff --git a/tools/metadata/testdata/file5.txt b/tools/metadata/testdata/file5.txt new file mode 100644 index 0000000000..d8de06457d --- /dev/null +++ b/tools/metadata/testdata/file5.txt @@ -0,0 +1,4 @@ + + +foo +Android.bp12345"a.java diff --git a/tools/metadata/testdata/file6.txt b/tools/metadata/testdata/file6.txt new file mode 100644 index 0000000000..9c7cdcd505 --- /dev/null +++ b/tools/metadata/testdata/file6.txt @@ -0,0 +1,4 @@ + + +bar +Android.bp12346"b.java diff --git a/tools/metadata/testdata/file7.txt b/tools/metadata/testdata/file7.txt new file mode 100644 index 0000000000..d8de06457d --- /dev/null +++ b/tools/metadata/testdata/file7.txt @@ -0,0 +1,4 @@ + + +foo +Android.bp12345"a.java diff --git a/tools/metadata/testdata/file8.txt b/tools/metadata/testdata/file8.txt new file mode 100644 index 0000000000..a931690022 --- /dev/null +++ b/tools/metadata/testdata/file8.txt @@ -0,0 +1,4 @@ + + +foo +Android.gp12346"a.java diff --git a/tools/metadata/testdata/generatedCodeMetadataOutput.txt b/tools/metadata/testdata/generatedCodeMetadataOutput.txt new file mode 100644 index 0000000000..755cf40a30 --- /dev/null +++ b/tools/metadata/testdata/generatedCodeMetadataOutput.txt @@ -0,0 +1,7 @@ + + +bar +Android.bp12346"b.java + +foo +Android.bp12345"a.java
\ No newline at end of file diff --git a/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt b/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt new file mode 100644 index 0000000000..755cf40a30 --- /dev/null +++ b/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt @@ -0,0 +1,7 @@ + + +bar +Android.bp12346"b.java + +foo +Android.bp12345"a.java
\ No newline at end of file diff --git a/tools/metadata/testdata/generatedEmptyOutputFile.txt b/tools/metadata/testdata/generatedEmptyOutputFile.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/metadata/testdata/generatedEmptyOutputFile.txt diff --git a/tools/metadata/testdata/generatedOutputFile.txt b/tools/metadata/testdata/generatedOutputFile.txt new file mode 100644 index 0000000000..b0d382f279 --- /dev/null +++ b/tools/metadata/testdata/generatedOutputFile.txt @@ -0,0 +1,22 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/metadata/testdata/inputCodeMetadata.txt b/tools/metadata/testdata/inputCodeMetadata.txt new file mode 100644 index 0000000000..7a81b7d523 --- /dev/null +++ b/tools/metadata/testdata/inputCodeMetadata.txt @@ -0,0 +1 @@ +file5.txt file6.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputCodeMetadataNegative.txt b/tools/metadata/testdata/inputCodeMetadataNegative.txt new file mode 100644 index 0000000000..26668e44a9 --- /dev/null +++ b/tools/metadata/testdata/inputCodeMetadataNegative.txt @@ -0,0 +1 @@ +file7.txt file8.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputFiles.txt b/tools/metadata/testdata/inputFiles.txt new file mode 100644 index 0000000000..e44bc94d32 --- /dev/null +++ b/tools/metadata/testdata/inputFiles.txt @@ -0,0 +1 @@ +file1.txt file2.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputFilesNegativeCase.txt b/tools/metadata/testdata/inputFilesNegativeCase.txt new file mode 100644 index 0000000000..a37aa3fd5d --- /dev/null +++ b/tools/metadata/testdata/inputFilesNegativeCase.txt @@ -0,0 +1 @@ +file3.txt file4.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/metadata_test.go b/tools/metadata/testdata/metadata_test.go new file mode 100644 index 0000000000..314add352f --- /dev/null +++ b/tools/metadata/testdata/metadata_test.go @@ -0,0 +1,119 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os/exec" + "strings" + "testing" +) + +func TestMetadata(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "test_spec", "-inputFile", "./inputFiles.txt", "-outputFile", + "./generatedOutputFile.txt", + ) + stderr, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) + } + + // Read the contents of the expected output file + expectedOutput, err := ioutil.ReadFile("./expectedOutputFile.txt") + if err != nil { + t.Fatalf("Error reading expected output file: %s", err) + } + + // Read the contents of the generated output file + generatedOutput, err := ioutil.ReadFile("./generatedOutputFile.txt") + if err != nil { + t.Fatalf("Error reading generated output file: %s", err) + } + + fmt.Println() + + // Compare the contents + if string(expectedOutput) != string(generatedOutput) { + t.Errorf("Generated file contents do not match the expected output") + } +} + +func TestMetadataNegativeCase(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "test_spec", "-inputFile", "./inputFilesNegativeCase.txt", "-outputFile", + "./generatedOutputFileNegativeCase.txt", + ) + stderr, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf( + "Expected an error, but the metadata command executed successfully. Output: %s", + stderr, + ) + } + + expectedError := "Conflicting trendy team IDs found for java-test-module" + + "-name-one at:\nAndroid.bp with teamId: 12346," + + "\nAndroid.bp with teamId: 12345" + if !strings.Contains( + strings.TrimSpace(string(stderr)), strings.TrimSpace(expectedError), + ) { + t.Errorf( + "Unexpected error message. Expected to contain: %s, Got: %s", + expectedError, stderr, + ) + } +} + +func TestEmptyInputFile(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "test_spec", "-inputFile", "./emptyInputFile.txt", "-outputFile", + "./generatedEmptyOutputFile.txt", + ) + stderr, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) + } + + // Read the contents of the generated output file + generatedOutput, err := ioutil.ReadFile("./generatedEmptyOutputFile.txt") + if err != nil { + t.Fatalf("Error reading generated output file: %s", err) + } + + fmt.Println() + + // Compare the contents + if string(generatedOutput) != "\n" { + t.Errorf("Generated file contents do not match the expected output") + } +} + +func TestCodeMetadata(t *testing.T) { + cmd := exec.Command( + "metadata", "-rule", "code_metadata", "-inputFile", "./inputCodeMetadata.txt", "-outputFile", + "./generatedCodeMetadataOutputFile.txt", + ) + stderr, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) + } + + // Read the contents of the expected output file + expectedOutput, err := ioutil.ReadFile("./expectedCodeMetadataOutput.txt") + if err != nil { + t.Fatalf("Error reading expected output file: %s", err) + } + + // Read the contents of the generated output file + generatedOutput, err := ioutil.ReadFile("./generatedCodeMetadataOutputFile.txt") + if err != nil { + t.Fatalf("Error reading generated output file: %s", err) + } + + fmt.Println() + + // Compare the contents + if string(expectedOutput) != string(generatedOutput) { + t.Errorf("Generated file contents do not match the expected output") + } +} diff --git a/tools/metadata/testdata/outputFile.txt b/tools/metadata/testdata/outputFile.txt new file mode 100644 index 0000000000..b0d382f279 --- /dev/null +++ b/tools/metadata/testdata/outputFile.txt @@ -0,0 +1,22 @@ + +. +java-test-module-name-one +Android.bp12345 +. +java-test-module-name-six +Android.bp12346 +. +java-test-module-name-six +Aqwerty.bp12346 +. +java-test-module-name-six +Apoiuyt.bp12346 +. +java-test-module-name-two +Android.bp12345 +. +java-test-module-name-two +Asdfghj.bp12345 +. +java-test-module-name-two +Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/overrideflags.sh b/tools/overrideflags.sh new file mode 100755 index 0000000000..b8605dc034 --- /dev/null +++ b/tools/overrideflags.sh @@ -0,0 +1,99 @@ +#!/bin/bash -e +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +source $(cd $(dirname $BASH_SOURCE) &> /dev/null && pwd)/../shell_utils.sh +require_top + +function print_help() { + echo -e "overrideflags is used to set default value for local build." + echo -e "\nOptions:" + echo -e "\t--release-config \tPath to release configuration directory. Required" + echo -e "\t--no-edit \tIf present, skip editing flag value file." + echo -e "\t-h/--help \tShow this help." +} + +function main() { + while (($# > 0)); do + case $1 in + --release-config) + if [[ $# -le 1 ]]; then + echo "--release-config requires a path" + return 1 + fi + local release_config_dir="$2" + shift 2 + ;; + --no-edit) + local no_edit="true" + shift 1 + ;; + -h|--help) + print_help + return + ;; + *) + echo "$1 is unrecognized" + print_help + return 1 + ;; + esac + done + + + + case $(uname -s) in + Darwin) + local host_arch=darwin-x86 + ;; + Linux) + local host_arch=linux-x86 + ;; + *) + >&2 echo Unknown host $(uname -s) + return + ;; + esac + + if [[ -z "${release_config_dir}" ]]; then + echo "Please provide release configuration path by --release-config" + exit 1 + elif [ ! -d "${release_config_dir}" ]; then + echo "${release_config_dir} is an invalid directory" + exit 1 + fi + local T="$(gettop)" + local aconfig_dir="${T}"/build/make/tools/aconfig/ + local overrideflag_py="${aconfig_dir}"/overrideflags/overrideflags.py + local overridefile="${release_config_dir}/aconfig/override_values.textproto" + + # Edit override file + if [[ -z "${no_edit}" ]]; then + editor="${EDITOR:-$(which vim)}" + + eval "${editor} ${overridefile}" + if [ $? -ne 0 ]; then + echo "Fail to set override values" + return 1 + fi + fi + + ${T}/prebuilts/build-tools/${host_arch}/bin/py3-cmd -u "${overrideflag_py}" \ + --overrides "${overridefile}" \ + --out "${release_config_dir}/aconfig" +} + + +main "$@" diff --git a/tools/perf/benchmarks b/tools/perf/benchmarks new file mode 100755 index 0000000000..6998ecd5c2 --- /dev/null +++ b/tools/perf/benchmarks @@ -0,0 +1,811 @@ +#!/usr/bin/env python3 +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +if __name__ == "__main__": + sys.dont_write_bytecode = True + +import argparse +import dataclasses +import datetime +import json +import os +import pathlib +import random +import re +import shutil +import subprocess +import time +import uuid +from typing import Optional + +import pretty +import utils + + +class FatalError(Exception): + def __init__(self): + pass + + +class OptionsError(Exception): + def __init__(self, message): + self.message = message + + +@dataclasses.dataclass(frozen=True) +class Lunch: + "Lunch combination" + + target_product: str + "TARGET_PRODUCT" + + target_release: str + "TARGET_RELEASE" + + target_build_variant: str + "TARGET_BUILD_VARIANT" + + def ToDict(self): + return { + "TARGET_PRODUCT": self.target_product, + "TARGET_RELEASE": self.target_release, + "TARGET_BUILD_VARIANT": self.target_build_variant, + } + + def Combine(self): + return f"{self.target_product}-{self.target_release}-{self.target_build_variant}" + + +@dataclasses.dataclass(frozen=True) +class Change: + "A change that we make to the tree, and how to undo it" + label: str + "String to print in the log when the change is made" + + change: callable + "Function to change the source tree" + + undo: callable + "Function to revert the source tree to its previous condition in the most minimal way possible." + +_DUMPVARS_VARS=[ + "COMMON_LUNCH_CHOICES", + "HOST_PREBUILT_TAG", + "print", + "PRODUCT_OUT", + "report_config", + "TARGET_ARCH", + "TARGET_BUILD_VARIANT", + "TARGET_DEVICE", + "TARGET_PRODUCT", +] + +_DUMPVARS_ABS_VARS =[ + "ANDROID_CLANG_PREBUILTS", + "ANDROID_JAVA_HOME", + "ANDROID_JAVA_TOOLCHAIN", + "ANDROID_PREBUILTS", + "HOST_OUT", + "HOST_OUT_EXECUTABLES", + "HOST_OUT_TESTCASES", + "OUT_DIR", + "print", + "PRODUCT_OUT", + "SOONG_HOST_OUT", + "SOONG_HOST_OUT_EXECUTABLES", + "TARGET_OUT_TESTCASES", +] + +@dataclasses.dataclass(frozen=True) +class Benchmark: + "Something we measure" + + id: str + "Short ID for the benchmark, for the command line" + + title: str + "Title for reports" + + change: Change + "Source tree modification for the benchmark that will be measured" + + dumpvars: Optional[bool] = False + "If specified, soong will run in dumpvars mode rather than build-mode." + + modules: Optional[list[str]] = None + "Build modules to build on soong command line" + + preroll: Optional[int] = 0 + "Number of times to run the build command to stabilize" + + postroll: Optional[int] = 3 + "Number of times to run the build command after reverting the action to stabilize" + + def build_description(self): + "Short description of the benchmark's Soong invocation." + if self.dumpvars: + return "dumpvars" + elif self.modules: + return " ".join(self.modules) + return "" + + + def soong_command(self, root): + "Command line args to soong_ui for this benchmark." + if self.dumpvars: + return [ + "--dumpvars-mode", + f"--vars=\"{' '.join(_DUMPVARS_VARS)}\"", + f"--abs-vars=\"{' '.join(_DUMPVARS_ABS_VARS)}\"", + "--var-prefix=var_cache_", + "--abs-var-prefix=abs_var_cache_", + ] + elif self.modules: + return [ + "--build-mode", + "--all-modules", + f"--dir={root}", + "--skip-metrics-upload", + ] + self.modules + else: + raise Exception("Benchmark must specify dumpvars or modules") + + +@dataclasses.dataclass(frozen=True) +class FileSnapshot: + "Snapshot of a file's contents." + + filename: str + "The file that was snapshottened" + + contents: str + "The contents of the file" + + def write(self): + "Write the contents back to the file" + with open(self.filename, "w") as f: + f.write(self.contents) + + +def Snapshot(filename): + """Return a FileSnapshot with the file's current contents.""" + with open(filename) as f: + contents = f.read() + return FileSnapshot(filename, contents) + + +def Clean(): + """Remove the out directory.""" + def remove_out(): + out_dir = utils.get_out_dir() + #only remove actual contents, in case out is a symlink (as is the case for cog) + if os.path.exists(out_dir): + for filename in os.listdir(out_dir): + p = os.path.join(out_dir, filename) + if os.path.isfile(p) or os.path.islink(p): + os.remove(p) + elif os.path.isdir(p): + shutil.rmtree(p) + return Change(label="Remove out", change=remove_out, undo=lambda: None) + + +def NoChange(): + """No change to the source tree.""" + return Change(label="No change", change=lambda: None, undo=lambda: None) + + +def Create(filename): + "Create an action to create `filename`. The parent directory must exist." + def create(): + with open(filename, "w") as f: + pass + def delete(): + os.remove(filename) + return Change( + label=f"Create {filename}", + change=create, + undo=delete, + ) + + +def Modify(filename, contents, before=None): + """Create an action to modify `filename` by appending the result of `contents` + before the last instances of `before` in the file. + + Raises an error if `before` doesn't appear in the file. + """ + orig = Snapshot(filename) + if before: + index = orig.contents.rfind(before) + if index < 0: + report_error(f"{filename}: Unable to find string '{before}' for modify operation.") + raise FatalError() + else: + index = len(orig.contents) + modified = FileSnapshot(filename, orig.contents[:index] + contents() + orig.contents[index:]) + if False: + print(f"Modify: {filename}") + x = orig.contents.replace("\n", "\n ORIG") + print(f" ORIG {x}") + x = modified.contents.replace("\n", "\n MODIFIED") + print(f" MODIFIED {x}") + + return Change( + label="Modify " + filename, + change=lambda: modified.write(), + undo=lambda: orig.write() + ) + +def ChangePublicApi(): + change = AddJavaField("frameworks/base/core/java/android/provider/Settings.java", + "@android.annotation.SuppressLint(\"UnflaggedApi\") public") + orig_current_text = Snapshot("frameworks/base/core/api/current.txt") + + def undo(): + change.undo() + orig_current_text.write() + + return Change( + label=change.label, + change=change.change, + undo=lambda: undo() + ) + +def AddJavaField(filename, prefix): + return Modify(filename, + lambda: f"{prefix} static final int BENCHMARK = {random.randint(0, 1000000)};\n", + before="}") + + +def Comment(prefix, suffix=""): + return lambda: prefix + " " + str(uuid.uuid4()) + suffix + + +class BenchmarkReport(): + "Information about a run of the benchmark" + + lunch: Lunch + "lunch combo" + + benchmark: Benchmark + "The benchmark object." + + iteration: int + "Which iteration of the benchmark" + + log_dir: str + "Path the the log directory, relative to the root of the reports directory" + + preroll_duration_ns: [int] + "Durations of the in nanoseconds." + + duration_ns: int + "Duration of the measured portion of the benchmark in nanoseconds." + + postroll_duration_ns: [int] + "Durations of the postrolls in nanoseconds." + + complete: bool + "Whether the benchmark made it all the way through the postrolls." + + def __init__(self, lunch, benchmark, iteration, log_dir): + self.lunch = lunch + self.benchmark = benchmark + self.iteration = iteration + self.log_dir = log_dir + self.preroll_duration_ns = [] + self.duration_ns = -1 + self.postroll_duration_ns = [] + self.complete = False + + def ToDict(self): + return { + "lunch": self.lunch.ToDict(), + "id": self.benchmark.id, + "title": self.benchmark.title, + "modules": self.benchmark.modules, + "dumpvars": self.benchmark.dumpvars, + "change": self.benchmark.change.label, + "iteration": self.iteration, + "log_dir": self.log_dir, + "preroll_duration_ns": self.preroll_duration_ns, + "duration_ns": self.duration_ns, + "postroll_duration_ns": self.postroll_duration_ns, + "complete": self.complete, + } + +class Runner(): + """Runs the benchmarks.""" + + def __init__(self, options): + self._options = options + self._reports = [] + self._complete = False + + def Run(self): + """Run all of the user-selected benchmarks.""" + # Clean out the log dir or create it if necessary + prepare_log_dir(self._options.LogDir()) + + try: + for lunch in self._options.Lunches(): + print(lunch) + for benchmark in self._options.Benchmarks(): + for iteration in range(self._options.Iterations()): + self._run_benchmark(lunch, benchmark, iteration) + self._complete = True + finally: + self._write_summary() + + + def _run_benchmark(self, lunch, benchmark, iteration): + """Run a single benchmark.""" + benchmark_log_subdir = self._benchmark_log_dir(lunch, benchmark, iteration) + benchmark_log_dir = self._options.LogDir().joinpath(benchmark_log_subdir) + + sys.stderr.write(f"STARTING BENCHMARK: {benchmark.id}\n") + sys.stderr.write(f" lunch: {lunch.Combine()}\n") + sys.stderr.write(f" iteration: {iteration}\n") + sys.stderr.write(f" benchmark_log_dir: {benchmark_log_dir}\n") + + report = BenchmarkReport(lunch, benchmark, iteration, benchmark_log_subdir) + self._reports.append(report) + + # Preroll builds + for i in range(benchmark.preroll): + ns = self._run_build(lunch, benchmark_log_dir.joinpath(f"pre_{i}"), benchmark) + report.preroll_duration_ns.append(ns) + + sys.stderr.write(f"PERFORMING CHANGE: {benchmark.change.label}\n") + if not self._options.DryRun(): + benchmark.change.change() + try: + + # Measured build + ns = self._run_build(lunch, benchmark_log_dir.joinpath("measured"), benchmark) + report.duration_ns = ns + + dist_one = self._options.DistOne() + if dist_one: + # If we're disting just one benchmark, save the logs and we can stop here. + self._dist(utils.get_dist_dir(), benchmark.dumpvars) + else: + self._dist(benchmark_log_dir, benchmark.dumpvars, store_metrics_only=True) + # Postroll builds + for i in range(benchmark.postroll): + ns = self._run_build(lunch, benchmark_log_dir.joinpath(f"post_{i}"), + benchmark) + report.postroll_duration_ns.append(ns) + + finally: + # Always undo, even if we crashed or the build failed and we stopped. + sys.stderr.write(f"UNDOING CHANGE: {benchmark.change.label}\n") + if not self._options.DryRun(): + benchmark.change.undo() + + self._write_summary() + sys.stderr.write(f"FINISHED BENCHMARK: {benchmark.id}\n") + + def _benchmark_log_dir(self, lunch, benchmark, iteration): + """Construct the log directory fir a benchmark run.""" + path = f"{lunch.Combine()}/{benchmark.id}" + # Zero pad to the correct length for correct alpha sorting + path += ("/%0" + str(len(str(self._options.Iterations()))) + "d") % iteration + return path + + def _run_build(self, lunch, build_log_dir, benchmark): + """Builds the modules. Saves interesting log files to log_dir. Raises FatalError + if the build fails. + """ + sys.stderr.write(f"STARTING BUILD {benchmark.build_description()}\n") + + before_ns = time.perf_counter_ns() + if not self._options.DryRun(): + cmd = [ + "build/soong/soong_ui.bash", + ] + benchmark.soong_command(self._options.root) + env = dict(os.environ) + env["TARGET_PRODUCT"] = lunch.target_product + env["TARGET_RELEASE"] = lunch.target_release + env["TARGET_BUILD_VARIANT"] = lunch.target_build_variant + returncode = subprocess.call(cmd, env=env) + if returncode != 0: + report_error(f"Build failed: {' '.join(cmd)}") + raise FatalError() + + after_ns = time.perf_counter_ns() + + # TODO: Copy some log files. + + sys.stderr.write(f"FINISHED BUILD {benchmark.build_description()}\n") + + return after_ns - before_ns + + def _dist(self, dist_dir, dumpvars, store_metrics_only=False): + out_dir = utils.get_out_dir() + dest_dir = dist_dir.joinpath("logs") + os.makedirs(dest_dir, exist_ok=True) + basenames = [ + "soong_build_metrics.pb", + "soong_metrics", + ] + if not store_metrics_only: + basenames.extend([ + "build.trace.gz", + "soong.log", + ]) + if dumpvars: + basenames = ['dumpvars-'+b for b in basenames] + for base in basenames: + src = out_dir.joinpath(base) + if src.exists(): + sys.stderr.write(f"DIST: copied {src} to {dest_dir}\n") + shutil.copy(src, dest_dir) + + def _write_summary(self): + # Write the results, even if the build failed or we crashed, including + # whether we finished all of the benchmarks. + data = { + "start_time": self._options.Timestamp().isoformat(), + "branch": self._options.Branch(), + "tag": self._options.Tag(), + "benchmarks": [report.ToDict() for report in self._reports], + "complete": self._complete, + } + with open(self._options.LogDir().joinpath("summary.json"), "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, sort_keys=True) + + +def benchmark_table(benchmarks): + rows = [("ID", "DESCRIPTION", "REBUILD"),] + rows += [(benchmark.id, benchmark.title, benchmark.build_description()) for benchmark in + benchmarks] + return rows + + +def prepare_log_dir(directory): + if os.path.exists(directory): + # If it exists and isn't a directory, fail. + if not os.path.isdir(directory): + report_error(f"Log directory already exists but isn't a directory: {directory}") + raise FatalError() + # Make sure the directory is empty. Do this rather than deleting it to handle + # symlinks cleanly. + for filename in os.listdir(directory): + entry = os.path.join(directory, filename) + if os.path.isdir(entry): + shutil.rmtree(entry) + else: + os.unlink(entry) + else: + # Create it + os.makedirs(directory) + + +class Options(): + def __init__(self): + self._had_error = False + + # Wall time clock when we started + self._timestamp = datetime.datetime.now(datetime.timezone.utc) + + # Move to the root of the tree right away. Everything must happen from there. + self.root = utils.get_root() + if not self.root: + report_error("Unable to find root of tree from cwd.") + raise FatalError() + os.chdir(self.root) + + # Initialize the Benchmarks. Note that this pre-loads all of the files, etc. + # Doing all that here forces us to fail fast if one of them can't load a required + # file, at the cost of a small startup speed. Don't make this do something slow + # like scan the whole tree. + self._init_benchmarks() + + # Argument parsing + epilog = f""" +benchmarks: +{pretty.FormatTable(benchmark_table(self._benchmarks), prefix=" ")} +""" + + parser = argparse.ArgumentParser( + prog="benchmarks", + allow_abbrev=False, # Don't let people write unsupportable scripts. + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=epilog, + description="Run build system performance benchmarks.") + self.parser = parser + + parser.add_argument("--log-dir", + help="Directory for logs. Default is $TOP/../benchmarks/.") + parser.add_argument("--dated-logs", action="store_true", + help="Append timestamp to log dir.") + parser.add_argument("-n", action="store_true", dest="dry_run", + help="Dry run. Don't run the build commands but do everything else.") + parser.add_argument("--tag", + help="Variant of the run, for when there are multiple perf runs.") + parser.add_argument("--lunch", nargs="*", + help="Lunch combos to test") + parser.add_argument("--iterations", type=int, default=1, + help="Number of iterations of each test to run.") + parser.add_argument("--branch", type=str, + help="Specify branch. Otherwise a guess will be made based on repo.") + parser.add_argument("--benchmark", nargs="*", default=[b.id for b in self._benchmarks], + metavar="BENCHMARKS", + help="Benchmarks to run. Default suite will be run if omitted.") + parser.add_argument("--dist-one", action="store_true", + help="Copy logs and metrics to the given dist dir. Requires that only" + + " one benchmark be supplied. Postroll steps will be skipped.") + + self._args = parser.parse_args() + + self._branch = self._branch() + self._log_dir = self._log_dir() + self._lunches = self._lunches() + + # Validate the benchmark ids + all_ids = [benchmark.id for benchmark in self._benchmarks] + bad_ids = [id for id in self._args.benchmark if id not in all_ids] + if bad_ids: + for id in bad_ids: + self._error(f"Invalid benchmark: {id}") + + # --dist-one requires that only one benchmark be supplied + if self._args.dist_one and len(self.Benchmarks()) != 1: + self._error("--dist-one requires that exactly one --benchmark.") + + if self._had_error: + raise FatalError() + + def Timestamp(self): + return self._timestamp + + def _branch(self): + """Return the branch, either from the command line or by guessing from repo.""" + if self._args.branch: + return self._args.branch + try: + branch = subprocess.check_output(f"cd {self.root}/.repo/manifests" + + " && git rev-parse --abbrev-ref --symbolic-full-name @{u}", + shell=True, encoding="utf-8") + return branch.strip().split("/")[-1] + except subprocess.CalledProcessError as ex: + report_error("Can't get branch from .repo dir. Specify --branch argument") + report_error(str(ex)) + raise FatalError() + + def Branch(self): + return self._branch + + def _log_dir(self): + "The log directory to use, based on the current options" + if self._args.log_dir: + d = pathlib.Path(self._args.log_dir).resolve().absolute() + else: + d = self.root.joinpath("..", utils.DEFAULT_REPORT_DIR) + if self._args.dated_logs: + d = d.joinpath(self._timestamp.strftime('%Y-%m-%d')) + d = d.joinpath(self._branch) + if self._args.tag: + d = d.joinpath(self._args.tag) + return d.resolve().absolute() + + def LogDir(self): + return self._log_dir + + def Benchmarks(self): + return [b for b in self._benchmarks if b.id in self._args.benchmark] + + def Tag(self): + return self._args.tag + + def DryRun(self): + return self._args.dry_run + + def _lunches(self): + def parse_lunch(lunch): + parts = lunch.split("-") + if len(parts) != 3: + raise OptionsError(f"Invalid lunch combo: {lunch}") + return Lunch(parts[0], parts[1], parts[2]) + # If they gave lunch targets on the command line use that + if self._args.lunch: + result = [] + # Split into Lunch objects + for lunch in self._args.lunch: + try: + result.append(parse_lunch(lunch)) + except OptionsError as ex: + self._error(ex.message) + return result + # Use whats in the environment + product = os.getenv("TARGET_PRODUCT") + release = os.getenv("TARGET_RELEASE") + variant = os.getenv("TARGET_BUILD_VARIANT") + if (not product) or (not release) or (not variant): + # If they didn't give us anything, fail rather than guessing. There's no good + # default for AOSP. + self._error("No lunch combo specified. Either pass --lunch argument or run lunch.") + return [] + return [Lunch(product, release, variant),] + + def Lunches(self): + return self._lunches + + def Iterations(self): + return self._args.iterations + + def DistOne(self): + return self._args.dist_one + + def _init_benchmarks(self): + """Initialize the list of benchmarks.""" + # Assumes that we've already chdired to the root of the tree. + self._benchmarks = [ + Benchmark( + id="full_lunch", + title="Lunch from clean out", + change=Clean(), + dumpvars=True, + preroll=0, + postroll=0, + ), + Benchmark( + id="noop_lunch", + title="Lunch with no change", + change=NoChange(), + dumpvars=True, + preroll=1, + postroll=0, + ), + Benchmark(id="full", + title="Full build", + change=Clean(), + modules=["droid"], + preroll=0, + postroll=3, + ), + Benchmark(id="nochange", + title="No change", + change=NoChange(), + modules=["droid"], + preroll=2, + postroll=3, + ), + Benchmark(id="unreferenced", + title="Create unreferenced file", + change=Create("bionic/unreferenced.txt"), + modules=["droid"], + preroll=1, + postroll=2, + ), + Benchmark(id="modify_bp", + title="Modify Android.bp", + change=Modify("bionic/libc/Android.bp", Comment("//")), + modules=["droid"], + preroll=1, + postroll=3, + ), + Benchmark(id="modify_stdio", + title="Modify stdio.cpp", + change=Modify("bionic/libc/stdio/stdio.cpp", Comment("//")), + modules=["libc"], + preroll=1, + postroll=2, + ), + Benchmark(id="modify_adbd", + title="Modify adbd", + change=Modify("packages/modules/adb/daemon/main.cpp", Comment("//")), + modules=["adbd"], + preroll=1, + postroll=2, + ), + Benchmark(id="services_private_field", + title="Add private field to ActivityManagerService.java", + change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java", + "private"), + modules=["services"], + preroll=1, + postroll=2, + ), + Benchmark(id="services_public_field", + title="Add public field to ActivityManagerService.java", + change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java", + "/** @hide */ public"), + modules=["services"], + preroll=1, + postroll=2, + ), + Benchmark(id="services_api", + title="Add API to ActivityManagerService.javaa", + change=AddJavaField("frameworks/base/services/core/java/com/android/server/am/ActivityManagerService.java", + "@android.annotation.SuppressLint(\"UnflaggedApi\") public"), + modules=["services"], + preroll=1, + postroll=2, + ), + Benchmark(id="framework_private_field", + title="Add private field to Settings.java", + change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java", + "private"), + modules=["framework-minus-apex"], + preroll=1, + postroll=2, + ), + Benchmark(id="framework_public_field", + title="Add public field to Settings.java", + change=AddJavaField("frameworks/base/core/java/android/provider/Settings.java", + "/** @hide */ public"), + modules=["framework-minus-apex"], + preroll=1, + postroll=2, + ), + Benchmark(id="framework_api", + title="Add API to Settings.java", + change=ChangePublicApi(), + modules=["api-stubs-docs-non-updatable-update-current-api", "framework-minus-apex"], + preroll=1, + postroll=2, + ), + Benchmark(id="modify_framework_resource", + title="Modify framework resource", + change=Modify("frameworks/base/core/res/res/values/config.xml", + lambda: str(uuid.uuid4()), + before="</string>"), + modules=["framework-minus-apex"], + preroll=1, + postroll=2, + ), + Benchmark(id="add_framework_resource", + title="Add framework resource", + change=Modify("frameworks/base/core/res/res/values/config.xml", + lambda: f"<string name=\"BENCHMARK\">{uuid.uuid4()}</string>", + before="</resources>"), + modules=["framework-minus-apex"], + preroll=1, + postroll=2, + ), + Benchmark(id="add_systemui_field", + title="Add SystemUI field", + change=AddJavaField("frameworks/base/packages/SystemUI/src/com/android/systemui/wmshell/WMShell.java", + "public"), + modules=["SystemUI"], + preroll=1, + postroll=2, + ), + ] + + def _error(self, message): + report_error(message) + self._had_error = True + + +def report_error(message): + sys.stderr.write(f"error: {message}\n") + + +def main(argv): + try: + options = Options() + runner = Runner(options) + runner.Run() + except FatalError: + sys.stderr.write(f"FAILED\n") + sys.exit(1) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/tools/perf/format_benchmarks b/tools/perf/format_benchmarks new file mode 100755 index 0000000000..162c5770a9 --- /dev/null +++ b/tools/perf/format_benchmarks @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +if __name__ == "__main__": + sys.dont_write_bytecode = True + +import argparse +import dataclasses +import datetime +import json +import os +import pathlib +import statistics +import zoneinfo + +import pretty +import utils + +# TODO: +# - Flag if the last postroll build was more than 15 seconds or something. That's +# an indicator that something is amiss. +# - Add a mode to print all of the values for multi-iteration runs +# - Add a flag to reorder the tags +# - Add a flag to reorder the headers in order to show grouping more clearly. + + +def FindSummaries(args): + def find_summaries(directory): + return [str(p.resolve()) for p in pathlib.Path(directory).glob("**/summary.json")] + if not args: + # If they didn't give an argument, use the default dir + root = utils.get_root() + if not root: + return [] + return find_summaries(root.joinpath("..", utils.DEFAULT_REPORT_DIR)) + results = list() + for arg in args: + if os.path.isfile(arg): + # If it's a file add that + results.append(arg) + elif os.path.isdir(arg): + # If it's a directory, find all of the files there + results += find_summaries(arg) + else: + sys.stderr.write(f"Invalid summary argument: {arg}\n") + sys.exit(1) + return sorted(list(results)) + + +def LoadSummary(filename): + with open(filename) as f: + return json.load(f) + +# Columns: +# Date +# Branch +# Tag +# -- +# Lunch +# Rows: +# Benchmark + +def lunch_str(d): + "Convert a lunch dict to a string" + return f"{d['TARGET_PRODUCT']}-{d['TARGET_RELEASE']}-{d['TARGET_BUILD_VARIANT']}" + +def group_by(l, key): + "Return a list of tuples, grouped by key, sorted by key" + result = {} + for item in l: + result.setdefault(key(item), []).append(item) + return [(k, v) for k, v in result.items()] + + +class Table: + def __init__(self, row_title, fixed_titles=[]): + self._data = {} + self._rows = [] + self._cols = [] + self._fixed_cols = {} + self._titles = [row_title] + fixed_titles + + def Set(self, column_key, row_key, data): + self._data[(column_key, row_key)] = data + if not column_key in self._cols: + self._cols.append(column_key) + if not row_key in self._rows: + self._rows.append(row_key) + + def SetFixedCol(self, row_key, columns): + self._fixed_cols[row_key] = columns + + def Write(self, out): + table = [] + # Expand the column items + for row in zip(*self._cols): + if row.count(row[0]) == len(row): + continue + table.append([""] * len(self._titles) + [col for col in row]) + if table: + # Update the last row of the header with title and add separator + for i in range(len(self._titles)): + table[len(table)-1][i] = self._titles[i] + table.append(pretty.SEPARATOR) + # Populate the data + for row in self._rows: + table.append([str(row)] + + self._fixed_cols[row] + + [str(self._data.get((col, row), "")) for col in self._cols]) + out.write(pretty.FormatTable(table, alignments="LL")) + + +def format_duration_sec(ns): + "Format a duration in ns to second precision" + sec = round(ns / 1000000000) + h, sec = divmod(sec, 60*60) + m, sec = divmod(sec, 60) + result = "" + if h > 0: + result += f"{h:2d}h " + if h > 0 or m > 0: + result += f"{m:2d}m " + return result + f"{sec:2d}s" + + +def main(argv): + parser = argparse.ArgumentParser( + prog="format_benchmarks", + allow_abbrev=False, # Don't let people write unsupportable scripts. + description="Print analysis tables for benchmarks") + + parser.add_argument("--tags", nargs="*", + help="The tags to print, in order.") + + parser.add_argument("summaries", nargs="*", + help="A summary.json file or a directory in which to look for summaries.") + + args = parser.parse_args() + + # Load the summaries + summaries = [(s, LoadSummary(s)) for s in FindSummaries(args.summaries)] + + # Convert to MTV time + for filename, s in summaries: + dt = datetime.datetime.fromisoformat(s["start_time"]) + dt = dt.astimezone(zoneinfo.ZoneInfo("America/Los_Angeles")) + s["datetime"] = dt + s["date"] = datetime.date(dt.year, dt.month, dt.day) + + # Filter out tags we don't want + if args.tags: + summaries = [(f, s) for f, s in summaries if s.get("tag", "") in args.tags] + + # If they supplied tags, sort in that order, otherwise sort by tag + if args.tags: + tagsort = lambda tag: args.tags.index(tag) + else: + tagsort = lambda tag: tag + + # Sort the summaries + summaries.sort(key=lambda s: (s[1]["date"], s[1]["branch"], tagsort(s[1]["tag"]))) + + # group the benchmarks by column and iteration + def bm_key(b): + return ( + lunch_str(b["lunch"]), + ) + for filename, summary in summaries: + summary["columns"] = [(key, group_by(bms, lambda b: b["id"])) for key, bms + in group_by(summary["benchmarks"], bm_key)] + + # Build the table + table = Table("Benchmark", ["Rebuild"]) + for filename, summary in summaries: + for key, column in summary["columns"]: + for id, cell in column: + duration_ns = statistics.median([b["duration_ns"] for b in cell]) + table.SetFixedCol(cell[0]["title"], [" ".join(cell[0]["modules"])]) + table.Set(tuple([summary["date"].strftime("%Y-%m-%d"), + summary["branch"], + summary["tag"]] + + list(key)), + cell[0]["title"], format_duration_sec(duration_ns)) + + table.Write(sys.stdout) + +if __name__ == "__main__": + main(sys.argv) + diff --git a/tools/perf/pretty.py b/tools/perf/pretty.py new file mode 100644 index 0000000000..14fdc9ed8d --- /dev/null +++ b/tools/perf/pretty.py @@ -0,0 +1,52 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Formatting utilities + +class Sentinel(): + pass + +SEPARATOR = Sentinel() + +def FormatTable(data, prefix="", alignments=[]): + """Pretty print a table. + + Prefixes each row with `prefix`. + """ + if not data: + return "" + widths = [max([len(x) if x else 0 for x in col]) for col + in zip(*[d for d in data if not isinstance(d, Sentinel)])] + result = "" + colsep = " " + for row in data: + result += prefix + if row == SEPARATOR: + for w in widths: + result += "-" * w + result += colsep + result += "\n" + else: + for i in range(len(row)): + cell = row[i] if row[i] else "" + if i >= len(alignments) or alignments[i] == "R": + result += " " * (widths[i] - len(cell)) + result += cell + if i < len(alignments) and alignments[i] == "L": + result += " " * (widths[i] - len(cell)) + result += colsep + result += "\n" + return result + + diff --git a/tools/perf/utils.py b/tools/perf/utils.py new file mode 100644 index 0000000000..0e66d4cd24 --- /dev/null +++ b/tools/perf/utils.py @@ -0,0 +1,44 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib + +DEFAULT_REPORT_DIR = "benchmarks" + +def get_root(): + top_dir = os.environ.get("ANDROID_BUILD_TOP") + d = pathlib.Path.cwd() + # with cog, someone may have a new workspace and new source tree top, but + # not run lunch yet, resulting in a misleading ANDROID_BUILD_TOP value + if top_dir and d.is_relative_to(top_dir): + return pathlib.Path(top_dir).resolve() + while True: + if d.joinpath("build", "soong", "soong_ui.bash").exists(): + return d.resolve().absolute() + d = d.parent + if d == pathlib.Path("/"): + return None + +def get_dist_dir(): + dist_dir = os.getenv("DIST_DIR") + if dist_dir: + return pathlib.Path(dist_dir).resolve() + return get_out_dir().joinpath("dist") + +def get_out_dir(): + out_dir = os.getenv("OUT_DIR") + if not out_dir: + out_dir = "out" + return pathlib.Path(out_dir).resolve() diff --git a/tools/post_process_props.py b/tools/post_process_props.py index 38d17a81db..6f429fa23e 100755 --- a/tools/post_process_props.py +++ b/tools/post_process_props.py @@ -17,6 +17,8 @@ import argparse import sys +from uffd_gc_utils import should_enable_uffd_gc + # Usage: post_process_props.py file.prop [disallowed_key, ...] # Disallowed keys are removed from the property file, if present @@ -27,7 +29,7 @@ PROP_VALUE_MAX = 91 # Put the modifications that you need to make into the */build.prop into this # function. -def mangle_build_prop(prop_list): +def mangle_build_prop(prop_list, kernel_version_file_for_uffd_gc): # If ro.debuggable is 1, then enable adb on USB by default # (this is for userdebug builds) if prop_list.get_value("ro.debuggable") == "1": @@ -38,13 +40,16 @@ def mangle_build_prop(prop_list): else: val = val + ",adb" prop_list.put("persist.sys.usb.config", val) + if prop_list.get_value("ro.dalvik.vm.enable_uffd_gc") == "default": + assert kernel_version_file_for_uffd_gc != "" + enable_uffd_gc = should_enable_uffd_gc(kernel_version_file_for_uffd_gc) + prop_list.put("ro.dalvik.vm.enable_uffd_gc", + "true" if enable_uffd_gc else "false") -def validate_grf_props(prop_list, sdk_version): +def validate_grf_props(prop_list): """Validate GRF properties if exist. - If ro.board.first_api_level is defined, check if its value is valid for the - sdk version. - Also, validate the value of ro.board.api_level if defined. + If ro.board.first_api_level is defined, check if its value is valid. Returns: True if the GRF properties are valid. @@ -52,28 +57,13 @@ def validate_grf_props(prop_list, sdk_version): grf_api_level = prop_list.get_value("ro.board.first_api_level") board_api_level = prop_list.get_value("ro.board.api_level") - if not grf_api_level: - if board_api_level: - sys.stderr.write("error: non-GRF device must not define " - "ro.board.api_level\n") - return False - # non-GRF device skips the GRF validation test - return True - - grf_api_level = int(grf_api_level) - if grf_api_level > sdk_version: - sys.stderr.write("error: ro.board.first_api_level(%d) must be less than " - "or equal to ro.build.version.sdk(%d)\n" - % (grf_api_level, sdk_version)) - return False - - if board_api_level: + if grf_api_level and board_api_level: + grf_api_level = int(grf_api_level) board_api_level = int(board_api_level) - if board_api_level < grf_api_level or board_api_level > sdk_version: - sys.stderr.write("error: ro.board.api_level(%d) must be neither less " - "than ro.board.first_api_level(%d) nor greater than " - "ro.build.version.sdk(%d)\n" - % (board_api_level, grf_api_level, sdk_version)) + if board_api_level < grf_api_level: + sys.stderr.write("error: ro.board.api_level(%d) must not be less than " + "ro.board.first_api_level(%d)\n" + % (board_api_level, grf_api_level)) return False return True @@ -250,6 +240,7 @@ def main(argv): parser.add_argument("filename") parser.add_argument("disallowed_keys", metavar="KEY", type=str, nargs="*") parser.add_argument("--sdk-version", type=int, required=True) + parser.add_argument("--kernel-version-file-for-uffd-gc", required=True) args = parser.parse_args() if not args.filename.endswith("/build.prop"): @@ -257,10 +248,10 @@ def main(argv): sys.exit(1) props = PropList(args.filename) - mangle_build_prop(props) + mangle_build_prop(props, args.kernel_version_file_for_uffd_gc) if not override_optional_props(props, args.allow_dup): sys.exit(1) - if not validate_grf_props(props, args.sdk_version): + if not validate_grf_props(props): sys.exit(1) if not validate(props): sys.exit(1) diff --git a/tools/protos/Android.bp b/tools/protos/Android.bp new file mode 100644 index 0000000000..c6ad19e644 --- /dev/null +++ b/tools/protos/Android.bp @@ -0,0 +1,32 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +python_library_host { + name: "metadata_file_proto_py", + version: { + py3: { + enabled: true, + }, + }, + srcs: [ + "metadata_file.proto", + ], + proto: { + canonical_path_from_root: false, + }, +} diff --git a/tools/protos/metadata_file.proto b/tools/protos/metadata_file.proto new file mode 100644 index 0000000000..5c8961812c --- /dev/null +++ b/tools/protos/metadata_file.proto @@ -0,0 +1,427 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package metadata_file; + +// Proto definition of METADATA files of packages in AOSP codebase. +message Metadata { + // Name of the package. + optional string name = 1; + + // A short description (a few lines) of the package. + // Example: "Handles location lookups, throttling, batching, etc." + optional string description = 2; + + // Specifies additional data about third-party packages. + optional ThirdParty third_party = 3; +} + +message ThirdParty { + // URL(s) associated with the package. + // + // At a minimum, all packages must specify a URL which identifies where it + // came from, containing a type of: ARCHIVE, GIT or OTHER. Typically, + // a package should contain only a single URL from these types. Occasionally, + // a package may be broken across multiple archive files for whatever reason, + // in which case having multiple ARCHIVE URLs is okay. However, this should + // not be used to combine different logical packages that are versioned and + // possibly licensed differently. + repeated URL url = 1; + + // The package version. In order of preference, this should contain: + // - If the package comes from Git or another source control system, + // a specific tag or revision in source control, such as "r123" or + // "58e27d2". This MUST NOT be a mutable ref such as a branch name. + // - a released package version such as "1.0", "2.3-beta", etc. + // - the date the package was retrieved, formatted as "As of YYYY-MM-DD". + optional string version = 2; + + // The date of the change in which the package was last upgraded from + // upstream. + // This should only identify package upgrades from upstream, not local + // modifications. This may identify the date of either the original or + // merged change. + // + // Note: this is NOT the date that this version of the package was released + // externally. + optional Date last_upgrade_date = 3; + + // License type that identifies how the package may be used. + optional LicenseType license_type = 4; + + // An additional note explaining the licensing of this package. This is most + // commonly used with commercial license. + optional string license_note = 5; + + // Description of local changes that have been made to the package. This does + // not need to (and in most cases should not) attempt to include an exhaustive + // list of all changes, but may instead direct readers to review the local + // commit history, a collection of patch files, a separate README.md (or + // similar) document, etc. + // Note: Use of this field to store IDs of advisories fixed with a backported + // patch is deprecated, use "security.mitigated_security_patch" instead. + optional string local_modifications = 6; + + // Security related metadata including risk category and any special + // instructions for using the package, as determined by an ISE-TPS review. + optional Security security = 7; + + // The type of directory this metadata represents. + optional DirectoryType type = 8 [default = PACKAGE]; + + // The homepage for the package. This will eventually replace + // `url { type: HOMEPAGE }` + optional string homepage = 9; + + // SBOM information of the package. It is mandatory for prebuilt packages. + oneof sbom { + // Reference to external SBOM document provided as URL. + SBOMRef sbom_ref = 10; + } + + // Identifiers for the package. + repeated Identifier identifier = 11; +} + +// URL associated with a third-party package. +message URL { + enum Type { + // The homepage for the package. For example, "https://bazel.io/". This URL + // is optional, but encouraged to help disambiguate similarly named packages + // or to get more information about the package. This is especially helpful + // when no other URLs provide human readable resources (such as git:// or + // sso:// URLs). + HOMEPAGE = 1; + + // The URL of the archive containing the source code for the package, for + // example a zip or tgz file. + ARCHIVE = 2; + + // The URL of the upstream git repository this package is retrieved from. + // For example: + // - https://github.com/git/git.git + // - git://git.kernel.org/pub/scm/git/git.git + // + // Use of a git URL requires that the package "version" value must specify a + // specific git tag or revision. + GIT = 3; + + // The URL of the upstream SVN repository this package is retrieved from. + // For example: + // - http://llvm.org/svn/llvm-project/llvm/ + // + // Use of an SVN URL requires that the package "version" value must specify + // a specific SVN tag or revision. + SVN = 4; + + // The URL of the upstream mercurial repository this package is retrieved + // from. For example: + // - https://mercurial-scm.org/repo/evolve + // + // Use of a mercurial URL requires that the package "version" value must + // specify a specific tag or revision. + HG = 5; + + // The URL of the upstream darcs repository this package is retrieved + // from. For example: + // - https://hub.darcs.net/hu.dwim/hu.dwim.util + // + // Use of a DARCS URL requires that the package "version" value must + // specify a specific tag or revision. + DARCS = 6; + + PIPER = 7; + + // A URL that does not fit any other type. This may also indicate that the + // source code was received via email or some other out-of-band way. This is + // most commonly used with commercial software received directly from the + // vendor. In the case of email, the URL value can be used to provide + // additional information about how it was received. + OTHER = 8; + + // The URL identifying where the local copy of the package source code can + // be found. + // + // Typically, the metadata files describing a package reside in the same + // directory as the source code for the package. In a few rare cases where + // they are separate, the LOCAL_SOURCE URL identifies where to find the + // source code. This only describes where to find the local copy of the + // source; there should always be an additional URL describing where the + // package was retrieved from. + // + // Examples: + // - https://android.googlesource.com/platform/external/apache-http/ + LOCAL_SOURCE = 9; + } + + // The type of resource this URL identifies. + optional Type type = 1; + + // The actual URL value. URLs should be absolute and start with 'http://' or + // 'https://' (or occasionally 'git://' or 'ftp://' where appropriate). + optional string value = 2; +} + +// License type that identifies how the packages may be used. +enum LicenseType { + BY_EXCEPTION_ONLY = 1; + NOTICE = 2; + PERMISSIVE = 3; + RECIPROCAL = 4; + RESTRICTED_IF_STATICALLY_LINKED = 5; + RESTRICTED = 6; + UNENCUMBERED = 7; +} + +// Identifies security related metadata including risk category and any special +// instructions for using the package. +message Security { + // Security risk category for a package, as determined by an ISE-TPS review. + enum Category { + CATEGORY_UNSPECIFIED = 0; + + // Package should only be used in a sandboxed environment. + // Package should have restricted visibility. + SANDBOXED_ONLY = 1; + + // Package should not be used to process user content. It is considered + // safe to use to process trusted data only. Package should have restricted + // visibility. + TRUSTED_DATA_ONLY = 2; + + // Package is considered safe to use. + REVIEWED_AND_SECURE = 3; + } + + // Identifies the security risk category for the package. This will be + // provided by the ISE-TPS team as the result of a security review of the + // package. + optional Category category = 1; + + // An additional security note for the package. + optional string note = 2; + + // Text tag to categorize the package. It's currently used by security to: + // - to disable OSV (https://osv.dev) + // support via the `OSV:disable` tag + // - to attach CPE to their corresponding packages, for vulnerability + // monitoring: + // + // Please do document your usecase here should you want to add one. + repeated string tag = 3; + + // ID of advisories fixed with a mitigated patch, for example CVE-2018-1111. + repeated string mitigated_security_patch = 4; +} + +enum DirectoryType { + UNDEFINED = 0; + + // This directory represents a package. + PACKAGE = 1; + + // This directory is designed to organize multiple third-party PACKAGE + // directories. + GROUP = 2; + + // This directory contains several PACKAGE directories representing + // different versions of the same third-party project. + VERSIONS = 3; +} + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values. +// * A month and day, with a zero year (for example, an anniversary). +// * A year on its own, with a zero month and a zero day. +// * A year and month, with a zero day (for example, a credit card expiration +// date). +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + optional int32 year = 1; + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + optional int32 month = 2; + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + optional int32 day = 3; +} + +// Reference to external SBOM document and element corresponding to the package. +// See https://spdx.github.io/spdx-spec/v2.3/document-creation-information/#66-external-document-references-field +message SBOMRef { + // The URL that points to the SBOM document of the upstream package of this + // third_party package. + optional string url = 1; + // Checksum of the SBOM document the url field points to. + // Format: e.g. SHA1:<checksum>, or any algorithm defined in + // https://spdx.github.io/spdx-spec/v2.3/file-information/#8.4 + optional string checksum = 2; + // SPDXID of the upstream package/file defined in the SBOM document the url field points to. + // Format: SPDXRef-[a-zA-Z0-9.-]+, see + // https://spdx.github.io/spdx-spec/v2.3/package-information/#72-package-spdx-identifier-field or + // https://spdx.github.io/spdx-spec/v2.3/file-information/#82-file-spdx-identifier-field + optional string element_id = 3; +} + +// Identifier for a third-party package. +// See go/tp-metadata-id. +message Identifier { + // The type of the identifier. Either an "ecosystem" value from + // https://ossf.github.io/osv-schema/#affectedpackage-field such as "Go", + // "npm" or "PyPI". The "value" and "version" fields follow the same rules as + // defined in the OSV spec. + + // Or one of: + // - "Git": The "value" field is the URL of the upstream git repository this + // package is retrieved from. + // For example: + // - https://github.com/git/git + // - git://git.kernel.org/pub/scm/git/git + // + // Use of a git URL requires that the package "version" value must specify a + // specific git tag or revision. This must not be a branch name. + // + // - "SVN": The "value" field is the URL of the upstream SVN repository this + // package is retrieved from. + // For example: + // - http://llvm.org/svn/llvm-project/llvm/ + // + // Use of an SVN URL requires that the package "version" value must specify + // a specific SVN tag or revision. This must not be a branch name. + // + // - "Hg": The "value" field is the URL of the upstream mercurial repository + // this package is retrieved from. + // For example: + // - https://mercurial-scm.org/repo/evolve + // + // Use of a mercurial URL requires that the package "version" value must + // specify a specific tag or revision. This must not be a branch name. + // + // - "Darcs": the "value" field is the URL of the upstream darcs repository + // this package is retrieved from. + // For example: + // - https://hub.darcs.net/hu.dwim/hu.dwim.util + // + // Use of a Darcs URL requires that the package "version" value must + // specify a specific tag or revision. This must not be a branch name. + // + // - "Piper": The "value" field is the URL of the upstream piper location. + // This is primarily used when a package is being migrated into third_party + // from elsewhere in Piper, or when a package is being newly developed in + // third_party. + // + // - "VCS": This is a generic fallback for an unlisted VCS system. The + // "value" field is the URL of the repository for this VCS. + // + // - "Archive": The "value" field is the URL of the archive containing the + // source code for the package, for example a zip or tgz file. + // + // - "PrebuiltByAlphabet": This type should be used for archives of primarily + // Google-owned source code (may contain non-Google-owned dependencies), + // which has been built using production Google infrastructure, and copied + // into Android. The "value" field is the URL of the prebuilt artifact or + // the relative path of the artifact to the root of a package. + // Example: + // identifier { + // type: "PrebuiltByAlphabet", + // version: "1", + // value: "v1/arm84_hdpi.apk", + // } + // identifier { + // type: "PrebuiltByAlphabet", + // version: "2", + // value: "v2/x86_64_xhdpi.apk", + // } + // + // - "LocalSource": The "value" field is the URL identifying where the local + // copy of the package source code can be found. + // Examples: + // - https://android.googlesource.com/platform/external/apache-http/ + // + // Typically, the metadata files describing a package reside in the same + // directory as the source code for the package. In a few rare cases where + // they are separate, the LocalSource URL identifies where to find the + // source code. This only describes where to find the local copy of the + // source; there should always be an additional URL describing where the + // package was retrieved from. + // + // - "Other": An identifier that does not fit any other type. This may also + // indicate that the Source code was received via email or some other + // out-of-band way. This is most commonly used with commercial software + // received directly from the Vendor. In the case of email, the "value" field + // can be used to provide additional information about how it was received. + optional string type = 1; + + // A human readable string to indicate why a third-package package does not + // have this identifier type set. + // Example: + // identifier { + // type: "PyPI" + // omission_reason: "Only on Git. Not published to PyPI." + // } + optional string omission_reason = 2; + + // The value of the package identifier as defined by the "type". + // Example: + // identifier { + // type: "PyPI" + // value: "django" + // version: "3.2.8" + // } + optional string value = 3; + + // The version associated with this package as defined by the "type". + // Example: + // identifier { + // type: "PyPI" + // value: "django" + // version: "3.2.8" + // } + optional string version = 4; + + // The closest version associated with this package as defined by the "type". + // This should only be set by automated infrastructure by applying automated + // heuristics, such as the closest git tag or package version from a package + // manifest file (e.g. pom.xml). + // + // For most identifier types, only one of `version` or `closest_version` + // should be set (not both). The exception is source repository types such as + // "Git", where `version` will refer to a git commit, and `closest_version` + // refers to a git tag. + // Example: + // identifier { + // type: "Git", + // value: "https://github.com/my/repo" + // version: "e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" + // closest_version: "v1.4" + // } + optional string closest_version = 5; + + // When `true`, this Identifier represents the location from which the source + // code for this package was originally obtained. This should only be set for + // *one* Identifier in a third_party package's METADATA. + + // For external packages, this is typically for the Identifier associated + // with the version control system or package manager that was used to + // check out or download the code. + optional bool primary_source = 6; +}
\ No newline at end of file diff --git a/tools/rbcrun/Android.bp b/tools/rbcrun/Android.bp index 90173ac193..4fab858ab4 100644 --- a/tools/rbcrun/Android.bp +++ b/tools/rbcrun/Android.bp @@ -19,7 +19,7 @@ package { blueprint_go_binary { name: "rbcrun", - srcs: ["cmd/rbcrun.go"], + srcs: ["rbcrun/rbcrun.go"], deps: ["rbcrun-module"], } @@ -34,6 +34,7 @@ bootstrap_go_package { pkgPath: "rbcrun", deps: [ "go-starlark-starlark", + "go-starlark-starlarkjson", "go-starlark-starlarkstruct", "go-starlark-starlarktest", ], diff --git a/tools/rbcrun/cmd/rbcrun.go b/tools/rbcrun/cmd/rbcrun.go deleted file mode 100644 index 4db6a0bf38..0000000000 --- a/tools/rbcrun/cmd/rbcrun.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "go.starlark.net/starlark" - "os" - "rbcrun" - "strings" -) - -var ( - execprog = flag.String("c", "", "execute program `prog`") - rootdir = flag.String("d", ".", "the value of // for load paths") - file = flag.String("f", "", "file to execute") - perfFile = flag.String("perf", "", "save performance data") -) - -func main() { - flag.Parse() - filename := *file - var src interface{} - var env []string - - rc := 0 - for _, arg := range flag.Args() { - if strings.Contains(arg, "=") { - env = append(env, arg) - } else if filename == "" { - filename = arg - } else { - quit("only one file can be executed\n") - } - } - if *execprog != "" { - if filename != "" { - quit("either -c or file name should be present\n") - } - filename = "<cmdline>" - src = *execprog - } - if filename == "" { - if len(env) > 0 { - fmt.Fprintln(os.Stderr, - "no file to run -- if your file's name contains '=', use -f to specify it") - } - flag.Usage() - os.Exit(1) - } - if stat, err := os.Stat(*rootdir); os.IsNotExist(err) || !stat.IsDir() { - quit("%s is not a directory\n", *rootdir) - } - if *perfFile != "" { - pprof, err := os.Create(*perfFile) - if err != nil { - quit("%s: err", *perfFile) - } - defer pprof.Close() - if err := starlark.StartProfile(pprof); err != nil { - quit("%s\n", err) - } - } - rbcrun.LoadPathRoot = *rootdir - err := rbcrun.Run(filename, src, env) - if *perfFile != "" { - if err2 := starlark.StopProfile(); err2 != nil { - fmt.Fprintln(os.Stderr, err2) - rc = 1 - } - } - if err != nil { - if evalErr, ok := err.(*starlark.EvalError); ok { - quit("%s\n", evalErr.Backtrace()) - } else { - quit("%s\n", err) - } - } - os.Exit(rc) -} - -func quit(format string, s ...interface{}) { - fmt.Fprintf(os.Stderr, format, s...) - os.Exit(2) -} diff --git a/tools/rbcrun/go.mod b/tools/rbcrun/go.mod index a029eb4ec2..6e99ce975c 100644 --- a/tools/rbcrun/go.mod +++ b/tools/rbcrun/go.mod @@ -1,10 +1,7 @@ module rbcrun -require ( - github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect - go.starlark.net v0.0.0-20201006213952-227f4aabceb5 -) +require go.starlark.net v0.0.0-20201006213952-227f4aabceb5 replace go.starlark.net => ../../../../external/starlark-go -go 1.15 +go 1.21 diff --git a/tools/rbcrun/go.sum b/tools/rbcrun/go.sum index db4d51e9dd..10761a86e6 100644 --- a/tools/rbcrun/go.sum +++ b/tools/rbcrun/go.sum @@ -1,11 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -26,8 +23,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -44,9 +39,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/tools/rbcrun/host.go b/tools/rbcrun/host.go index 32afa458b9..8cd2845fae 100644 --- a/tools/rbcrun/host.go +++ b/tools/rbcrun/host.go @@ -27,10 +27,16 @@ import ( "go.starlark.net/starlarkstruct" ) -const callerDirKey = "callerDir" +type ExecutionMode int +const ( + ExecutionModeRbc ExecutionMode = iota + ExecutionModeScl ExecutionMode = iota +) -var LoadPathRoot = "." -var shellPath string +const allowExternalEntrypointKey = "allowExternalEntrypoint" +const callingFileKey = "callingFile" +const executionModeKey = "executionMode" +const shellKey = "shell" type modentry struct { globals starlark.StringDict @@ -39,20 +45,75 @@ type modentry struct { var moduleCache = make(map[string]*modentry) -var builtins starlark.StringDict +var rbcBuiltins starlark.StringDict = starlark.StringDict{ + "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), + // To convert find-copy-subdir and product-copy-files-by pattern + "rblf_find_files": starlark.NewBuiltin("rblf_find_files", find), + // To convert makefile's $(shell cmd) + "rblf_shell": starlark.NewBuiltin("rblf_shell", shell), + // Output to stderr + "rblf_log": starlark.NewBuiltin("rblf_log", log), + // To convert makefile's $(wildcard foo*) + "rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard), +} + +var sclBuiltins starlark.StringDict = starlark.StringDict{ + "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), +} + +func isSymlink(filepath string) (bool, error) { + if info, err := os.Lstat(filepath); err == nil { + return info.Mode() & os.ModeSymlink != 0, nil + } else { + return false, err + } +} + +// Takes a module name (the first argument to the load() function) and returns the path +// it's trying to load, stripping out leading //, and handling leading :s. +func cleanModuleName(moduleName string, callerDir string, allowExternalPaths bool) (string, error) { + if strings.Count(moduleName, ":") > 1 { + return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName) + } -func moduleName2AbsPath(moduleName string, callerDir string) (string, error) { - path := moduleName - if ix := strings.LastIndex(path, ":"); ix >= 0 { - path = path[0:ix] + string(os.PathSeparator) + path[ix+1:] + // We don't have full support for external repositories, but at least support skylib's dicts. + if moduleName == "@bazel_skylib//lib:dicts.bzl" { + return "external/bazel-skylib/lib/dicts.bzl", nil } - if strings.HasPrefix(path, "//") { - return filepath.Abs(filepath.Join(LoadPathRoot, path[2:])) + + localLoad := false + if strings.HasPrefix(moduleName, "@//") { + moduleName = moduleName[3:] + } else if strings.HasPrefix(moduleName, "//") { + moduleName = moduleName[2:] } else if strings.HasPrefix(moduleName, ":") { - return filepath.Abs(filepath.Join(callerDir, path[1:])) - } else { - return filepath.Abs(path) + moduleName = moduleName[1:] + localLoad = true + } else if !allowExternalPaths { + return "", fmt.Errorf("load path must start with // or :") + } + + if ix := strings.LastIndex(moduleName, ":"); ix >= 0 { + moduleName = moduleName[:ix] + string(os.PathSeparator) + moduleName[ix+1:] + } + + if filepath.Clean(moduleName) != moduleName { + return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName)) } + if !allowExternalPaths { + if strings.HasPrefix(moduleName, "../") { + return "", fmt.Errorf("load path must not start with ../: %s", moduleName) + } + if strings.HasPrefix(moduleName, "/") { + return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName) + } + } + + if localLoad { + return filepath.Join(callerDir, moduleName), nil + } + + return moduleName, nil } // loader implements load statement. The format of the loaded module URI is @@ -61,14 +122,20 @@ func moduleName2AbsPath(moduleName string, callerDir string) (string, error) { // The presence of `|symbol` indicates that the loader should return a single 'symbol' // bound to None if file is missing. func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) { - pipePos := strings.LastIndex(module, "|") - mustLoad := pipePos < 0 + mode := thread.Local(executionModeKey).(ExecutionMode) + allowExternalEntrypoint := thread.Local(allowExternalEntrypointKey).(bool) var defaultSymbol string - if !mustLoad { - defaultSymbol = module[pipePos+1:] - module = module[:pipePos] + mustLoad := true + if mode == ExecutionModeRbc { + pipePos := strings.LastIndex(module, "|") + if pipePos >= 0 { + mustLoad = false + defaultSymbol = module[pipePos+1:] + module = module[:pipePos] + } } - modulePath, err := moduleName2AbsPath(module, thread.Local(callerDirKey).(string)) + callingFile := thread.Local(callingFileKey).(string) + modulePath, err := cleanModuleName(module, filepath.Dir(callingFile), allowExternalEntrypoint) if err != nil { return nil, err } @@ -90,6 +157,20 @@ func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) // Load or return default if mustLoad { + if strings.HasSuffix(callingFile, ".scl") && !strings.HasSuffix(modulePath, ".scl") { + return nil, fmt.Errorf(".scl files can only load other .scl files: %q loads %q", callingFile, modulePath) + } + // Switch into scl mode from here on + if strings.HasSuffix(modulePath, ".scl") { + mode = ExecutionModeScl + } + + if sym, err := isSymlink(modulePath); sym && err == nil { + return nil, fmt.Errorf("symlinks to starlark files are not allowed. Instead, load the target file and re-export its symbols: %s", modulePath) + } else if err != nil { + return nil, err + } + childThread := &starlark.Thread{Name: "exec " + module, Load: thread.Load} // Cheating for the sake of testing: // propagate starlarktest's Reporter key, otherwise testing @@ -99,9 +180,20 @@ func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) childThread.SetLocal(testReporterKey, v) } - childThread.SetLocal(callerDirKey, filepath.Dir(modulePath)) - globals, err := starlark.ExecFile(childThread, modulePath, nil, builtins) - e = &modentry{globals, err} + // Only the entrypoint starlark file allows external loads. + childThread.SetLocal(allowExternalEntrypointKey, false) + childThread.SetLocal(callingFileKey, modulePath) + childThread.SetLocal(executionModeKey, mode) + childThread.SetLocal(shellKey, thread.Local(shellKey)) + if mode == ExecutionModeRbc { + globals, err := starlark.ExecFile(childThread, modulePath, nil, rbcBuiltins) + e = &modentry{globals, err} + } else if mode == ExecutionModeScl { + globals, err := starlark.ExecFile(childThread, modulePath, nil, sclBuiltins) + e = &modentry{globals, err} + } else { + return nil, fmt.Errorf("unknown executionMode %d", mode) + } } else { e = &modentry{starlark.StringDict{defaultSymbol: starlark.None}, nil} } @@ -189,12 +281,13 @@ func find(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, // its output the same way as Make's $(shell ) function. The end-of-lines // ("\n" or "\r\n") are replaced with " " in the result, and the trailing // end-of-line is removed. -func shell(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, +func shell(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var command string if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &command); err != nil { return starlark.None, err } + shellPath := thread.Local(shellKey).(string) if shellPath == "" { return starlark.None, fmt.Errorf("cannot run shell, /bin/sh is missing (running on Windows?)") @@ -223,16 +316,6 @@ func makeStringList(items []string) *starlark.List { return starlark.NewList(elems) } -// propsetFromEnv constructs a propset from the array of KEY=value strings -func structFromEnv(env []string) *starlarkstruct.Struct { - sd := make(map[string]starlark.Value, len(env)) - for _, x := range env { - kv := strings.SplitN(x, "=", 2) - sd[kv[0]] = starlark.String(kv[1]) - } - return starlarkstruct.FromStringDict(starlarkstruct.Default, sd) -} - func log(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { sep := " " if err := starlark.UnpackArgs("print", nil, kwargs, "sep?", &sep); err != nil { @@ -255,50 +338,79 @@ func log(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwa return starlark.None, nil } -func setup(env []string) { - // Create the symbols that aid makefile conversion. See README.md - builtins = starlark.StringDict{ - "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), - "rblf_cli": structFromEnv(env), - "rblf_env": structFromEnv(os.Environ()), - // To convert find-copy-subdir and product-copy-files-by pattern - "rblf_find_files": starlark.NewBuiltin("rblf_find_files", find), - // To convert makefile's $(shell cmd) - "rblf_shell": starlark.NewBuiltin("rblf_shell", shell), - // Output to stderr - "rblf_log": starlark.NewBuiltin("rblf_log", log), - // To convert makefile's $(wildcard foo*) - "rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard), - } - - // NOTE(asmundak): OS-specific. Behave similar to Linux `system` call, - // which always uses /bin/sh to run the command - shellPath = "/bin/sh" - if _, err := os.Stat(shellPath); err != nil { - shellPath = "" - } -} - // Parses, resolves, and executes a Starlark file. // filename and src parameters are as for starlark.ExecFile: // * filename is the name of the file to execute, // and the name that appears in error messages; // * src is an optional source of bytes to use instead of filename // (it can be a string, or a byte array, or an io.Reader instance) -// * commandVars is an array of "VAR=value" items. They are accessible from -// the starlark script as members of the `rblf_cli` propset. -func Run(filename string, src interface{}, commandVars []string) error { - setup(commandVars) +// Returns the top-level starlark variables, the list of starlark files loaded, and an error +func Run(filename string, src interface{}, mode ExecutionMode, allowExternalEntrypoint bool) (starlark.StringDict, []string, error) { + // NOTE(asmundak): OS-specific. Behave similar to Linux `system` call, + // which always uses /bin/sh to run the command + shellPath := "/bin/sh" + if _, err := os.Stat(shellPath); err != nil { + shellPath = "" + } mainThread := &starlark.Thread{ Name: "main", - Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) }, + Print: func(_ *starlark.Thread, msg string) { + if mode == ExecutionModeRbc { + // In rbc mode, rblf_log is used to print to stderr + fmt.Println(msg) + } else if mode == ExecutionModeScl { + fmt.Fprintln(os.Stderr, msg) + } + }, Load: loader, } - absPath, err := filepath.Abs(filename) - if err == nil { - mainThread.SetLocal(callerDirKey, filepath.Dir(absPath)) - _, err = starlark.ExecFile(mainThread, absPath, src, builtins) + filename, err := filepath.Abs(filename) + if err != nil { + return nil, nil, err + } + if wd, err := os.Getwd(); err == nil { + filename, err = filepath.Rel(wd, filename) + if err != nil { + return nil, nil, err + } + if !allowExternalEntrypoint && strings.HasPrefix(filename, "../") { + return nil, nil, fmt.Errorf("path could not be made relative to workspace root: %s", filename) + } + } else { + return nil, nil, err + } + + if sym, err := isSymlink(filename); sym && err == nil { + return nil, nil, fmt.Errorf("symlinks to starlark files are not allowed. Instead, load the target file and re-export its symbols: %s", filename) + } else if err != nil { + return nil, nil, err } - return err + + if mode == ExecutionModeScl && !strings.HasSuffix(filename, ".scl") { + return nil, nil, fmt.Errorf("filename must end in .scl: %s", filename) + } + + // Add top-level file to cache for cycle detection purposes + moduleCache[filename] = nil + + var results starlark.StringDict + mainThread.SetLocal(allowExternalEntrypointKey, allowExternalEntrypoint) + mainThread.SetLocal(callingFileKey, filename) + mainThread.SetLocal(executionModeKey, mode) + mainThread.SetLocal(shellKey, shellPath) + if mode == ExecutionModeRbc { + results, err = starlark.ExecFile(mainThread, filename, src, rbcBuiltins) + } else if mode == ExecutionModeScl { + results, err = starlark.ExecFile(mainThread, filename, src, sclBuiltins) + } else { + return results, nil, fmt.Errorf("unknown executionMode %d", mode) + } + loadedStarlarkFiles := make([]string, 0, len(moduleCache)) + for file := range moduleCache { + loadedStarlarkFiles = append(loadedStarlarkFiles, file) + } + sort.Strings(loadedStarlarkFiles) + + return results, loadedStarlarkFiles, err } diff --git a/tools/rbcrun/host_test.go b/tools/rbcrun/host_test.go index 97f6ce9e93..38b292376a 100644 --- a/tools/rbcrun/host_test.go +++ b/tools/rbcrun/host_test.go @@ -19,6 +19,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "go.starlark.net/resolve" @@ -53,8 +54,7 @@ func starlarktestSetup() { } // Common setup for the tests: create thread, change to the test directory -func testSetup(t *testing.T, env []string) *starlark.Thread { - setup(env) +func testSetup(t *testing.T) *starlark.Thread { thread := &starlark.Thread{ Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) { if module == "assert.star" { @@ -72,14 +72,15 @@ func testSetup(t *testing.T, env []string) *starlark.Thread { func dataDir() string { _, thisSrcFile, _, _ := runtime.Caller(0) return filepath.Join(filepath.Dir(thisSrcFile), "testdata") - } func exerciseStarlarkTestFile(t *testing.T, starFile string) { // In order to use "assert.star" from go/starlark.net/starlarktest in the tests, provide: // * load function that handles "assert.star" // * starlarktest.DataFile function that finds its location - setup(nil) + if err := os.Chdir(dataDir()); err != nil { + t.Fatal(err) + } thread := &starlark.Thread{ Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) { if module == "assert.star" { @@ -90,21 +91,9 @@ func exerciseStarlarkTestFile(t *testing.T, starFile string) { starlarktest.SetReporter(thread, t) _, thisSrcFile, _, _ := runtime.Caller(0) filename := filepath.Join(filepath.Dir(thisSrcFile), starFile) - if _, err := starlark.ExecFile(thread, filename, nil, builtins); err != nil { - if err, ok := err.(*starlark.EvalError); ok { - t.Fatal(err.Backtrace()) - } - t.Fatal(err) - } -} - -func TestCliAndEnv(t *testing.T) { - // TODO(asmundak): convert this to use exerciseStarlarkTestFile - if err := os.Setenv("TEST_ENVIRONMENT_FOO", "test_environment_foo"); err != nil { - t.Fatal(err) - } - thread := testSetup(t, []string{"CLI_FOO=foo"}) - if _, err := starlark.ExecFile(thread, "cli_and_env.star", nil, builtins); err != nil { + thread.SetLocal(executionModeKey, ExecutionModeRbc) + thread.SetLocal(shellKey, "/bin/sh") + if _, err := starlark.ExecFile(thread, filename, nil, rbcBuiltins); err != nil { if err, ok := err.(*starlark.EvalError); ok { t.Fatal(err.Backtrace()) } @@ -114,11 +103,8 @@ func TestCliAndEnv(t *testing.T) { func TestFileOps(t *testing.T) { // TODO(asmundak): convert this to use exerciseStarlarkTestFile - if err := os.Setenv("TEST_DATA_DIR", dataDir()); err != nil { - t.Fatal(err) - } - thread := testSetup(t, nil) - if _, err := starlark.ExecFile(thread, "file_ops.star", nil, builtins); err != nil { + thread := testSetup(t) + if _, err := starlark.ExecFile(thread, "file_ops.star", nil, rbcBuiltins); err != nil { if err, ok := err.(*starlark.EvalError); ok { t.Fatal(err.Backtrace()) } @@ -128,7 +114,7 @@ func TestFileOps(t *testing.T) { func TestLoad(t *testing.T) { // TODO(asmundak): convert this to use exerciseStarlarkTestFile - thread := testSetup(t, nil) + thread := testSetup(t) thread.Load = func(thread *starlark.Thread, module string) (starlark.StringDict, error) { if module == "assert.star" { return starlarktest.LoadAssertModule() @@ -137,9 +123,13 @@ func TestLoad(t *testing.T) { } } dir := dataDir() - thread.SetLocal(callerDirKey, dir) - LoadPathRoot = filepath.Dir(dir) - if _, err := starlark.ExecFile(thread, "load.star", nil, builtins); err != nil { + if err := os.Chdir(filepath.Dir(dir)); err != nil { + t.Fatal(err) + } + thread.SetLocal(allowExternalEntrypointKey, false) + thread.SetLocal(callingFileKey, "testdata/load.star") + thread.SetLocal(executionModeKey, ExecutionModeRbc) + if _, err := starlark.ExecFile(thread, "testdata/load.star", nil, rbcBuiltins); err != nil { if err, ok := err.(*starlark.EvalError); ok { t.Fatal(err.Backtrace()) } @@ -147,9 +137,70 @@ func TestLoad(t *testing.T) { } } -func TestShell(t *testing.T) { - if err := os.Setenv("TEST_DATA_DIR", dataDir()); err != nil { +func TestBzlLoadsScl(t *testing.T) { + moduleCache = make(map[string]*modentry) + dir := dataDir() + if err := os.Chdir(filepath.Dir(dir)); err != nil { + t.Fatal(err) + } + vars, _, err := Run("testdata/bzl_loads_scl.bzl", nil, ExecutionModeRbc, false) + if err != nil { + t.Fatal(err) + } + if val, ok := vars["foo"]; !ok { + t.Fatalf("Failed to load foo variable") + } else if val.(starlark.String) != "bar" { + t.Fatalf("Expected \"bar\", got %q", val) + } +} + +func TestNonEntrypointBzlLoadsScl(t *testing.T) { + moduleCache = make(map[string]*modentry) + dir := dataDir() + if err := os.Chdir(filepath.Dir(dir)); err != nil { + t.Fatal(err) + } + vars, _, err := Run("testdata/bzl_loads_scl_2.bzl", nil, ExecutionModeRbc, false) + if err != nil { + t.Fatal(err) + } + if val, ok := vars["foo"]; !ok { + t.Fatalf("Failed to load foo variable") + } else if val.(starlark.String) != "bar" { + t.Fatalf("Expected \"bar\", got %q", val) + } +} + +func TestSclLoadsBzl(t *testing.T) { + moduleCache = make(map[string]*modentry) + dir := dataDir() + if err := os.Chdir(filepath.Dir(dir)); err != nil { t.Fatal(err) } + _, _, err := Run("testdata/scl_incorrectly_loads_bzl.scl", nil, ExecutionModeScl, false) + if err == nil { + t.Fatal("Expected failure") + } + if !strings.Contains(err.Error(), ".scl files can only load other .scl files") { + t.Fatalf("Expected error to contain \".scl files can only load other .scl files\": %q", err.Error()) + } +} + +func TestCantLoadSymlink(t *testing.T) { + moduleCache = make(map[string]*modentry) + dir := dataDir() + if err := os.Chdir(filepath.Dir(dir)); err != nil { + t.Fatal(err) + } + _, _, err := Run("testdata/test_scl_symlink.scl", nil, ExecutionModeScl, false) + if err == nil { + t.Fatal("Expected failure") + } + if !strings.Contains(err.Error(), "symlinks to starlark files are not allowed") { + t.Fatalf("Expected error to contain \"symlinks to starlark files are not allowed\": %q", err.Error()) + } +} + +func TestShell(t *testing.T) { exerciseStarlarkTestFile(t, "testdata/shell.star") } diff --git a/tools/rbcrun/rbcrun/rbcrun.go b/tools/rbcrun/rbcrun/rbcrun.go new file mode 100644 index 0000000000..8c372c71d3 --- /dev/null +++ b/tools/rbcrun/rbcrun/rbcrun.go @@ -0,0 +1,190 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "rbcrun" + "regexp" + "strings" + + "go.starlark.net/starlark" +) + +var ( + allowExternalEntrypoint = flag.Bool("allow_external_entrypoint", false, "allow the entrypoint starlark file to be outside of the source tree") + modeFlag = flag.String("mode", "", "the general behavior of rbcrun. Can be \"rbc\" or \"make\". Required.") + rootdir = flag.String("d", ".", "the value of // for load paths") + perfFile = flag.String("perf", "", "save performance data") + identifierRe = regexp.MustCompile("[a-zA-Z_][a-zA-Z0-9_]*") +) + +func getEntrypointStarlarkFile() string { + filename := "" + + for _, arg := range flag.Args() { + if filename == "" { + filename = arg + } else { + quit("only one file can be executed\n") + } + } + if filename == "" { + flag.Usage() + os.Exit(1) + } + return filename +} + +func getMode() rbcrun.ExecutionMode { + switch *modeFlag { + case "rbc": + return rbcrun.ExecutionModeRbc + case "make": + return rbcrun.ExecutionModeScl + case "": + quit("-mode flag is required.") + default: + quit("Unknown -mode value %q, expected 1 of \"rbc\", \"make\"", *modeFlag) + } + return rbcrun.ExecutionModeScl +} + +var makeStringReplacer = strings.NewReplacer("#", "\\#", "$", "$$") + +func cleanStringForMake(s string) (string, error) { + if strings.ContainsAny(s, "\\\n") { + // \\ in make is literally \\, not a single \, so we can't allow them. + // \<newline> in make will produce a space, not a newline. + return "", fmt.Errorf("starlark strings exported to make cannot contain backslashes or newlines") + } + return makeStringReplacer.Replace(s), nil +} + +func getValueInMakeFormat(value starlark.Value, allowLists bool) (string, error) { + switch v := value.(type) { + case starlark.String: + if cleanedValue, err := cleanStringForMake(v.GoString()); err == nil { + return cleanedValue, nil + } else { + return "", err + } + case starlark.Int: + return v.String(), nil + case *starlark.List: + if !allowLists { + return "", fmt.Errorf("nested lists are not allowed to be exported from starlark to make, flatten the list in starlark first") + } + result := "" + for i := 0; i < v.Len(); i++ { + value, err := getValueInMakeFormat(v.Index(i), false) + if err != nil { + return "", err + } + if i > 0 { + result += " " + } + result += value + } + return result, nil + default: + return "", fmt.Errorf("only starlark strings, ints, and lists of strings/ints can be exported to make. Please convert all other types in starlark first. Found type: %s", value.Type()) + } +} + +func printVarsInMakeFormat(globals starlark.StringDict) error { + // We could just directly export top level variables by name instead of going through + // a variables_to_export_to_make dictionary, but that wouldn't allow for exporting a + // runtime-defined number of variables to make. This can be important because dictionaries + // in make are often represented by a unique variable for every key in the dictionary. + variablesValue, ok := globals["variables_to_export_to_make"] + if !ok { + return fmt.Errorf("expected top-level starlark file to have a \"variables_to_export_to_make\" variable") + } + variables, ok := variablesValue.(*starlark.Dict) + if !ok { + return fmt.Errorf("expected variables_to_export_to_make to be a dict, got %s", variablesValue.Type()) + } + + for _, varTuple := range variables.Items() { + varNameStarlark, ok := varTuple.Index(0).(starlark.String) + if !ok { + return fmt.Errorf("all keys in variables_to_export_to_make must be strings, but got %q", varTuple.Index(0).Type()) + } + varName := varNameStarlark.GoString() + if !identifierRe.MatchString(varName) { + return fmt.Errorf("all variables at the top level starlark file must be valid c identifiers, but got %q", varName) + } + if varName == "LOADED_STARLARK_FILES" { + return fmt.Errorf("the name LOADED_STARLARK_FILES is reserved for use by the starlark interpreter") + } + valueMake, err := getValueInMakeFormat(varTuple.Index(1), true) + if err != nil { + return err + } + // The :=$= is special Kati syntax that means "set and make readonly" + fmt.Printf("%s :=$= %s\n", varName, valueMake) + } + return nil +} + +func main() { + flag.Parse() + filename := getEntrypointStarlarkFile() + mode := getMode() + + if os.Chdir(*rootdir) != nil { + quit("could not chdir to %s\n", *rootdir) + } + if *perfFile != "" { + pprof, err := os.Create(*perfFile) + if err != nil { + quit("%s: err", *perfFile) + } + defer pprof.Close() + if err := starlark.StartProfile(pprof); err != nil { + quit("%s\n", err) + } + } + variables, loadedStarlarkFiles, err := rbcrun.Run(filename, nil, mode, *allowExternalEntrypoint) + rc := 0 + if *perfFile != "" { + if err2 := starlark.StopProfile(); err2 != nil { + fmt.Fprintln(os.Stderr, err2) + rc = 1 + } + } + if err != nil { + if evalErr, ok := err.(*starlark.EvalError); ok { + quit("%s\n", evalErr.Backtrace()) + } else { + quit("%s\n", err) + } + } + if mode == rbcrun.ExecutionModeScl { + if err := printVarsInMakeFormat(variables); err != nil { + quit("%s\n", err) + } + fmt.Printf("LOADED_STARLARK_FILES := %s\n", strings.Join(loadedStarlarkFiles, " ")) + } + os.Exit(rc) +} + +func quit(format string, s ...interface{}) { + fmt.Fprintf(os.Stderr, format, s...) + os.Exit(2) +} diff --git a/tools/rbcrun/testdata/bzl_loads_scl.bzl b/tools/rbcrun/testdata/bzl_loads_scl.bzl new file mode 100644 index 0000000000..e8deca3d9c --- /dev/null +++ b/tools/rbcrun/testdata/bzl_loads_scl.bzl @@ -0,0 +1,3 @@ +load(":test_scl.scl", _foo = "foo") + +foo = _foo diff --git a/tools/rbcrun/testdata/bzl_loads_scl_2.bzl b/tools/rbcrun/testdata/bzl_loads_scl_2.bzl new file mode 100644 index 0000000000..9a680edc03 --- /dev/null +++ b/tools/rbcrun/testdata/bzl_loads_scl_2.bzl @@ -0,0 +1,3 @@ +load(":bzl_loads_scl.bzl", _foo = "foo") + +foo = _foo diff --git a/tools/rbcrun/testdata/cli_and_env.star b/tools/rbcrun/testdata/cli_and_env.star deleted file mode 100644 index d6f464aa79..0000000000 --- a/tools/rbcrun/testdata/cli_and_env.star +++ /dev/null @@ -1,11 +0,0 @@ -# Tests rblf_env access -load("assert.star", "assert") - - -def test(): - assert.eq(rblf_env.TEST_ENVIRONMENT_FOO, "test_environment_foo") - assert.fails(lambda: rblf_env.FOO_BAR_BAZ, ".*struct has no .FOO_BAR_BAZ attribute$") - assert.eq(rblf_cli.CLI_FOO, "foo") - - -test() diff --git a/tools/rbcrun/testdata/file_ops.star b/tools/rbcrun/testdata/file_ops.star index 2ee78fcc7e..b2b907c2c9 100644 --- a/tools/rbcrun/testdata/file_ops.star +++ b/tools/rbcrun/testdata/file_ops.star @@ -1,22 +1,21 @@ # Tests file ops builtins load("assert.star", "assert") - def test(): myname = "file_ops.star" files = rblf_wildcard("*.star") assert.true(myname in files, "expected %s in %s" % (myname, files)) - files = rblf_wildcard("*.star", rblf_env.TEST_DATA_DIR) + files = rblf_wildcard("*.star") assert.true(myname in files, "expected %s in %s" % (myname, files)) files = rblf_wildcard("*.xxx") assert.true(len(files) == 0, "expansion should be empty but contains %s" % files) mydir = "testdata" myrelname = "%s/%s" % (mydir, myname) - files = rblf_find_files(rblf_env.TEST_DATA_DIR + "/../", "*") + files = rblf_find_files("../", "*") assert.true(mydir in files and myrelname in files, "expected %s and %s in %s" % (mydir, myrelname, files)) - files = rblf_find_files(rblf_env.TEST_DATA_DIR + "/../", "*", only_files=1) + files = rblf_find_files("../", "*", only_files=1) assert.true(mydir not in files, "did not expect %s in %s" % (mydir, files)) assert.true(myrelname in files, "expected %s in %s" % (myrelname, files)) - files = rblf_find_files(rblf_env.TEST_DATA_DIR + "/../", "*.star") + files = rblf_find_files("../", "*.star") assert.true(myrelname in files, "expected %s in %s" % (myrelname, files)) test() diff --git a/tools/rbcrun/testdata/module1.star b/tools/rbcrun/testdata/module1.star index be04f75b04..02919a0e7d 100644 --- a/tools/rbcrun/testdata/module1.star +++ b/tools/rbcrun/testdata/module1.star @@ -2,6 +2,6 @@ load("assert.star", "assert") # Make sure that builtins are defined for the loaded module, too -assert.true(rblf_wildcard("module1.star")) -assert.true(not rblf_wildcard("no_such file")) +assert.true(rblf_wildcard("testdata/module1.star")) +assert.true(not rblf_wildcard("testdata/no_such file")) test = "module1" diff --git a/tools/rbcrun/testdata/scl_incorrectly_loads_bzl.scl b/tools/rbcrun/testdata/scl_incorrectly_loads_bzl.scl new file mode 100644 index 0000000000..9a680edc03 --- /dev/null +++ b/tools/rbcrun/testdata/scl_incorrectly_loads_bzl.scl @@ -0,0 +1,3 @@ +load(":bzl_loads_scl.bzl", _foo = "foo") + +foo = _foo diff --git a/tools/rbcrun/testdata/shell.star b/tools/rbcrun/testdata/shell.star index ad106974c4..dd173751b9 100644 --- a/tools/rbcrun/testdata/shell.star +++ b/tools/rbcrun/testdata/shell.star @@ -1,5 +1,5 @@ # Tests "queue" data type load("assert.star", "assert") -assert.eq("load.star shell.star", rblf_shell("cd %s && ls -1 shell.star load.star 2>&1" % rblf_env.TEST_DATA_DIR)) -assert.eq("shell.star", rblf_shell("cd %s && echo shell.sta*" % rblf_env.TEST_DATA_DIR)) +assert.eq("load.star shell.star", rblf_shell("ls -1 shell.star load.star 2>&1")) +assert.eq("shell.star", rblf_shell("echo shell.sta*")) diff --git a/tools/rbcrun/testdata/test_scl.scl b/tools/rbcrun/testdata/test_scl.scl new file mode 100644 index 0000000000..6360ccb715 --- /dev/null +++ b/tools/rbcrun/testdata/test_scl.scl @@ -0,0 +1,2 @@ + +foo = "bar" diff --git a/tools/rbcrun/testdata/test_scl_symlink.scl b/tools/rbcrun/testdata/test_scl_symlink.scl new file mode 120000 index 0000000000..3f5aef488e --- /dev/null +++ b/tools/rbcrun/testdata/test_scl_symlink.scl @@ -0,0 +1 @@ +test_scl.scl
\ No newline at end of file diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp index 29fc771b86..9b134f22d4 100644 --- a/tools/releasetools/Android.bp +++ b/tools/releasetools/Android.bp @@ -99,9 +99,8 @@ python_defaults { "releasetools_common", ], required: [ + "apexd_host", "checkvintf", - "deapexer", - "dump_apex_info", ], } @@ -166,10 +165,9 @@ python_defaults { "ota_utils_lib", ], required: [ + "apexd_host", "brillo_update_payload", "checkvintf", - "generate_gki_certificate", - "minigzip", "lz4", "toybox", "unpack_bootimg", @@ -236,6 +234,9 @@ python_library_host { "rangelib.py", "sparse_img.py", ], + data: [ + ":zip2zip", + ], // Only the tools that are referenced directly are listed as required modules. For example, // `avbtool` is not here, as the script always uses the one from info_dict['avb_avbtool']. required: [ @@ -243,9 +244,6 @@ python_library_host { "boot_signer", "brotli", "bsdiff", - "generate_gki_certificate", - "imgdiff", - "minigzip", "lz4", "mkbootfs", "signapk", @@ -309,9 +307,6 @@ python_defaults { "brotli", "bsdiff", "deapexer", - "generate_gki_certificate", - "imgdiff", - "minigzip", "lz4", "mkbootfs", "signapk", @@ -333,6 +328,10 @@ python_library_host { srcs: [ "ota_utils.py", "payload_signer.py", + "ota_signing_utils.py", + ], + libs: [ + "releasetools_common", ], } @@ -356,6 +355,21 @@ python_binary_host { } python_binary_host { + name: "create_brick_ota", + version: { + py3: { + embedded_launcher: true, + }, + }, + srcs: [ + "create_brick_ota.py", + ], + libs: [ + "ota_utils_lib", + ], +} + +python_binary_host { name: "build_image", defaults: [ "releasetools_binary_defaults", @@ -479,6 +493,25 @@ python_binary_host { } python_binary_host { + name: "ota_from_raw_img", + srcs: [ + "ota_from_raw_img.py", + ], + main: "ota_from_raw_img.py", + defaults: [ + "releasetools_binary_defaults", + ], + required: [ + "delta_generator", + ], + libs: [ + "ota_metadata_proto", + "releasetools_common", + "ota_utils_lib", + ], +} + +python_binary_host { name: "ota_package_parser", defaults: ["releasetools_binary_defaults"], srcs: [ @@ -513,6 +546,8 @@ python_binary_host { defaults: ["releasetools_binary_defaults"], srcs: [ "sign_target_files_apks.py", + "payload_signer.py", + "ota_signing_utils.py", ], libs: [ "releasetools_add_img_to_target_files", @@ -597,7 +632,7 @@ python_defaults { data: [ "testdata/**/*", ":com.android.apex.compressed.v1", - ":com.android.apex.compressed.v1_original", + ":com.android.apex.vendor.foo.with_vintf" ], target: { darwin: { @@ -606,6 +641,7 @@ python_defaults { }, }, required: [ + "apexd_host", "deapexer", ], } diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py index e154a0f60a..b39a82cf45 100644 --- a/tools/releasetools/add_img_to_target_files.py +++ b/tools/releasetools/add_img_to_target_files.py @@ -42,6 +42,10 @@ Usage: add_img_to_target_files [flag] target_files --is_signing Skip building & adding the images for "userdata" and "cache" if we are signing the target files. + + --avb-resolve-rollback-index-location-conflict + If provided, resolve the conflict AVB rollback index location when + necessary. """ from __future__ import print_function @@ -65,9 +69,10 @@ import verity_utils import ota_metadata_pb2 import rangelib import sparse_img - +from concurrent.futures import ThreadPoolExecutor from apex_utils import GetApexInfoFromTargetFiles from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite +from build_image import FIXED_FILE_TIMESTAMP if sys.hexversion < 0x02070000: print("Python 2.7 or newer is required.", file=sys.stderr) @@ -80,12 +85,7 @@ OPTIONS.add_missing = False OPTIONS.rebuild_recovery = False OPTIONS.replace_updated_files_list = [] OPTIONS.is_signing = False - -# Use a fixed timestamp (01/01/2009 00:00:00 UTC) for files when packaging -# images. (b/24377993, b/80600931) -FIXED_FILE_TIMESTAMP = int(( - datetime.datetime(2009, 1, 1, 0, 0, 0, 0, None) - - datetime.datetime.utcfromtimestamp(0)).total_seconds()) +OPTIONS.avb_resolve_rollback_index_location_conflict = False def ParseAvbFooter(img_path) -> avbtool.AvbFooter: @@ -522,12 +522,14 @@ def AddPvmfw(output_zip): return img.name -def AddCustomImages(output_zip, partition_name): - """Adds and signs custom images in IMAGES/. +def AddCustomImages(output_zip, partition_name, image_list): + """Adds and signs avb custom images as needed in IMAGES/. Args: output_zip: The output zip file (needs to be already open), or None to write images to OPTIONS.input_tmp/. + partition_name: The custom image partition name. + image_list: The image list of the custom image partition. Uses the image under IMAGES/ if it already exists. Otherwise looks for the image under PREBUILT_IMAGES/, signs it as needed, and returns the image name. @@ -536,19 +538,20 @@ def AddCustomImages(output_zip, partition_name): AssertionError: If image can't be found. """ + builder = None key_path = OPTIONS.info_dict.get("avb_{}_key_path".format(partition_name)) - algorithm = OPTIONS.info_dict.get("avb_{}_algorithm".format(partition_name)) - extra_args = OPTIONS.info_dict.get( - "avb_{}_add_hashtree_footer_args".format(partition_name)) - partition_size = OPTIONS.info_dict.get( - "avb_{}_partition_size".format(partition_name)) - - builder = verity_utils.CreateCustomImageBuilder( - OPTIONS.info_dict, partition_name, partition_size, - key_path, algorithm, extra_args) - - for img_name in OPTIONS.info_dict.get( - "avb_{}_image_list".format(partition_name)).split(): + if key_path is not None: + algorithm = OPTIONS.info_dict.get("avb_{}_algorithm".format(partition_name)) + extra_args = OPTIONS.info_dict.get( + "avb_{}_add_hashtree_footer_args".format(partition_name)) + partition_size = OPTIONS.info_dict.get( + "avb_{}_partition_size".format(partition_name)) + + builder = verity_utils.CreateCustomImageBuilder( + OPTIONS.info_dict, partition_name, partition_size, + key_path, algorithm, extra_args) + + for img_name in image_list: custom_image = OutputFile( output_zip, OPTIONS.input_tmp, "IMAGES", img_name) if os.path.exists(custom_image.name): @@ -594,15 +597,6 @@ def CreateImage(input_dir, info_dict, what, output_file, block_list=None): if block_list: image_props["block_list"] = block_list.name - # Use repeatable ext4 FS UUID and hash_seed UUID (based on partition name and - # build fingerprint). Also use the legacy build id, because the vbmeta digest - # isn't available at this point. - build_info = common.BuildInfo(info_dict, use_legacy_id=True) - uuid_seed = what + "-" + build_info.GetPartitionFingerprint(what) - image_props["uuid"] = str(uuid.uuid5(uuid.NAMESPACE_URL, uuid_seed)) - hash_seed = "hash_seed-" + uuid_seed - image_props["hash_seed"] = str(uuid.uuid5(uuid.NAMESPACE_URL, hash_seed)) - build_image.BuildImage( os.path.join(input_dir, what.upper()), image_props, output_file.name) @@ -693,39 +687,12 @@ def AddVBMeta(output_zip, partitions, name, needed_partitions): logger.info("%s.img already exists; not rebuilding...", name) return img.name - common.BuildVBMeta(img.name, partitions, name, needed_partitions) + common.BuildVBMeta(img.name, partitions, name, needed_partitions, + OPTIONS.avb_resolve_rollback_index_location_conflict) img.Write() return img.name -def AddPartitionTable(output_zip): - """Create a partition table image and store it in output_zip.""" - - img = OutputFile( - output_zip, OPTIONS.input_tmp, "IMAGES", "partition-table.img") - bpt = OutputFile( - output_zip, OPTIONS.input_tmp, "META", "partition-table.bpt") - - # use BPTTOOL from environ, or "bpttool" if empty or not set. - bpttool = os.getenv("BPTTOOL") or "bpttool" - cmd = [bpttool, "make_table", "--output_json", bpt.name, - "--output_gpt", img.name] - input_files_str = OPTIONS.info_dict["board_bpt_input_files"] - input_files = input_files_str.split() - for i in input_files: - cmd.extend(["--input", i]) - disk_size = OPTIONS.info_dict.get("board_bpt_disk_size") - if disk_size: - cmd.extend(["--disk_size", disk_size]) - args = OPTIONS.info_dict.get("board_bpt_make_table_args") - if args: - cmd.extend(shlex.split(args)) - common.RunAndCheckOutput(cmd) - - img.Write() - bpt.Write() - - def AddCache(output_zip): """Create an empty cache image and store it in output_zip.""" @@ -818,6 +785,9 @@ def AddSuperEmpty(output_zip): """Create a super_empty.img and store it in output_zip.""" img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "super_empty.img") + if os.path.exists(img.name): + logger.info("super_empty.img already exists; no need to rebuild...") + return build_super_image.BuildSuperImage(OPTIONS.info_dict, img.name) img.Write() @@ -842,13 +812,14 @@ def ReplaceUpdatedFiles(zip_filename, files_list): SYSTEM/ after rebuilding recovery. """ common.ZipDelete(zip_filename, files_list) - with zipfile.ZipFile(zip_filename, "a", + output_zip = zipfile.ZipFile(zip_filename, "a", compression=zipfile.ZIP_DEFLATED, - allowZip64=True) as output_zip: - for item in files_list: - file_path = os.path.join(OPTIONS.input_tmp, item) - assert os.path.exists(file_path) - common.ZipWrite(output_zip, file_path, arcname=item) + allowZip64=True) + for item in files_list: + file_path = os.path.join(OPTIONS.input_tmp, item) + assert os.path.exists(file_path) + common.ZipWrite(output_zip, file_path, arcname=item) + common.ZipClose(output_zip) def HasPartition(partition_name): @@ -864,8 +835,7 @@ def HasPartition(partition_name): def AddApexInfo(output_zip): - apex_infos = GetApexInfoFromTargetFiles(OPTIONS.input_tmp, 'system', - compressed_only=False) + apex_infos = GetApexInfoFromTargetFiles(OPTIONS.input_tmp) apex_metadata_proto = ota_metadata_pb2.ApexMetadata() apex_metadata_proto.apex_info.extend(apex_infos) apex_info_bytes = apex_metadata_proto.SerializeToString() @@ -1079,8 +1049,15 @@ def AddImagesToTargetFiles(filename): ("system_dlkm", has_system_dlkm, AddSystemDlkm, []), ("system_other", has_system_other, AddSystemOther, []), ) - for call in add_partition_calls: - add_partition(*call) + # If output_zip exists, each add_partition_calls writes bytes to the same output_zip, + # which is not thread-safe. So, run them in serial if output_zip exists. + if output_zip: + for call in add_partition_calls: + add_partition(*call) + else: + with ThreadPoolExecutor(max_workers=len(add_partition_calls)) as executor: + for future in [executor.submit(add_partition, *call) for call in add_partition_calls]: + future.result() AddApexInfo(output_zip) @@ -1090,10 +1067,6 @@ def AddImagesToTargetFiles(filename): banner("cache") AddCache(output_zip) - if OPTIONS.info_dict.get("board_bpt_enable") == "true": - banner("partition-table") - AddPartitionTable(output_zip) - add_partition("dtbo", OPTIONS.info_dict.get("has_dtbo") == "true", AddDtbo, []) add_partition("pvmfw", @@ -1101,18 +1074,29 @@ def AddImagesToTargetFiles(filename): # Custom images. custom_partitions = OPTIONS.info_dict.get( - "avb_custom_images_partition_list", "").strip().split() + "custom_images_partition_list", "").strip().split() for partition_name in custom_partitions: partition_name = partition_name.strip() banner("custom images for " + partition_name) - partitions[partition_name] = AddCustomImages(output_zip, partition_name) + image_list = OPTIONS.info_dict.get( + "{}_image_list".format(partition_name)).split() + partitions[partition_name] = AddCustomImages(output_zip, partition_name, image_list) + + avb_custom_partitions = OPTIONS.info_dict.get( + "avb_custom_images_partition_list", "").strip().split() + for partition_name in avb_custom_partitions: + partition_name = partition_name.strip() + banner("avb custom images for " + partition_name) + image_list = OPTIONS.info_dict.get( + "avb_{}_image_list".format(partition_name)).split() + partitions[partition_name] = AddCustomImages(output_zip, partition_name, image_list) if OPTIONS.info_dict.get("avb_enable") == "true": # vbmeta_partitions includes the partitions that should be included into # top-level vbmeta.img, which are the ones that are not included in any # chained VBMeta image plus the chained VBMeta images themselves. - # Currently custom_partitions are all chained to VBMeta image. - vbmeta_partitions = common.AVB_PARTITIONS[:] + tuple(custom_partitions) + # Currently avb_custom_partitions are all chained to VBMeta image. + vbmeta_partitions = common.AVB_PARTITIONS[:] + tuple(avb_custom_partitions) vbmeta_system = OPTIONS.info_dict.get("avb_vbmeta_system", "").strip() if vbmeta_system: @@ -1133,14 +1117,18 @@ def AddImagesToTargetFiles(filename): item for item in vbmeta_partitions if item not in vbmeta_vendor.split()] vbmeta_partitions.append("vbmeta_vendor") - custom_avb_partitions = OPTIONS.info_dict.get("avb_custom_vbmeta_images_partition_list", "").strip().split() + custom_avb_partitions = OPTIONS.info_dict.get( + "avb_custom_vbmeta_images_partition_list", "").strip().split() if custom_avb_partitions: for avb_part in custom_avb_partitions: partition_name = "vbmeta_" + avb_part - included_partitions = OPTIONS.info_dict.get("avb_vbmeta_{}".format(avb_part), "").strip().split() - assert included_partitions, "Custom vbmeta partition {0} missing avb_vbmeta_{0} prop".format(avb_part) + included_partitions = OPTIONS.info_dict.get( + "avb_vbmeta_{}".format(avb_part), "").strip().split() + assert included_partitions, "Custom vbmeta partition {0} missing avb_vbmeta_{0} prop".format( + avb_part) banner(partition_name) - logger.info("VBMeta partition {} needs {}".format(partition_name, included_partitions)) + logger.info("VBMeta partition {} needs {}".format( + partition_name, included_partitions)) partitions[partition_name] = AddVBMeta( output_zip, partitions, partition_name, included_partitions) vbmeta_partitions = [ @@ -1148,7 +1136,6 @@ def AddImagesToTargetFiles(filename): if item not in included_partitions] vbmeta_partitions.append(partition_name) - if OPTIONS.info_dict.get("avb_building_vbmeta_image") == "true": banner("vbmeta") AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions) @@ -1191,7 +1178,7 @@ def AddImagesToTargetFiles(filename): AddVbmetaDigest(output_zip) if output_zip: - output_zip.close() + common.ZipClose(output_zip) if OPTIONS.replace_updated_files_list: ReplaceUpdatedFiles(output_zip.filename, OPTIONS.replace_updated_files_list) @@ -1242,6 +1229,8 @@ def main(argv): " please switch to AVB") elif o == "--is_signing": OPTIONS.is_signing = True + elif o == "--avb_resolve_rollback_index_location_conflict": + OPTIONS.avb_resolve_rollback_index_location_conflict = True else: return False return True @@ -1251,7 +1240,8 @@ def main(argv): extra_long_opts=["add_missing", "rebuild_recovery", "replace_verity_public_key=", "replace_verity_private_key=", - "is_signing"], + "is_signing", + "avb_resolve_rollback_index_location_conflict"], extra_option_handler=option_handler) if len(args) != 1: diff --git a/tools/releasetools/apex_utils.py b/tools/releasetools/apex_utils.py index 40f7c92deb..3abef3bece 100644 --- a/tools/releasetools/apex_utils.py +++ b/tools/releasetools/apex_utils.py @@ -65,12 +65,10 @@ class ApexApkSigner(object): OPTIONS.search_path, "bin", "debugfs_static") self.fsckerofs_path = os.path.join( OPTIONS.search_path, "bin", "fsck.erofs") - self.blkid_path = os.path.join( - OPTIONS.search_path, "bin", "blkid_static") self.avbtool = avbtool if avbtool else "avbtool" self.sign_tool = sign_tool - def ProcessApexFile(self, apk_keys, payload_key, signing_args=None, is_sepolicy=False): + def ProcessApexFile(self, apk_keys, payload_key, signing_args=None): """Scans and signs the payload files and repack the apex Args: @@ -88,13 +86,9 @@ class ApexApkSigner(object): 'list', self.apex_path] entries_names = common.RunAndCheckOutput(list_cmd).split() apk_entries = [name for name in entries_names if name.endswith('.apk')] - sepolicy_entries = [] - if is_sepolicy: - sepolicy_entries = [name for name in entries_names if - name.startswith('./etc/SEPolicy') and name.endswith('.zip')] # No need to sign and repack, return the original apex path. - if not apk_entries and not sepolicy_entries and self.sign_tool is None: + if not apk_entries and self.sign_tool is None: logger.info('No apk file to sign in %s', self.apex_path) return self.apex_path @@ -110,14 +104,14 @@ class ApexApkSigner(object): ' %s', entry) payload_dir, has_signed_content = self.ExtractApexPayloadAndSignContents( - apk_entries, sepolicy_entries, apk_keys, payload_key, signing_args) + apk_entries, apk_keys, payload_key, signing_args) if not has_signed_content: - logger.info('No contents have been signed in %s', self.apex_path) + logger.info('No contents has been signed in %s', self.apex_path) return self.apex_path return self.RepackApexPayload(payload_dir, payload_key, signing_args) - def ExtractApexPayloadAndSignContents(self, apk_entries, sepolicy_entries, apk_keys, payload_key, signing_args): + def ExtractApexPayloadAndSignContents(self, apk_entries, apk_keys, payload_key, signing_args): """Extracts the payload image and signs the containing apk files.""" if not os.path.exists(self.debugfs_path): raise ApexSigningError( @@ -129,22 +123,17 @@ class ApexApkSigner(object): "Couldn't find location of fsck.erofs: " + "Path {} does not exist. ".format(self.fsckerofs_path) + "Make sure bin/fsck.erofs can be found in -p <path>") - if not os.path.exists(self.blkid_path): - raise ApexSigningError( - "Couldn't find location of blkid: " + - "Path {} does not exist. ".format(self.blkid_path) + - "Make sure bin/blkid can be found in -p <path>") payload_dir = common.MakeTempDir() extract_cmd = ['deapexer', '--debugfs_path', self.debugfs_path, '--fsckerofs_path', self.fsckerofs_path, - '--blkid_path', self.blkid_path, 'extract', + 'extract', self.apex_path, payload_dir] common.RunAndCheckOutput(extract_cmd) - assert os.path.exists(self.apex_path) has_signed_content = False for entry in apk_entries: apk_path = os.path.join(payload_dir, entry) + assert os.path.exists(self.apex_path) key_name = apk_keys.get(os.path.basename(entry)) if key_name in common.SPECIAL_CERT_STRINGS: @@ -161,37 +150,6 @@ class ApexApkSigner(object): codename_to_api_level_map=self.codename_to_api_level_map) has_signed_content = True - for entry in sepolicy_entries: - sepolicy_path = os.path.join(payload_dir, entry) - - if not 'etc' in entry: - logger.warning('Sepolicy path does not contain the intended directory name etc:' - ' %s', entry) - - key_name = apk_keys.get(os.path.basename(entry)) - if key_name is None: - logger.warning('Failed to find signing keys for {} in' - ' apex {}, payload key will be used instead.' - ' Use "-e <name>=" to specify a key' - .format(entry, self.apex_path)) - key_name = payload_key - - if key_name in common.SPECIAL_CERT_STRINGS: - logger.info('Not signing: %s due to special cert string', sepolicy_path) - continue - - if OPTIONS.sign_sepolicy_path is not None: - sig_path = os.path.join(payload_dir, sepolicy_path + '.sig') - fsv_sig_path = os.path.join(payload_dir, sepolicy_path + '.fsv_sig') - old_sig = common.MakeTempFile() - old_fsv_sig = common.MakeTempFile() - os.rename(sig_path, old_sig) - os.rename(fsv_sig_path, old_fsv_sig) - - logger.info('Signing sepolicy file %s in apex %s', sepolicy_path, self.apex_path) - if common.SignSePolicy(sepolicy_path, key_name, self.key_passwords.get(key_name)): - has_signed_content = True - if self.sign_tool: logger.info('Signing payload contents in apex %s with %s', self.apex_path, self.sign_tool) # Pass avbtool to the custom signing tool @@ -375,8 +333,7 @@ def ParseApexPayloadInfo(avbtool, payload_path): def SignUncompressedApex(avbtool, apex_file, payload_key, container_key, container_pw, apk_keys, codename_to_api_level_map, - no_hashtree, signing_args=None, sign_tool=None, - is_sepolicy=False): + no_hashtree, signing_args=None, sign_tool=None): """Signs the current uncompressed APEX with the given payload/container keys. Args: @@ -389,7 +346,6 @@ def SignUncompressedApex(avbtool, apex_file, payload_key, container_key, no_hashtree: Don't include hashtree in the signed APEX. signing_args: Additional args to be passed to the payload signer. sign_tool: A tool to sign the contents of the APEX. - is_sepolicy: Indicates if the apex is a sepolicy.apex Returns: The path to the signed APEX file. @@ -399,8 +355,7 @@ def SignUncompressedApex(avbtool, apex_file, payload_key, container_key, apk_signer = ApexApkSigner(apex_file, container_pw, codename_to_api_level_map, avbtool, sign_tool) - apex_file = apk_signer.ProcessApexFile( - apk_keys, payload_key, signing_args, is_sepolicy) + apex_file = apk_signer.ProcessApexFile(apk_keys, payload_key, signing_args) # 2a. Extract and sign the APEX_PAYLOAD_IMAGE entry with the given # payload_key. @@ -431,7 +386,7 @@ def SignUncompressedApex(avbtool, apex_file, payload_key, container_key, apex_zip = zipfile.ZipFile(apex_file, 'a', allowZip64=True) common.ZipWrite(apex_zip, payload_file, arcname=APEX_PAYLOAD_IMAGE) common.ZipWrite(apex_zip, payload_public_key, arcname=APEX_PUBKEY) - apex_zip.close() + common.ZipClose(apex_zip) # 3. Sign the APEX container with container_key. signed_apex = common.MakeTempFile(prefix='apex-container-', suffix='.apex') @@ -454,8 +409,7 @@ def SignUncompressedApex(avbtool, apex_file, payload_key, container_key, def SignCompressedApex(avbtool, apex_file, payload_key, container_key, container_pw, apk_keys, codename_to_api_level_map, - no_hashtree, signing_args=None, sign_tool=None, - is_sepolicy=False): + no_hashtree, signing_args=None, sign_tool=None): """Signs the current compressed APEX with the given payload/container keys. Args: @@ -467,7 +421,6 @@ def SignCompressedApex(avbtool, apex_file, payload_key, container_key, codename_to_api_level_map: A dict that maps from codename to API level. no_hashtree: Don't include hashtree in the signed APEX. signing_args: Additional args to be passed to the payload signer. - is_sepolicy: Indicates if the apex is a sepolicy.apex Returns: The path to the signed APEX file. @@ -494,8 +447,7 @@ def SignCompressedApex(avbtool, apex_file, payload_key, container_key, codename_to_api_level_map, no_hashtree, signing_args, - sign_tool, - is_sepolicy) + sign_tool) # 3. Compress signed original apex. compressed_apex_file = common.MakeTempFile(prefix='apex-container-', @@ -522,8 +474,8 @@ def SignCompressedApex(avbtool, apex_file, payload_key, container_key, def SignApex(avbtool, apex_data, payload_key, container_key, container_pw, - apk_keys, codename_to_api_level_map, no_hashtree, - signing_args=None, sign_tool=None, is_sepolicy=False): + apk_keys, codename_to_api_level_map, + no_hashtree, signing_args=None, sign_tool=None): """Signs the current APEX with the given payload/container keys. Args: @@ -535,7 +487,6 @@ def SignApex(avbtool, apex_data, payload_key, container_key, container_pw, codename_to_api_level_map: A dict that maps from codename to API level. no_hashtree: Don't include hashtree in the signed APEX. signing_args: Additional args to be passed to the payload signer. - is_sepolicy: Indicates if the apex is a sepolicy.apex Returns: The path to the signed APEX file. @@ -561,8 +512,7 @@ def SignApex(avbtool, apex_data, payload_key, container_key, container_pw, no_hashtree=no_hashtree, apk_keys=apk_keys, signing_args=signing_args, - sign_tool=sign_tool, - is_sepolicy=is_sepolicy) + sign_tool=sign_tool) elif apex_type == 'COMPRESSED': return SignCompressedApex( avbtool, @@ -574,8 +524,7 @@ def SignApex(avbtool, apex_data, payload_key, container_key, container_pw, no_hashtree=no_hashtree, apk_keys=apk_keys, signing_args=signing_args, - sign_tool=sign_tool, - is_sepolicy=is_sepolicy) + sign_tool=sign_tool) else: # TODO(b/172912232): support signing compressed apex raise ApexInfoError('Unsupported apex type {}'.format(apex_type)) @@ -585,22 +534,28 @@ def SignApex(avbtool, apex_data, payload_key, container_key, container_pw, 'Failed to get type for {}:\n{}'.format(apex_file, e)) -def GetApexInfoFromTargetFiles(input_file, partition, compressed_only=True): +def GetApexInfoFromTargetFiles(input_file): """ - Get information about system APEX stored in the input_file zip + Get information about APEXes stored in the input_file zip Args: input_file: The filename of the target build target-files zip or directory. Return: A list of ota_metadata_pb2.ApexInfo() populated using the APEX stored in - /system partition of the input_file + each partition of the input_file """ # Extract the apex files so that we can run checks on them if not isinstance(input_file, str): raise RuntimeError("must pass filepath to target-files zip or directory") + apex_infos = [] + for partition in ['system', 'system_ext', 'product', 'vendor']: + apex_infos.extend(GetApexInfoForPartition(input_file, partition)) + return apex_infos + +def GetApexInfoForPartition(input_file, partition): apex_subdir = os.path.join(partition.upper(), 'apex') if os.path.isdir(input_file): tmp_dir = input_file @@ -626,7 +581,7 @@ def GetApexInfoFromTargetFiles(input_file, partition, compressed_only=True): if os.path.isfile(deapexer_path): deapexer = deapexer_path - for apex_filename in os.listdir(target_dir): + for apex_filename in sorted(os.listdir(target_dir)): apex_filepath = os.path.join(target_dir, apex_filename) if not os.path.isfile(apex_filepath) or \ not zipfile.is_zipfile(apex_filepath): @@ -658,7 +613,6 @@ def GetApexInfoFromTargetFiles(input_file, partition, compressed_only=True): '--output', decompressed_file_path]) apex_info.decompressed_size = os.path.getsize(decompressed_file_path) - if not compressed_only or apex_info.is_compressed: - apex_infos.append(apex_info) + apex_infos.append(apex_info) return apex_infos diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index 90641360e6..464ad9b4cc 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2011 The Android Open Source Project # @@ -22,25 +22,36 @@ Usage: build_image input_directory properties_file output_image \\ target_output_directory """ -from __future__ import print_function +import datetime +import argparse import glob import logging import os import os.path import re +import shlex import shutil import sys +import uuid +import tempfile import common import verity_utils + logger = logging.getLogger(__name__) OPTIONS = common.OPTIONS BLOCK_SIZE = common.BLOCK_SIZE BYTES_IN_MB = 1024 * 1024 +# Use a fixed timestamp (01/01/2009 00:00:00 UTC) for files when packaging +# images. (b/24377993, b/80600931) +FIXED_FILE_TIMESTAMP = int(( + datetime.datetime(2009, 1, 1, 0, 0, 0, 0, None) - + datetime.datetime.utcfromtimestamp(0)).total_seconds()) + class BuildImageError(Exception): """An Exception raised during image building.""" @@ -249,7 +260,7 @@ def CalculateSizeAndReserved(prop_dict, size): if fs_type.startswith("ext4") and partition_headroom > reserved_size: reserved_size = partition_headroom - return size + reserved_size + return int(size * 1.1) + reserved_size def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config): @@ -283,7 +294,7 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config): build_command = [prop_dict["ext_mkuserimg"]] if "extfs_sparse_flag" in prop_dict and not disable_sparse: build_command.append(prop_dict["extfs_sparse_flag"]) - run_e2fsck = RunE2fsck + run_fsck = RunE2fsck build_command.extend([in_dir, out_file, fs_type, prop_dict["mount_point"]]) build_command.append(prop_dict["image_size"]) @@ -342,6 +353,8 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config): if compress_hints: build_command.extend(["--compress-hints", compress_hints]) + build_command.extend(["-b", prop_dict.get("erofs_blocksize", "4096")]) + build_command.extend(["--mount-point", prop_dict["mount_point"]]) if target_out: build_command.extend(["--product-out", target_out]) @@ -424,6 +437,8 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config): sldc_flags = sldc_flags_str.split() build_command.append(str(len(sldc_flags))) build_command.extend(sldc_flags) + f2fs_blocksize = prop_dict.get("f2fs_blocksize", "4096") + build_command.extend(["-b", f2fs_blocksize]) else: raise BuildImageError( "Error: unknown filesystem type: {}".format(fs_type)) @@ -487,6 +502,20 @@ def RunErofsFsck(out_file): raise +def SetUUIDIfNotExist(image_props): + + # Use repeatable ext4 FS UUID and hash_seed UUID (based on partition name and + # build fingerprint). Also use the legacy build id, because the vbmeta digest + # isn't available at this point. + what = image_props["mount_point"] + fingerprint = image_props.get("fingerprint", "") + uuid_seed = what + "-" + fingerprint + logger.info("Using fingerprint %s for partition %s", fingerprint, what) + image_props["uuid"] = str(uuid.uuid5(uuid.NAMESPACE_URL, uuid_seed)) + hash_seed = "hash_seed-" + uuid_seed + image_props["hash_seed"] = str(uuid.uuid5(uuid.NAMESPACE_URL, hash_seed)) + + def BuildImage(in_dir, prop_dict, out_file, target_out=None): """Builds an image for the files under in_dir and writes it to out_file. @@ -504,6 +533,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): BuildImageError: On build image failures. """ in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict) + SetUUIDIfNotExist(prop_dict) build_command = [] fs_type = prop_dict.get("fs_type", "") @@ -606,7 +636,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): size = verity_image_builder.CalculateDynamicPartitionSize(size) prop_dict["partition_size"] = str(size) logger.info( - "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file) + "Allocating %d MB for %s", size // BYTES_IN_MB, out_file) prop_dict["image_size"] = prop_dict["partition_size"] @@ -635,6 +665,19 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): verity_image_builder.Build(out_file) +def TryParseFingerprint(glob_dict: dict): + for (key, val) in glob_dict.items(): + if not key.endswith("_add_hashtree_footer_args") and not key.endswith("_add_hash_footer_args"): + continue + for arg in shlex.split(val): + m = re.match(r"^com\.android\.build\.\w+\.fingerprint:", arg) + if m is None: + continue + fingerprint = arg[len(m.group()):] + glob_dict["fingerprint"] = fingerprint + return + + def ImagePropFromGlobalDict(glob_dict, mount_point): """Build an image property dictionary from the global dictionary. @@ -643,7 +686,11 @@ def ImagePropFromGlobalDict(glob_dict, mount_point): mount_point: such as "system", "data" etc. """ d = {} + TryParseFingerprint(glob_dict) + # Set fixed timestamp for building the OTA package. + if "use_fixed_timestamp" in glob_dict: + d["timestamp"] = FIXED_FILE_TIMESTAMP if "build.prop" in glob_dict: timestamp = glob_dict["build.prop"].GetProp("ro.build.date.utc") if timestamp: @@ -668,6 +715,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point): "erofs_default_compressor", "erofs_default_compress_hints", "erofs_pcluster_size", + "erofs_blocksize", "erofs_share_dup_blocks", "erofs_sparse_flag", "erofs_use_legacy_compression", @@ -675,11 +723,13 @@ def ImagePropFromGlobalDict(glob_dict, mount_point): "system_f2fs_compress", "system_f2fs_sldc_flags", "f2fs_sparse_flag", + "f2fs_blocksize", "skip_fsck", "ext_mkuserimg", "avb_enable", "avb_avbtool", "use_dynamic_partition_size", + "fingerprint", ) for p in common_props: copy_prop(p, p) @@ -718,10 +768,12 @@ def ImagePropFromGlobalDict(glob_dict, mount_point): (True, "{}_erofs_compressor", "erofs_compressor"), (True, "{}_erofs_compress_hints", "erofs_compress_hints"), (True, "{}_erofs_pcluster_size", "erofs_pcluster_size"), + (True, "{}_erofs_blocksize", "erofs_blocksize"), (True, "{}_erofs_share_dup_blocks", "erofs_share_dup_blocks"), (True, "{}_extfs_inode_count", "extfs_inode_count"), (True, "{}_f2fs_compress", "f2fs_compress"), (True, "{}_f2fs_sldc_flags", "f2fs_sldc_flags"), + (True, "{}_f2fs_blocksize", "f2fs_block_size"), (True, "{}_reserved_size", "partition_reserved_size"), (True, "{}_squashfs_block_size", "squashfs_block_size"), (True, "{}_squashfs_compressor", "squashfs_compressor"), @@ -769,7 +821,6 @@ def ImagePropFromGlobalDict(glob_dict, mount_point): d["mount_point"] = mount_point if mount_point == "system": copy_prop("system_headroom", "partition_headroom") - copy_prop("system_root_image", "system_root_image") copy_prop("root_dir", "root_dir") copy_prop("root_fs_config", "root_fs_config") elif mount_point == "data": @@ -870,34 +921,79 @@ def BuildVBMeta(in_dir, glob_dict, output_path): if item not in vbmeta_vendor.split()] vbmeta_partitions.append("vbmeta_vendor") - partitions = {part: os.path.join(in_dir, part + ".img") for part in vbmeta_partitions} - partitions = {part:path for (part, path) in partitions.items() if os.path.exists(path)} + partitions = {part: path for (part, path) in partitions.items() if os.path.exists(path)} common.BuildVBMeta(output_path, partitions, name, vbmeta_partitions) -def main(argv): - args = common.ParseOptions(argv, __doc__) +def BuildImageOrVBMeta(input_directory, target_out, glob_dict, image_properties, out_file): + try: + if "vbmeta" in os.path.basename(out_file): + OPTIONS.info_dict = glob_dict + BuildVBMeta(input_directory, glob_dict, out_file) + else: + BuildImage(input_directory, image_properties, out_file, target_out) + except: + logger.error("Failed to build %s from %s", out_file, input_directory) + raise - if len(args) != 4: - print(__doc__) - sys.exit(1) - common.InitLogging() +def CopyInputDirectory(src, dst, filter_file): + with open(filter_file, 'r') as f: + for line in f: + line = line.strip() + if not line: + return + if line != os.path.normpath(line): + sys.exit(f"{line}: not normalized") + if line.startswith("../") or line.startswith('/'): + sys.exit(f"{line}: escapes staging directory by starting with ../ or /") + full_src = os.path.join(src, line) + full_dst = os.path.join(dst, line) + if os.path.isdir(full_src): + os.makedirs(full_dst, exist_ok=True) + else: + os.makedirs(os.path.dirname(full_dst), exist_ok=True) + os.link(full_src, full_dst, follow_symlinks=False) - in_dir = args[0] - glob_dict_file = args[1] - out_file = args[2] - target_out = args[3] - glob_dict = LoadGlobalDict(glob_dict_file) +def main(argv): + parser = argparse.ArgumentParser( + description="Builds output_image from the given input_directory and properties_file, and " + "writes the image to target_output_directory.") + parser.add_argument("--input-directory-filter-file", + help="the path to a file that contains a list of all files in the input_directory. If this " + "option is provided, all files under the input_directory that are not listed in this file will " + "be deleted before building the image. This is to work around the fact that building a module " + "will install in by default, so there could be files in the input_directory that are not " + "actually supposed to be part of the partition. The paths in this file must be relative to " + "input_directory.") + parser.add_argument("input_directory", + help="the staging directory to be converted to an image file") + parser.add_argument("properties_file", + help="a file containing the 'global dictionary' of properties that affect how the image is " + "built") + parser.add_argument("out_file", + help="the output file to write") + parser.add_argument("target_out", + help="the path to $(TARGET_OUT). Certain tools will use this to look through multiple staging " + "directories for fs config files.") + parser.add_argument("-v", action="store_true", + help="Enable verbose logging", dest="verbose") + args = parser.parse_args() + if args.verbose: + OPTIONS.verbose = True + + common.InitLogging() + + glob_dict = LoadGlobalDict(args.properties_file) if "mount_point" in glob_dict: # The caller knows the mount point and provides a dictionary needed by # BuildImage(). image_properties = glob_dict else: - image_filename = os.path.basename(out_file) + image_filename = os.path.basename(args.out_file) mount_point = "" if image_filename == "system.img": mount_point = "system" @@ -932,15 +1028,12 @@ def main(argv): if "vbmeta" != mount_point: image_properties = ImagePropFromGlobalDict(glob_dict, mount_point) - try: - if "vbmeta" in os.path.basename(out_file): - OPTIONS.info_dict = glob_dict - BuildVBMeta(in_dir, glob_dict, out_file) - else: - BuildImage(in_dir, image_properties, out_file, target_out) - except: - logger.error("Failed to build %s from %s", out_file, in_dir) - raise + if args.input_directory_filter_file and not os.environ.get("BUILD_BROKEN_INCORRECT_PARTITION_IMAGES"): + with tempfile.TemporaryDirectory(dir=os.path.dirname(args.input_directory)) as new_input_directory: + CopyInputDirectory(args.input_directory, new_input_directory, args.input_directory_filter_file) + BuildImageOrVBMeta(new_input_directory, args.target_out, glob_dict, image_properties, args.out_file) + else: + BuildImageOrVBMeta(args.input_directory, args.target_out, glob_dict, image_properties, args.out_file) if __name__ == '__main__': diff --git a/tools/releasetools/check_ota_package_signature.py b/tools/releasetools/check_ota_package_signature.py index 97957be28f..b395c196d0 100755 --- a/tools/releasetools/check_ota_package_signature.py +++ b/tools/releasetools/check_ota_package_signature.py @@ -142,7 +142,7 @@ def VerifyAbOtaPayload(cert, package): """Verifies the payload and metadata signatures in an A/B OTA payload.""" package_zip = zipfile.ZipFile(package, 'r', allowZip64=True) if 'payload.bin' not in package_zip.namelist(): - package_zip.close() + common.ZipClose(package_zip) return print('Verifying A/B OTA payload signatures...') @@ -160,7 +160,7 @@ def VerifyAbOtaPayload(cert, package): '--in_file=' + payload_file, '--public_key=' + pubkey] common.RunAndCheckOutput(cmd) - package_zip.close() + common.ZipClose(package_zip) # Verified successfully upon reaching here. print('\nPayload signatures VERIFIED\n\n') diff --git a/tools/releasetools/check_target_files_signatures.py b/tools/releasetools/check_target_files_signatures.py index d935607e43..cdafb4b2e5 100755 --- a/tools/releasetools/check_target_files_signatures.py +++ b/tools/releasetools/check_target_files_signatures.py @@ -58,22 +58,6 @@ if sys.hexversion < 0x02070000: logger = logging.getLogger(__name__) -# Work around a bug in Python's zipfile module that prevents opening of zipfiles -# if any entry has an extra field of between 1 and 3 bytes (which is common with -# zipaligned APKs). This overrides the ZipInfo._decodeExtra() method (which -# contains the bug) with an empty version (since we don't need to decode the -# extra field anyway). -# Issue #14315: https://bugs.python.org/issue14315, fixed in Python 2.7.8 and -# Python 3.5.0 alpha 1. - - -class MyZipInfo(zipfile.ZipInfo): - def _decodeExtra(self): - pass - - -zipfile.ZipInfo = MyZipInfo - OPTIONS = common.OPTIONS @@ -241,7 +225,8 @@ class APK(object): # Signer (minSdkVersion=24, maxSdkVersion=32) certificate SHA-1 digest: 19da94896ce4078c38ca695701f1dec741ec6d67 # ... certs_info = {} - certificate_regex = re.compile(r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)") + certificate_regex = re.compile( + r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)") for line in output.splitlines(): m = certificate_regex.match(line) if not m: @@ -312,7 +297,7 @@ class TargetFiles(object): # This is the list of wildcards of files we extract from |filename|. apk_extensions = ['*.apk', '*.apex'] - with zipfile.ZipFile(filename) as input_zip: + with zipfile.ZipFile(filename, "r") as input_zip: self.certmap, compressed_extension = common.ReadApkCerts(input_zip) if compressed_extension: apk_extensions.append('*.apk' + compressed_extension) diff --git a/tools/releasetools/check_target_files_vintf.py b/tools/releasetools/check_target_files_vintf.py index 5b71c726a8..b8dcd8465c 100755 --- a/tools/releasetools/check_target_files_vintf.py +++ b/tools/releasetools/check_target_files_vintf.py @@ -129,8 +129,9 @@ def CheckVintfFromExtractedTargetFiles(input_tmp, info_dict=None): dirmap = GetDirmap(input_tmp) - # Simulate apexd from target-files. - dirmap['/apex'] = PrepareApexDirectory(input_tmp) + # Simulate apexd with target-files. + # add a mapping('/apex' => ${input_tmp}/APEX) to dirmap + PrepareApexDirectory(input_tmp, dirmap) args_for_skus = GetArgsForSkus(info_dict) shipping_api_level_args = GetArgsForShippingApiLevel(info_dict) @@ -204,7 +205,8 @@ def GetVintfApexUnzipPatterns(): return patterns -def PrepareApexDirectory(inp): + +def PrepareApexDirectory(inp, dirmap): """ Prepare /apex directory before running checkvintf Apex binaries do not support dirmaps, in order to use these binaries we @@ -212,96 +214,26 @@ def PrepareApexDirectory(inp): expected device locations. This simulates how apexd activates APEXes. - 1. create {inp}/APEX which is treated as a "/" on device. - 2. copy apexes from target-files to {root}/{partition}/apex. - 3. mount apexes under {root}/{partition}/apex at {root}/apex. - 4. generate info files with dump_apex_info. - - We'll get the following layout - {inp}/APEX/apex # Activated APEXes + some info files - {inp}/APEX/system/apex # System APEXes - {inp}/APEX/vendor/apex # Vendor APEXes - ... - - Args: - inp: path to the directory that contains the extracted target files archive. - - Returns: - directory representing /apex on device + 1. create {inp}/APEX which is treated as a "/apex" on device. + 2. invoke apexd_host with APEXes. """ - deapexer = 'deapexer' - debugfs_path = 'debugfs' - blkid_path = 'blkid' - fsckerofs_path = 'fsck.erofs' - if OPTIONS.search_path: - debugfs_path = os.path.join(OPTIONS.search_path, 'bin', 'debugfs_static') - deapexer_path = os.path.join(OPTIONS.search_path, 'bin', 'deapexer') - blkid_path = os.path.join(OPTIONS.search_path, 'bin', 'blkid_static') - fsckerofs_path = os.path.join(OPTIONS.search_path, 'bin', 'fsck.erofs') - if os.path.isfile(deapexer_path): - deapexer = deapexer_path - - def ExtractApexes(path, outp): - # Extract all APEXes found in input path. - logger.info('Extracting APEXs in %s', path) - for f in os.listdir(path): - logger.info(' adding APEX %s', os.path.basename(f)) - apex = os.path.join(path, f) - if os.path.isdir(apex) and os.path.isfile(os.path.join(apex, 'apex_manifest.pb')): - info = ParseApexManifest(os.path.join(apex, 'apex_manifest.pb')) - # Flattened APEXes may have symlinks for libs (linked to /system/lib) - # We need to blindly copy them all. - shutil.copytree(apex, os.path.join(outp, info.name), symlinks=True) - elif os.path.isfile(apex) and apex.endswith(('.apex', '.capex')): - cmd = [deapexer, - '--debugfs_path', debugfs_path, - 'info', - apex] - info = json.loads(common.RunAndCheckOutput(cmd)) - - cmd = [deapexer, - '--debugfs_path', debugfs_path, - '--fsckerofs_path', fsckerofs_path, - '--blkid_path', blkid_path, - 'extract', - apex, - os.path.join(outp, info['name'])] - common.RunAndCheckOutput(cmd) - else: - logger.info(' .. skipping %s (is it APEX?)', path) - - root_dir_name = 'APEX' - root_dir = os.path.join(inp, root_dir_name) - extracted_root = os.path.join(root_dir, 'apex') + apex_dir = common.MakeTempDir('APEX') + # checkvintf needs /apex dirmap + dirmap['/apex'] = apex_dir # Always create /apex directory for dirmap - os.makedirs(extracted_root) + os.makedirs(apex_dir, exist_ok=True) - create_info_file = False + # Invoke apexd_host to activate APEXes for checkvintf + apex_host = os.path.join(OPTIONS.search_path, 'bin', 'apexd_host') + cmd = [apex_host, '--tool_path', OPTIONS.search_path] + cmd += ['--apex_path', dirmap['/apex']] + for p in ['system', 'system_ext', 'product', 'vendor']: + if '/' + p in dirmap: + cmd += ['--' + p + '_path', dirmap['/' + p]] + common.RunAndCheckOutput(cmd) - # Loop through search path looking for and processing apex/ directories. - for device_path, target_files_rel_paths in DIR_SEARCH_PATHS.items(): - # checkvintf only needs vendor apexes. skip other partitions for efficiency - if device_path not in ['/vendor', '/odm']: - continue - # First, copy VENDOR/apex/foo.apex to APEX/vendor/apex/foo.apex - # Then, extract the contents to APEX/apex/foo/ - for target_files_rel_path in target_files_rel_paths: - inp_partition = os.path.join(inp, target_files_rel_path,"apex") - if os.path.exists(inp_partition): - apex_dir = root_dir + os.path.join(device_path + "/apex"); - os.makedirs(root_dir + device_path) - shutil.copytree(inp_partition, apex_dir, symlinks=True) - ExtractApexes(apex_dir, extracted_root) - create_info_file = True - - if create_info_file: - ### Dump apex info files - dump_cmd = ['dump_apex_info', '--root_dir', root_dir] - common.RunAndCheckOutput(dump_cmd) - - return extracted_root def CheckVintfFromTargetFiles(inp, info_dict=None): """ diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 8d16ca0d44..2367691e43 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -20,7 +20,6 @@ import copy import datetime import errno import fnmatch -from genericpath import isdir import getopt import getpass import gzip @@ -34,21 +33,29 @@ import re import shlex import shutil import subprocess +import stat import sys import tempfile import threading import time import zipfile + +from typing import Iterable, Callable +from dataclasses import dataclass from hashlib import sha1, sha256 import images -import rangelib import sparse_img from blockimgdiff import BlockImageDiff logger = logging.getLogger(__name__) +@dataclass +class OptionHandler: + extra_long_opts: Iterable[str] + handler: Callable + class Options(object): def __init__(self): @@ -73,9 +80,7 @@ class Options(object): if "ANDROID_HOST_OUT" in os.environ: self.search_path = os.environ["ANDROID_HOST_OUT"] self.signapk_shared_library_path = "lib64" # Relative to search_path - self.sign_sepolicy_path = None self.extra_signapk_args = [] - self.extra_sign_sepolicy_args = [] self.aapt2_path = "aapt2" self.java_path = "java" # Use the one on the path by default. self.java_args = ["-Xmx4096m"] # The default JVM args. @@ -95,8 +100,6 @@ class Options(object): self.cache_size = None self.stash_threshold = 0.8 self.logfile = None - self.host_tools = {} - self.sepolicy_name = 'sepolicy.apex' OPTIONS = Options() @@ -112,13 +115,18 @@ SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") # descriptor into vbmeta.img. When adding a new entry here, the # AVB_FOOTER_ARGS_BY_PARTITION in sign_target_files_apks need to be updated # accordingly. -AVB_PARTITIONS = ('boot', 'init_boot', 'dtbo', 'odm', 'product', 'pvmfw', 'recovery', - 'system', 'system_ext', 'vendor', 'vendor_boot', 'vendor_kernel_boot', - 'vendor_dlkm', 'odm_dlkm', 'system_dlkm') +AVB_PARTITIONS = ('boot', 'init_boot', 'dtbo', 'odm', 'product', 'pvmfw', + 'recovery', 'system', 'system_ext', 'vendor', 'vendor_boot', + 'vendor_kernel_boot', 'vendor_dlkm', 'odm_dlkm', + 'system_dlkm') # Chained VBMeta partitions. AVB_VBMETA_PARTITIONS = ('vbmeta_system', 'vbmeta_vendor') +# avbtool arguments name +AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG = '--include_descriptors_from_image' +AVB_ARG_NAME_CHAIN_PARTITION = '--chain_partition' + # Partitions that should have their care_map added to META/care_map.pb PARTITIONS_WITH_CARE_MAP = [ 'system', @@ -139,6 +147,19 @@ PARTITIONS_WITH_BUILD_PROP = PARTITIONS_WITH_CARE_MAP + ['boot', 'init_boot'] RAMDISK_BUILD_PROP_REL_PATHS = ['system/etc/ramdisk/build.prop'] +@dataclass +class AvbChainedPartitionArg: + """The required arguments for avbtool --chain_partition.""" + partition: str + rollback_index_location: int + pubkey_path: str + + def to_string(self): + """Convert to string command arguments.""" + return '{}:{}:{}'.format( + self.partition, self.rollback_index_location, self.pubkey_path) + + class ErrorCode(object): """Define error_codes for failures that happen during the actual update package installation. @@ -194,7 +215,7 @@ def InitLogging(): '': { 'handlers': ['default'], 'propagate': True, - 'level': 'INFO', + 'level': 'NOTSET', } } } @@ -224,23 +245,15 @@ def InitLogging(): logging.config.dictConfig(config) -def SetHostToolLocation(tool_name, location): - OPTIONS.host_tools[tool_name] = location - - def FindHostToolPath(tool_name): """Finds the path to the host tool. Args: tool_name: name of the tool to find Returns: - path to the tool if found under either one of the host_tools map or under - the same directory as this binary is located at. If not found, tool_name - is returned. + path to the tool if found under the same directory as this binary is located at. If not found, + tool_name is returned. """ - if tool_name in OPTIONS.host_tools: - return OPTIONS.host_tools[tool_name] - my_dir = os.path.dirname(os.path.realpath(sys.argv[0])) tool_path = os.path.join(my_dir, tool_name) if os.path.exists(tool_path): @@ -450,10 +463,7 @@ class BuildInfo(object): @property def is_vabc(self): - vendor_prop = self.info_dict.get("vendor.build.prop") - vabc_enabled = vendor_prop and \ - vendor_prop.GetProp("ro.virtual_ab.compression.enabled") == "true" - return vabc_enabled + return self.info_dict.get("virtual_ab_compression") == "true" @property def is_android_r(self): @@ -461,6 +471,37 @@ class BuildInfo(object): return system_prop and system_prop.GetProp("ro.build.version.release") == "11" @property + def is_release_key(self): + system_prop = self.info_dict.get("build.prop") + return system_prop and system_prop.GetProp("ro.build.tags") == "release-key" + + @property + def vabc_compression_param(self): + return self.get("virtual_ab_compression_method", "") + + @property + def vabc_cow_version(self): + return self.get("virtual_ab_cow_version", "") + + @property + def vendor_api_level(self): + vendor_prop = self.info_dict.get("vendor.build.prop") + if not vendor_prop: + return -1 + + props = [ + "ro.board.first_api_level", + "ro.product.first_api_level", + ] + for prop in props: + value = vendor_prop.GetProp(prop) + try: + return int(value) + except: + pass + return -1 + + @property def is_vabc_xor(self): vendor_prop = self.info_dict.get("vendor.build.prop") vabc_xor_enabled = vendor_prop and \ @@ -698,26 +739,73 @@ class BuildInfo(object): script.AssertOemProperty(prop, values, oem_no_mount) -def ReadFromInputFile(input_file, fn): - """Reads the contents of fn from input zipfile or directory.""" +def DoesInputFileContain(input_file, fn): + """Check whether the input target_files.zip contain an entry `fn`""" if isinstance(input_file, zipfile.ZipFile): - return input_file.read(fn).decode() + return fn in input_file.namelist() elif zipfile.is_zipfile(input_file): with zipfile.ZipFile(input_file, "r", allowZip64=True) as zfp: - return zfp.read(fn).decode() + return fn in zfp.namelist() + else: + if not os.path.isdir(input_file): + raise ValueError( + "Invalid input_file, accepted inputs are ZipFile object, path to .zip file on disk, or path to extracted directory. Actual: " + input_file) + path = os.path.join(input_file, *fn.split("/")) + return os.path.exists(path) + + +def ReadBytesFromInputFile(input_file, fn): + """Reads the bytes of fn from input zipfile or directory.""" + if isinstance(input_file, zipfile.ZipFile): + return input_file.read(fn) + elif zipfile.is_zipfile(input_file): + with zipfile.ZipFile(input_file, "r", allowZip64=True) as zfp: + return zfp.read(fn) else: if not os.path.isdir(input_file): raise ValueError( "Invalid input_file, accepted inputs are ZipFile object, path to .zip file on disk, or path to extracted directory. Actual: " + input_file) path = os.path.join(input_file, *fn.split("/")) try: - with open(path) as f: + with open(path, "rb") as f: return f.read() except IOError as e: if e.errno == errno.ENOENT: raise KeyError(fn) +def ReadFromInputFile(input_file, fn): + """Reads the str contents of fn from input zipfile or directory.""" + return ReadBytesFromInputFile(input_file, fn).decode() + + +def WriteBytesToInputFile(input_file, fn, data): + """Write bytes |data| contents to fn of input zipfile or directory.""" + if isinstance(input_file, zipfile.ZipFile): + with input_file.open(fn, "w") as entry_fp: + return entry_fp.write(data) + elif zipfile.is_zipfile(input_file): + with zipfile.ZipFile(input_file, "r", allowZip64=True) as zfp: + with zfp.open(fn, "w") as entry_fp: + return entry_fp.write(data) + else: + if not os.path.isdir(input_file): + raise ValueError( + "Invalid input_file, accepted inputs are ZipFile object, path to .zip file on disk, or path to extracted directory. Actual: " + input_file) + path = os.path.join(input_file, *fn.split("/")) + try: + with open(path, "wb") as f: + return f.write(data) + except IOError as e: + if e.errno == errno.ENOENT: + raise KeyError(fn) + + +def WriteToInputFile(input_file, fn, str: str): + """Write str content to fn of input file or directory""" + return WriteBytesToInputFile(input_file, fn, str.encode()) + + def ExtractFromInputFile(input_file, fn): """Extracts the contents of fn from input zipfile or directory into a file.""" if isinstance(input_file, zipfile.ZipFile): @@ -865,20 +953,14 @@ def LoadInfoDict(input_file, repacking=False): input_file, partition, ramdisk_format=ramdisk_format) d["build.prop"] = d["system.build.prop"] - # Set up the salt (based on fingerprint) that will be used when adding AVB - # hash / hashtree footers. if d.get("avb_enable") == "true": build_info = BuildInfo(d, use_legacy_id=True) - for partition in PARTITIONS_WITH_BUILD_PROP: - fingerprint = build_info.GetPartitionFingerprint(partition) - if fingerprint: - d["avb_{}_salt".format(partition)] = sha256( - fingerprint.encode()).hexdigest() - # Set up the salt for partitions without build.prop if build_info.fingerprint: - d["avb_salt"] = sha256(build_info.fingerprint.encode()).hexdigest() - + if "fingerprint" not in d: + d["fingerprint"] = build_info.fingerprint + if "avb_salt" not in d: + d["avb_salt"] = sha256(build_info.fingerprint.encode()).hexdigest() # Set the vbmeta digest if exists try: d["vbmeta_digest"] = read_helper("META/vbmeta_digest.txt").rstrip() @@ -934,7 +1016,7 @@ class PartitionBuildProps(object): each of the variables. ramdisk_format: If name is "boot", the format of ramdisk inside the boot image. Otherwise, its value is ignored. - Use lz4 to decompress by default. If its value is gzip, use minigzip. + Use lz4 to decompress by default. If its value is gzip, use gzip. """ def __init__(self, input_file, name, placeholder_values=None): @@ -1084,8 +1166,7 @@ class PartitionBuildProps(object): return self.build_props.get(prop) -def LoadRecoveryFSTab(read_helper, fstab_version, recovery_fstab_path, - system_root_image=False): +def LoadRecoveryFSTab(read_helper, fstab_version, recovery_fstab_path): class Partition(object): def __init__(self, mount_point, fs_type, device, length, context, slotselect): self.mount_point = mount_point @@ -1144,12 +1225,6 @@ def LoadRecoveryFSTab(read_helper, fstab_version, recovery_fstab_path, device=pieces[0], length=length, context=context, slotselect=slotselect) - # / is used for the system mount point when the root directory is included in - # system. Other areas assume system is always at "/system" so point /system - # at /. - if system_root_image: - assert '/system' not in d and '/' in d - d["/system"] = d["/"] return d @@ -1165,32 +1240,19 @@ def _FindAndLoadRecoveryFstab(info_dict, input_file, read_helper): # ../RAMDISK/system/etc/recovery.fstab. This function has to handle both # cases, since it may load the info_dict from an old build (e.g. when # generating incremental OTAs from that build). - system_root_image = info_dict.get('system_root_image') == 'true' if info_dict.get('no_recovery') != 'true': recovery_fstab_path = 'RECOVERY/RAMDISK/system/etc/recovery.fstab' - if isinstance(input_file, zipfile.ZipFile): - if recovery_fstab_path not in input_file.namelist(): - recovery_fstab_path = 'RECOVERY/RAMDISK/etc/recovery.fstab' - else: - path = os.path.join(input_file, *recovery_fstab_path.split('/')) - if not os.path.exists(path): - recovery_fstab_path = 'RECOVERY/RAMDISK/etc/recovery.fstab' + if not DoesInputFileContain(input_file, recovery_fstab_path): + recovery_fstab_path = 'RECOVERY/RAMDISK/etc/recovery.fstab' return LoadRecoveryFSTab( - read_helper, info_dict['fstab_version'], recovery_fstab_path, - system_root_image) + read_helper, info_dict['fstab_version'], recovery_fstab_path) if info_dict.get('recovery_as_boot') == 'true': recovery_fstab_path = 'BOOT/RAMDISK/system/etc/recovery.fstab' - if isinstance(input_file, zipfile.ZipFile): - if recovery_fstab_path not in input_file.namelist(): - recovery_fstab_path = 'BOOT/RAMDISK/etc/recovery.fstab' - else: - path = os.path.join(input_file, *recovery_fstab_path.split('/')) - if not os.path.exists(path): - recovery_fstab_path = 'BOOT/RAMDISK/etc/recovery.fstab' + if not DoesInputFileContain(input_file, recovery_fstab_path): + recovery_fstab_path = 'BOOT/RAMDISK/etc/recovery.fstab' return LoadRecoveryFSTab( - read_helper, info_dict['fstab_version'], recovery_fstab_path, - system_root_image) + read_helper, info_dict['fstab_version'], recovery_fstab_path) return None @@ -1257,7 +1319,11 @@ def MergeDynamicPartitionInfoDicts(framework_dict, vendor_dict): key = "super_%s_partition_list" % partition_group merged_dict[key] = uniq_concat( framework_dict.get(key, ""), vendor_dict.get(key, "")) - + # in the case that vendor is on s build, but is taking a v3 -> v3 vabc ota, we want to fallback to v2 + if "vabc_cow_version" not in vendor_dict or "vabc_cow_version" not in framework_dict: + merged_dict["vabc_cow_version"] = '2' + else: + merged_dict["vabc_cow_version"] = min(vendor_dict["vabc_cow_version"], framework_dict["vabc_cow_version"]) # Various other flags should be copied from the vendor dict, if defined. for key in ("virtual_ab", "virtual_ab_retrofit", "lpmake", "super_metadata_device", "super_partition_error_limit", @@ -1354,25 +1420,50 @@ def RunHostInitVerifier(product_out, partition_map): return RunAndCheckOutput(cmd) -def AppendAVBSigningArgs(cmd, partition): +def AppendAVBSigningArgs(cmd, partition, avb_salt=None): """Append signing arguments for avbtool.""" # e.g., "--key path/to/signing_key --algorithm SHA256_RSA4096" - key_path = OPTIONS.info_dict.get("avb_" + partition + "_key_path") - if key_path and not os.path.exists(key_path) and OPTIONS.search_path: - new_key_path = os.path.join(OPTIONS.search_path, key_path) - if os.path.exists(new_key_path): - key_path = new_key_path + key_path = ResolveAVBSigningPathArgs( + OPTIONS.info_dict.get("avb_" + partition + "_key_path")) algorithm = OPTIONS.info_dict.get("avb_" + partition + "_algorithm") if key_path and algorithm: cmd.extend(["--key", key_path, "--algorithm", algorithm]) - avb_salt = OPTIONS.info_dict.get("avb_salt") + if avb_salt is None: + avb_salt = OPTIONS.info_dict.get("avb_salt") # make_vbmeta_image doesn't like "--salt" (and it's not needed). if avb_salt and not partition.startswith("vbmeta"): cmd.extend(["--salt", avb_salt]) +def ResolveAVBSigningPathArgs(split_args): + + def ResolveBinaryPath(path): + if os.path.exists(path): + return path + if OPTIONS.search_path: + new_path = os.path.join(OPTIONS.search_path, path) + if os.path.exists(new_path): + return new_path + raise ExternalError( + "Failed to find {}".format(path)) + + if not split_args: + return split_args + + if isinstance(split_args, list): + for index, arg in enumerate(split_args[:-1]): + if arg == '--signing_helper': + signing_helper_path = split_args[index + 1] + split_args[index + 1] = ResolveBinaryPath(signing_helper_path) + break + elif isinstance(split_args, str): + split_args = ResolveBinaryPath(split_args) + + return split_args + + def GetAvbPartitionArg(partition, image, info_dict=None): - """Returns the VBMeta arguments for partition. + """Returns the VBMeta arguments for one partition. It sets up the VBMeta argument by including the partition descriptor from the given 'image', or by configuring the partition as a chained partition. @@ -1384,7 +1475,7 @@ def GetAvbPartitionArg(partition, image, info_dict=None): OPTIONS.info_dict if None has been given. Returns: - A list of VBMeta arguments. + A list of VBMeta arguments for one partition. """ if info_dict is None: info_dict = OPTIONS.info_dict @@ -1392,7 +1483,7 @@ def GetAvbPartitionArg(partition, image, info_dict=None): # Check if chain partition is used. key_path = info_dict.get("avb_" + partition + "_key_path") if not key_path: - return ["--include_descriptors_from_image", image] + return [AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG, image] # For a non-A/B device, we don't chain /recovery nor include its descriptor # into vbmeta.img. The recovery image will be configured on an independent @@ -1404,7 +1495,62 @@ def GetAvbPartitionArg(partition, image, info_dict=None): # Otherwise chain the partition into vbmeta. chained_partition_arg = GetAvbChainedPartitionArg(partition, info_dict) - return ["--chain_partition", chained_partition_arg] + return [AVB_ARG_NAME_CHAIN_PARTITION, chained_partition_arg] + + +def GetAvbPartitionsArg(partitions, + resolve_rollback_index_location_conflict=False, + info_dict=None): + """Returns the VBMeta arguments for all AVB partitions. + + It sets up the VBMeta argument by calling GetAvbPartitionArg of all + partitions. + + Args: + partitions: A dict of all AVB partitions. + resolve_rollback_index_location_conflict: If true, resolve conflicting avb + rollback index locations by assigning the smallest unused value. + info_dict: A dict returned by common.LoadInfoDict(). + + Returns: + A list of VBMeta arguments for all partitions. + """ + # An AVB partition will be linked into a vbmeta partition by either + # AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG or AVB_ARG_NAME_CHAIN_PARTITION, there + # should be no other cases. + valid_args = { + AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG: [], + AVB_ARG_NAME_CHAIN_PARTITION: [] + } + + for partition, path in sorted(partitions.items()): + avb_partition_arg = GetAvbPartitionArg(partition, path, info_dict) + if not avb_partition_arg: + continue + arg_name, arg_value = avb_partition_arg + assert arg_name in valid_args + valid_args[arg_name].append(arg_value) + + # Copy the arguments for non-chained AVB partitions directly without + # intervention. + avb_args = [] + for image in valid_args[AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG]: + avb_args.extend([AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG, image]) + + # Handle chained AVB partitions. The rollback index location might be + # adjusted if two partitions use the same value. This may happen when mixing + # a shared system image with other vendor images. + used_index_loc = set() + for chained_partition_arg in valid_args[AVB_ARG_NAME_CHAIN_PARTITION]: + if resolve_rollback_index_location_conflict: + while chained_partition_arg.rollback_index_location in used_index_loc: + chained_partition_arg.rollback_index_location += 1 + + used_index_loc.add(chained_partition_arg.rollback_index_location) + avb_args.extend([AVB_ARG_NAME_CHAIN_PARTITION, + chained_partition_arg.to_string()]) + + return avb_args def GetAvbChainedPartitionArg(partition, info_dict, key=None): @@ -1418,69 +1564,23 @@ def GetAvbChainedPartitionArg(partition, info_dict, key=None): the key listed in info_dict. Returns: - A string of form "partition:rollback_index_location:key" that can be used to - build or verify vbmeta image. + An AvbChainedPartitionArg object with rollback_index_location and + pubkey_path that can be used to build or verify vbmeta image. """ if key is None: key = info_dict["avb_" + partition + "_key_path"] - if key and not os.path.exists(key) and OPTIONS.search_path: - new_key_path = os.path.join(OPTIONS.search_path, key) - if os.path.exists(new_key_path): - key = new_key_path + key = ResolveAVBSigningPathArgs(key) pubkey_path = ExtractAvbPublicKey(info_dict["avb_avbtool"], key) rollback_index_location = info_dict[ "avb_" + partition + "_rollback_index_location"] - return "{}:{}:{}".format(partition, rollback_index_location, pubkey_path) - + return AvbChainedPartitionArg( + partition=partition, + rollback_index_location=int(rollback_index_location), + pubkey_path=pubkey_path) -def _HasGkiCertificationArgs(): - return ("gki_signing_key_path" in OPTIONS.info_dict and - "gki_signing_algorithm" in OPTIONS.info_dict) - -def _GenerateGkiCertificate(image, image_name): - key_path = OPTIONS.info_dict.get("gki_signing_key_path") - algorithm = OPTIONS.info_dict.get("gki_signing_algorithm") - - if not os.path.exists(key_path) and OPTIONS.search_path: - new_key_path = os.path.join(OPTIONS.search_path, key_path) - if os.path.exists(new_key_path): - key_path = new_key_path - - # Checks key_path exists, before processing --gki_signing_* args. - if not os.path.exists(key_path): - raise ExternalError( - 'gki_signing_key_path: "{}" not found'.format(key_path)) - - output_certificate = tempfile.NamedTemporaryFile() - cmd = [ - "generate_gki_certificate", - "--name", image_name, - "--algorithm", algorithm, - "--key", key_path, - "--output", output_certificate.name, - image, - ] - - signature_args = OPTIONS.info_dict.get("gki_signing_signature_args", "") - signature_args = signature_args.strip() - if signature_args: - cmd.extend(["--additional_avb_args", signature_args]) - - args = OPTIONS.info_dict.get("avb_boot_add_hash_footer_args", "") - args = args.strip() - if args: - cmd.extend(["--additional_avb_args", args]) - - RunAndCheckOutput(cmd) - - output_certificate.seek(os.SEEK_SET, 0) - data = output_certificate.read() - output_certificate.close() - return data - - -def BuildVBMeta(image_path, partitions, name, needed_partitions): +def BuildVBMeta(image_path, partitions, name, needed_partitions, + resolve_rollback_index_location_conflict=False): """Creates a VBMeta image. It generates the requested VBMeta image. The requested image could be for @@ -1495,6 +1595,8 @@ def BuildVBMeta(image_path, partitions, name, needed_partitions): name: Name of the VBMeta partition, e.g. 'vbmeta', 'vbmeta_system'. needed_partitions: Partitions whose descriptors should be included into the generated VBMeta image. + resolve_rollback_index_location_conflict: If true, resolve conflicting avb + rollback index locations by assigning the smallest unused value. Raises: AssertionError: On invalid input args. @@ -1505,9 +1607,11 @@ def BuildVBMeta(image_path, partitions, name, needed_partitions): custom_partitions = OPTIONS.info_dict.get( "avb_custom_images_partition_list", "").strip().split() - custom_avb_partitions = ["vbmeta_" + part for part in OPTIONS.info_dict.get("avb_custom_vbmeta_images_partition_list", "").strip().split()] + custom_avb_partitions = ["vbmeta_" + part for part in OPTIONS.info_dict.get( + "avb_custom_vbmeta_images_partition_list", "").strip().split()] - for partition, path in partitions.items(): + avb_partitions = {} + for partition, path in sorted(partitions.items()): if partition not in needed_partitions: continue assert (partition in AVB_PARTITIONS or @@ -1517,7 +1621,9 @@ def BuildVBMeta(image_path, partitions, name, needed_partitions): 'Unknown partition: {}'.format(partition) assert os.path.exists(path), \ 'Failed to find {} for {}'.format(path, partition) - cmd.extend(GetAvbPartitionArg(partition, path)) + avb_partitions[partition] = path + cmd.extend(GetAvbPartitionsArg(avb_partitions, + resolve_rollback_index_location_conflict)) args = OPTIONS.info_dict.get("avb_{}_args".format(name)) if args and args.strip(): @@ -1528,7 +1634,7 @@ def BuildVBMeta(image_path, partitions, name, needed_partitions): # same location when running this script (we have the input target_files # zip only). For such cases, we additionally scan other locations (e.g. # IMAGES/, RADIO/, etc) before bailing out. - if arg == '--include_descriptors_from_image': + if arg == AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG: chained_image = split_args[index + 1] if os.path.exists(chained_image): continue @@ -1541,6 +1647,8 @@ def BuildVBMeta(image_path, partitions, name, needed_partitions): found = True break assert found, 'Failed to find {}'.format(chained_image) + + split_args = ResolveAVBSigningPathArgs(split_args) cmd.extend(split_args) RunAndCheckOutput(cmd) @@ -1566,9 +1674,9 @@ def _MakeRamdisk(sourcedir, fs_config_file=None, p2 = Run(["lz4", "-l", "-12", "--favor-decSpeed"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) elif ramdisk_format == RamdiskFormat.GZ: - p2 = Run(["minigzip"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) + p2 = Run(["gzip"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) else: - raise ValueError("Only support lz4 or minigzip ramdisk format.") + raise ValueError("Only support lz4 or gzip ramdisk format.") p2.wait() p1.wait() @@ -1693,29 +1801,6 @@ def _BuildBootableImage(image_name, sourcedir, fs_config_file, RunAndCheckOutput(cmd) - if _HasGkiCertificationArgs(): - if not os.path.exists(img.name): - raise ValueError("Cannot find GKI boot.img") - if kernel_path is None or not os.path.exists(kernel_path): - raise ValueError("Cannot find GKI kernel.img") - - # Certify GKI images. - boot_signature_bytes = b'' - boot_signature_bytes += _GenerateGkiCertificate(img.name, "boot") - boot_signature_bytes += _GenerateGkiCertificate( - kernel_path, "generic_kernel") - - BOOT_SIGNATURE_SIZE = 16 * 1024 - if len(boot_signature_bytes) > BOOT_SIGNATURE_SIZE: - raise ValueError( - f"GKI boot_signature size must be <= {BOOT_SIGNATURE_SIZE}") - boot_signature_bytes += ( - b'\0' * (BOOT_SIGNATURE_SIZE - len(boot_signature_bytes))) - assert len(boot_signature_bytes) == BOOT_SIGNATURE_SIZE - - with open(img.name, 'ab') as f: - f.write(boot_signature_bytes) - # Sign the image if vboot is non-empty. if info_dict.get("vboot"): path = "/" + partition_name @@ -1748,10 +1833,15 @@ def _BuildBootableImage(image_name, sourcedir, fs_config_file, cmd = [avbtool, "add_hash_footer", "--image", img.name, "--partition_size", str(part_size), "--partition_name", partition_name] - AppendAVBSigningArgs(cmd, partition_name) + salt = None + if kernel_path is not None: + with open(kernel_path, "rb") as fp: + salt = sha256(fp.read()).hexdigest() + AppendAVBSigningArgs(cmd, partition_name, salt) args = info_dict.get("avb_" + partition_name + "_add_hash_footer_args") if args and args.strip(): - cmd.extend(shlex.split(args)) + split_args = ResolveAVBSigningPathArgs(shlex.split(args)) + cmd.extend(split_args) RunAndCheckOutput(cmd) img.seek(os.SEEK_SET, 0) @@ -1789,10 +1879,22 @@ def _SignBootableImage(image_path, prebuilt_name, partition_name, cmd = [avbtool, "add_hash_footer", "--image", image_path, "--partition_size", str(part_size), "--partition_name", partition_name] - AppendAVBSigningArgs(cmd, partition_name) + # Use sha256 of the kernel as salt for reproducible builds + with tempfile.TemporaryDirectory() as tmpdir: + RunAndCheckOutput(["unpack_bootimg", "--boot_img", image_path, "--out", tmpdir]) + for filename in ["kernel", "ramdisk", "vendor_ramdisk00"]: + path = os.path.join(tmpdir, filename) + if os.path.exists(path) and os.path.getsize(path): + print("Using {} as salt for avb footer of {}".format( + filename, partition_name)) + with open(path, "rb") as fp: + salt = sha256(fp.read()).hexdigest() + break + AppendAVBSigningArgs(cmd, partition_name, salt) args = info_dict.get("avb_" + partition_name + "_add_hash_footer_args") if args and args.strip(): - cmd.extend(shlex.split(args)) + split_args = ResolveAVBSigningPathArgs(shlex.split(args)) + cmd.extend(split_args) RunAndCheckOutput(cmd) @@ -1815,11 +1917,6 @@ def HasRamdisk(partition_name, info_dict=None): if info_dict.get("gki_boot_image_without_ramdisk") == "true": return False # A GKI boot.img has no ramdisk since Android-13. - if info_dict.get("system_root_image") == "true": - # The ramdisk content is merged into the system.img, so there is NO - # ramdisk in the boot.img or boot-<kernel version>.img. - return False - if info_dict.get("init_boot") == "true": # The ramdisk is moved to the init_boot.img, so there is NO # ramdisk in the boot.img or boot-<kernel version>.img. @@ -1867,7 +1964,7 @@ def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, data = _BuildBootableImage(prebuilt_name, os.path.join(unpack_dir, tree_subdir), os.path.join(unpack_dir, fs_config), os.path.join(unpack_dir, 'META/ramdisk_node_list') - if dev_nodes else None, + if dev_nodes else None, info_dict, has_ramdisk, two_step_image) if data: return File(name, data) @@ -1972,7 +2069,8 @@ def _BuildVendorBootImage(sourcedir, partition_name, info_dict=None): AppendAVBSigningArgs(cmd, partition_name) args = info_dict.get(f'avb_{partition_name}_add_hash_footer_args') if args and args.strip(): - cmd.extend(shlex.split(args)) + split_args = ResolveAVBSigningPathArgs(shlex.split(args)) + cmd.extend(split_args) RunAndCheckOutput(cmd) img.seek(os.SEEK_SET, 0) @@ -2041,6 +2139,39 @@ def Gunzip(in_filename, out_filename): shutil.copyfileobj(in_file, out_file) +def UnzipSingleFile(input_zip: zipfile.ZipFile, info: zipfile.ZipInfo, dirname: str): + # According to https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip/6297838#6297838 + # higher bits of |external_attr| are unix file permission and types + unix_filetype = info.external_attr >> 16 + file_perm = unix_filetype & 0o777 + + def CheckMask(a, mask): + return (a & mask) == mask + + def IsSymlink(a): + return CheckMask(a, stat.S_IFLNK) + + def IsDir(a): + return CheckMask(a, stat.S_IFDIR) + # python3.11 zipfile implementation doesn't handle symlink correctly + if not IsSymlink(unix_filetype): + target = input_zip.extract(info, dirname) + # We want to ensure that the file is at least read/writable by owner and readable by all users + if IsDir(unix_filetype): + os.chmod(target, file_perm | 0o755) + else: + os.chmod(target, file_perm | 0o644) + return target + if dirname is None: + dirname = os.getcwd() + target = os.path.join(dirname, info.filename) + os.makedirs(os.path.dirname(target), exist_ok=True) + if os.path.exists(target): + os.unlink(target) + os.symlink(input_zip.read(info).decode(), target) + return target + + def UnzipToDir(filename, dirname, patterns=None): """Unzips the archive to the given directory. @@ -2051,20 +2182,46 @@ def UnzipToDir(filename, dirname, patterns=None): archvie. Non-matching patterns will be filtered out. If there's no match after the filtering, no file will be unzipped. """ - cmd = ["unzip", "-o", "-q", filename, "-d", dirname] - if patterns is not None: + with zipfile.ZipFile(filename, allowZip64=True, mode="r") as input_zip: # Filter out non-matching patterns. unzip will complain otherwise. - with zipfile.ZipFile(filename, allowZip64=True) as input_zip: - names = input_zip.namelist() - filtered = [ - pattern for pattern in patterns if fnmatch.filter(names, pattern)] - - # There isn't any matching files. Don't unzip anything. - if not filtered: - return - cmd.extend(filtered) + entries = input_zip.infolist() + # b/283033491 + # Per https://en.wikipedia.org/wiki/ZIP_(file_format)#Central_directory_file_header + # In zip64 mode, central directory record's header_offset field might be + # set to 0xFFFFFFFF if header offset is > 2^32. In this case, the extra + # fields will contain an 8 byte little endian integer at offset 20 + # to indicate the actual local header offset. + # As of python3.11, python does not handle zip64 central directories + # correctly, so we will manually do the parsing here. + + # ZIP64 central directory extra field has two required fields: + # 2 bytes header ID and 2 bytes size field. Thes two require fields have + # a total size of 4 bytes. Then it has three other 8 bytes field, followed + # by a 4 byte disk number field. The last disk number field is not required + # to be present, but if it is present, the total size of extra field will be + # divisible by 8(because 2+2+4+8*n is always going to be multiple of 8) + # Most extra fields are optional, but when they appear, their must appear + # in the order defined by zip64 spec. Since file header offset is the 2nd + # to last field in zip64 spec, it will only be at last 8 bytes or last 12-4 + # bytes, depending on whether disk number is present. + for entry in entries: + if entry.header_offset == 0xFFFFFFFF: + if len(entry.extra) % 8 == 0: + entry.header_offset = int.from_bytes(entry.extra[-12:-4], "little") + else: + entry.header_offset = int.from_bytes(entry.extra[-8:], "little") + if patterns is not None: + filtered = [info for info in entries if any( + [fnmatch.fnmatch(info.filename, p) for p in patterns])] - RunAndCheckOutput(cmd) + # There isn't any matching files. Don't unzip anything. + if not filtered: + return + for info in filtered: + UnzipSingleFile(input_zip, info, dirname) + else: + for info in entries: + UnzipSingleFile(input_zip, info, dirname) def UnzipTemp(filename, patterns=None): @@ -2121,9 +2278,7 @@ def GetUserImage(which, tmpdir, input_zip, if info_dict is None: info_dict = LoadInfoDict(input_zip) - is_sparse = info_dict.get("extfs_sparse_flag") - if info_dict.get(which + "_disable_sparse"): - is_sparse = False + is_sparse = IsSparseImage(os.path.join(tmpdir, "IMAGES", which + ".img")) # When target uses 'BOARD_EXT4_SHARE_DUP_BLOCKS := true', images may contain # shared blocks (i.e. some blocks will show up in multiple files' block @@ -2306,11 +2461,12 @@ def GetMinSdkVersion(apk_name): apk_name, proc.returncode, stdoutdata, stderrdata)) for line in stdoutdata.split("\n"): - # Looking for lines such as sdkVersion:'23' or sdkVersion:'M'. - m = re.match(r'sdkVersion:\'([^\']*)\'', line) + # Due to ag/24161708, looking for lines such as minSdkVersion:'23',minSdkVersion:'M' + # or sdkVersion:'23', sdkVersion:'M'. + m = re.match(r'(?:minSdkVersion|sdkVersion):\'([^\']*)\'', line) if m: return m.group(1) - raise ExternalError("No minSdkVersion returned by aapt2") + raise ExternalError("No minSdkVersion returned by aapt2 for apk: {}".format(apk_name)) def GetMinSdkVersionInt(apk_name, codename_to_api_level_map): @@ -2332,12 +2488,22 @@ def GetMinSdkVersionInt(apk_name, codename_to_api_level_map): try: return int(version) except ValueError: - # Not a decimal number. Codename? - if version in codename_to_api_level_map: - return codename_to_api_level_map[version] + # Not a decimal number. + # + # It could be either a straight codename, e.g. + # UpsideDownCake + # + # Or a codename with API fingerprint SHA, e.g. + # UpsideDownCake.e7d3947f14eb9dc4fec25ff6c5f8563e + # + # Extract the codename and try and map it to a version number. + split = version.split(".") + codename = split[0] + if codename in codename_to_api_level_map: + return codename_to_api_level_map[codename] raise ExternalError( - "Unknown minSdkVersion: '{}'. Known codenames: {}".format( - version, codename_to_api_level_map)) + "Unknown codename: '{}' from minSdkVersion: '{}'. Known codenames: {}".format( + codename, version, codename_to_api_level_map)) def SignFile(input_name, output_name, key, password, min_api_level=None, @@ -2398,38 +2564,6 @@ def SignFile(input_name, output_name, key, password, min_api_level=None, proc.returncode, stdoutdata)) -def SignSePolicy(sepolicy, key, password): - """Sign the sepolicy zip, producing an fsverity .fsv_sig and - an RSA .sig signature files. - """ - - if OPTIONS.sign_sepolicy_path is None: - logger.info("No sign_sepolicy_path specified, %s was not signed", sepolicy) - return False - - java_library_path = os.path.join( - OPTIONS.search_path, OPTIONS.signapk_shared_library_path) - - cmd = ([OPTIONS.java_path] + OPTIONS.java_args + - ["-Djava.library.path=" + java_library_path, - "-jar", os.path.join(OPTIONS.search_path, OPTIONS.sign_sepolicy_path)] + - OPTIONS.extra_sign_sepolicy_args) - - cmd.extend([key + OPTIONS.public_key_suffix, - key + OPTIONS.private_key_suffix, - sepolicy, os.path.dirname(sepolicy)]) - - proc = Run(cmd, stdin=subprocess.PIPE) - if password is not None: - password += "\n" - stdoutdata, _ = proc.communicate(password) - if proc.returncode != 0: - raise ExternalError( - "Failed to run sign sepolicy: return code {}:\n{}".format( - proc.returncode, stdoutdata)) - return True - - def CheckSize(data, target, info_dict): """Checks the data string passed against the max size limit. @@ -2457,7 +2591,9 @@ def CheckSize(data, target, info_dict): device = p.device if "/" in device: device = device[device.rfind("/")+1:] - limit = info_dict.get(device + "_size") + limit = info_dict.get(device + "_size", 0) + if isinstance(limit, str): + limit = int(limit, 0) if not fs_type or not limit: return @@ -2594,19 +2730,25 @@ def Usage(docstring): def ParseOptions(argv, docstring, extra_opts="", extra_long_opts=(), - extra_option_handler=None): + extra_option_handler: Iterable[OptionHandler] = None): """Parse the options in argv and return any arguments that aren't flags. docstring is the calling module's docstring, to be displayed for errors and -h. extra_opts and extra_long_opts are for flags defined by the caller, which are processed by passing them to extra_option_handler.""" + extra_long_opts = list(extra_long_opts) + if not isinstance(extra_option_handler, Iterable): + extra_option_handler = [extra_option_handler] + + for handler in extra_option_handler: + if isinstance(handler, OptionHandler): + extra_long_opts.extend(handler.extra_long_opts) try: opts, args = getopt.getopt( argv, "hvp:s:x:" + extra_opts, ["help", "verbose", "path=", "signapk_path=", - "signapk_shared_library_path=", "extra_signapk_args=", - "sign_sepolicy_path=", "extra_sign_sepolicy_args=", "aapt2_path=", + "signapk_shared_library_path=", "extra_signapk_args=", "aapt2_path=", "java_path=", "java_args=", "android_jar_path=", "public_key_suffix=", "private_key_suffix=", "boot_signer_path=", "boot_signer_args=", "verity_signer_path=", "verity_signer_args=", "device_specific=", @@ -2630,10 +2772,6 @@ def ParseOptions(argv, OPTIONS.signapk_shared_library_path = a elif o in ("--extra_signapk_args",): OPTIONS.extra_signapk_args = shlex.split(a) - elif o in ("--sign_sepolicy_path",): - OPTIONS.sign_sepolicy_path = a - elif o in ("--extra_sign_sepolicy_args",): - OPTIONS.extra_sign_sepolicy_args = shlex.split(a) elif o in ("--aapt2_path",): OPTIONS.aapt2_path = a elif o in ("--java_path",): @@ -2666,8 +2804,19 @@ def ParseOptions(argv, elif o in ("--logfile",): OPTIONS.logfile = a else: - if extra_option_handler is None or not extra_option_handler(o, a): - assert False, "unknown option \"%s\"" % (o,) + if extra_option_handler is None: + raise ValueError("unknown option \"%s\"" % (o,)) + success = False + for handler in extra_option_handler: + if isinstance(handler, OptionHandler): + if handler.handler(o, a): + success = True + break + elif handler(o, a): + success = True + if not success: + raise ValueError("unknown option \"%s\"" % (o,)) + if OPTIONS.search_path: os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + @@ -2698,6 +2847,8 @@ def MakeTempDir(prefix='tmp', suffix=''): def Cleanup(): for i in OPTIONS.tempfiles: + if not os.path.exists(i): + continue if os.path.isdir(i): shutil.rmtree(i, ignore_errors=True) else: @@ -2811,6 +2962,18 @@ class PasswordManager(object): def ZipWrite(zip_file, filename, arcname=None, perms=0o644, compress_type=None): + # http://b/18015246 + # Python 2.7's zipfile implementation wrongly thinks that zip64 is required + # for files larger than 2GiB. We can work around this by adjusting their + # limit. Note that `zipfile.writestr()` will not work for strings larger than + # 2GiB. The Python interpreter sometimes rejects strings that large (though + # it isn't clear to me exactly what circumstances cause this). + # `zipfile.write()` must be used directly to work around this. + # + # This mess can be avoided if we port to python3. + saved_zip64_limit = zipfile.ZIP64_LIMIT + zipfile.ZIP64_LIMIT = (1 << 32) - 1 + if compress_type is None: compress_type = zip_file.compression if arcname is None: @@ -2836,13 +2999,14 @@ def ZipWrite(zip_file, filename, arcname=None, perms=0o644, finally: os.chmod(filename, saved_stat.st_mode) os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime)) + zipfile.ZIP64_LIMIT = saved_zip64_limit def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None, compress_type=None): """Wrap zipfile.writestr() function to work around the zip64 limit. - Python's zip implementation won't allow writing a string + Even with the ZIP64_LIMIT workaround, it won't allow writing a string longer than 2GiB. It gives 'OverflowError: size does not fit in an int' when calling crc32(bytes). @@ -2851,6 +3015,9 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None, when we know the string won't be too long. """ + saved_zip64_limit = zipfile.ZIP64_LIMIT + zipfile.ZIP64_LIMIT = (1 << 32) - 1 + if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname) zinfo.compress_type = zip_file.compression @@ -2883,9 +3050,9 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None, zinfo.date_time = (2009, 1, 1, 0, 0, 0) zip_file.writestr(zinfo, data) + zipfile.ZIP64_LIMIT = saved_zip64_limit - -def ZipDelete(zip_filename, entries, force=False): +def ZipExclude(input_zip, output_zip, entries, force=False): """Deletes entries from a ZIP file. Args: @@ -2896,23 +3063,50 @@ def ZipDelete(zip_filename, entries, force=False): entries = [entries] # If list is empty, nothing to do if not entries: + shutil.copy(input_zip, output_zip) return - with zipfile.ZipFile(zip_filename, 'r') as zin: + with zipfile.ZipFile(input_zip, 'r') as zin: if not force and len(set(zin.namelist()).intersection(entries)) == 0: raise ExternalError( "Failed to delete zip entries, name not matched: %s" % entries) - fd, new_zipfile = tempfile.mkstemp(dir=os.path.dirname(zip_filename)) + fd, new_zipfile = tempfile.mkstemp(dir=os.path.dirname(input_zip)) os.close(fd) - cmd = ["zip2zip", "-i", zip_filename, "-o", new_zipfile] + cmd = ["zip2zip", "-i", input_zip, "-o", new_zipfile] for entry in entries: cmd.append("-x") cmd.append(entry) RunAndCheckOutput(cmd) + os.replace(new_zipfile, output_zip) + + +def ZipDelete(zip_filename, entries, force=False): + """Deletes entries from a ZIP file. + + Args: + zip_filename: The name of the ZIP file. + entries: The name of the entry, or the list of names to be deleted. + """ + if isinstance(entries, str): + entries = [entries] + # If list is empty, nothing to do + if not entries: + return + + ZipExclude(zip_filename, zip_filename, entries, force) + +def ZipClose(zip_file): + # http://b/18015246 + # zipfile also refers to ZIP64_LIMIT during close() when it writes out the + # central directory. + saved_zip64_limit = zipfile.ZIP64_LIMIT + zipfile.ZIP64_LIMIT = (1 << 32) - 1 - os.replace(new_zipfile, zip_filename) + zip_file.close() + + zipfile.ZIP64_LIMIT = saved_zip64_limit class DeviceSpecificParams(object): @@ -3606,12 +3800,11 @@ def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, output_sink(recovery_img_path, recovery_img.data) else: - system_root_image = info_dict.get("system_root_image") == "true" + include_recovery_dtbo = info_dict.get("include_recovery_dtbo") == "true" + include_recovery_acpio = info_dict.get("include_recovery_acpio") == "true" path = os.path.join(input_dir, recovery_resource_dat_path) - # With system-root-image, boot and recovery images will have mismatching - # entries (only recovery has the ramdisk entry) (Bug: 72731506). Use bsdiff - # to handle such a case. - if system_root_image: + # Use bsdiff to handle mismatching entries (Bug: 72731506) + if include_recovery_dtbo or include_recovery_acpio: diff_program = ["bsdiff"] bonus_args = "" assert not os.path.exists(path) @@ -3914,7 +4107,7 @@ def GetBootImageBuildProp(boot_img, ramdisk_format=RamdiskFormat.LZ4): Get build.prop from ramdisk within the boot image Args: - boot_img: the boot image file. Ramdisk must be compressed with lz4 or minigzip format. + boot_img: the boot image file. Ramdisk must be compressed with lz4 or gzip format. Return: An extracted file that stores properties in the boot image. @@ -3933,11 +4126,11 @@ def GetBootImageBuildProp(boot_img, ramdisk_format=RamdiskFormat.LZ4): elif ramdisk_format == RamdiskFormat.GZ: with open(ramdisk, 'rb') as input_stream: with open(uncompressed_ramdisk, 'wb') as output_stream: - p2 = Run(['minigzip', '-d'], stdin=input_stream.fileno(), + p2 = Run(['gzip', '-d'], stdin=input_stream.fileno(), stdout=output_stream.fileno()) p2.wait() else: - logger.error('Only support lz4 or minigzip ramdisk format.') + logger.error('Only support lz4 or gzip ramdisk format.') return None abs_uncompressed_ramdisk = os.path.abspath(uncompressed_ramdisk) @@ -4001,3 +4194,38 @@ def IsSparseImage(filepath): # Magic for android sparse image format # https://source.android.com/devices/bootloader/images return fp.read(4) == b'\x3A\xFF\x26\xED' + + +def UnsparseImage(filepath, target_path=None): + if not IsSparseImage(filepath): + return + if target_path is None: + tmp_img = MakeTempFile(suffix=".img") + RunAndCheckOutput(["simg2img", filepath, tmp_img]) + os.rename(tmp_img, filepath) + else: + RunAndCheckOutput(["simg2img", filepath, target_path]) + + +def ParseUpdateEngineConfig(path: str): + """Parse the update_engine config stored in file `path` + Args + path: Path to update_engine_config.txt file in target_files + + Returns + A tuple of (major, minor) version number . E.g. (2, 8) + """ + with open(path, "r") as fp: + # update_engine_config.txt is only supposed to contain two lines, + # PAYLOAD_MAJOR_VERSION and PAYLOAD_MINOR_VERSION. 1024 should be more than + # sufficient. If the length is more than that, something is wrong. + data = fp.read(1024) + major = re.search(r"PAYLOAD_MAJOR_VERSION=(\d+)", data) + if not major: + raise ValueError( + f"{path} is an invalid update_engine config, missing PAYLOAD_MAJOR_VERSION {data}") + minor = re.search(r"PAYLOAD_MINOR_VERSION=(\d+)", data) + if not minor: + raise ValueError( + f"{path} is an invalid update_engine config, missing PAYLOAD_MINOR_VERSION {data}") + return (int(major.group(1)), int(minor.group(1))) diff --git a/tools/releasetools/create_brick_ota.py b/tools/releasetools/create_brick_ota.py new file mode 100644 index 0000000000..bf50f71049 --- /dev/null +++ b/tools/releasetools/create_brick_ota.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +from pathlib import Path +import zipfile +from typing import List +import common +import tempfile +import shutil + +PARTITIONS_TO_WIPE = ["/dev/block/by-name/vbmeta", + "/dev/block/by-name/vbmeta_a", + "/dev/block/by-name/vbmeta_b", + "/dev/block/by-name/vbmeta_system_a", + "/dev/block/by-name/vbmeta_system_b", + "/dev/block/by-name/boot", + "/dev/block/by-name/boot_a", + "/dev/block/by-name/boot_b", + "/dev/block/by-name/vendor_boot", + "/dev/block/by-name/vendor_boot_a", + "/dev/block/by-name/vendor_boot_b", + "/dev/block/by-name/init_boot_a", + "/dev/block/by-name/init_boot_b", + "/dev/block/by-name/metadata", + "/dev/block/by-name/super", + "/dev/block/by-name/userdata"] + + +def CreateBrickOta(product_name: str, output_path: Path, extra_wipe_partitions: str, serialno: str): + partitions_to_wipe = PARTITIONS_TO_WIPE + if extra_wipe_partitions is not None: + partitions_to_wipe = PARTITIONS_TO_WIPE + extra_wipe_partitions.split(",") + ota_metadata = ["ota-type=BRICK", "post-timestamp=9999999999", + "pre-device=" + product_name] + if serialno is not None: + ota_metadata.append("serialno=" + serialno) + # recovery requiers product name to be a | separated list + product_name = product_name.replace(",", "|") + with zipfile.ZipFile(output_path, "w") as zfp: + zfp.writestr("recovery.wipe", "\n".join(partitions_to_wipe)) + zfp.writestr("payload.bin", "") + zfp.writestr("META-INF/com/android/metadata", "\n".join( + ota_metadata)) + + +def main(argv): + parser = argparse.ArgumentParser(description='Android Brick OTA generator') + parser.add_argument('otafile', metavar='PAYLOAD', type=str, + help='The output OTA package file.') + parser.add_argument('--product', type=str, + help='The product name of the device, for example, bramble, redfin.', required=True) + parser.add_argument('--serialno', type=str, + help='The serial number of devices that are allowed to install this OTA package. This can be a | separated list.') + parser.add_argument('--extra_wipe_partitions', type=str, + help='Additional partitions on device which should be wiped.') + parser.add_argument('-v', action="store_true", + help="Enable verbose logging", dest="verbose") + parser.add_argument('--package_key', type=str, + help='Paths to private key for signing payload') + parser.add_argument('--search_path', type=str, + help='Search path for framework/signapk.jar') + parser.add_argument('--private_key_suffix', type=str, + help='Suffix to be appended to package_key path', default=".pk8") + args = parser.parse_args(argv[1:]) + if args.search_path: + common.OPTIONS.search_path = args.search_path + if args.verbose: + common.OPTIONS.verbose = args.verbose + CreateBrickOta(args.product, args.otafile, + args.extra_wipe_partitions, args.serialno) + if args.package_key: + common.OPTIONS.private_key_suffix = args.private_key_suffix + with tempfile.NamedTemporaryFile() as tmpfile: + common.SignFile(args.otafile, tmpfile.name, + args.package_key, None, whole_file=True) + shutil.copy(tmpfile.name, args.otafile) + + +if __name__ == "__main__": + import sys + main(sys.argv) diff --git a/tools/releasetools/img_from_target_files.py b/tools/releasetools/img_from_target_files.py index f8bdd81ce6..b7a5ad8b74 100755 --- a/tools/releasetools/img_from_target_files.py +++ b/tools/releasetools/img_from_target_files.py @@ -64,6 +64,8 @@ OPTIONS.super_device_list = None OPTIONS.retrofit_dap = None OPTIONS.build_super = None OPTIONS.sparse_userimages = None +OPTIONS.use_fastboot_info = True +OPTIONS.build_super_image = None def LoadOptions(input_file): @@ -104,6 +106,13 @@ def CopyZipEntries(input_file, output_file, entries): common.RunAndCheckOutput(cmd) +def LocatePartitionEntry(partition_name, namelist): + for subdir in ["IMAGES", "PREBUILT_IMAGES", "RADIO"]: + entry_name = os.path.join(subdir, partition_name + ".img") + if entry_name in namelist: + return entry_name + + def EntriesForUserImages(input_file): """Returns the user images entries to be copied. @@ -119,12 +128,23 @@ def EntriesForUserImages(input_file): entries = [ 'OTA/android-info.txt:android-info.txt', ] + if OPTIONS.use_fastboot_info: + entries.append('META/fastboot-info.txt:fastboot-info.txt') + ab_partitions = [] with zipfile.ZipFile(input_file) as input_zip: namelist = input_zip.namelist() - + if "META/ab_partitions.txt" in namelist: + ab_partitions = input_zip.read( + "META/ab_partitions.txt").decode().strip().split() + if 'PREBUILT_IMAGES/kernel_16k' in namelist: + entries.append('PREBUILT_IMAGES/kernel_16k:kernel_16k') + if 'PREBUILT_IMAGES/ramdisk_16k.img' in namelist: + entries.append('PREBUILT_IMAGES/ramdisk_16k.img:ramdisk_16k.img') + + visited_partitions = set(OPTIONS.dynamic_partition_list) for image_path in [name for name in namelist if name.startswith('IMAGES/')]: image = os.path.basename(image_path) - if OPTIONS.bootable_only and image not in('boot.img', 'recovery.img', 'bootloader', 'init_boot.img'): + if OPTIONS.bootable_only and image not in ('boot.img', 'recovery.img', 'bootloader', 'init_boot.img'): continue if not image.endswith('.img') and image != 'bootloader': continue @@ -136,7 +156,14 @@ def EntriesForUserImages(input_file): continue if image in dynamic_images: continue + partition_name = image.rstrip(".img") + visited_partitions.add(partition_name) entries.append('{}:{}'.format(image_path, image)) + for part in [part for part in ab_partitions if part not in visited_partitions]: + entry = LocatePartitionEntry(part, namelist) + image = os.path.basename(entry) + if entry is not None: + entries.append('{}:{}'.format(entry, image)) return entries @@ -168,12 +195,18 @@ def RebuildAndWriteSuperImages(input_file, output_file): input_tmp = common.UnzipTemp(input_file) super_file = common.MakeTempFile('super_', '.img') - BuildSuperImage(input_tmp, super_file) + + # Allow overriding the BUILD_SUPER_IMAGE binary + if OPTIONS.build_super_image: + command = [OPTIONS.build_super_image, input_tmp, super_file] + common.RunAndCheckOutput(command) + else: + BuildSuperImage(input_tmp, super_file) logger.info('Writing super.img to archive...') with zipfile.ZipFile( - output_file, 'a', compression=zipfile.ZIP_DEFLATED, - allowZip64=True) as output_zip: + output_file, 'a', compression=zipfile.ZIP_DEFLATED, + allowZip64=True) as output_zip: common.ZipWrite(output_zip, super_file, 'super.img') @@ -225,6 +258,8 @@ def main(argv): OPTIONS.bootable_only = True elif o == '--additional': OPTIONS.additional_entries.append(a) + elif o == '--build_super_image': + OPTIONS.build_super_image = a else: return False return True @@ -234,6 +269,7 @@ def main(argv): extra_long_opts=[ 'additional=', 'bootable_zip', + 'build_super_image=', ], extra_option_handler=option_handler) if len(args) != 2: diff --git a/tools/releasetools/merge/Android.bp b/tools/releasetools/merge/Android.bp index 219acf8dbe..96ec73e637 100644 --- a/tools/releasetools/merge/Android.bp +++ b/tools/releasetools/merge/Android.bp @@ -50,6 +50,7 @@ python_binary_host { "releasetools_ota_from_target_files", ], required: [ + "apexd_host", "checkvintf", "host_init_verifier", "secilc", diff --git a/tools/releasetools/merge/merge_builds.py b/tools/releasetools/merge/merge_builds.py index 3ac4ec48a3..032278c9fd 100644 --- a/tools/releasetools/merge/merge_builds.py +++ b/tools/releasetools/merge/merge_builds.py @@ -47,6 +47,10 @@ Usage: merge_builds.py [args] The optional path to a newline-separated config file containing keys to obtain from the framework instance of misc_info.txt, used for creating vbmeta.img. The remaining keys come from the vendor instance. + + --avb_resolve_rollback_index_location_conflict + If provided, resolve the conflict AVB rollback index location when + necessary. """ from __future__ import print_function @@ -65,6 +69,7 @@ OPTIONS.product_out_framework = None OPTIONS.product_out_vendor = None OPTIONS.build_vbmeta = False OPTIONS.framework_misc_info_keys = None +OPTIONS.avb_resolve_rollback_index_location_conflict = False def CreateImageSymlinks(): @@ -140,7 +145,8 @@ def BuildVBMeta(): output_vbmeta_path = os.path.join(OPTIONS.product_out_vendor, "vbmeta.img") OPTIONS.info_dict = merged_dict common.BuildVBMeta(output_vbmeta_path, partitions, "vbmeta", - vbmeta_partitions) + vbmeta_partitions, + OPTIONS.avb_resolve_rollback_index_location_conflict) def MergeBuilds(): @@ -164,6 +170,8 @@ def main(): OPTIONS.build_vbmeta = True elif o == "--framework_misc_info_keys": OPTIONS.framework_misc_info_keys = a + elif o == "--avb_resolve_rollback_index_location_conflict": + OPTIONS.avb_resolve_rollback_index_location_conflict = True else: return False return True @@ -177,6 +185,7 @@ def main(): "product_out_vendor=", "build_vbmeta", "framework_misc_info_keys=", + "avb_resolve_rollback_index_location_conflict" ], extra_option_handler=option_handler) diff --git a/tools/releasetools/merge/merge_compatibility_checks.py b/tools/releasetools/merge/merge_compatibility_checks.py index 207abe2555..8c9993f2e2 100644 --- a/tools/releasetools/merge/merge_compatibility_checks.py +++ b/tools/releasetools/merge/merge_compatibility_checks.py @@ -190,8 +190,8 @@ def CheckApexDuplicatePackages(target_files_dir, partition_map): apex_packages = set() for partition in partition_map.keys(): try: - apex_info = apex_utils.GetApexInfoFromTargetFiles( - target_files_dir, partition, compressed_only=False) + apex_info = apex_utils.GetApexInfoForPartition( + target_files_dir, partition) except RuntimeError as err: errors.append(str(err)) apex_info = [] diff --git a/tools/releasetools/merge/merge_dexopt.py b/tools/releasetools/merge/merge_dexopt.py index 16182b5cbb..1c0c7433e5 100644 --- a/tools/releasetools/merge/merge_dexopt.py +++ b/tools/releasetools/merge/merge_dexopt.py @@ -72,7 +72,6 @@ def MergeDexopt(temp_dir, output_target_files_dir): # <contents of vendor dexpreopt_config.zip> # system -> output/SYSTEM # vendor -> output/VENDOR - # apex -> output/SYSTEM/apex (only for flattened APEX builds) # apex/ (extracted updatable APEX) # <apex 1>/ # ... @@ -114,70 +113,20 @@ def MergeDexopt(temp_dir, output_target_files_dir): os.path.join(output_target_files_dir, 'VENDOR'), os.path.join(temp_dir, 'vendor')) - # The directory structure for flatteded APEXes is: - # - # SYSTEM - # apex - # <APEX name, e.g., com.android.wifi> - # apex_manifest.pb - # apex_pubkey - # etc/ - # javalib/ - # lib/ - # lib64/ - # priv-app/ - # - # The directory structure for updatable APEXes is: - # - # SYSTEM - # apex - # com.android.adbd.apex - # com.android.appsearch.apex - # com.android.art.apex - # ... - apex_root = os.path.join(output_target_files_dir, 'SYSTEM', 'apex') - - # Check for flattended versus updatable APEX. - if OPTIONS.framework_misc_info.get('target_flatten_apex') == 'false': - # Extract APEX. - logging.info('extracting APEX') + # Extract APEX. + logging.info('extracting APEX') + apex_extract_root_dir = os.path.join(temp_dir, 'apex') + os.makedirs(apex_extract_root_dir) - apex_extract_root_dir = os.path.join(temp_dir, 'apex') - os.makedirs(apex_extract_root_dir) - - for apex in (glob.glob(os.path.join(apex_root, '*.apex')) + - glob.glob(os.path.join(apex_root, '*.capex'))): - logging.info(' apex: %s', apex) - # deapexer is in the same directory as the merge_target_files binary extracted - # from otatools.zip. - apex_json_info = subprocess.check_output(['deapexer', 'info', apex]) - logging.info(' info: %s', apex_json_info) - apex_info = json.loads(apex_json_info) - apex_name = apex_info['name'] - logging.info(' name: %s', apex_name) - - apex_extract_dir = os.path.join(apex_extract_root_dir, apex_name) - os.makedirs(apex_extract_dir) - - # deapexer uses debugfs_static, which is part of otatools.zip. - command = [ - 'deapexer', - '--debugfs_path', - 'debugfs_static', - '--blkid_path', - 'blkid', - '--fsckerofs_path', - 'fsck.erofs', - 'extract', - apex, - apex_extract_dir, - ] - logging.info(' running %s', command) - subprocess.check_call(command) - else: - # Flattened APEXes don't need to be extracted since they have the necessary - # directory structure. - os.symlink(os.path.join(apex_root), os.path.join(temp_dir, 'apex')) + command = [ + 'apexd_host', + '--system_path', + os.path.join(temp_dir, 'system'), + '--apex_path', + apex_extract_root_dir, + ] + logging.info(' running %s', command) + subprocess.check_call(command) # Modify system config to point to the tools that have been extracted. # Absolute or .. paths are not allowed by the dexpreopt_gen tool in diff --git a/tools/releasetools/merge/merge_meta.py b/tools/releasetools/merge/merge_meta.py index 3288ef7c6a..76582c0946 100644 --- a/tools/releasetools/merge/merge_meta.py +++ b/tools/releasetools/merge/merge_meta.py @@ -29,6 +29,7 @@ import common import merge_utils import sparse_img import verity_utils +from ota_utils import ParseUpdateEngineConfig from common import ExternalError @@ -52,63 +53,47 @@ PARTITION_TAG_PATTERN = re.compile(r'partition="(.*?)"') MODULE_KEY_PATTERN = re.compile(r'name="(.+)\.(apex|apk)"') -def ParseUpdateEngineConfig(path: str): - """Parse the update_engine config stored in file `path` - Args - path: Path to update_engine_config.txt file in target_files +def MergeUpdateEngineConfig(framework_meta_dir, vendor_meta_dir, + merged_meta_dir): + """Merges META/update_engine_config.txt. - Returns - A tuple of (major, minor) version number . E.g. (2, 8) + The output is the configuration for maximum compatibility. """ - with open(path, "r") as fp: - # update_engine_config.txt is only supposed to contain two lines, - # PAYLOAD_MAJOR_VERSION and PAYLOAD_MINOR_VERSION. 1024 should be more than - # sufficient. If the length is more than that, something is wrong. - data = fp.read(1024) - major = re.search(r"PAYLOAD_MAJOR_VERSION=(\d+)", data) - if not major: - raise ValueError( - f"{path} is an invalid update_engine config, missing PAYLOAD_MAJOR_VERSION {data}") - minor = re.search(r"PAYLOAD_MINOR_VERSION=(\d+)", data) - if not minor: - raise ValueError( - f"{path} is an invalid update_engine config, missing PAYLOAD_MINOR_VERSION {data}") - return (int(major.group(1)), int(minor.group(1))) - - -def MergeUpdateEngineConfig(input_metadir1, input_metadir2, merged_meta_dir): - UPDATE_ENGINE_CONFIG_NAME = "update_engine_config.txt" - config1_path = os.path.join( - input_metadir1, UPDATE_ENGINE_CONFIG_NAME) - config2_path = os.path.join( - input_metadir2, UPDATE_ENGINE_CONFIG_NAME) - config1 = ParseUpdateEngineConfig(config1_path) - config2 = ParseUpdateEngineConfig(config2_path) - # Copy older config to merged target files for maximum compatibility - # update_engine in system partition is from system side, but - # update_engine_sideload in recovery is from vendor side. - if config1 < config2: - shutil.copy(config1_path, os.path.join( - merged_meta_dir, UPDATE_ENGINE_CONFIG_NAME)) + _CONFIG_NAME = 'update_engine_config.txt' + framework_config_path = os.path.join(framework_meta_dir, _CONFIG_NAME) + vendor_config_path = os.path.join(vendor_meta_dir, _CONFIG_NAME) + merged_config_path = os.path.join(merged_meta_dir, _CONFIG_NAME) + + if os.path.exists(framework_config_path): + framework_config = ParseUpdateEngineConfig(framework_config_path) + vendor_config = ParseUpdateEngineConfig(vendor_config_path) + # Copy older config to merged target files for maximum compatibility + # update_engine in system partition is from system side, but + # update_engine_sideload in recovery is from vendor side. + if framework_config < vendor_config: + shutil.copy(framework_config_path, merged_config_path) + else: + shutil.copy(vendor_config_path, merged_config_path) else: - shutil.copy(config2_path, os.path.join( - merged_meta_dir, UPDATE_ENGINE_CONFIG_NAME)) + if not OPTIONS.allow_partial_ab: + raise FileNotFoundError(framework_config_path) + shutil.copy(vendor_config_path, merged_config_path) -def MergeMetaFiles(temp_dir, merged_dir): +def MergeMetaFiles(temp_dir, merged_dir, framework_partitions): """Merges various files in META/*.""" framework_meta_dir = os.path.join(temp_dir, 'framework_meta', 'META') - merge_utils.ExtractItems( - input_zip=OPTIONS.framework_target_files, + merge_utils.CollectTargetFiles( + input_zipfile_or_dir=OPTIONS.framework_target_files, output_dir=os.path.dirname(framework_meta_dir), - extract_item_list=('META/*',)) + item_list=('META/*',)) vendor_meta_dir = os.path.join(temp_dir, 'vendor_meta', 'META') - merge_utils.ExtractItems( - input_zip=OPTIONS.vendor_target_files, + merge_utils.CollectTargetFiles( + input_zipfile_or_dir=OPTIONS.vendor_target_files, output_dir=os.path.dirname(vendor_meta_dir), - extract_item_list=('META/*',)) + item_list=('META/*',)) merged_meta_dir = os.path.join(merged_dir, 'META') @@ -135,7 +120,8 @@ def MergeMetaFiles(temp_dir, merged_dir): MergeAbPartitions( framework_meta_dir=framework_meta_dir, vendor_meta_dir=vendor_meta_dir, - merged_meta_dir=merged_meta_dir) + merged_meta_dir=merged_meta_dir, + framework_partitions=framework_partitions) UpdateCareMapImageSizeProps(images_dir=os.path.join(merged_dir, 'IMAGES')) for file_name in ('apkcerts.txt', 'apexkeys.txt'): @@ -145,10 +131,9 @@ def MergeMetaFiles(temp_dir, merged_dir): merged_meta_dir=merged_meta_dir, file_name=file_name) - MergeUpdateEngineConfig( - framework_meta_dir, - vendor_meta_dir, merged_meta_dir, - ) + if OPTIONS.merged_misc_info.get('ab_update') == 'true': + MergeUpdateEngineConfig( + framework_meta_dir, vendor_meta_dir, merged_meta_dir) # Write the now-finalized OPTIONS.merged_misc_info. merge_utils.WriteSortedData( @@ -156,13 +141,30 @@ def MergeMetaFiles(temp_dir, merged_dir): path=os.path.join(merged_meta_dir, 'misc_info.txt')) -def MergeAbPartitions(framework_meta_dir, vendor_meta_dir, merged_meta_dir): +def MergeAbPartitions(framework_meta_dir, vendor_meta_dir, merged_meta_dir, + framework_partitions): """Merges META/ab_partitions.txt. The output contains the union of the partition names. """ - with open(os.path.join(framework_meta_dir, 'ab_partitions.txt')) as f: - framework_ab_partitions = f.read().splitlines() + framework_ab_partitions = [] + framework_ab_config = os.path.join(framework_meta_dir, 'ab_partitions.txt') + if os.path.exists(framework_ab_config): + with open(framework_ab_config) as f: + # Filter out some partitions here to support the case that the + # ab_partitions.txt of framework-target-files has non-framework + # partitions. This case happens when we use a complete merged target + # files package as the framework-target-files. + framework_ab_partitions.extend([ + partition + for partition in f.read().splitlines() + if partition in framework_partitions + ]) + else: + if not OPTIONS.allow_partial_ab: + raise FileNotFoundError(framework_ab_config) + logger.info('Use partial AB because framework ab_partitions.txt does not ' + 'exist.') with open(os.path.join(vendor_meta_dir, 'ab_partitions.txt')) as f: vendor_ab_partitions = f.read().splitlines() diff --git a/tools/releasetools/merge/merge_target_files.py b/tools/releasetools/merge/merge_target_files.py index 54122b0b05..fdba927db9 100755 --- a/tools/releasetools/merge/merge_target_files.py +++ b/tools/releasetools/merge/merge_target_files.py @@ -26,9 +26,9 @@ This script produces a complete, merged target files package: Usage: merge_target_files [args] - --framework-target-files framework-target-files-zip-archive + --framework-target-files framework-target-files-package The input target files package containing framework bits. This is a zip - archive. + archive or a directory. --framework-item-list framework-item-list-file The optional path to a newline-separated config file of items that @@ -38,14 +38,18 @@ Usage: merge_target_files [args] The optional path to a newline-separated config file of keys to extract from the framework META/misc_info.txt file. - --vendor-target-files vendor-target-files-zip-archive + --vendor-target-files vendor-target-files-package The input target files package containing vendor bits. This is a zip - archive. + archive or a directory. --vendor-item-list vendor-item-list-file The optional path to a newline-separated config file of items that are extracted as-is from the vendor target files package. + --boot-image-dir-path + The input boot image directory path. This path contains IMAGES/boot.img + file. + --output-target-files output-target-files-package If provided, the output merged target files package. Also a zip archive. @@ -90,6 +94,14 @@ Usage: merge_target_files [args] --keep-tmp Keep tempoary files for debugging purposes. + --avb-resolve-rollback-index-location-conflict + If provided, resolve the conflict AVB rollback index location when + necessary. + + --allow-partial-ab + If provided, allow merging non-AB framework target files with AB vendor + target files, which means that only the vendor has AB partitions. + The following only apply when using the VSDK to perform dexopt on vendor apps: --framework-dexpreopt-config @@ -132,6 +144,7 @@ OPTIONS.framework_item_list = [] OPTIONS.framework_misc_info_keys = [] OPTIONS.vendor_target_files = None OPTIONS.vendor_item_list = [] +OPTIONS.boot_image_dir_path = None OPTIONS.output_target_files = None OPTIONS.output_dir = None OPTIONS.output_item_list = [] @@ -144,6 +157,8 @@ OPTIONS.allow_duplicate_apkapex_keys = False OPTIONS.vendor_otatools = None OPTIONS.rebuild_sepolicy = False OPTIONS.keep_tmp = False +OPTIONS.avb_resolve_rollback_index_location_conflict = False +OPTIONS.allow_partial_ab = False OPTIONS.framework_dexpreopt_config = None OPTIONS.framework_dexpreopt_tools = None OPTIONS.vendor_dexpreopt_config = None @@ -165,6 +180,26 @@ def remove_file_if_exists(file_name): pass +def include_extra_in_list(item_list): + """ + 1. Include all `META/*` files in the item list. + + To ensure that `AddImagesToTargetFiles` can still be used with vendor item + list that do not specify all of the required META/ files, those files should + be included by default. This preserves the backward compatibility of + `rebuild_image_with_sepolicy`. + + 2. Include `SYSTEM/build.prop` file in the item list. + + To ensure that `AddImagesToTargetFiles` for GRF vendor images, can still + access SYSTEM/build.prop to pass GetPartitionFingerprint check in BuildInfo + constructor. + """ + if not item_list: + return None + return list(item_list) + ['META/*'] + ['SYSTEM/build.prop'] + + def create_merged_package(temp_dir): """Merges two target files packages into one target files structure. @@ -172,24 +207,32 @@ def create_merged_package(temp_dir): Path to merged package under temp directory. """ # Extract "as is" items from the input framework and vendor partial target - # files packages directly into the output temporary directory, since these items - # do not need special case processing. + # files packages directly into the output temporary directory, since these + # items do not need special case processing. output_target_files_temp_dir = os.path.join(temp_dir, 'output') - merge_utils.ExtractItems( - input_zip=OPTIONS.framework_target_files, + merge_utils.CollectTargetFiles( + input_zipfile_or_dir=OPTIONS.framework_target_files, output_dir=output_target_files_temp_dir, - extract_item_list=OPTIONS.framework_item_list) - merge_utils.ExtractItems( - input_zip=OPTIONS.vendor_target_files, + item_list=OPTIONS.framework_item_list) + merge_utils.CollectTargetFiles( + input_zipfile_or_dir=OPTIONS.vendor_target_files, output_dir=output_target_files_temp_dir, - extract_item_list=OPTIONS.vendor_item_list) + item_list=OPTIONS.vendor_item_list) + + if OPTIONS.boot_image_dir_path: + merge_utils.CollectTargetFiles( + input_zipfile_or_dir=OPTIONS.boot_image_dir_path, + output_dir=output_target_files_temp_dir, + item_list=['IMAGES/boot.img']) # Perform special case processing on META/* items. # After this function completes successfully, all the files we need to create # the output target files package are in place. merge_meta.MergeMetaFiles( - temp_dir=temp_dir, merged_dir=output_target_files_temp_dir) + temp_dir=temp_dir, + merged_dir=output_target_files_temp_dir, + framework_partitions=OPTIONS.framework_partition_set) merge_dexopt.MergeDexopt( temp_dir=temp_dir, output_target_files_dir=output_target_files_temp_dir) @@ -208,6 +251,8 @@ def generate_missing_images(target_files_dir): ] if OPTIONS.rebuild_recovery: add_img_args.append('--rebuild_recovery') + if OPTIONS.avb_resolve_rollback_index_location_conflict: + add_img_args.append('--avb_resolve_rollback_index_location_conflict') add_img_args.append(target_files_dir) add_img_to_target_files.main(add_img_args) @@ -231,7 +276,8 @@ def rebuild_image_with_sepolicy(target_files_dir): def copy_selinux_file(input_path, output_filename): input_filename = os.path.join(target_files_dir, input_path) if not os.path.exists(input_filename): - input_filename = input_filename.replace('SYSTEM_EXT/', 'SYSTEM/system_ext/') \ + input_filename = input_filename.replace('SYSTEM_EXT/', + 'SYSTEM/system_ext/') \ .replace('PRODUCT/', 'SYSTEM/product/') if not os.path.exists(input_filename): logger.info('Skipping copy_selinux_file for %s', input_filename) @@ -272,7 +318,10 @@ def rebuild_image_with_sepolicy(target_files_dir): vendor_target_files_dir = common.MakeTempDir( prefix='merge_target_files_vendor_target_files_') common.UnzipToDir(OPTIONS.vendor_otatools, vendor_otatools_dir) - common.UnzipToDir(OPTIONS.vendor_target_files, vendor_target_files_dir) + merge_utils.CollectTargetFiles( + input_zipfile_or_dir=OPTIONS.vendor_target_files, + output_dir=vendor_target_files_dir, + item_list=include_extra_in_list(OPTIONS.vendor_item_list)) # Copy the partition contents from the merged target-files archive to the # vendor target-files archive. @@ -303,8 +352,9 @@ def rebuild_image_with_sepolicy(target_files_dir): shutil.move( os.path.join(vendor_target_files_dir, 'IMAGES', partition_img), os.path.join(target_files_dir, 'IMAGES', partition_img)) - move_only_exists(os.path.join(vendor_target_files_dir, 'IMAGES', partition_map), - os.path.join(target_files_dir, 'IMAGES', partition_map)) + move_only_exists( + os.path.join(vendor_target_files_dir, 'IMAGES', partition_map), + os.path.join(target_files_dir, 'IMAGES', partition_map)) def copy_recovery_file(filename): for subdir in ('VENDOR', 'SYSTEM/vendor'): @@ -505,6 +555,8 @@ def main(): OPTIONS.vendor_item_list = a elif o == '--vendor-item-list': OPTIONS.vendor_item_list = a + elif o == '--boot-image-dir-path': + OPTIONS.boot_image_dir_path = a elif o == '--output-target-files': OPTIONS.output_target_files = a elif o == '--output-dir': @@ -527,6 +579,10 @@ def main(): OPTIONS.rebuild_sepolicy = True elif o == '--keep-tmp': OPTIONS.keep_tmp = True + elif o == '--avb-resolve-rollback-index-location-conflict': + OPTIONS.avb_resolve_rollback_index_location_conflict = True + elif o == '--allow-partial-ab': + OPTIONS.allow_partial_ab = True elif o == '--framework-dexpreopt-config': OPTIONS.framework_dexpreopt_config = a elif o == '--framework-dexpreopt-tools': @@ -551,6 +607,7 @@ def main(): 'vendor-target-files=', 'other-item-list=', 'vendor-item-list=', + 'boot-image-dir-path=', 'output-target-files=', 'output-dir=', 'output-item-list=', @@ -566,6 +623,8 @@ def main(): 'vendor-otatools=', 'rebuild-sepolicy', 'keep-tmp', + 'avb-resolve-rollback-index-location-conflict', + 'allow-partial-ab', ], extra_option_handler=option_handler) @@ -578,10 +637,10 @@ def main(): common.Usage(__doc__) sys.exit(1) - with zipfile.ZipFile(OPTIONS.framework_target_files, allowZip64=True) as fz: - framework_namelist = fz.namelist() - with zipfile.ZipFile(OPTIONS.vendor_target_files, allowZip64=True) as vz: - vendor_namelist = vz.namelist() + framework_namelist = merge_utils.GetTargetFilesItems( + OPTIONS.framework_target_files) + vendor_namelist = merge_utils.GetTargetFilesItems( + OPTIONS.vendor_target_files) if OPTIONS.framework_item_list: OPTIONS.framework_item_list = common.LoadListFromFile( diff --git a/tools/releasetools/merge/merge_utils.py b/tools/releasetools/merge/merge_utils.py index e056195733..d446fc020d 100644 --- a/tools/releasetools/merge/merge_utils.py +++ b/tools/releasetools/merge/merge_utils.py @@ -49,28 +49,80 @@ def ExtractItems(input_zip, output_dir, extract_item_list): common.UnzipToDir(input_zip, output_dir, filtered_extract_item_list) -def CopyItems(from_dir, to_dir, patterns): - """Similar to ExtractItems() except uses an input dir instead of zip.""" - file_paths = [] - for dirpath, _, filenames in os.walk(from_dir): - file_paths.extend( - os.path.relpath(path=os.path.join(dirpath, filename), start=from_dir) - for filename in filenames) - - filtered_file_paths = set() - for pattern in patterns: - filtered_file_paths.update(fnmatch.filter(file_paths, pattern)) - - for file_path in filtered_file_paths: - original_file_path = os.path.join(from_dir, file_path) - copied_file_path = os.path.join(to_dir, file_path) - copied_file_dir = os.path.dirname(copied_file_path) - if not os.path.exists(copied_file_dir): - os.makedirs(copied_file_dir) - if os.path.islink(original_file_path): - os.symlink(os.readlink(original_file_path), copied_file_path) +def CopyItems(from_dir, to_dir, copy_item_list): + """Copies the items in copy_item_list from source to destination directory. + + copy_item_list may include files and directories. Will copy the matched + files and create the matched directories. + + Args: + from_dir: The source directory. + to_dir: The destination directory. + copy_item_list: Items to be copied. + """ + item_paths = [] + for root, dirs, files in os.walk(from_dir): + item_paths.extend( + os.path.relpath(path=os.path.join(root, item_name), start=from_dir) + for item_name in files + dirs) + + filtered = set() + for pattern in copy_item_list: + filtered.update(fnmatch.filter(item_paths, pattern)) + + for item in filtered: + original_path = os.path.join(from_dir, item) + copied_path = os.path.join(to_dir, item) + copied_parent_path = os.path.dirname(copied_path) + if not os.path.exists(copied_parent_path): + os.makedirs(copied_parent_path) + if os.path.islink(original_path): + os.symlink(os.readlink(original_path), copied_path) + elif os.path.isdir(original_path): + if not os.path.exists(copied_path): + os.makedirs(copied_path) else: - shutil.copyfile(original_file_path, copied_file_path) + shutil.copyfile(original_path, copied_path) + + +def GetTargetFilesItems(target_files_zipfile_or_dir): + """Gets a list of target files items.""" + if zipfile.is_zipfile(target_files_zipfile_or_dir): + with zipfile.ZipFile(target_files_zipfile_or_dir, allowZip64=True) as fz: + return fz.namelist() + elif os.path.isdir(target_files_zipfile_or_dir): + item_list = [] + for root, dirs, files in os.walk(target_files_zipfile_or_dir): + item_list.extend( + os.path.relpath(path=os.path.join(root, item), + start=target_files_zipfile_or_dir) + for item in dirs + files) + return item_list + else: + raise ValueError('Target files should be either zipfile or directory.') + + +def CollectTargetFiles(input_zipfile_or_dir, output_dir, item_list=None): + """Extracts input zipfile or copy input directory to output directory. + + Extracts the input zipfile if `input_zipfile_or_dir` is a zip archive, or + copies the items if `input_zipfile_or_dir` is a directory. + + Args: + input_zipfile_or_dir: The input target files, could be either a zipfile to + extract or a directory to copy. + output_dir: The output directory that the input files are either extracted + or copied. + item_list: Files to be extracted or copied. Will extract or copy all files + if omitted. + """ + patterns = item_list if item_list else ('*',) + if zipfile.is_zipfile(input_zipfile_or_dir): + ExtractItems(input_zipfile_or_dir, output_dir, patterns) + elif os.path.isdir(input_zipfile_or_dir): + CopyItems(input_zipfile_or_dir, output_dir, patterns) + else: + raise ValueError('Target files should be either zipfile or directory.') def WriteSortedData(data, path): @@ -129,6 +181,7 @@ def ValidateConfigLists(): _PARTITION_ITEM_PATTERN = re.compile(r'^([A-Z_]+)/.*$') _IMAGE_PARTITION_PATTERN = re.compile(r'^IMAGES/(.*)\.img$') +_PREBUILT_IMAGE_PARTITION_PATTERN = re.compile(r'^PREBUILT_IMAGES/(.*)\.img$') def ItemListToPartitionSet(item_list): @@ -151,12 +204,12 @@ def ItemListToPartitionSet(item_list): partition_set = set() for item in item_list: - for pattern in (_PARTITION_ITEM_PATTERN, _IMAGE_PARTITION_PATTERN): + for pattern in (_PARTITION_ITEM_PATTERN, _IMAGE_PARTITION_PATTERN, _PREBUILT_IMAGE_PARTITION_PATTERN): partition_match = pattern.search(item.strip()) if partition_match: partition = partition_match.group(1).lower() # These directories in target-files are not actual partitions. - if partition not in ('meta', 'images'): + if partition not in ('meta', 'images', 'prebuilt_images'): partition_set.add(partition) return partition_set @@ -164,8 +217,8 @@ def ItemListToPartitionSet(item_list): # Partitions that are grabbed from the framework partial build by default. _FRAMEWORK_PARTITIONS = { - 'system', 'product', 'system_ext', 'system_other', 'root', 'system_dlkm', - 'vbmeta_system' + 'system', 'product', 'system_ext', 'system_other', 'root', + 'vbmeta_system', 'pvmfw' } @@ -201,7 +254,7 @@ def InferItemList(input_namelist, framework): if partition == 'meta': continue - if partition == 'images': + if partition in ('images', 'prebuilt_images'): image_partition, extension = os.path.splitext(os.path.basename(namelist)) if image_partition == 'vbmeta': # Always regenerate vbmeta.img since it depends on hash information diff --git a/tools/releasetools/merge/test_merge_utils.py b/tools/releasetools/merge/test_merge_utils.py index 1ae1f54e81..b4c47aeb89 100644 --- a/tools/releasetools/merge/test_merge_utils.py +++ b/tools/releasetools/merge/test_merge_utils.py @@ -35,22 +35,27 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase): open(path, 'a').close() return path + def createEmptyFolder(path): + os.makedirs(path) + return path + def createSymLink(source, dest): os.symlink(source, dest) return dest def getRelPaths(start, filepaths): return set( - os.path.relpath(path=filepath, start=start) for filepath in filepaths) + os.path.relpath(path=filepath, start=start) + for filepath in filepaths) input_dir = common.MakeTempDir() output_dir = common.MakeTempDir() expected_copied_items = [] actual_copied_items = [] - patterns = ['*.cpp', 'subdir/*.txt'] + patterns = ['*.cpp', 'subdir/*.txt', 'subdir/empty_dir'] - # Create various files that we expect to get copied because they - # match one of the patterns. + # Create various files and empty directories that we expect to get copied + # because they match one of the patterns. expected_copied_items.extend([ createEmptyFile(os.path.join(input_dir, 'a.cpp')), createEmptyFile(os.path.join(input_dir, 'b.cpp')), @@ -58,6 +63,7 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase): createEmptyFile(os.path.join(input_dir, 'subdir', 'd.txt')), createEmptyFile( os.path.join(input_dir, 'subdir', 'subsubdir', 'e.txt')), + createEmptyFolder(os.path.join(input_dir, 'subdir', 'empty_dir')), createSymLink('a.cpp', os.path.join(input_dir, 'a_link.cpp')), ]) # Create some more files that we expect to not get copied. @@ -70,9 +76,13 @@ class MergeUtilsTest(test_utils.ReleaseToolsTestCase): merge_utils.CopyItems(input_dir, output_dir, patterns) # Assert the actual copied items match the ones we expected. - for dirpath, _, filenames in os.walk(output_dir): + for root_dir, dirs, files in os.walk(output_dir): actual_copied_items.extend( - os.path.join(dirpath, filename) for filename in filenames) + os.path.join(root_dir, filename) for filename in files) + for dirname in dirs: + dir_path = os.path.join(root_dir, dirname) + if not os.listdir(dir_path): + actual_copied_items.append(dir_path) self.assertEqual( getRelPaths(output_dir, actual_copied_items), getRelPaths(input_dir, expected_copied_items)) diff --git a/tools/releasetools/merge_ota.py b/tools/releasetools/merge_ota.py index 7d3d3a3fc3..fb5957a857 100644 --- a/tools/releasetools/merge_ota.py +++ b/tools/releasetools/merge_ota.py @@ -30,10 +30,12 @@ from update_payload import Payload from payload_signer import PayloadSigner from ota_utils import PayloadGenerator, METADATA_PROTO_NAME, FinalizeMetadata +from ota_signing_utils import AddSigningArgumentParse logger = logging.getLogger(__name__) CARE_MAP_ENTRY = "care_map.pb" +APEX_INFO_ENTRY = "apex_info.pb" def WriteDataBlob(payload: Payload, outfp: BinaryIO, read_size=1024*64): @@ -124,7 +126,7 @@ def MergeManifests(payloads: List[Payload]) -> DeltaArchiveManifest: ExtendPartitionUpdates(output_manifest.partitions, manifest.partitions) try: MergeDynamicPartitionMetadata( - output_manifest.dynamic_partition_metadata, manifest.dynamic_partition_metadata) + output_manifest.dynamic_partition_metadata, manifest.dynamic_partition_metadata) except DuplicatePartitionError: logger.error( "OTA %s has duplicate partition with some of the previous OTAs", payload.name) @@ -188,21 +190,34 @@ def CheckDuplicatePartitions(payloads: List[Payload]): f"OTA {partition_to_ota[part].name} and {payload.name} have duplicating partition {part}") partition_to_ota[part] = payload + +def ApexInfo(file_paths): + if len(file_paths) > 1: + logger.info("More than one target file specified, will ignore " + "apex_info.pb (if any)") + return None + with zipfile.ZipFile(file_paths[0], "r", allowZip64=True) as zfp: + if APEX_INFO_ENTRY in zfp.namelist(): + apex_info_bytes = zfp.read(APEX_INFO_ENTRY) + return apex_info_bytes + return None + + def main(argv): parser = argparse.ArgumentParser(description='Merge multiple partial OTAs') parser.add_argument('packages', type=str, nargs='+', help='Paths to OTA packages to merge') - parser.add_argument('--package_key', type=str, - help='Paths to private key for signing payload') - parser.add_argument('--search_path', type=str, - help='Search path for framework/signapk.jar') parser.add_argument('--output', type=str, help='Paths to output merged ota', required=True) parser.add_argument('--metadata_ota', type=str, help='Output zip will use build metadata from this OTA package, if unspecified, use the last OTA package in merge list') - parser.add_argument('--private_key_suffix', type=str, - help='Suffix to be appended to package_key path', default=".pk8") - parser.add_argument('-v', action="store_true", help="Enable verbose logging", dest="verbose") + parser.add_argument('-v', action="store_true", + help="Enable verbose logging", dest="verbose") + AddSigningArgumentParse(parser) + + parser.epilog = ('This tool can also be used to resign a regular OTA. For a single regular OTA, ' + 'apex_info.pb will be written to output. When merging multiple OTAs, ' + 'apex_info.pb will not be written.') args = parser.parse_args(argv[1:]) file_paths = args.packages @@ -225,6 +240,11 @@ def main(argv): merged_manifest = MergeManifests(payloads) + # Get signing keys + key_passwords = common.GetKeyPasswords([args.package_key]) + + apex_info_bytes = ApexInfo(file_paths) + with tempfile.NamedTemporaryFile() as unsigned_payload: WriteHeaderAndManifest(merged_manifest, unsigned_payload) ConcatBlobs(payloads, unsigned_payload) @@ -236,27 +256,36 @@ def main(argv): if args.package_key: logger.info("Signing payload...") - signer = PayloadSigner(args.package_key, args.private_key_suffix) + # TODO: remove OPTIONS when no longer used as fallback in payload_signer + common.OPTIONS.payload_signer_args = None + common.OPTIONS.payload_signer_maximum_signature_size = None + signer = PayloadSigner(args.package_key, args.private_key_suffix, + key_passwords[args.package_key], + payload_signer=args.payload_signer, + payload_signer_args=args.payload_signer_args, + payload_signer_maximum_signature_size=args.payload_signer_maximum_signature_size) generator.payload_file = unsigned_payload.name generator.Sign(signer) logger.info("Payload size: %d", os.path.getsize(generator.payload_file)) logger.info("Writing to %s", args.output) + key_passwords = common.GetKeyPasswords([args.package_key]) with tempfile.NamedTemporaryFile(prefix="signed_ota", suffix=".zip") as signed_ota: with zipfile.ZipFile(signed_ota, "w") as zfp: generator.WriteToZip(zfp) care_map_bytes = MergeCareMap(args.packages) if care_map_bytes: - zfp.writestr(CARE_MAP_ENTRY, care_map_bytes) + common.ZipWriteStr(zfp, CARE_MAP_ENTRY, care_map_bytes) + if apex_info_bytes: + logger.info("Writing %s", APEX_INFO_ENTRY) + common.ZipWriteStr(zfp, APEX_INFO_ENTRY, apex_info_bytes) AddOtaMetadata(signed_ota.name, metadata_ota, args.output, args.package_key, key_passwords[args.package_key]) return 0 - - if __name__ == '__main__': logging.basicConfig() sys.exit(main(sys.argv)) diff --git a/tools/releasetools/non_ab_ota.py b/tools/releasetools/non_ab_ota.py index 7078d67ef7..667891c65e 100644 --- a/tools/releasetools/non_ab_ota.py +++ b/tools/releasetools/non_ab_ota.py @@ -23,6 +23,7 @@ import verity_utils from check_target_files_vintf import CheckVintfIfTrebleEnabled, HasPartition from common import OPTIONS from ota_utils import UNZIP_PATTERN, FinalizeMetadata, GetPackageMetadata, PropertyFiles +import subprocess logger = logging.getLogger(__name__) @@ -272,12 +273,13 @@ endif; # We haven't written the metadata entry, which will be done in # FinalizeMetadata. - output_zip.close() + common.ZipClose(output_zip) needed_property_files = ( NonAbOtaPropertyFiles(), ) - FinalizeMetadata(metadata, staging_file, output_file, needed_property_files, package_key=OPTIONS.package_key) + FinalizeMetadata(metadata, staging_file, output_file, + needed_property_files, package_key=OPTIONS.package_key) def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): @@ -526,13 +528,14 @@ endif; # We haven't written the metadata entry yet, which will be handled in # FinalizeMetadata(). - output_zip.close() + common.ZipClose(output_zip) # Sign the generated zip package unless no_signing is specified. needed_property_files = ( NonAbOtaPropertyFiles(), ) - FinalizeMetadata(metadata, staging_file, output_file, needed_property_files, package_key=OPTIONS.package_key) + FinalizeMetadata(metadata, staging_file, output_file, + needed_property_files, package_key=OPTIONS.package_key) def GenerateNonAbOtaPackage(target_file, output_file, source_file=None): @@ -555,8 +558,18 @@ def GenerateNonAbOtaPackage(target_file, output_file, source_file=None): if OPTIONS.extracted_input is not None: OPTIONS.input_tmp = OPTIONS.extracted_input else: - logger.info("unzipping target target-files...") - OPTIONS.input_tmp = common.UnzipTemp(target_file, UNZIP_PATTERN) + if not os.path.isdir(target_file): + logger.info("unzipping target target-files...") + OPTIONS.input_tmp = common.UnzipTemp(target_file, UNZIP_PATTERN) + else: + OPTIONS.input_tmp = target_file + tmpfile = common.MakeTempFile(suffix=".zip") + os.unlink(tmpfile) + common.RunAndCheckOutput( + ["zip", tmpfile, "-r", ".", "-0"], cwd=target_file) + assert zipfile.is_zipfile(tmpfile) + target_file = tmpfile + OPTIONS.target_tmp = OPTIONS.input_tmp # If the caller explicitly specified the device-specific extensions path via diff --git a/tools/releasetools/ota_from_raw_img.py b/tools/releasetools/ota_from_raw_img.py new file mode 100644 index 0000000000..c186940c25 --- /dev/null +++ b/tools/releasetools/ota_from_raw_img.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Copyright (C) 2008 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Given a series of .img files, produces an OTA package that installs thoese images +""" + +import sys +import os +import argparse +import subprocess +import tempfile +import logging +import zipfile + +import common +from payload_signer import PayloadSigner +from ota_utils import PayloadGenerator +from ota_signing_utils import AddSigningArgumentParse + + +logger = logging.getLogger(__name__) + + +def ResolveBinaryPath(filename, search_path): + if not search_path: + return filename + if not os.path.exists(search_path): + return filename + path = os.path.join(search_path, "bin", filename) + if os.path.exists(path): + return path + path = os.path.join(search_path, filename) + if os.path.exists(path): + return path + return path + + +def main(argv): + parser = argparse.ArgumentParser( + prog=argv[0], description="Given a series of .img files, produces a full OTA package that installs thoese images") + parser.add_argument("images", nargs="+", type=str, + help="List of images to generate OTA") + parser.add_argument("--partition_names", nargs='+', type=str, + help="Partition names to install the images, default to basename of the image(no file name extension)") + parser.add_argument('--output', type=str, + help='Paths to output merged ota', required=True) + parser.add_argument('--max_timestamp', type=int, + help='Maximum build timestamp allowed to install this OTA') + parser.add_argument("-v", action="store_true", + help="Enable verbose logging", dest="verbose") + AddSigningArgumentParse(parser) + + args = parser.parse_args(argv[1:]) + if args.verbose: + logger.setLevel(logging.INFO) + logger.info(args) + old_imgs = [""] * len(args.images) + for (i, img) in enumerate(args.images): + if ":" in img: + old_imgs[i], args.images[i] = img.split(":", maxsplit=1) + + if not args.partition_names: + args.partition_names = [os.path.os.path.splitext(os.path.basename(path))[ + 0] for path in args.images] + with tempfile.NamedTemporaryFile() as unsigned_payload, tempfile.NamedTemporaryFile() as dynamic_partition_info_file: + dynamic_partition_info_file.writelines( + [b"virtual_ab=true\n", b"super_partition_groups=\n"]) + dynamic_partition_info_file.flush() + cmd = [ResolveBinaryPath("delta_generator", args.search_path)] + cmd.append("--partition_names=" + ",".join(args.partition_names)) + cmd.append("--dynamic_partition_info_file=" + + dynamic_partition_info_file.name) + cmd.append("--old_partitions=" + ",".join(old_imgs)) + cmd.append("--new_partitions=" + ",".join(args.images)) + cmd.append("--out_file=" + unsigned_payload.name) + cmd.append("--is_partial_update") + if args.max_timestamp: + cmd.append("--max_timestamp=" + str(args.max_timestamp)) + cmd.append("--partition_timestamps=boot:" + str(args.max_timestamp)) + logger.info("Running %s", cmd) + + subprocess.check_call(cmd) + generator = PayloadGenerator() + generator.payload_file = unsigned_payload.name + logger.info("Payload size: %d", os.path.getsize(generator.payload_file)) + + # Get signing keys + key_passwords = common.GetKeyPasswords([args.package_key]) + + if args.package_key: + logger.info("Signing payload...") + # TODO: remove OPTIONS when no longer used as fallback in payload_signer + common.OPTIONS.payload_signer_args = None + common.OPTIONS.payload_signer_maximum_signature_size = None + signer = PayloadSigner(args.package_key, args.private_key_suffix, + key_passwords[args.package_key], + payload_signer=args.payload_signer, + payload_signer_args=args.payload_signer_args, + payload_signer_maximum_signature_size=args.payload_signer_maximum_signature_size) + generator.payload_file = unsigned_payload.name + generator.Sign(signer) + + logger.info("Payload size: %d", os.path.getsize(generator.payload_file)) + + logger.info("Writing to %s", args.output) + with zipfile.ZipFile(args.output, "w") as zfp: + generator.WriteToZip(zfp) + + +if __name__ == "__main__": + logging.basicConfig() + main(sys.argv) diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index 78c1c2f653..432ea199bb 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -147,7 +147,7 @@ Non-A/B OTA specific options A/B OTA specific options --disable_fec_computation - Disable the on device FEC data computation for incremental updates. + Disable the on device FEC data computation for incremental updates. OTA will be larger but installation will be faster. --include_secondary Additionally include the payload for secondary slot images (default: @@ -195,6 +195,8 @@ A/B OTA specific options ro.product.* properties are overridden by the 'import' statement. The file expects one property per line, and each line has the following format: 'prop_name=value1,value2'. e.g. 'ro.boot.product.sku=std,pro' + The path specified can either be relative to the current working directory + or the path to a file inside of input_target_files. --skip_postinstall Skip the postinstall hooks when generating an A/B OTA package (default: @@ -224,7 +226,7 @@ A/B OTA specific options wait time in recovery. --enable_vabc_xor - Enable the VABC xor feature. Will reduce space requirements for OTA + Enable the VABC xor feature. Will reduce space requirements for OTA, but OTA installation will be slower. --force_minor_version Override the update_engine minor version for delta generation. @@ -233,7 +235,10 @@ A/B OTA specific options A colon ':' separated list of compressors. Allowed values are bz2 and brotli. --enable_zucchini - Whether to enable to zucchini feature. Will generate smaller OTA but uses more memory. + Whether to enable to zucchini feature. Will generate smaller OTA but uses more memory, OTA generation will take longer. + + --enable_puffdiff + Whether to enable to puffdiff feature. Will generate smaller OTA but uses more memory, OTA generation will take longer. --enable_lz4diff Whether to enable lz4diff feature. Will generate smaller OTA for EROFS but @@ -244,10 +249,21 @@ A/B OTA specific options older SPL. --vabc_compression_param - Compression algorithm to be used for VABC. Available options: gz, brotli, none + Compression algorithm to be used for VABC. Available options: gz, lz4, zstd, brotli, none. + Compression level can be specified by appending ",$LEVEL" to option. + e.g. --vabc_compression_param=gz,9 specifies level 9 compression with gz algorithm --security_patch_level Override the security patch level in target files + + --max_threads + Specify max number of threads allowed when generating A/B OTA + + --vabc_cow_version + Specify the VABC cow version to be used + + --compression_factor + Specify the maximum block size to be compressed at once during OTA. supported options: 4k, 8k, 16k, 32k, 64k, 128k, 256k """ from __future__ import print_function @@ -257,7 +273,6 @@ import multiprocessing import os import os.path import re -import shlex import shutil import subprocess import sys @@ -266,11 +281,11 @@ import zipfile import care_map_pb2 import common import ota_utils -from ota_utils import (UNZIP_PATTERN, FinalizeMetadata, GetPackageMetadata, - PayloadGenerator, SECURITY_PATCH_LEVEL_PROP_NAME) -from common import IsSparseImage +import payload_signer +from ota_utils import (VABC_COMPRESSION_PARAM_SUPPORT, FinalizeMetadata, GetPackageMetadata, + PayloadGenerator, SECURITY_PATCH_LEVEL_PROP_NAME, ExtractTargetFiles, CopyTargetFilesDir) +from common import DoesInputFileContain, IsSparseImage import target_files_diff -from check_target_files_vintf import CheckVintfIfTrebleEnabled from non_ab_ota import GenerateNonAbOtaPackage from payload_signer import PayloadSigner @@ -301,9 +316,6 @@ OPTIONS.full_bootloader = False OPTIONS.cache_size = None OPTIONS.stash_threshold = 0.8 OPTIONS.log_diff = None -OPTIONS.payload_signer = None -OPTIONS.payload_signer_args = [] -OPTIONS.payload_signer_maximum_signature_size = None OPTIONS.extracted_input = None OPTIONS.skip_postinstall = False OPTIONS.skip_compatibility_check = False @@ -318,12 +330,18 @@ OPTIONS.enable_vabc_xor = True OPTIONS.force_minor_version = None OPTIONS.compressor_types = None OPTIONS.enable_zucchini = True +OPTIONS.enable_puffdiff = None OPTIONS.enable_lz4diff = False OPTIONS.vabc_compression_param = None OPTIONS.security_patch_level = None +OPTIONS.max_threads = None +OPTIONS.vabc_cow_version = None +OPTIONS.compression_factor = None + POSTINSTALL_CONFIG = 'META/postinstall_config.txt' DYNAMIC_PARTITION_INFO = 'META/dynamic_partitions_info.txt' +MISC_INFO = 'META/misc_info.txt' AB_PARTITIONS = 'META/ab_partitions.txt' # Files to be unzipped for target diffing purpose. @@ -350,6 +368,25 @@ def _LoadOemDicts(oem_source): oem_dicts.append(common.LoadDictionaryFromFile(oem_file)) return oem_dicts +def ModifyKeyvalueList(content: str, key: str, value: str): + """ Update update the key value list with specified key and value + Args: + content: The string content of dynamic_partitions_info.txt. Each line + should be a key valur pair, where string before the first '=' are keys, + remaining parts are values. + key: the key of the key value pair to modify + value: the new value to replace with + + Returns: + Updated content of the key value list + """ + output_list = [] + for line in content.splitlines(): + if line.startswith(key+"="): + continue + output_list.append(line) + output_list.append("{}={}".format(key, value)) + return "\n".join(output_list) def ModifyVABCCompressionParam(content, algo): """ Update update VABC Compression Param in dynamic_partitions_info.txt @@ -360,13 +397,7 @@ def ModifyVABCCompressionParam(content, algo): Returns: Updated content of dynamic_partitions_info.txt , with custom compression algo """ - output_list = [] - for line in content.splitlines(): - if line.startswith("virtual_ab_compression_method="): - continue - output_list.append(line) - output_list.append("virtual_ab_compression_method="+algo) - return "\n".join(output_list) + return ModifyKeyvalueList(content, "virtual_ab_compression_method", algo) def UpdatesInfoForSpecialUpdates(content, partitions_filter, @@ -423,6 +454,13 @@ def GetTargetFilesZipForSecondaryImages(input_file, skip_postinstall=False): slot will be used. This is to ensure that we always have valid boot, vbmeta, bootloader images in the inactive slot. + After writing system_other to inactive slot's system partiiton, + PackageManagerService will read `ro.cp_system_other_odex`, and set + `sys.cppreopt` to "requested". Then, according to + system/extras/cppreopts/cppreopts.rc , init will mount system_other at + /postinstall, and execute `cppreopts` to copy optimized APKs from + /postinstall to /data . + Args: input_file: The input target-files.zip file. skip_postinstall: Whether to skip copying the postinstall config file. @@ -445,50 +483,53 @@ def GetTargetFilesZipForSecondaryImages(input_file, skip_postinstall=False): target_file = common.MakeTempFile(prefix="targetfiles-", suffix=".zip") target_zip = zipfile.ZipFile(target_file, 'w', allowZip64=True) - with zipfile.ZipFile(input_file, 'r', allowZip64=True) as input_zip: - infolist = input_zip.infolist() + fileslist = [] + for (root, dirs, files) in os.walk(input_file): + root = root.lstrip(input_file).lstrip("/") + fileslist.extend([os.path.join(root, d) for d in dirs]) + fileslist.extend([os.path.join(root, d) for d in files]) - input_tmp = common.UnzipTemp(input_file, UNZIP_PATTERN) - for info in infolist: - unzipped_file = os.path.join(input_tmp, *info.filename.split('/')) - if info.filename == 'IMAGES/system_other.img': + input_tmp = input_file + for filename in fileslist: + unzipped_file = os.path.join(input_tmp, *filename.split('/')) + if filename == 'IMAGES/system_other.img': common.ZipWrite(target_zip, unzipped_file, arcname='IMAGES/system.img') # Primary images and friends need to be skipped explicitly. - elif info.filename in ('IMAGES/system.img', - 'IMAGES/system.map'): + elif filename in ('IMAGES/system.img', + 'IMAGES/system.map'): pass # Copy images that are not in SECONDARY_PAYLOAD_SKIPPED_IMAGES. - elif info.filename.startswith(('IMAGES/', 'RADIO/')): - image_name = os.path.basename(info.filename) + elif filename.startswith(('IMAGES/', 'RADIO/')): + image_name = os.path.basename(filename) if image_name not in ['{}.img'.format(partition) for partition in SECONDARY_PAYLOAD_SKIPPED_IMAGES]: - common.ZipWrite(target_zip, unzipped_file, arcname=info.filename) + common.ZipWrite(target_zip, unzipped_file, arcname=filename) # Skip copying the postinstall config if requested. - elif skip_postinstall and info.filename == POSTINSTALL_CONFIG: + elif skip_postinstall and filename == POSTINSTALL_CONFIG: pass - elif info.filename.startswith('META/'): + elif filename.startswith('META/'): # Remove the unnecessary partitions for secondary images from the # ab_partitions file. - if info.filename == AB_PARTITIONS: + if filename == AB_PARTITIONS: with open(unzipped_file) as f: partition_list = f.read().splitlines() partition_list = [partition for partition in partition_list if partition and partition not in SECONDARY_PAYLOAD_SKIPPED_IMAGES] - common.ZipWriteStr(target_zip, info.filename, + common.ZipWriteStr(target_zip, filename, '\n'.join(partition_list)) # Remove the unnecessary partitions from the dynamic partitions list. - elif (info.filename == 'META/misc_info.txt' or - info.filename == DYNAMIC_PARTITION_INFO): + elif (filename == 'META/misc_info.txt' or + filename == DYNAMIC_PARTITION_INFO): modified_info = GetInfoForSecondaryImages(unzipped_file) - common.ZipWriteStr(target_zip, info.filename, modified_info) + common.ZipWriteStr(target_zip, filename, modified_info) else: - common.ZipWrite(target_zip, unzipped_file, arcname=info.filename) + common.ZipWrite(target_zip, unzipped_file, arcname=filename) - target_zip.close() + common.ZipClose(target_zip) return target_file @@ -507,23 +548,16 @@ def GetTargetFilesZipWithoutPostinstallConfig(input_file): Returns: The filename of target-files.zip that doesn't contain postinstall config. """ - # We should only make a copy if postinstall_config entry exists. - with zipfile.ZipFile(input_file, 'r', allowZip64=True) as input_zip: - if POSTINSTALL_CONFIG not in input_zip.namelist(): - return input_file - - target_file = common.MakeTempFile(prefix="targetfiles-", suffix=".zip") - shutil.copyfile(input_file, target_file) - common.ZipDelete(target_file, POSTINSTALL_CONFIG) - return target_file + config_path = os.path.join(input_file, POSTINSTALL_CONFIG) + if os.path.exists(config_path): + os.unlink(config_path) + return input_file def ParseInfoDict(target_file_path): - with zipfile.ZipFile(target_file_path, 'r', allowZip64=True) as zfp: - return common.LoadInfoDict(zfp) - + return common.LoadInfoDict(target_file_path) -def GetTargetFilesZipForCustomVABCCompression(input_file, vabc_compression_param): +def ModifyTargetFilesDynamicPartitionInfo(input_file, key, value): """Returns a target-files.zip with a custom VABC compression param. Args: input_file: The input target-files.zip path @@ -532,24 +566,46 @@ def GetTargetFilesZipForCustomVABCCompression(input_file, vabc_compression_param Returns: The path to modified target-files.zip """ + if os.path.isdir(input_file): + dynamic_partition_info_path = os.path.join( + input_file, *DYNAMIC_PARTITION_INFO.split("/")) + with open(dynamic_partition_info_path, "r") as fp: + dynamic_partition_info = fp.read() + dynamic_partition_info = ModifyKeyvalueList( + dynamic_partition_info, key, value) + with open(dynamic_partition_info_path, "w") as fp: + fp.write(dynamic_partition_info) + return input_file + target_file = common.MakeTempFile(prefix="targetfiles-", suffix=".zip") shutil.copyfile(input_file, target_file) common.ZipDelete(target_file, DYNAMIC_PARTITION_INFO) with zipfile.ZipFile(input_file, 'r', allowZip64=True) as zfp: dynamic_partition_info = zfp.read(DYNAMIC_PARTITION_INFO).decode() - dynamic_partition_info = ModifyVABCCompressionParam( - dynamic_partition_info, vabc_compression_param) + dynamic_partition_info = ModifyKeyvalueList( + dynamic_partition_info, key, value) with zipfile.ZipFile(target_file, "a", allowZip64=True) as output_zip: output_zip.writestr(DYNAMIC_PARTITION_INFO, dynamic_partition_info) return target_file +def GetTargetFilesZipForCustomVABCCompression(input_file, vabc_compression_param): + """Returns a target-files.zip with a custom VABC compression param. + Args: + input_file: The input target-files.zip path + vabc_compression_param: Custom Virtual AB Compression algorithm + + Returns: + The path to modified target-files.zip + """ + return ModifyTargetFilesDynamicPartitionInfo(input_file, "virtual_ab_compression_method", vabc_compression_param) + def GetTargetFilesZipForPartialUpdates(input_file, ab_partitions): """Returns a target-files.zip for partial ota update package generation. This function modifies ab_partitions list with the desired partitions before calling the brillo_update_payload script. It also cleans up the reference to - the excluded partitions in the info file, e.g misc_info.txt. + the excluded partitions in the info file, e.g. misc_info.txt. Args: input_file: The input target-files.zip filename. @@ -559,23 +615,7 @@ def GetTargetFilesZipForPartialUpdates(input_file, ab_partitions): The filename of target-files.zip used for partial ota update. """ - def AddImageForPartition(partition_name): - """Add the archive name for a given partition to the copy list.""" - for prefix in ['IMAGES', 'RADIO']: - image_path = '{}/{}.img'.format(prefix, partition_name) - if image_path in namelist: - copy_entries.append(image_path) - map_path = '{}/{}.map'.format(prefix, partition_name) - if map_path in namelist: - copy_entries.append(map_path) - return - - raise ValueError("Cannot find {} in input zipfile".format(partition_name)) - - with zipfile.ZipFile(input_file, allowZip64=True) as input_zip: - original_ab_partitions = input_zip.read( - AB_PARTITIONS).decode().splitlines() - namelist = input_zip.namelist() + original_ab_partitions = common.ReadFromInputFile(input_file, AB_PARTITIONS) unrecognized_partitions = [partition for partition in ab_partitions if partition not in original_ab_partitions] @@ -584,50 +624,68 @@ def GetTargetFilesZipForPartialUpdates(input_file, ab_partitions): unrecognized_partitions) logger.info("Generating partial updates for %s", ab_partitions) + for subdir in ["IMAGES", "RADIO", "PREBUILT_IMAGES"]: + image_dir = os.path.join(subdir) + if not os.path.exists(image_dir): + continue + for filename in os.listdir(image_dir): + filepath = os.path.join(image_dir, filename) + if filename.endswith(".img"): + partition_name = filename.removesuffix(".img") + if partition_name not in ab_partitions: + os.unlink(filepath) + + common.WriteToInputFile(input_file, 'META/ab_partitions.txt', + '\n'.join(ab_partitions)) + CARE_MAP_ENTRY = "META/care_map.pb" + if DoesInputFileContain(input_file, CARE_MAP_ENTRY): + caremap = care_map_pb2.CareMap() + caremap.ParseFromString( + common.ReadBytesFromInputFile(input_file, CARE_MAP_ENTRY)) + filtered = [ + part for part in caremap.partitions if part.name in ab_partitions] + del caremap.partitions[:] + caremap.partitions.extend(filtered) + common.WriteBytesToInputFile(input_file, CARE_MAP_ENTRY, + caremap.SerializeToString()) + + for info_file in ['META/misc_info.txt', DYNAMIC_PARTITION_INFO]: + if not DoesInputFileContain(input_file, info_file): + logger.warning('Cannot find %s in input zipfile', info_file) + continue - copy_entries = ['META/update_engine_config.txt'] - for partition_name in ab_partitions: - AddImageForPartition(partition_name) - - # Use zip2zip to avoid extracting the zipfile. - partial_target_file = common.MakeTempFile(suffix='.zip') - cmd = ['zip2zip', '-i', input_file, '-o', partial_target_file] - cmd.extend(['{}:{}'.format(name, name) for name in copy_entries]) - common.RunAndCheckOutput(cmd) - - partial_target_zip = zipfile.ZipFile(partial_target_file, 'a', - allowZip64=True) - with zipfile.ZipFile(input_file, allowZip64=True) as input_zip: - common.ZipWriteStr(partial_target_zip, 'META/ab_partitions.txt', - '\n'.join(ab_partitions)) - CARE_MAP_ENTRY = "META/care_map.pb" - if CARE_MAP_ENTRY in input_zip.namelist(): - caremap = care_map_pb2.CareMap() - caremap.ParseFromString(input_zip.read(CARE_MAP_ENTRY)) - filtered = [ - part for part in caremap.partitions if part.name in ab_partitions] - del caremap.partitions[:] - caremap.partitions.extend(filtered) - common.ZipWriteStr(partial_target_zip, CARE_MAP_ENTRY, - caremap.SerializeToString()) - - for info_file in ['META/misc_info.txt', DYNAMIC_PARTITION_INFO]: - if info_file not in input_zip.namelist(): - logger.warning('Cannot find %s in input zipfile', info_file) - continue - content = input_zip.read(info_file).decode() - modified_info = UpdatesInfoForSpecialUpdates( - content, lambda p: p in ab_partitions) - if OPTIONS.vabc_compression_param and info_file == DYNAMIC_PARTITION_INFO: - modified_info = ModifyVABCCompressionParam( - modified_info, OPTIONS.vabc_compression_param) - common.ZipWriteStr(partial_target_zip, info_file, modified_info) - - # TODO(xunchang) handle META/postinstall_config.txt' + content = common.ReadFromInputFile(input_file, info_file) + modified_info = UpdatesInfoForSpecialUpdates( + content, lambda p: p in ab_partitions) + if OPTIONS.vabc_compression_param and info_file == DYNAMIC_PARTITION_INFO: + modified_info = ModifyVABCCompressionParam( + modified_info, OPTIONS.vabc_compression_param) + common.WriteToInputFile(input_file, info_file, modified_info) + + def IsInPartialList(postinstall_line: str): + idx = postinstall_line.find("=") + if idx < 0: + return False + key = postinstall_line[:idx] + logger.info("%s %s", key, ab_partitions) + for part in ab_partitions: + if key.endswith("_" + part): + return True + return False - partial_target_zip.close() + if common.DoesInputFileContain(input_file, POSTINSTALL_CONFIG): + postinstall_config = common.ReadFromInputFile( + input_file, POSTINSTALL_CONFIG) + postinstall_config = [ + line for line in postinstall_config.splitlines() if IsInPartialList(line)] + if postinstall_config: + postinstall_config = "\n".join(postinstall_config) + common.WriteToInputFile( + input_file, POSTINSTALL_CONFIG, postinstall_config) + else: + os.unlink(os.path.join(input_file, POSTINSTALL_CONFIG)) - return partial_target_file + return input_file def GetTargetFilesZipForRetrofitDynamicPartitions(input_file, @@ -652,21 +710,12 @@ def GetTargetFilesZipForRetrofitDynamicPartitions(input_file, replace = {'OTA/super_{}.img'.format(dev): 'IMAGES/{}.img'.format(dev) for dev in super_block_devices} - target_file = common.MakeTempFile(prefix="targetfiles-", suffix=".zip") - shutil.copyfile(input_file, target_file) - - with zipfile.ZipFile(input_file, allowZip64=True) as input_zip: - namelist = input_zip.namelist() - - input_tmp = common.UnzipTemp(input_file, RETROFIT_DAP_UNZIP_PATTERN) - # Remove partitions from META/ab_partitions.txt that is in # dynamic_partition_list but not in super_block_devices so that # brillo_update_payload won't generate update for those logical partitions. - ab_partitions_file = os.path.join(input_tmp, *AB_PARTITIONS.split('/')) - with open(ab_partitions_file) as f: - ab_partitions_lines = f.readlines() - ab_partitions = [line.strip() for line in ab_partitions_lines] + ab_partitions_lines = common.ReadFromInputFile( + input_file, AB_PARTITIONS).split("\n") + ab_partitions = [line.strip() for line in ab_partitions_lines] # Assert that all super_block_devices are in ab_partitions super_device_not_updated = [partition for partition in super_block_devices if partition not in ab_partitions] @@ -674,15 +723,6 @@ def GetTargetFilesZipForRetrofitDynamicPartitions(input_file, "{} is in super_block_devices but not in {}".format( super_device_not_updated, AB_PARTITIONS) # ab_partitions -= (dynamic_partition_list - super_block_devices) - new_ab_partitions = common.MakeTempFile( - prefix="ab_partitions", suffix=".txt") - with open(new_ab_partitions, 'w') as f: - for partition in ab_partitions: - if (partition in dynamic_partition_list and - partition not in super_block_devices): - logger.info("Dropping %s from ab_partitions.txt", partition) - continue - f.write(partition + "\n") to_delete = [AB_PARTITIONS] # Always skip postinstall for a retrofit update. @@ -695,67 +735,57 @@ def GetTargetFilesZipForRetrofitDynamicPartitions(input_file, # Remove the existing partition images as well as the map files. to_delete += list(replace.values()) to_delete += ['IMAGES/{}.map'.format(dev) for dev in super_block_devices] - - common.ZipDelete(target_file, to_delete) - - target_zip = zipfile.ZipFile(target_file, 'a', allowZip64=True) + for item in to_delete: + os.unlink(os.path.join(input_file, item)) # Write super_{foo}.img as {foo}.img. for src, dst in replace.items(): - assert src in namelist, \ + assert DoesInputFileContain(input_file, src), \ 'Missing {} in {}; {} cannot be written'.format(src, input_file, dst) - unzipped_file = os.path.join(input_tmp, *src.split('/')) - common.ZipWrite(target_zip, unzipped_file, arcname=dst) + source_path = os.path.join(input_file, *src.split("/")) + target_path = os.path.join(input_file, *dst.split("/")) + os.rename(source_path, target_path) # Write new ab_partitions.txt file - common.ZipWrite(target_zip, new_ab_partitions, arcname=AB_PARTITIONS) - - target_zip.close() + new_ab_partitions = os.path.join(input_file, AB_PARTITIONS) + with open(new_ab_partitions, 'w') as f: + for partition in ab_partitions: + if (partition in dynamic_partition_list and + partition not in super_block_devices): + logger.info("Dropping %s from ab_partitions.txt", partition) + continue + f.write(partition + "\n") - return target_file + return input_file -def GetTargetFilesZipForCustomImagesUpdates(input_file, custom_images): +def GetTargetFilesZipForCustomImagesUpdates(input_file, custom_images: dict): """Returns a target-files.zip for custom partitions update. This function modifies ab_partitions list with the desired custom partitions and puts the custom images into the target target-files.zip. Args: - input_file: The input target-files.zip filename. + input_file: The input target-files extracted directory custom_images: A map of custom partitions and custom images. Returns: - The filename of a target-files.zip which has renamed the custom images in - the IMAGES/ to their partition names. + The extracted dir of a target-files.zip which has renamed the custom images + in the IMAGES/ to their partition names. """ + for custom_image in custom_images.values(): + if not os.path.exists(os.path.join(input_file, "IMAGES", custom_image)): + raise ValueError("Specified custom image {} not found in target files {}, available images are {}", + custom_image, input_file, os.listdir(os.path.join(input_file, "IMAGES"))) - # First pass: use zip2zip to copy the target files contents, excluding - # the "custom" images that will be replaced. - target_file = common.MakeTempFile(prefix="targetfiles-", suffix=".zip") - cmd = ['zip2zip', '-i', input_file, '-o', target_file] - - images = {} for custom_partition, custom_image in custom_images.items(): default_custom_image = '{}.img'.format(custom_partition) if default_custom_image != custom_image: - src = 'IMAGES/' + custom_image - dst = 'IMAGES/' + default_custom_image - cmd.extend(['-x', dst]) - images[dst] = src - - common.RunAndCheckOutput(cmd) - - # Second pass: write {custom_image}.img as {custom_partition}.img. - with zipfile.ZipFile(input_file, allowZip64=True) as input_zip: - with zipfile.ZipFile(target_file, 'a', allowZip64=True) as output_zip: - for dst, src in images.items(): - data = input_zip.read(src) - logger.info("Update custom partition '%s'", dst) - common.ZipWriteStr(output_zip, dst, data) - output_zip.close() + src = os.path.join(input_file, 'IMAGES', custom_image) + dst = os.path.join(input_file, 'IMAGES', default_custom_image) + os.rename(src, dst) - return target_file + return input_file def GeneratePartitionTimestampFlags(partition_state): @@ -821,8 +851,36 @@ def SupportsMainlineGkiUpdates(target_file): return pattern.search(output) is not None +def ExtractOrCopyTargetFiles(target_file): + if os.path.isdir(target_file): + return CopyTargetFilesDir(target_file) + else: + return ExtractTargetFiles(target_file) + + +def ValidateCompressionParam(target_info): + vabc_compression_param = OPTIONS.vabc_compression_param + if vabc_compression_param: + minimum_api_level_required = VABC_COMPRESSION_PARAM_SUPPORT[vabc_compression_param.split(",")[0]] + if target_info.vendor_api_level < minimum_api_level_required: + raise ValueError("Specified VABC compression param {} is only supported for API level >= {}, device is on API level {}".format( + vabc_compression_param, minimum_api_level_required, target_info.vendor_api_level)) + + def GenerateAbOtaPackage(target_file, output_file, source_file=None): """Generates an Android OTA package that has A/B update payload.""" + # If input target_files are directories, create a copy so that we can modify + # them directly + target_info = common.BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts) + if OPTIONS.disable_vabc and target_info.is_release_key: + raise ValueError("Disabling VABC on release-key builds is not supported.") + + ValidateCompressionParam(target_info) + vabc_compression_param = target_info.vabc_compression_param + + target_file = ExtractOrCopyTargetFiles(target_file) + if source_file is not None: + source_file = ExtractOrCopyTargetFiles(source_file) # Stage the output zip package for package signing. if not OPTIONS.no_signing: staging_file = common.MakeTempFile(suffix='.zip') @@ -833,6 +891,7 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): allowZip64=True) if source_file is not None: + source_file = ExtractTargetFiles(source_file) assert "ab_partitions" in OPTIONS.source_info_dict, \ "META/ab_partitions.txt is required for ab_update." assert "ab_partitions" in OPTIONS.target_info_dict, \ @@ -846,6 +905,21 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): if not source_info.is_vabc or not target_info.is_vabc: logger.info("Either source or target does not support VABC, disabling.") OPTIONS.disable_vabc = True + if OPTIONS.vabc_compression_param is None and \ + source_info.vabc_compression_param != target_info.vabc_compression_param: + logger.info("Source build and target build use different compression methods {} vs {}, default to source builds parameter {}".format( + source_info.vabc_compression_param, target_info.vabc_compression_param, source_info.vabc_compression_param)) + vabc_compression_param = source_info.vabc_compression_param + # Virtual AB Cow version 3 is introduced in Android U with improved memory + # and install time performance. All OTA's with + # both the source build and target build with VIRTUAL_AB_COW_VERSION = 3 + # can support the new format. Otherwise, fallback on older versions + if not source_info.vabc_cow_version or not target_info.vabc_cow_version: + logger.info("Source or Target doesn't have VABC_COW_VERSION specified, default to version 2") + OPTIONS.vabc_cow_version = 2 + elif source_info.vabc_cow_version != target_info.vabc_cow_version: + logger.info("Source and Target have different cow VABC_COW_VERSION specified, default to minimum version") + OPTIONS.vabc_cow_version = min(source_info.vabc_cow_version, target_info.vabc_cow_version) # Virtual AB Compression was introduced in Androd S. # Later, we backported VABC to Android R. But verity support was not @@ -858,8 +932,36 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): else: assert "ab_partitions" in OPTIONS.info_dict, \ "META/ab_partitions.txt is required for ab_update." - target_info = common.BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts) source_info = None + if not target_info.vabc_cow_version: + OPTIONS.vabc_cow_version = 2 + elif target_info.vabc_cow_version >= "3" and target_info.vendor_api_level < 35: + logger.warning( + "This full OTA is configured to use VABC cow version" + " 3 which is supported since" + " Android API level 35, but device is " + "launched with {} . If this full OTA is" + " served to a device running old build, OTA might fail due to " + "unsupported vabc cow version. For safety, version 2 is used because " + "it's supported since day 1.".format( + target_info.vendor_api_level)) + OPTIONS.vabc_cow_version = 2 + if OPTIONS.vabc_compression_param is None and vabc_compression_param: + minimum_api_level_required = VABC_COMPRESSION_PARAM_SUPPORT[ + vabc_compression_param] + if target_info.vendor_api_level < minimum_api_level_required: + logger.warning( + "This full OTA is configured to use VABC compression algorithm" + " {}, which is supported since" + " Android API level {}, but device is " + "launched with {} . If this full OTA is" + " served to a device running old build, OTA might fail due to " + "unsupported compression parameter. For safety, gz is used because " + "it's supported since day 1.".format( + vabc_compression_param, + minimum_api_level_required, + target_info.vendor_api_level)) + vabc_compression_param = "gz" if OPTIONS.partial == []: logger.info( @@ -897,6 +999,27 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): (source_info is not None and not source_info.is_vabc_xor): logger.info("VABC XOR Not supported, disabling") OPTIONS.enable_vabc_xor = False + + if OPTIONS.vabc_compression_param == "none": + logger.info( + "VABC Compression algorithm is set to 'none', disabling VABC xor") + OPTIONS.enable_vabc_xor = False + + if OPTIONS.enable_vabc_xor: + api_level = -1 + if source_info is not None: + api_level = source_info.vendor_api_level + if api_level == -1: + api_level = target_info.vendor_api_level + + # XOR is only supported on T and higher. + if api_level < 33: + logger.error("VABC XOR not supported on this vendor, disabling") + OPTIONS.enable_vabc_xor = False + + if OPTIONS.vabc_compression_param: + vabc_compression_param = OPTIONS.vabc_compression_param + additional_args = [] # Prepare custom images. @@ -911,28 +1034,40 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): elif OPTIONS.partial: target_file = GetTargetFilesZipForPartialUpdates(target_file, OPTIONS.partial) - additional_args += ["--is_partial_update", "true"] - elif OPTIONS.vabc_compression_param: + if vabc_compression_param != target_info.vabc_compression_param: target_file = GetTargetFilesZipForCustomVABCCompression( - target_file, OPTIONS.vabc_compression_param) - elif OPTIONS.skip_postinstall: + target_file, vabc_compression_param) + if OPTIONS.vabc_cow_version: + target_file = ModifyTargetFilesDynamicPartitionInfo(target_file, "virtual_ab_cow_version", OPTIONS.vabc_cow_version) + if OPTIONS.compression_factor: + target_file = ModifyTargetFilesDynamicPartitionInfo(target_file, "virtual_ab_compression_factor", OPTIONS.compression_factor) + if OPTIONS.skip_postinstall: target_file = GetTargetFilesZipWithoutPostinstallConfig(target_file) # Target_file may have been modified, reparse ab_partitions - with zipfile.ZipFile(target_file, allowZip64=True) as zfp: - target_info.info_dict['ab_partitions'] = zfp.read( - AB_PARTITIONS).decode().strip().split("\n") + target_info.info_dict['ab_partitions'] = common.ReadFromInputFile(target_file, + AB_PARTITIONS).strip().split("\n") + from check_target_files_vintf import CheckVintfIfTrebleEnabled CheckVintfIfTrebleEnabled(target_file, target_info) + # Allow boot_variable_file to also exist in target-files + if OPTIONS.boot_variable_file: + if not os.path.isfile(OPTIONS.boot_variable_file): + OPTIONS.boot_variable_file = os.path.join(target_file, OPTIONS.boot_variable_file) # Metadata to comply with Android OTA package format. metadata = GetPackageMetadata(target_info, source_info) # Generate payload. - payload = PayloadGenerator(OPTIONS.include_secondary, OPTIONS.wipe_user_data) + payload = PayloadGenerator( + wipe_user_data=OPTIONS.wipe_user_data, minor_version=OPTIONS.force_minor_version, is_partial_update=OPTIONS.partial, spl_downgrade=OPTIONS.spl_downgrade) partition_timestamps_flags = [] # Enforce a max timestamp this payload can be applied on top of. if OPTIONS.downgrade: - max_timestamp = source_info.GetBuildProp("ro.build.date.utc") + # When generating ota between merged target-files, partition build date can + # decrease in target, at the same time as ro.build.date.utc increases, + # so always pick largest value. + max_timestamp = max(source_info.GetBuildProp("ro.build.date.utc"), + str(metadata.postcondition.timestamp)) partition_timestamps_flags = GeneratePartitionTimestampFlagsDowngrade( metadata.precondition.partition_state, metadata.postcondition.partition_state @@ -954,15 +1089,21 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): additional_args += ["--security_patch_level", security_patch_level] - additional_args += ["--enable_zucchini", + if OPTIONS.max_threads: + additional_args += ["--max_threads", OPTIONS.max_threads] + + additional_args += ["--enable_zucchini=" + str(OPTIONS.enable_zucchini).lower()] + if OPTIONS.enable_puffdiff is not None: + additional_args += ["--enable_puffdiff=" + + str(OPTIONS.enable_puffdiff).lower()] if not ota_utils.IsLz4diffCompatible(source_file, target_file): logger.warning( "Source build doesn't support lz4diff, or source/target don't have compatible lz4diff versions. Disabling lz4diff.") OPTIONS.enable_lz4diff = False - additional_args += ["--enable_lz4diff", + additional_args += ["--enable_lz4diff=" + str(OPTIONS.enable_lz4diff).lower()] if source_file and OPTIONS.enable_lz4diff: @@ -978,20 +1119,13 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): additional_args += ["--erofs_compression_param", erofs_compression_param] if OPTIONS.disable_vabc: - additional_args += ["--disable_vabc", "true"] + additional_args += ["--disable_vabc=true"] if OPTIONS.enable_vabc_xor: - additional_args += ["--enable_vabc_xor", "true"] - if OPTIONS.force_minor_version: - additional_args += ["--force_minor_version", OPTIONS.force_minor_version] + additional_args += ["--enable_vabc_xor=true"] if OPTIONS.compressor_types: additional_args += ["--compressor_types", OPTIONS.compressor_types] additional_args += ["--max_timestamp", max_timestamp] - if SupportsMainlineGkiUpdates(source_file): - logger.warning( - "Detected build with mainline GKI, include full boot image.") - additional_args.extend(["--full_boot", "true"]) - payload.Generate( target_file, source_file, @@ -1024,34 +1158,32 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): # If dm-verity is supported for the device, copy contents of care_map # into A/B OTA package. - target_zip = zipfile.ZipFile(target_file, "r", allowZip64=True) if target_info.get("avb_enable") == "true": - care_map_list = [x for x in ["care_map.pb", "care_map.txt"] if - "META/" + x in target_zip.namelist()] - # Adds care_map if either the protobuf format or the plain text one exists. - if care_map_list: - care_map_name = care_map_list[0] - care_map_data = target_zip.read("META/" + care_map_name) + for care_map_name in ["care_map.pb", "care_map.txt"]: + if not DoesInputFileContain(target_file, "META/" + care_map_name): + continue + care_map_data = common.ReadBytesFromInputFile( + target_file, "META/" + care_map_name) # In order to support streaming, care_map needs to be packed as # ZIP_STORED. common.ZipWriteStr(output_zip, care_map_name, care_map_data, compress_type=zipfile.ZIP_STORED) + # break here to avoid going into else when care map has been handled + break else: logger.warning("Cannot find care map file in target_file package") # Add the source apex version for incremental ota updates, and write the # result apex info to the ota package. - ota_apex_info = ota_utils.ConstructOtaApexInfo(target_zip, source_file) + ota_apex_info = ota_utils.ConstructOtaApexInfo(target_file, source_file) if ota_apex_info is not None: common.ZipWriteStr(output_zip, "apex_info.pb", ota_apex_info, compress_type=zipfile.ZIP_STORED) - target_zip.close() - # We haven't written the metadata entry yet, which will be handled in # FinalizeMetadata(). - output_zip.close() + common.ZipClose(output_zip) FinalizeMetadata(metadata, staging_file, output_file, package_key=OPTIONS.package_key) @@ -1060,9 +1192,7 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): def main(argv): def option_handler(o, a): - if o in ("-k", "--package_key"): - OPTIONS.package_key = a - elif o in ("-i", "--incremental_from"): + if o in ("-i", "--incremental_from"): OPTIONS.incremental_source = a elif o == "--full_radio": OPTIONS.full_radio = True @@ -1107,17 +1237,6 @@ def main(argv): "a float" % (a, o)) elif o == "--log_diff": OPTIONS.log_diff = a - elif o == "--payload_signer": - OPTIONS.payload_signer = a - elif o == "--payload_signer_args": - OPTIONS.payload_signer_args = shlex.split(a) - elif o == "--payload_signer_maximum_signature_size": - OPTIONS.payload_signer_maximum_signature_size = a - elif o == "--payload_signer_key_size": - # TODO(Xunchang) remove this option after cleaning up the callers. - logger.warning("The option '--payload_signer_key_size' is deprecated." - " Use '--payload_signer_maximum_signature_size' instead.") - OPTIONS.payload_signer_maximum_signature_size = a elif o == "--extracted_input_target_files": OPTIONS.extracted_input = a elif o == "--skip_postinstall": @@ -1164,13 +1283,41 @@ def main(argv): elif o == "--enable_zucchini": assert a.lower() in ["true", "false"] OPTIONS.enable_zucchini = a.lower() != "false" + elif o == "--enable_puffdiff": + assert a.lower() in ["true", "false"] + OPTIONS.enable_puffdiff = a.lower() != "false" elif o == "--enable_lz4diff": assert a.lower() in ["true", "false"] OPTIONS.enable_lz4diff = a.lower() != "false" elif o == "--vabc_compression_param": + words = a.split(",") + assert len(words) >= 1 and len(words) <= 2 OPTIONS.vabc_compression_param = a.lower() + if len(words) == 2: + if not words[1].lstrip("-").isdigit(): + raise ValueError("Cannot parse value %r for option $COMPRESSION_LEVEL - only " + "integers are allowed." % words[1]) elif o == "--security_patch_level": OPTIONS.security_patch_level = a + elif o in ("--max_threads"): + if a.isdigit(): + OPTIONS.max_threads = a + else: + raise ValueError("Cannot parse value %r for option %r - only " + "integers are allowed." % (a, o)) + elif o in ("--compression_factor"): + values = ["4k", "8k", "16k", "32k", "64k", "128k", "256k"] + if a[:-1].isdigit() and a in values and a.endswith("k"): + OPTIONS.compression_factor = str(int(a[:-1]) * 1024) + else: + raise ValueError("Please specify value from following options: 4k, 8k, 16k, 32k, 64k, 128k", "256k") + + elif o == "--vabc_cow_version": + if a.isdigit(): + OPTIONS.vabc_cow_version = a + else: + raise ValueError("Cannot parse value %r for option %r - only " + "integers are allowed." % (a, o)) else: return False return True @@ -1178,7 +1325,6 @@ def main(argv): args = common.ParseOptions(argv, __doc__, extra_opts="b:k:i:d:e:t:2o:", extra_long_opts=[ - "package_key=", "incremental_from=", "full_radio", "full_bootloader", @@ -1197,10 +1343,6 @@ def main(argv): "verify", "stash_threshold=", "log_diff=", - "payload_signer=", - "payload_signer_args=", - "payload_signer_maximum_signature_size=", - "payload_signer_key_size=", "extracted_input_target_files=", "skip_postinstall", "retrofit_dynamic_partitions", @@ -1219,10 +1361,14 @@ def main(argv): "force_minor_version=", "compressor_types=", "enable_zucchini=", + "enable_puffdiff=", "enable_lz4diff=", "vabc_compression_param=", "security_patch_level=", - ], extra_option_handler=option_handler) + "max_threads=", + "vabc_cow_version=", + "compression_factor=", + ], extra_option_handler=[option_handler, payload_signer.signer_options]) common.InitLogging() if len(args) != 2: @@ -1239,7 +1385,7 @@ def main(argv): if OPTIONS.extracted_input is not None: OPTIONS.info_dict = common.LoadInfoDict(OPTIONS.extracted_input) else: - OPTIONS.info_dict = ParseInfoDict(args[0]) + OPTIONS.info_dict = common.LoadInfoDict(args[0]) if OPTIONS.wipe_user_data: if not OPTIONS.vabc_downgrade: @@ -1352,7 +1498,8 @@ def main(argv): "what(even if data wipe is done), so SPL downgrade on any " "release-keys build is not allowed.".format(target_spl, source_spl)) - logger.info("SPL downgrade on %s", target_build_prop.GetProp("ro.build.tags")) + logger.info("SPL downgrade on %s", + target_build_prop.GetProp("ro.build.tags")) if is_spl_downgrade and not OPTIONS.spl_downgrade and not OPTIONS.downgrade: raise common.ExternalError( "Target security patch level {} is older than source SPL {} applying " diff --git a/tools/releasetools/ota_signing_utils.py b/tools/releasetools/ota_signing_utils.py new file mode 100644 index 0000000000..60c8c94f91 --- /dev/null +++ b/tools/releasetools/ota_signing_utils.py @@ -0,0 +1,38 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import shlex + + +def ParseSignerArgs(args): + if args is None: + return None + return shlex.split(args) + + +def AddSigningArgumentParse(parser: argparse.ArgumentParser): + parser.add_argument('--package_key', type=str, + help='Paths to private key for signing payload') + parser.add_argument('--search_path', '--path', type=str, + help='Search path for framework/signapk.jar') + parser.add_argument('--payload_signer', type=str, + help='Path to custom payload signer') + parser.add_argument('--payload_signer_args', type=ParseSignerArgs, + help='Arguments for payload signer if necessary') + parser.add_argument('--payload_signer_maximum_signature_size', type=str, + help='Maximum signature size (in bytes) that would be ' + 'generated by the given payload signer') + parser.add_argument('--private_key_suffix', type=str, + help='Suffix to be appended to package_key path', default=".pk8") diff --git a/tools/releasetools/ota_utils.py b/tools/releasetools/ota_utils.py index e2ce31d590..048a497585 100644 --- a/tools/releasetools/ota_utils.py +++ b/tools/releasetools/ota_utils.py @@ -22,11 +22,13 @@ import zipfile import ota_metadata_pb2 import common -from common import (ZipDelete, OPTIONS, MakeTempFile, +import fnmatch +from common import (ZipDelete, DoesInputFileContain, ReadBytesFromInputFile, OPTIONS, MakeTempFile, ZipWriteStr, BuildInfo, LoadDictionaryFromFile, SignFile, PARTITIONS_WITH_BUILD_PROP, PartitionBuildProps, - GetRamdiskFormat) -from payload_signer import PayloadSigner + GetRamdiskFormat, ParseUpdateEngineConfig) +import payload_signer +from payload_signer import PayloadSigner, AddSigningArgumentParse, GeneratePayloadProperties logger = logging.getLogger(__name__) @@ -36,7 +38,6 @@ OPTIONS.force_non_ab = False OPTIONS.wipe_user_data = False OPTIONS.downgrade = False OPTIONS.key_passwords = {} -OPTIONS.package_key = None OPTIONS.incremental_source = None OPTIONS.retrofit_dynamic_partitions = False OPTIONS.output_metadata_path = None @@ -44,8 +45,23 @@ OPTIONS.boot_variable_file = None METADATA_NAME = 'META-INF/com/android/metadata' METADATA_PROTO_NAME = 'META-INF/com/android/metadata.pb' -UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'OTA/*', 'RADIO/*'] +UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'OTA/*', + 'RADIO/*', '*/build.prop', '*/default.prop', '*/build.default', "*/etc/vintf/*"] SECURITY_PATCH_LEVEL_PROP_NAME = "ro.build.version.security_patch" +TARGET_FILES_IMAGES_SUBDIR = ["IMAGES", "PREBUILT_IMAGES", "RADIO"] + + +# Key is the compression algorithm, value is minimum API level required to +# use this compression algorithm for VABC OTA on device. +VABC_COMPRESSION_PARAM_SUPPORT = { + "gz": 31, + "brotli": 31, + "none": 31, + # lz4 support is added in Android U + "lz4": 34, + # zstd support is added in Android V + "zstd": 35, +} def FinalizeMetadata(metadata, input_file, output_file, needed_property_files=None, package_key=None, pw=None): @@ -135,7 +151,8 @@ def FinalizeMetadata(metadata, input_file, output_file, needed_property_files=No logger.info(f"Signing disabled for output file {output_file}") shutil.copy(prelim_signing, output_file) else: - logger.info(f"Signing the output file {output_file} with key {package_key}") + logger.info( + f"Signing the output file {output_file} with key {package_key}") SignOutput(prelim_signing, output_file, package_key, pw) # Reopen the final signed zip to double check the streaming metadata. @@ -347,26 +364,66 @@ def HandleDowngradeMetadata(metadata_proto, target_info, source_info): # Only incremental OTAs are allowed to reach here. assert OPTIONS.incremental_source is not None + # used for logging upon errors + log_downgrades = [] + log_upgrades = [] + post_timestamp = target_info.GetBuildProp("ro.build.date.utc") pre_timestamp = source_info.GetBuildProp("ro.build.date.utc") - is_downgrade = int(post_timestamp) < int(pre_timestamp) + if int(post_timestamp) < int(pre_timestamp): + logger.info(f"ro.build.date.utc pre timestamp: {pre_timestamp}, " + f"post timestamp: {post_timestamp}. Downgrade detected.") + log_downgrades.append(f"ro.build.date.utc pre: {pre_timestamp} post: {post_timestamp}") + else: + logger.info(f"ro.build.date.utc pre timestamp: {pre_timestamp}, " + f"post timestamp: {post_timestamp}.") + log_upgrades.append(f"ro.build.date.utc pre: {pre_timestamp} post: {post_timestamp}") + + # When merging system and vendor target files, it is not enough + # to check ro.build.date.utc, the timestamp for each partition must + # be checked. + if source_info.is_ab: + ab_partitions = set(source_info.get("ab_partitions")) + for partition in sorted(set(PARTITIONS_WITH_BUILD_PROP) & ab_partitions): + + partition_prop = source_info.get('{}.build.prop'.format(partition)) + # Skip if the partition is missing, or it doesn't have a build.prop + if not partition_prop or not partition_prop.build_props: + continue + partition_prop = target_info.get('{}.build.prop'.format(partition)) + # Skip if the partition is missing, or it doesn't have a build.prop + if not partition_prop or not partition_prop.build_props: + continue + + post_timestamp = target_info.GetPartitionBuildProp( + 'ro.build.date.utc', partition) + pre_timestamp = source_info.GetPartitionBuildProp( + 'ro.build.date.utc', partition) + if int(post_timestamp) < int(pre_timestamp): + logger.info(f"Partition {partition} pre timestamp: {pre_timestamp}, " + f"post time: {post_timestamp}. Downgrade detected.") + log_downgrades.append(f"{partition} pre: {pre_timestamp} post: {post_timestamp}") + else: + logger.info(f"Partition {partition} pre timestamp: {pre_timestamp}, " + f"post timestamp: {post_timestamp}.") + log_upgrades.append(f"{partition} pre: {pre_timestamp} post: {post_timestamp}") if OPTIONS.spl_downgrade: metadata_proto.spl_downgrade = True if OPTIONS.downgrade: - if not is_downgrade: + if len(log_downgrades) == 0: raise RuntimeError( "--downgrade or --override_timestamp specified but no downgrade " - "detected: pre: %s, post: %s" % (pre_timestamp, post_timestamp)) + "detected. Current values for ro.build.date.utc: " + ', '.join(log_upgrades)) metadata_proto.downgrade = True else: - if is_downgrade: + if len(log_downgrades) != 0: raise RuntimeError( - "Downgrade detected based on timestamp check: pre: %s, post: %s. " + "Downgrade detected based on timestamp check in ro.build.date.utc. " "Need to specify --override_timestamp OR --downgrade to allow " - "building the incremental." % (pre_timestamp, post_timestamp)) - + "building the incremental. Downgrades detected for: " + + ', '.join(log_downgrades)) def ComputeRuntimeBuildInfos(default_build_info, boot_variable_values): """Returns a set of build info objects that may exist during runtime.""" @@ -625,12 +682,10 @@ def ConstructOtaApexInfo(target_zip, source_file=None): """If applicable, add the source version to the apex info.""" def _ReadApexInfo(input_zip): - if "META/apex_info.pb" not in input_zip.namelist(): + if not DoesInputFileContain(input_zip, "META/apex_info.pb"): logger.warning("target_file doesn't contain apex_info.pb %s", input_zip) return None - - with input_zip.open("META/apex_info.pb", "r") as zfp: - return zfp.read() + return ReadBytesFromInputFile(input_zip, "META/apex_info.pb") target_apex_string = _ReadApexInfo(target_zip) # Return early if the target apex info doesn't exist or is empty. @@ -641,8 +696,7 @@ def ConstructOtaApexInfo(target_zip, source_file=None): if not source_file: return target_apex_string - with zipfile.ZipFile(source_file, "r", allowZip64=True) as source_zip: - source_apex_string = _ReadApexInfo(source_zip) + source_apex_string = _ReadApexInfo(source_file) if not source_apex_string: return target_apex_string @@ -721,15 +775,63 @@ def IsZucchiniCompatible(source_file: str, target_file: str): return sourceEntry and targetEntry and sourceEntry == targetEntry +def ExtractTargetFiles(path: str): + if os.path.isdir(path): + logger.info("target files %s is already extracted", path) + return path + extracted_dir = common.MakeTempDir("target_files") + logger.info(f"Extracting target files {path} to {extracted_dir}") + common.UnzipToDir(path, extracted_dir, UNZIP_PATTERN + [""]) + for subdir in TARGET_FILES_IMAGES_SUBDIR: + image_dir = os.path.join(extracted_dir, subdir) + if not os.path.exists(image_dir): + continue + for filename in os.listdir(image_dir): + if not filename.endswith(".img"): + continue + common.UnsparseImage(os.path.join(image_dir, filename)) + + return extracted_dir + + +def LocatePartitionPath(target_files_dir: str, partition: str, allow_empty): + for subdir in TARGET_FILES_IMAGES_SUBDIR: + path = os.path.join(target_files_dir, subdir, partition + ".img") + if os.path.exists(path): + return path + if allow_empty: + return "" + raise common.ExternalError( + "Partition {} not found in target files {}".format(partition, target_files_dir)) + + +def GetPartitionImages(target_files_dir: str, ab_partitions, allow_empty=True): + assert os.path.isdir(target_files_dir) + return ":".join([LocatePartitionPath(target_files_dir, partition, allow_empty) for partition in ab_partitions]) + + +def LocatePartitionMap(target_files_dir: str, partition: str): + for subdir in TARGET_FILES_IMAGES_SUBDIR: + path = os.path.join(target_files_dir, subdir, partition + ".map") + if os.path.exists(path): + return path + return "" + + +def GetPartitionMaps(target_files_dir: str, ab_partitions): + assert os.path.isdir(target_files_dir) + return ":".join([LocatePartitionMap(target_files_dir, partition) for partition in ab_partitions]) + + class PayloadGenerator(object): """Manages the creation and the signing of an A/B OTA Payload.""" - PAYLOAD_BIN = 'payload.bin' - PAYLOAD_PROPERTIES_TXT = 'payload_properties.txt' + PAYLOAD_BIN = payload_signer.PAYLOAD_BIN + PAYLOAD_PROPERTIES_TXT = payload_signer.PAYLOAD_PROPERTIES_TXT SECONDARY_PAYLOAD_BIN = 'secondary/payload.bin' SECONDARY_PAYLOAD_PROPERTIES_TXT = 'secondary/payload_properties.txt' - def __init__(self, secondary=False, wipe_user_data=False): + def __init__(self, secondary=False, wipe_user_data=False, minor_version=None, is_partial_update=False, spl_downgrade=False): """Initializes a Payload instance. Args: @@ -739,6 +841,9 @@ class PayloadGenerator(object): self.payload_properties = None self.secondary = secondary self.wipe_user_data = wipe_user_data + self.minor_version = minor_version + self.is_partial_update = is_partial_update + self.spl_downgrade = spl_downgrade def _Run(self, cmd): # pylint: disable=no-self-use # Don't pipe (buffer) the output if verbose is set. Let @@ -757,21 +862,61 @@ class PayloadGenerator(object): source_file: The filename of the source build target-files zip; or None if generating a full OTA. additional_args: A list of additional args that should be passed to - brillo_update_payload script; or None. + delta_generator binary; or None. """ if additional_args is None: additional_args = [] payload_file = common.MakeTempFile(prefix="payload-", suffix=".bin") - cmd = ["brillo_update_payload", "generate", - "--payload", payload_file, - "--target_image", target_file] + target_dir = ExtractTargetFiles(target_file) + cmd = ["delta_generator", + "--out_file", payload_file] + with open(os.path.join(target_dir, "META", "ab_partitions.txt"), "r") as fp: + ab_partitions = fp.read().strip().splitlines() + cmd.extend(["--partition_names", ":".join(ab_partitions)]) + cmd.extend( + ["--new_partitions", GetPartitionImages(target_dir, ab_partitions, False)]) + cmd.extend( + ["--new_mapfiles", GetPartitionMaps(target_dir, ab_partitions)]) if source_file is not None: - cmd.extend(["--source_image", source_file]) + source_dir = ExtractTargetFiles(source_file) + cmd.extend( + ["--old_partitions", GetPartitionImages(source_dir, ab_partitions, True)]) + cmd.extend( + ["--old_mapfiles", GetPartitionMaps(source_dir, ab_partitions)]) + if OPTIONS.disable_fec_computation: - cmd.extend(["--disable_fec_computation", "true"]) + cmd.extend(["--disable_fec_computation=true"]) if OPTIONS.disable_verity_computation: - cmd.extend(["--disable_verity_computation", "true"]) + cmd.extend(["--disable_verity_computation=true"]) + postinstall_config = os.path.join( + target_dir, "META", "postinstall_config.txt") + + if os.path.exists(postinstall_config): + cmd.extend(["--new_postinstall_config_file", postinstall_config]) + dynamic_partition_info = os.path.join( + target_dir, "META", "dynamic_partitions_info.txt") + + if os.path.exists(dynamic_partition_info): + cmd.extend(["--dynamic_partition_info_file", dynamic_partition_info]) + + apex_info = os.path.join( + target_dir, "META", "apex_info.pb") + if os.path.exists(apex_info): + cmd.extend(["--apex_info_file", apex_info]) + + major_version, minor_version = ParseUpdateEngineConfig( + os.path.join(target_dir, "META", "update_engine_config.txt")) + if source_file: + major_version, minor_version = ParseUpdateEngineConfig( + os.path.join(source_dir, "META", "update_engine_config.txt")) + if self.minor_version: + minor_version = self.minor_version + cmd.extend(["--major_version", str(major_version)]) + if source_file is not None or self.is_partial_update: + cmd.extend(["--minor_version", str(minor_version)]) + if self.is_partial_update: + cmd.extend(["--is_partial_update=true"]) cmd.extend(additional_args) self._Run(cmd) @@ -789,30 +934,7 @@ class PayloadGenerator(object): """ assert isinstance(payload_signer, PayloadSigner) - # 1. Generate hashes of the payload and metadata files. - payload_sig_file = common.MakeTempFile(prefix="sig-", suffix=".bin") - metadata_sig_file = common.MakeTempFile(prefix="sig-", suffix=".bin") - cmd = ["brillo_update_payload", "hash", - "--unsigned_payload", self.payload_file, - "--signature_size", str(payload_signer.maximum_signature_size), - "--metadata_hash_file", metadata_sig_file, - "--payload_hash_file", payload_sig_file] - self._Run(cmd) - - # 2. Sign the hashes. - signed_payload_sig_file = payload_signer.SignHashFile(payload_sig_file) - signed_metadata_sig_file = payload_signer.SignHashFile(metadata_sig_file) - - # 3. Insert the signatures back into the payload file. - signed_payload_file = common.MakeTempFile(prefix="signed-payload-", - suffix=".bin") - cmd = ["brillo_update_payload", "sign", - "--unsigned_payload", self.payload_file, - "--payload", signed_payload_file, - "--signature_size", str(payload_signer.maximum_signature_size), - "--metadata_signature_file", signed_metadata_sig_file, - "--payload_signature_file", signed_payload_sig_file] - self._Run(cmd) + signed_payload_file = payload_signer.SignPayload(self.payload_file) self.payload_file = signed_payload_file @@ -824,20 +946,17 @@ class PayloadGenerator(object): """ assert self.payload_file is not None # 4. Dump the signed payload properties. - properties_file = common.MakeTempFile(prefix="payload-properties-", - suffix=".txt") - cmd = ["brillo_update_payload", "properties", - "--payload", self.payload_file, - "--properties_file", properties_file] - self._Run(cmd) + properties_file = GeneratePayloadProperties(self.payload_file) - if self.secondary: - with open(properties_file, "a") as f: - f.write("SWITCH_SLOT_ON_REBOOT=0\n") - if self.wipe_user_data: - with open(properties_file, "a") as f: + with open(properties_file, "a") as f: + if self.wipe_user_data: f.write("POWERWASH=1\n") + if self.secondary: + f.write("SWITCH_SLOT_ON_REBOOT=0\n") + if self.spl_downgrade: + f.write("SPL_DOWNGRADE=1\n") + self.payload_properties = properties_file @@ -963,3 +1082,38 @@ class AbOtaPropertyFiles(StreamingPropertyFiles): assert metadata_total <= payload_size return (payload_offset, metadata_total) + + +def Fnmatch(filename, pattersn): + return any([fnmatch.fnmatch(filename, pat) for pat in pattersn]) + + +def CopyTargetFilesDir(input_dir): + output_dir = common.MakeTempDir("target_files") + + def SymlinkIfNotSparse(src, dst): + if common.IsSparseImage(src): + return common.UnsparseImage(src, dst) + else: + return os.symlink(os.path.realpath(src), dst) + + for subdir in TARGET_FILES_IMAGES_SUBDIR: + if not os.path.exists(os.path.join(input_dir, subdir)): + continue + shutil.copytree(os.path.join(input_dir, subdir), os.path.join( + output_dir, subdir), dirs_exist_ok=True, copy_function=SymlinkIfNotSparse) + shutil.copytree(os.path.join(input_dir, "META"), os.path.join( + output_dir, "META"), dirs_exist_ok=True) + + for (dirpath, _, filenames) in os.walk(input_dir): + for filename in filenames: + path = os.path.join(dirpath, filename) + relative_path = path.removeprefix(input_dir).removeprefix("/") + if not Fnmatch(relative_path, UNZIP_PATTERN): + continue + if filename.endswith(".prop") or filename == "prop.default" or "/etc/vintf/" in relative_path: + target_path = os.path.join( + output_dir, relative_path) + os.makedirs(os.path.dirname(target_path), exist_ok=True) + shutil.copy(path, target_path) + return output_dir diff --git a/tools/releasetools/payload_signer.py b/tools/releasetools/payload_signer.py index 4f342ac9fe..e85d64cee0 100644 --- a/tools/releasetools/payload_signer.py +++ b/tools/releasetools/payload_signer.py @@ -16,10 +16,58 @@ import common import logging -from common import OPTIONS +import shlex +import argparse +import tempfile +import zipfile +import shutil +from common import OPTIONS, OptionHandler +from ota_signing_utils import AddSigningArgumentParse logger = logging.getLogger(__name__) +OPTIONS.payload_signer = None +OPTIONS.payload_signer_args = [] +OPTIONS.payload_signer_maximum_signature_size = None +OPTIONS.package_key = None + +PAYLOAD_BIN = 'payload.bin' +PAYLOAD_PROPERTIES_TXT = 'payload_properties.txt' + +class SignerOptions(OptionHandler): + + @staticmethod + def ParseOptions(o, a): + if o in ("-k", "--package_key"): + OPTIONS.package_key = a + elif o == "--payload_signer": + OPTIONS.payload_signer = a + elif o == "--payload_signer_args": + OPTIONS.payload_signer_args = shlex.split(a) + elif o == "--payload_signer_maximum_signature_size": + OPTIONS.payload_signer_maximum_signature_size = a + elif o == "--payload_signer_key_size": + # TODO(xunchang) remove this option after cleaning up the callers. + logger.warning("The option '--payload_signer_key_size' is deprecated." + " Use '--payload_signer_maximum_signature_size' instead.") + OPTIONS.payload_signer_maximum_signature_size = a + else: + return False + return True + + def __init__(self): + super().__init__( + ["payload_signer=", + "package_key=", + "payload_signer_args=", + "payload_signer_maximum_signature_size=", + "payload_signer_key_size="], + SignerOptions.ParseOptions + ) + + +signer_options = SignerOptions() + class PayloadSigner(object): """A class that wraps the payload signing works. @@ -36,11 +84,16 @@ class PayloadSigner(object): (OPTIONS.package_key) and calls openssl for the signing works. """ - def __init__(self, package_key=None, private_key_suffix=None, pw=None, payload_signer=None): + def __init__(self, package_key=None, private_key_suffix=None, pw=None, payload_signer=None, + payload_signer_args=None, payload_signer_maximum_signature_size=None): if package_key is None: package_key = OPTIONS.package_key if private_key_suffix is None: private_key_suffix = OPTIONS.private_key_suffix + if payload_signer_args is None: + payload_signer_args = OPTIONS.payload_signer_args + if payload_signer_maximum_signature_size is None: + payload_signer_maximum_signature_size = OPTIONS.payload_signer_maximum_signature_size if payload_signer is None: # Prepare the payload signing key. @@ -59,10 +112,10 @@ class PayloadSigner(object): signing_key) else: self.signer = payload_signer - self.signer_args = OPTIONS.payload_signer_args - if OPTIONS.payload_signer_maximum_signature_size: + self.signer_args = payload_signer_args + if payload_signer_maximum_signature_size: self.maximum_signature_size = int( - OPTIONS.payload_signer_maximum_signature_size) + payload_signer_maximum_signature_size) else: # The legacy config uses RSA2048 keys. logger.warning("The maximum signature size for payload signer is not" @@ -90,11 +143,11 @@ class PayloadSigner(object): # 1. Generate hashes of the payload and metadata files. payload_sig_file = common.MakeTempFile(prefix="sig-", suffix=".bin") metadata_sig_file = common.MakeTempFile(prefix="sig-", suffix=".bin") - cmd = ["brillo_update_payload", "hash", - "--unsigned_payload", unsigned_payload, - "--signature_size", str(self.maximum_signature_size), - "--metadata_hash_file", metadata_sig_file, - "--payload_hash_file", payload_sig_file] + cmd = ["delta_generator", + "--in_file=" + unsigned_payload, + "--signature_size=" + str(self.maximum_signature_size), + "--out_metadata_hash_file=" + metadata_sig_file, + "--out_hash_file=" + payload_sig_file] self._Run(cmd) # 2. Sign the hashes. @@ -104,19 +157,67 @@ class PayloadSigner(object): # 3. Insert the signatures back into the payload file. signed_payload_file = common.MakeTempFile(prefix="signed-payload-", suffix=".bin") - cmd = ["brillo_update_payload", "sign", - "--unsigned_payload", unsigned_payload, - "--payload", signed_payload_file, - "--signature_size", str(self.maximum_signature_size), - "--metadata_signature_file", signed_metadata_sig_file, - "--payload_signature_file", signed_payload_sig_file] + cmd = ["delta_generator", + "--in_file=" + unsigned_payload, + "--out_file=" + signed_payload_file, + "--signature_size=" + str(self.maximum_signature_size), + "--metadata_signature_file=" + signed_metadata_sig_file, + "--payload_signature_file=" + signed_payload_sig_file] self._Run(cmd) return signed_payload_file - def SignHashFile(self, in_file): """Signs the given input file. Returns the output filename.""" out_file = common.MakeTempFile(prefix="signed-", suffix=".bin") cmd = [self.signer] + self.signer_args + ['-in', in_file, '-out', out_file] common.RunAndCheckOutput(cmd) return out_file + +def GeneratePayloadProperties(payload_file): + properties_file = common.MakeTempFile(prefix="payload-properties-", + suffix=".txt") + cmd = ["delta_generator", + "--in_file=" + payload_file, + "--properties_file=" + properties_file] + common.RunAndCheckOutput(cmd) + return properties_file + +def SignOtaPackage(input_path, output_path): + payload_signer = PayloadSigner( + OPTIONS.package_key, OPTIONS.private_key_suffix, + None, OPTIONS.payload_signer, OPTIONS.payload_signer_args) + common.ZipExclude(input_path, output_path, [PAYLOAD_BIN, PAYLOAD_PROPERTIES_TXT]) + with tempfile.NamedTemporaryFile() as unsigned_payload, zipfile.ZipFile(input_path, "r", allowZip64=True) as zfp: + with zfp.open("payload.bin") as payload_fp: + shutil.copyfileobj(payload_fp, unsigned_payload) + signed_payload = payload_signer.SignPayload(unsigned_payload.name) + properties_file = GeneratePayloadProperties(signed_payload) + with zipfile.ZipFile(output_path, "a", compression=zipfile.ZIP_STORED, allowZip64=True) as output_zfp: + common.ZipWrite(output_zfp, signed_payload, PAYLOAD_BIN) + common.ZipWrite(output_zfp, properties_file, PAYLOAD_PROPERTIES_TXT) + + +def main(argv): + parser = argparse.ArgumentParser( + prog=argv[0], description="Given a series of .img files, produces a full OTA package that installs thoese images") + parser.add_argument("input_ota", type=str, + help="Input OTA for signing") + parser.add_argument('output_ota', type=str, + help='Output OTA for the signed package') + parser.add_argument("-v", action="store_true", + help="Enable verbose logging", dest="verbose") + AddSigningArgumentParse(parser) + args = parser.parse_args(argv[1:]) + input_ota = args.input_ota + output_ota = args.output_ota + if args.verbose: + OPTIONS.verbose = True + common.InitLogging() + if args.package_key: + OPTIONS.package_key = args.package_key + logger.info("Re-signing OTA package {}".format(input_ota)) + SignOtaPackage(input_ota, output_ota) + +if __name__ == "__main__": + import sys + main(sys.argv)
\ No newline at end of file diff --git a/tools/releasetools/sign_apex.py b/tools/releasetools/sign_apex.py index d73998247a..a0a94f67aa 100755 --- a/tools/releasetools/sign_apex.py +++ b/tools/releasetools/sign_apex.py @@ -56,7 +56,6 @@ import apex_utils import common logger = logging.getLogger(__name__) -OPTIONS = common.OPTIONS def SignApexFile(avbtool, apex_file, payload_key, container_key, no_hashtree, @@ -75,8 +74,7 @@ def SignApexFile(avbtool, apex_file, payload_key, container_key, no_hashtree, no_hashtree=no_hashtree, apk_keys=apk_keys, signing_args=signing_args, - sign_tool=sign_tool, - is_sepolicy=apex_file.endswith(OPTIONS.sepolicy_name)) + sign_tool=sign_tool) def main(argv): diff --git a/tools/releasetools/sign_target_files_apks.py b/tools/releasetools/sign_target_files_apks.py index 4a12e748da..b8f848fb2b 100755 --- a/tools/releasetools/sign_target_files_apks.py +++ b/tools/releasetools/sign_target_files_apks.py @@ -83,9 +83,8 @@ Usage: sign_target_files_apks [flags] input_target_files output_target_files --replace_verity_public_key <key> Replace the certificate (public key) used for verity verification. The - key file replaces the one at BOOT/RAMDISK/verity_key (or ROOT/verity_key - for devices using system_root_image). It expects the key filename WITH - the extension (e.g. verity_key.pub). + key file replaces the one at BOOT/RAMDISK/verity_key. It expects the key + filename WITH the extension (e.g. verity_key.pub). --replace_verity_keyid <path_to_X509_PEM_cert_file> Replace the veritykeyid in BOOT/cmdline of input_target_file_zip @@ -125,14 +124,8 @@ Usage: sign_target_files_apks [flags] input_target_files output_target_files --gki_signing_algorithm <algorithm> --gki_signing_key <key> - Use the specified algorithm (e.g. SHA256_RSA4096) and the key to generate - 'boot signature' in a v4 boot.img. Otherwise it uses the existing values - in info dict. - --gki_signing_extra_args <args> - Specify any additional args that are needed to generate 'boot signature' - (e.g. --prop foo:bar). The args will be appended to the existing ones - in info dict. + DEPRECATED Does nothing. --android_jar_path <path> Path to the android.jar to repack the apex file. @@ -147,6 +140,34 @@ Usage: sign_target_files_apks [flags] input_target_files output_target_files --override_apex_keys <path> Replace all APEX keys with this private key + + -k (--package_key) <key> + Key to use to sign the package (default is the value of + default_system_dev_certificate from the input target-files's + META/misc_info.txt, or "build/make/target/product/security/testkey" if + that value is not specified). + + For incremental OTAs, the default value is based on the source + target-file, not the target build. + + --payload_signer <signer> + Specify the signer when signing the payload and metadata for A/B OTAs. + By default (i.e. without this flag), it calls 'openssl pkeyutl' to sign + with the package private key. If the private key cannot be accessed + directly, a payload signer that knows how to do that should be specified. + The signer will be supplied with "-inkey <path_to_key>", + "-in <input_file>" and "-out <output_file>" parameters. + + --payload_signer_args <args> + Specify the arguments needed for payload signer. + + --payload_signer_maximum_signature_size <signature_size> + The maximum signature size (in bytes) that would be generated by the given + payload signer. Only meaningful when custom payload signer is specified + via '--payload_signer'. + If the signer uses a RSA key, this should be the number of bytes to + represent the modulus. If it uses an EC key, this is the size of a + DER-encoded ECDSA signature. """ from __future__ import print_function @@ -162,7 +183,6 @@ import os import re import shutil import stat -import subprocess import sys import tempfile import zipfile @@ -171,6 +191,8 @@ from xml.etree import ElementTree import add_img_to_target_files import apex_utils import common +import payload_signer +from payload_signer import SignOtaPackage, PAYLOAD_BIN if sys.hexversion < 0x02070000: @@ -193,9 +215,6 @@ OPTIONS.tag_changes = ("-test-keys", "-dev-keys", "+release-keys") OPTIONS.avb_keys = {} OPTIONS.avb_algorithms = {} OPTIONS.avb_extra_args = {} -OPTIONS.gki_signing_key = None -OPTIONS.gki_signing_algorithm = None -OPTIONS.gki_signing_extra_args = None OPTIONS.android_jar_path = None OPTIONS.vendor_partitions = set() OPTIONS.vendor_otatools = None @@ -241,6 +260,24 @@ def IsApexFile(filename): return filename.endswith(".apex") or filename.endswith(".capex") +def IsOtaPackage(fp): + with zipfile.ZipFile(fp) as zfp: + if not PAYLOAD_BIN in zfp.namelist(): + return False + with zfp.open(PAYLOAD_BIN, "r") as payload: + magic = payload.read(4) + return magic == b"CrAU" + + +def IsEntryOtaPackage(input_zip, filename): + with input_zip.open(filename, "r") as fp: + external_attr = input_zip.getinfo(filename).external_attr + if stat.S_ISLNK(external_attr >> 16): + return IsEntryOtaPackage(input_zip, + os.path.join(os.path.dirname(filename), fp.read().decode())) + return IsOtaPackage(fp) + + def GetApexFilename(filename): name = os.path.basename(filename) # Replace the suffix for compressed apex @@ -515,6 +552,7 @@ def SignApk(data, keyname, pw, platform_api_level, codename_to_api_level_map, return data + def IsBuildPropFile(filename): return filename in ( "SYSTEM/etc/prop.default", @@ -541,7 +579,7 @@ def IsBuildPropFile(filename): filename.endswith("/prop.default") -def ProcessTargetFiles(input_tf_zip, output_tf_zip, misc_info, +def ProcessTargetFiles(input_tf_zip: zipfile.ZipFile, output_tf_zip, misc_info, apk_keys, apex_keys, key_passwords, platform_api_level, codename_to_api_level_map, compressed_extension): @@ -552,11 +590,9 @@ def ProcessTargetFiles(input_tf_zip, output_tf_zip, misc_info, [len(os.path.basename(i.filename)) for i in input_tf_zip.infolist() if GetApkFileInfo(i.filename, compressed_extension, [])[0]]) except ValueError: - # Sets this to zero for targets without APK files, e.g., gki_arm64. + # Sets this to zero for targets without APK files. maxsize = 0 - system_root_image = misc_info.get("system_root_image") == "true" - for info in input_tf_zip.infolist(): filename = info.filename if filename.startswith("IMAGES/"): @@ -631,6 +667,15 @@ def ProcessTargetFiles(input_tf_zip, output_tf_zip, misc_info, " (skipped due to special cert string)" % (name,)) common.ZipWriteStr(output_tf_zip, out_info, data) + elif filename.endswith(".zip") and IsEntryOtaPackage(input_tf_zip, filename): + logger.info("Re-signing OTA package {}".format(filename)) + with tempfile.NamedTemporaryFile() as input_ota, tempfile.NamedTemporaryFile() as output_ota: + with input_tf_zip.open(filename, "r") as in_fp: + shutil.copyfileobj(in_fp, input_ota) + input_ota.flush() + SignOtaPackage(input_ota.name, output_ota.name) + common.ZipWrite(output_tf_zip, output_ota.name, filename, + compress_type=zipfile.ZIP_STORED) # System properties. elif IsBuildPropFile(filename): print("Rewriting %s:" % (filename,)) @@ -754,6 +799,16 @@ def ProcessTargetFiles(input_tf_zip, output_tf_zip, misc_info, # Copy it verbatim if we allow the file to exist. common.ZipWriteStr(output_tf_zip, out_info, data) + # Sign microdroid_vendor.img. + elif filename == "VENDOR/etc/avf/microdroid/microdroid_vendor.img": + vendor_key = OPTIONS.avb_keys.get("vendor") + vendor_algorithm = OPTIONS.avb_algorithms.get("vendor") + with tempfile.NamedTemporaryFile() as image: + image.write(data) + image.flush() + ReplaceKeyInAvbHashtreeFooter(image, vendor_key, vendor_algorithm, + misc_info) + common.ZipWrite(output_tf_zip, image.name, filename) # A non-APK file; copy it verbatim. else: common.ZipWriteStr(output_tf_zip, out_info, data) @@ -768,12 +823,111 @@ def ProcessTargetFiles(input_tf_zip, output_tf_zip, misc_info, if misc_info.get('avb_enable') == 'true': RewriteAvbProps(misc_info) - # Replace the GKI signing key for boot.img, if any. - ReplaceGkiSigningKey(misc_info) - # Write back misc_info with the latest values. ReplaceMiscInfoTxt(input_tf_zip, output_tf_zip, misc_info) +# Parse string output of `avbtool info_image`. +def ParseAvbInfo(info_raw): + # line_matcher is for parsing each output line of `avbtool info_image`. + # example string input: " Hash Algorithm: sha1" + # example matched input: (" ", "Hash Algorithm", "sha1") + line_matcher = re.compile(r'^(\s*)([^:]+):\s*(.*)$') + # prop_matcher is for parsing value part of 'Prop' in `avbtool info_image`. + # example string input: "example_prop_key -> 'example_prop_value'" + # example matched output: ("example_prop_key", "example_prop_value") + prop_matcher = re.compile(r"(.+)\s->\s'(.+)'") + info = {} + indent_stack = [[-1, info]] + for line_info_raw in info_raw.split('\n'): + # Parse the line + line_info_parsed = line_matcher.match(line_info_raw) + if not line_info_parsed: + continue + indent = len(line_info_parsed.group(1)) + key = line_info_parsed.group(2).strip() + value = line_info_parsed.group(3).strip() + + # Pop indentation stack + while indent <= indent_stack[-1][0]: + del indent_stack[-1] + + # Insert information into 'info'. + cur_info = indent_stack[-1][1] + if value == "": + if key == "Descriptors": + empty_list = [] + cur_info[key] = empty_list + indent_stack.append([indent, empty_list]) + else: + empty_dict = {} + cur_info.append({key:empty_dict}) + indent_stack.append([indent, empty_dict]) + elif key == "Prop": + prop_parsed = prop_matcher.match(value) + if not prop_parsed: + raise ValueError( + "Failed to parse prop while getting avb information.") + cur_info.append({key:{prop_parsed.group(1):prop_parsed.group(2)}}) + else: + cur_info[key] = value + return info + +def ReplaceKeyInAvbHashtreeFooter(image, new_key, new_algorithm, misc_info): + # Get avb information about the image by parsing avbtool info_image. + def GetAvbInfo(avbtool, image_name): + # Get information with raw string by `avbtool info_image`. + info_raw = common.RunAndCheckOutput([ + avbtool, 'info_image', + '--image', image_name + ]) + return ParseAvbInfo(info_raw) + + # Get hashtree descriptor from info + def GetAvbHashtreeDescriptor(avb_info): + hashtree_descriptors = tuple(filter(lambda x: "Hashtree descriptor" in x, + info.get('Descriptors'))) + if len(hashtree_descriptors) != 1: + raise ValueError("The number of hashtree descriptor is not 1.") + return hashtree_descriptors[0]["Hashtree descriptor"] + + # Get avb info + avbtool = misc_info['avb_avbtool'] + info = GetAvbInfo(avbtool, image.name) + hashtree_descriptor = GetAvbHashtreeDescriptor(info) + + # Generate command + cmd = [avbtool, 'add_hashtree_footer', + '--key', new_key, + '--algorithm', new_algorithm, + '--partition_name', hashtree_descriptor.get("Partition Name"), + '--partition_size', info.get("Image size").removesuffix(" bytes"), + '--hash_algorithm', hashtree_descriptor.get("Hash Algorithm"), + '--salt', hashtree_descriptor.get("Salt"), + '--do_not_generate_fec', + '--image', image.name + ] + + # Append properties into command + props = map(lambda x: x.get("Prop"), filter(lambda x: "Prop" in x, + info.get('Descriptors'))) + for prop_wrapped in props: + prop = tuple(prop_wrapped.items()) + if len(prop) != 1: + raise ValueError("The number of property is not 1.") + cmd.append('--prop') + cmd.append(prop[0][0] + ':' + prop[0][1]) + + # Replace Hashtree Footer with new key + common.RunAndCheckOutput(cmd) + + # Check root digest is not changed + new_info = GetAvbInfo(avbtool, image.name) + new_hashtree_descriptor = GetAvbHashtreeDescriptor(info) + root_digest = hashtree_descriptor.get("Root Digest") + new_root_digest = new_hashtree_descriptor.get("Root Digest") + assert root_digest == new_root_digest, \ + ("Root digest in hashtree descriptor shouldn't be changed. Old: {}, New: " + "{}").format(root_digest, new_root_digest) def ReplaceCerts(data): """Replaces all the occurences of X.509 certs with the new ones. @@ -908,7 +1062,7 @@ def WriteOtacerts(output_zip, filename, keys): certs_zip = zipfile.ZipFile(temp_file, "w", allowZip64=True) for k in keys: common.ZipWrite(certs_zip, k) - certs_zip.close() + common.ZipClose(certs_zip) common.ZipWriteStr(output_zip, filename, temp_file.getvalue()) @@ -1052,27 +1206,6 @@ def RewriteAvbProps(misc_info): misc_info[args_key] = result -def ReplaceGkiSigningKey(misc_info): - """Replaces the GKI signing key.""" - - key = OPTIONS.gki_signing_key - if not key: - return - - algorithm = OPTIONS.gki_signing_algorithm - if not algorithm: - raise ValueError("Missing --gki_signing_algorithm") - - print('Replacing GKI signing key with "%s" (%s)' % (key, algorithm)) - misc_info["gki_signing_algorithm"] = algorithm - misc_info["gki_signing_key_path"] = key - - extra_args = OPTIONS.gki_signing_extra_args - if extra_args: - print('Setting GKI signing args: "%s"' % (extra_args)) - misc_info["gki_signing_signature_args"] = extra_args - - def BuildKeyMap(misc_info, key_mapping_options): for s, d in key_mapping_options: if s is None: # -d option @@ -1087,6 +1220,7 @@ def BuildKeyMap(misc_info, key_mapping_options): devkeydir + "/shared": d + "/shared", devkeydir + "/platform": d + "/platform", devkeydir + "/networkstack": d + "/networkstack", + devkeydir + "/sdk_sandbox": d + "/sdk_sandbox", }) else: OPTIONS.key_map[s] = d @@ -1234,11 +1368,11 @@ def BuildVendorPartitions(output_zip_path): vendor_misc_info["no_recovery"] = "true" # recovery vendor_misc_info["avb_enable"] = "false" # vbmeta - vendor_misc_info["board_bpt_enable"] = "false" # partition-table vendor_misc_info["has_dtbo"] = "false" # dtbo vendor_misc_info["has_pvmfw"] = "false" # pvmfw - vendor_misc_info["avb_custom_images_partition_list"] = "" # custom images + vendor_misc_info["avb_custom_images_partition_list"] = "" # avb custom images vendor_misc_info["avb_building_vbmeta_image"] = "false" # skip building vbmeta + vendor_misc_info["custom_images_partition_list"] = "" # custom images vendor_misc_info["use_dynamic_partitions"] = "false" # super_empty vendor_misc_info["build_super_partition"] = "false" # super split vendor_misc_info["avb_vbmeta_system"] = "" # skip building vbmeta_system @@ -1426,12 +1560,6 @@ def main(argv): # 'oem=--signing_helper_with_files=/tmp/avbsigner.sh'. partition, extra_args = a.split("=", 1) OPTIONS.avb_extra_args[partition] = extra_args - elif o == "--gki_signing_key": - OPTIONS.gki_signing_key = a - elif o == "--gki_signing_algorithm": - OPTIONS.gki_signing_algorithm = a - elif o == "--gki_signing_extra_args": - OPTIONS.gki_signing_extra_args = a elif o == "--vendor_otatools": OPTIONS.vendor_otatools = a elif o == "--vendor_partitions": @@ -1442,6 +1570,8 @@ def main(argv): OPTIONS.override_apk_keys = a elif o == "--override_apex_keys": OPTIONS.override_apex_keys = a + elif o in ("--gki_signing_key", "--gki_signing_algorithm", "--gki_signing_extra_args"): + print(f"{o} is deprecated and does nothing") else: return False return True @@ -1504,7 +1634,7 @@ def main(argv): "override_apk_keys=", "override_apex_keys=", ], - extra_option_handler=option_handler) + extra_option_handler=[option_handler, payload_signer.signer_options]) if len(args) != 2: common.Usage(__doc__) @@ -1518,6 +1648,10 @@ def main(argv): allowZip64=True) misc_info = common.LoadInfoDict(input_zip) + if OPTIONS.package_key is None: + OPTIONS.package_key = misc_info.get( + "default_system_dev_certificate", + "build/make/target/product/security/testkey") BuildKeyMap(misc_info, key_mapping_options) @@ -1545,8 +1679,8 @@ def main(argv): platform_api_level, codename_to_api_level_map, compressed_extension) - input_zip.close() - output_zip.close() + common.ZipClose(input_zip) + common.ZipClose(output_zip) if OPTIONS.vendor_partitions and OPTIONS.vendor_otatools: BuildVendorPartitions(args[1]) diff --git a/tools/releasetools/test_build_image.py b/tools/releasetools/test_build_image.py index cfae7a5236..d4f7ccc98f 100644 --- a/tools/releasetools/test_build_image.py +++ b/tools/releasetools/test_build_image.py @@ -99,11 +99,10 @@ class BuildImageTest(test_utils.ReleaseToolsTestCase): } self.assertRaises(BuildImageError, CheckHeadroom, ext4fs_output, prop_dict) - def test_SetUpInDirAndFsConfig_SystemRootImageTrue_NonSystem(self): + def test_SetUpInDirAndFsConfig_NonSystem(self): prop_dict = { 'fs_config': 'fs-config', 'mount_point': 'vendor', - 'system_root_image': 'true', } in_dir, fs_config = SetUpInDirAndFsConfig('/path/to/in_dir', prop_dict) self.assertEqual('/path/to/in_dir', in_dir) diff --git a/tools/releasetools/test_check_target_files_vintf.py b/tools/releasetools/test_check_target_files_vintf.py index 8725dd6922..7c154d7461 100644 --- a/tools/releasetools/test_check_target_files_vintf.py +++ b/tools/releasetools/test_check_target_files_vintf.py @@ -15,6 +15,7 @@ # import os.path +import shutil import common import test_utils @@ -86,6 +87,28 @@ class CheckTargetFilesVintfTest(test_utils.ReleaseToolsTestCase): return test_dir + # Prepare test dir with required HAL for APEX testing + def prepare_apex_test_dir(self, test_delta_rel_path): + test_dir = self.prepare_test_dir(test_delta_rel_path) + write_string_to_file( + """<compatibility-matrix version="1.0" level="1" type="framework"> + <hal format="aidl" optional="false" updatable-via-apex="true"> + <name>android.apex.foo</name> + <version>1</version> + <interface> + <name>IApex</name> + <instance>default</instance> + </interface> + </hal> + <sepolicy> + <sepolicy-version>0.0</sepolicy-version> + <kernel-sepolicy-version>0</kernel-sepolicy-version> + </sepolicy> + </compatibility-matrix>""", + os.path.join(test_dir, 'SYSTEM/etc/vintf/compatibility_matrix.1.xml')) + + return test_dir + @test_utils.SkipIfExternalToolsUnavailable() def test_CheckVintf_skeleton(self): msg = 'vintf check with skeleton target files failed.' @@ -143,3 +166,25 @@ class CheckTargetFilesVintfTest(test_utils.ReleaseToolsTestCase): os.path.join(test_dir, 'VENDOR/etc/vintf/manifest.xml')) # Should raise an error because a file has invalid format. self.assertRaises(common.ExternalError, CheckVintf, test_dir) + + @test_utils.SkipIfExternalToolsUnavailable() + def test_CheckVintf_apex_compat(self): + apex_file_name = 'com.android.apex.vendor.foo.with_vintf.apex' + msg = 'vintf/apex_compat should be compatible because ' \ + 'APEX %s has the required HALs' % (apex_file_name) + test_dir = self.prepare_apex_test_dir('vintf/apex_compat') + # Copy APEX under VENDOR/apex + apex_file = os.path.join(test_utils.get_current_dir(), apex_file_name) + apex_dir = os.path.join(test_dir, 'VENDOR/apex') + os.makedirs(apex_dir) + shutil.copy(apex_file, apex_dir) + # Should find required HAL via APEX + self.assertTrue(CheckVintf(test_dir), msg=msg) + + @test_utils.SkipIfExternalToolsUnavailable() + def test_CheckVintf_apex_incompat(self): + msg = 'vintf/apex_incompat should be incompatible because ' \ + 'no APEX data' + test_dir = self.prepare_apex_test_dir('vintf/apex_incompat') + # Should not find required HAL + self.assertFalse(CheckVintf(test_dir), msg=msg) diff --git a/tools/releasetools/test_common.py b/tools/releasetools/test_common.py index 8c9655ad0d..89933a00fc 100644 --- a/tools/releasetools/test_common.py +++ b/tools/releasetools/test_common.py @@ -15,14 +15,13 @@ # import copy -import json import os import subprocess import tempfile -import time import unittest import zipfile from hashlib import sha1 +from typing import BinaryIO import common import test_utils @@ -36,14 +35,24 @@ MiB = 1024 * KiB GiB = 1024 * MiB -def get_2gb_string(): +def get_2gb_file(): size = int(2 * GiB + 1) block_size = 4 * KiB step_size = 4 * MiB - # Generate a long string with holes, e.g. 'xyz\x00abc\x00...'. + tmpfile = tempfile.NamedTemporaryFile() + tmpfile.truncate(size) for _ in range(0, size, step_size): - yield os.urandom(block_size) - yield b'\0' * (step_size - block_size) + tmpfile.write(os.urandom(block_size)) + tmpfile.seek(step_size - block_size, os.SEEK_CUR) + return tmpfile + + +def hash_file(filename): + sha1_hash = sha1() + with open(filename, "rb") as fp: + for data in iter(lambda: fp.read(4*MiB), b''): + sha1_hash.update(data) + return sha1_hash class BuildInfoTest(test_utils.ReleaseToolsTestCase): @@ -429,6 +438,13 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): self.assertIsNone(zip_file.testzip()) def _test_ZipWrite(self, contents, extra_zipwrite_args=None): + with tempfile.NamedTemporaryFile() as test_file: + test_file_name = test_file.name + for data in contents: + test_file.write(bytes(data)) + return self._test_ZipWriteFile(test_file_name, extra_zipwrite_args) + + def _test_ZipWriteFile(self, test_file_name, extra_zipwrite_args=None): extra_zipwrite_args = dict(extra_zipwrite_args or {}) test_file = tempfile.NamedTemporaryFile(delete=False) @@ -441,31 +457,27 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): arcname = extra_zipwrite_args.get("arcname", test_file_name) if arcname[0] == "/": arcname = arcname[1:] + sha1_hash = hash_file(test_file_name) zip_file.close() zip_file = zipfile.ZipFile(zip_file_name, "w", allowZip64=True) try: - sha1_hash = sha1() - for data in contents: - sha1_hash.update(bytes(data)) - test_file.write(bytes(data)) - test_file.close() - - expected_stat = os.stat(test_file_name) expected_mode = extra_zipwrite_args.get("perms", 0o644) expected_compress_type = extra_zipwrite_args.get("compress_type", zipfile.ZIP_STORED) - time.sleep(5) # Make sure the atime/mtime will change measurably. + # Arbitrary timestamp, just to make sure common.ZipWrite() restores + # the timestamp after writing. + os.utime(test_file_name, (1234567, 1234567)) + expected_stat = os.stat(test_file_name) common.ZipWrite(zip_file, test_file_name, **extra_zipwrite_args) - zip_file.close() + common.ZipClose(zip_file) self._verify(zip_file, zip_file_name, arcname, sha1_hash.hexdigest(), test_file_name, expected_stat, expected_mode, expected_compress_type) finally: - os.remove(test_file_name) os.remove(zip_file_name) def _test_ZipWriteStr(self, zinfo_or_arcname, contents, extra_args=None): @@ -480,8 +492,6 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): try: expected_compress_type = extra_args.get("compress_type", zipfile.ZIP_STORED) - time.sleep(5) # Make sure the atime/mtime will change measurably. - if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): arcname = zinfo_or_arcname expected_mode = extra_args.get("perms", 0o644) @@ -494,7 +504,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): expected_mode = extra_args.get("perms", zinfo_perms) common.ZipWriteStr(zip_file, zinfo_or_arcname, contents, **extra_args) - zip_file.close() + common.ZipClose(zip_file) self._verify(zip_file, zip_file_name, arcname, sha1(contents).hexdigest(), expected_mode=expected_mode, @@ -502,14 +512,13 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): finally: os.remove(zip_file_name) - def _test_ZipWriteStr_large_file(self, large, small, extra_args=None): + def _test_ZipWriteStr_large_file(self, large_file: BinaryIO, small, extra_args=None): extra_args = dict(extra_args or {}) zip_file = tempfile.NamedTemporaryFile(delete=False) zip_file_name = zip_file.name - test_file = tempfile.NamedTemporaryFile(delete=False) - test_file_name = test_file.name + test_file_name = large_file.name arcname_large = test_file_name arcname_small = "bar" @@ -522,21 +531,19 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): zip_file = zipfile.ZipFile(zip_file_name, "w", allowZip64=True) try: - sha1_hash = sha1() - for data in large: - sha1_hash.update(data) - test_file.write(data) - test_file.close() + sha1_hash = hash_file(test_file_name) + # Arbitrary timestamp, just to make sure common.ZipWrite() restores + # the timestamp after writing. + os.utime(test_file_name, (1234567, 1234567)) expected_stat = os.stat(test_file_name) expected_mode = 0o644 expected_compress_type = extra_args.get("compress_type", zipfile.ZIP_STORED) - time.sleep(5) # Make sure the atime/mtime will change measurably. common.ZipWrite(zip_file, test_file_name, **extra_args) common.ZipWriteStr(zip_file, arcname_small, small, **extra_args) - zip_file.close() + common.ZipClose(zip_file) # Verify the contents written by ZipWrite(). self._verify(zip_file, zip_file_name, arcname_large, @@ -549,7 +556,12 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): expected_compress_type=expected_compress_type) finally: os.remove(zip_file_name) - os.remove(test_file_name) + + def _test_reset_ZIP64_LIMIT(self, func, *args): + default_limit = (1 << 31) - 1 + self.assertEqual(default_limit, zipfile.ZIP64_LIMIT) + func(*args) + self.assertEqual(default_limit, zipfile.ZIP64_LIMIT) def test_ZipWrite(self): file_contents = os.urandom(1024) @@ -569,13 +581,13 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): }) def test_ZipWrite_large_file(self): - file_contents = get_2gb_string() - self._test_ZipWrite(file_contents, { - "compress_type": zipfile.ZIP_DEFLATED, - }) + with get_2gb_file() as tmpfile: + self._test_ZipWriteFile(tmpfile.name, { + "compress_type": zipfile.ZIP_DEFLATED, + }) def test_ZipWrite_resets_ZIP64_LIMIT(self): - self._test_ZipWrite("") + self._test_reset_ZIP64_LIMIT(self._test_ZipWrite, "") def test_ZipWriteStr(self): random_string = os.urandom(1024) @@ -619,16 +631,16 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): # zipfile.writestr() doesn't work when the str size is over 2GiB even with # the workaround. We will only test the case of writing a string into a # large archive. - long_string = get_2gb_string() short_string = os.urandom(1024) - self._test_ZipWriteStr_large_file(long_string, short_string, { - "compress_type": zipfile.ZIP_DEFLATED, - }) + with get_2gb_file() as large_file: + self._test_ZipWriteStr_large_file(large_file, short_string, { + "compress_type": zipfile.ZIP_DEFLATED, + }) def test_ZipWriteStr_resets_ZIP64_LIMIT(self): - self._test_ZipWriteStr('foo', b'') + self._test_reset_ZIP64_LIMIT(self._test_ZipWriteStr, 'foo', b'') zinfo = zipfile.ZipInfo(filename="foo") - self._test_ZipWriteStr(zinfo, b'') + self._test_reset_ZIP64_LIMIT(self._test_ZipWriteStr, zinfo, b'') def test_bug21309935(self): zip_file = tempfile.NamedTemporaryFile(delete=False) @@ -650,7 +662,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): zinfo = zipfile.ZipInfo(filename="qux") zinfo.external_attr = 0o700 << 16 common.ZipWriteStr(zip_file, zinfo, random_string, perms=0o400) - zip_file.close() + common.ZipClose(zip_file) self._verify(zip_file, zip_file_name, "foo", sha1(random_string).hexdigest(), @@ -677,7 +689,7 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): common.ZipWrite(output_zip, entry_file.name, arcname='Test1') common.ZipWrite(output_zip, entry_file.name, arcname='Test2') common.ZipWrite(output_zip, entry_file.name, arcname='Test3') - output_zip.close() + common.ZipClose(output_zip) zip_file.close() try: @@ -725,8 +737,8 @@ class CommonZipTest(test_utils.ReleaseToolsTestCase): common.ZipWrite(output_zip, entry_file.name, arcname='Foo3') common.ZipWrite(output_zip, entry_file.name, arcname='Bar4') common.ZipWrite(output_zip, entry_file.name, arcname='Dir5/Baz5') - output_zip.close() - output_zip.close() + common.ZipClose(output_zip) + common.ZipClose(output_zip) return zip_file @test_utils.SkipIfExternalToolsUnavailable() @@ -1287,11 +1299,11 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): 'avb_system_key_path': pubkey, 'avb_system_rollback_index_location': 2, } - args = common.GetAvbChainedPartitionArg('system', info_dict).split(':') - self.assertEqual(3, len(args)) - self.assertEqual('system', args[0]) - self.assertEqual('2', args[1]) - self.assertTrue(os.path.exists(args[2])) + chained_partition_args = common.GetAvbChainedPartitionArg( + 'system', info_dict) + self.assertEqual('system', chained_partition_args.partition) + self.assertEqual(2, chained_partition_args.rollback_index_location) + self.assertTrue(os.path.exists(chained_partition_args.pubkey_path)) @test_utils.SkipIfExternalToolsUnavailable() def test_GetAvbChainedPartitionArg_withPrivateKey(self): @@ -1301,11 +1313,11 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): 'avb_product_key_path': key, 'avb_product_rollback_index_location': 2, } - args = common.GetAvbChainedPartitionArg('product', info_dict).split(':') - self.assertEqual(3, len(args)) - self.assertEqual('product', args[0]) - self.assertEqual('2', args[1]) - self.assertTrue(os.path.exists(args[2])) + chained_partition_args = common.GetAvbChainedPartitionArg( + 'product', info_dict) + self.assertEqual('product', chained_partition_args.partition) + self.assertEqual(2, chained_partition_args.rollback_index_location) + self.assertTrue(os.path.exists(chained_partition_args.pubkey_path)) @test_utils.SkipIfExternalToolsUnavailable() def test_GetAvbChainedPartitionArg_withSpecifiedKey(self): @@ -1315,12 +1327,11 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): 'avb_system_rollback_index_location': 2, } pubkey = os.path.join(self.testdata_dir, 'testkey.pubkey.pem') - args = common.GetAvbChainedPartitionArg( - 'system', info_dict, pubkey).split(':') - self.assertEqual(3, len(args)) - self.assertEqual('system', args[0]) - self.assertEqual('2', args[1]) - self.assertTrue(os.path.exists(args[2])) + chained_partition_args = common.GetAvbChainedPartitionArg( + 'system', info_dict, pubkey) + self.assertEqual('system', chained_partition_args.partition) + self.assertEqual(2, chained_partition_args.rollback_index_location) + self.assertTrue(os.path.exists(chained_partition_args.pubkey_path)) @test_utils.SkipIfExternalToolsUnavailable() def test_GetAvbChainedPartitionArg_invalidKey(self): @@ -1337,7 +1348,6 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): INFO_DICT_DEFAULT = { 'recovery_api_version': 3, 'fstab_version': 2, - 'system_root_image': 'true', 'no_recovery': 'true', 'recovery_as_boot': 'true', } @@ -1366,14 +1376,8 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): info_values = ''.join( ['{}={}\n'.format(k, v) for k, v in sorted(info_dict.items())]) common.ZipWriteStr(target_files_zip, 'META/misc_info.txt', info_values) - - FSTAB_TEMPLATE = "/dev/block/system {} ext4 ro,barrier=1 defaults" - if info_dict.get('system_root_image') == 'true': - fstab_values = FSTAB_TEMPLATE.format('/') - else: - fstab_values = FSTAB_TEMPLATE.format('/system') - common.ZipWriteStr(target_files_zip, fstab_path, fstab_values) - + common.ZipWriteStr(target_files_zip, fstab_path, + "/dev/block/system /system ext4 ro,barrier=1 defaults") common.ZipWriteStr( target_files_zip, 'META/file_contexts', 'file-contexts') return target_files @@ -1386,7 +1390,6 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): loaded_dict = common.LoadInfoDict(target_files_zip) self.assertEqual(3, loaded_dict['recovery_api_version']) self.assertEqual(2, loaded_dict['fstab_version']) - self.assertIn('/', loaded_dict['fstab']) self.assertIn('/system', loaded_dict['fstab']) def test_LoadInfoDict_legacyRecoveryFstabPath(self): @@ -1397,7 +1400,6 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): loaded_dict = common.LoadInfoDict(target_files_zip) self.assertEqual(3, loaded_dict['recovery_api_version']) self.assertEqual(2, loaded_dict['fstab_version']) - self.assertIn('/', loaded_dict['fstab']) self.assertIn('/system', loaded_dict['fstab']) @test_utils.SkipIfExternalToolsUnavailable() @@ -1409,7 +1411,6 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): loaded_dict = common.LoadInfoDict(unzipped) self.assertEqual(3, loaded_dict['recovery_api_version']) self.assertEqual(2, loaded_dict['fstab_version']) - self.assertIn('/', loaded_dict['fstab']) self.assertIn('/system', loaded_dict['fstab']) @test_utils.SkipIfExternalToolsUnavailable() @@ -1421,29 +1422,9 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): loaded_dict = common.LoadInfoDict(unzipped) self.assertEqual(3, loaded_dict['recovery_api_version']) self.assertEqual(2, loaded_dict['fstab_version']) - self.assertIn('/', loaded_dict['fstab']) self.assertIn('/system', loaded_dict['fstab']) - def test_LoadInfoDict_systemRootImageFalse(self): - # Devices not using system-as-root nor recovery-as-boot. Non-A/B devices - # launched prior to P will likely have this config. - info_dict = copy.copy(self.INFO_DICT_DEFAULT) - del info_dict['no_recovery'] - del info_dict['system_root_image'] - del info_dict['recovery_as_boot'] - target_files = self._test_LoadInfoDict_createTargetFiles( - info_dict, - 'RECOVERY/RAMDISK/system/etc/recovery.fstab') - with zipfile.ZipFile(target_files, 'r', allowZip64=True) as target_files_zip: - loaded_dict = common.LoadInfoDict(target_files_zip) - self.assertEqual(3, loaded_dict['recovery_api_version']) - self.assertEqual(2, loaded_dict['fstab_version']) - self.assertNotIn('/', loaded_dict['fstab']) - self.assertIn('/system', loaded_dict['fstab']) - def test_LoadInfoDict_recoveryAsBootFalse(self): - # Devices using system-as-root, but with standalone recovery image. Non-A/B - # devices launched since P will likely have this config. info_dict = copy.copy(self.INFO_DICT_DEFAULT) del info_dict['no_recovery'] del info_dict['recovery_as_boot'] @@ -1454,7 +1435,7 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): loaded_dict = common.LoadInfoDict(target_files_zip) self.assertEqual(3, loaded_dict['recovery_api_version']) self.assertEqual(2, loaded_dict['fstab_version']) - self.assertIn('/', loaded_dict['fstab']) + self.assertNotIn('/', loaded_dict['fstab']) self.assertIn('/system', loaded_dict['fstab']) def test_LoadInfoDict_noRecoveryTrue(self): @@ -1488,7 +1469,6 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): loaded_dict = common.LoadInfoDict(unzipped, True) self.assertEqual(3, loaded_dict['recovery_api_version']) self.assertEqual(2, loaded_dict['fstab_version']) - self.assertIn('/', loaded_dict['fstab']) self.assertIn('/system', loaded_dict['fstab']) self.assertEqual( os.path.join(unzipped, 'ROOT'), loaded_dict['root_dir']) @@ -1535,6 +1515,7 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): 'super_group_a_group_size': '1000', 'super_group_b_partition_list': 'product', 'super_group_b_group_size': '2000', + 'vabc_cow_version': '2', } self.assertEqual(merged_dict, expected_merged_dict) @@ -1545,6 +1526,7 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): 'dynamic_partition_list': 'system', 'super_group_a_partition_list': 'system', 'super_group_a_group_size': '5000', + 'vabc_cow_version': '3', } vendor_dict = { 'use_dynamic_partitions': 'true', @@ -1566,6 +1548,7 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): 'super_group_a_group_size': '1000', 'super_group_b_partition_list': 'product', 'super_group_b_group_size': '2000', + 'vabc_cow_version': '2', } self.assertEqual(merged_dict, expected_merged_dict) @@ -1573,7 +1556,8 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): info_dict = {} cmd = common.GetAvbPartitionArg('system', '/path/to/system.img', info_dict) self.assertEqual( - ['--include_descriptors_from_image', '/path/to/system.img'], cmd) + [common.AVB_ARG_NAME_INCLUDE_DESC_FROM_IMG, '/path/to/system.img'], + cmd) @test_utils.SkipIfExternalToolsUnavailable() def test_AppendVBMetaArgsForPartition_vendorAsChainedPartition(self): @@ -1586,12 +1570,11 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): } cmd = common.GetAvbPartitionArg('vendor', '/path/to/vendor.img', info_dict) self.assertEqual(2, len(cmd)) - self.assertEqual('--chain_partition', cmd[0]) - chained_partition_args = cmd[1].split(':') - self.assertEqual(3, len(chained_partition_args)) - self.assertEqual('vendor', chained_partition_args[0]) - self.assertEqual('5', chained_partition_args[1]) - self.assertTrue(os.path.exists(chained_partition_args[2])) + self.assertEqual(common.AVB_ARG_NAME_CHAIN_PARTITION, cmd[0]) + chained_partition_args = cmd[1] + self.assertEqual('vendor', chained_partition_args.partition) + self.assertEqual(5, chained_partition_args.rollback_index_location) + self.assertTrue(os.path.exists(chained_partition_args.pubkey_path)) @test_utils.SkipIfExternalToolsUnavailable() def test_AppendVBMetaArgsForPartition_recoveryAsChainedPartition_nonAb(self): @@ -1619,45 +1602,11 @@ class CommonUtilsTest(test_utils.ReleaseToolsTestCase): cmd = common.GetAvbPartitionArg( 'recovery', '/path/to/recovery.img', info_dict) self.assertEqual(2, len(cmd)) - self.assertEqual('--chain_partition', cmd[0]) - chained_partition_args = cmd[1].split(':') - self.assertEqual(3, len(chained_partition_args)) - self.assertEqual('recovery', chained_partition_args[0]) - self.assertEqual('3', chained_partition_args[1]) - self.assertTrue(os.path.exists(chained_partition_args[2])) - - def test_GenerateGkiCertificate_KeyPathNotFound(self): - pubkey = os.path.join(self.testdata_dir, 'no_testkey_gki.pem') - self.assertFalse(os.path.exists(pubkey)) - - common.OPTIONS.info_dict = { - 'gki_signing_key_path': pubkey, - 'gki_signing_algorithm': 'SHA256_RSA4096', - 'gki_signing_signature_args': '--prop foo:bar', - } - test_file = tempfile.NamedTemporaryFile() - self.assertRaises(common.ExternalError, common._GenerateGkiCertificate, - test_file.name, 'generic_kernel') - - def test_GenerateGkiCertificate_SearchKeyPathNotFound(self): - pubkey = 'no_testkey_gki.pem' - self.assertFalse(os.path.exists(pubkey)) - - # Tests it should raise ExternalError if no key found under - # OPTIONS.search_path. - search_path_dir = common.MakeTempDir() - search_pubkey = os.path.join(search_path_dir, pubkey) - self.assertFalse(os.path.exists(search_pubkey)) - - common.OPTIONS.search_path = search_path_dir - common.OPTIONS.info_dict = { - 'gki_signing_key_path': pubkey, - 'gki_signing_algorithm': 'SHA256_RSA4096', - 'gki_signing_signature_args': '--prop foo:bar', - } - test_file = tempfile.NamedTemporaryFile() - self.assertRaises(common.ExternalError, common._GenerateGkiCertificate, - test_file.name, 'generic_kernel') + self.assertEqual(common.AVB_ARG_NAME_CHAIN_PARTITION, cmd[0]) + chained_partition_args = cmd[1] + self.assertEqual('recovery', chained_partition_args.partition) + self.assertEqual(3, chained_partition_args.rollback_index_location) + self.assertTrue(os.path.exists(chained_partition_args.pubkey_path)) class InstallRecoveryScriptFormatTest(test_utils.ReleaseToolsTestCase): diff --git a/tools/releasetools/test_ota_from_target_files.py b/tools/releasetools/test_ota_from_target_files.py index ad0f7a8eba..b6fcb1841e 100644 --- a/tools/releasetools/test_ota_from_target_files.py +++ b/tools/releasetools/test_ota_from_target_files.py @@ -163,6 +163,20 @@ class OtaFromTargetFilesTest(test_utils.ReleaseToolsTestCase): 'oem_fingerprint_properties': 'ro.product.device ro.product.brand', } + TEST_TARGET_VENDOR_INFO_DICT = common.PartitionBuildProps.FromDictionary( + 'vendor', { + 'ro.vendor.build.date.utc' : '87654321', + 'ro.product.vendor.device':'vendor-device', + 'ro.vendor.build.fingerprint': 'build-fingerprint-vendor'} + ) + + TEST_SOURCE_VENDOR_INFO_DICT = common.PartitionBuildProps.FromDictionary( + 'vendor', { + 'ro.vendor.build.date.utc' : '12345678', + 'ro.product.vendor.device':'vendor-device', + 'ro.vendor.build.fingerprint': 'build-fingerprint-vendor'} + ) + def setUp(self): self.testdata_dir = test_utils.get_testdata_dir() self.assertTrue(os.path.exists(self.testdata_dir)) @@ -285,7 +299,7 @@ class OtaFromTargetFilesTest(test_utils.ReleaseToolsTestCase): @test_utils.SkipIfExternalToolsUnavailable() def test_GetApexInfoFromTargetFiles(self): target_files = construct_target_files(compressedApex=True) - apex_infos = GetApexInfoFromTargetFiles(target_files, 'system') + apex_infos = GetApexInfoFromTargetFiles(target_files) self.assertEqual(len(apex_infos), 1) self.assertEqual(apex_infos[0].package_name, "com.android.apex.compressed") self.assertEqual(apex_infos[0].version, 1) @@ -351,6 +365,13 @@ class OtaFromTargetFilesTest(test_utils.ReleaseToolsTestCase): source_info['build.prop'].build_props['ro.build.date.utc'], target_info['build.prop'].build_props['ro.build.date.utc']) + @staticmethod + def _test_GetPackageMetadata_swapVendorBuildTimestamps(target_info, source_info): + (target_info['vendor.build.prop'].build_props['ro.vendor.build.date.utc'], + source_info['vendor.build.prop'].build_props['ro.vendor.build.date.utc']) = ( + source_info['vendor.build.prop'].build_props['ro.vendor.build.date.utc'], + target_info['vendor.build.prop'].build_props['ro.vendor.build.date.utc']) + def test_GetPackageMetadata_unintentionalDowngradeDetected(self): target_info_dict = copy.deepcopy(self.TEST_TARGET_INFO_DICT) source_info_dict = copy.deepcopy(self.TEST_SOURCE_INFO_DICT) @@ -363,6 +384,24 @@ class OtaFromTargetFilesTest(test_utils.ReleaseToolsTestCase): self.assertRaises(RuntimeError, self.GetLegacyOtaMetadata, target_info, source_info) + def test_GetPackageMetadata_unintentionalVendorDowngradeDetected(self): + target_info_dict = copy.deepcopy(self.TEST_TARGET_INFO_DICT) + target_info_dict['ab_update'] = 'true' + target_info_dict['ab_partitions'] = ['vendor'] + target_info_dict["vendor.build.prop"] = copy.deepcopy(self.TEST_TARGET_VENDOR_INFO_DICT) + source_info_dict = copy.deepcopy(self.TEST_SOURCE_INFO_DICT) + source_info_dict['ab_update'] = 'true' + source_info_dict['ab_partitions'] = ['vendor'] + source_info_dict["vendor.build.prop"] = copy.deepcopy(self.TEST_SOURCE_VENDOR_INFO_DICT) + self._test_GetPackageMetadata_swapVendorBuildTimestamps( + target_info_dict, source_info_dict) + + target_info = common.BuildInfo(target_info_dict, None) + source_info = common.BuildInfo(source_info_dict, None) + common.OPTIONS.incremental_source = '' + self.assertRaises(RuntimeError, self.GetLegacyOtaMetadata, target_info, + source_info) + def test_GetPackageMetadata_downgrade(self): target_info_dict = copy.deepcopy(self.TEST_TARGET_INFO_DICT) source_info_dict = copy.deepcopy(self.TEST_SOURCE_INFO_DICT) @@ -397,6 +436,55 @@ class OtaFromTargetFilesTest(test_utils.ReleaseToolsTestCase): }, metadata) + def test_GetPackageMetadata_vendorDowngrade(self): + target_info_dict = copy.deepcopy(self.TEST_TARGET_INFO_DICT) + target_info_dict['ab_update'] = 'true' + target_info_dict['ab_partitions'] = ['vendor'] + target_info_dict["vendor.build.prop"] = copy.deepcopy(self.TEST_TARGET_VENDOR_INFO_DICT) + source_info_dict = copy.deepcopy(self.TEST_SOURCE_INFO_DICT) + source_info_dict['ab_update'] = 'true' + source_info_dict['ab_partitions'] = ['vendor'] + source_info_dict["vendor.build.prop"] = copy.deepcopy(self.TEST_SOURCE_VENDOR_INFO_DICT) + self._test_GetPackageMetadata_swapVendorBuildTimestamps( + target_info_dict, source_info_dict) + + target_info = common.BuildInfo(target_info_dict, None) + source_info = common.BuildInfo(source_info_dict, None) + common.OPTIONS.incremental_source = '' + common.OPTIONS.downgrade = True + common.OPTIONS.wipe_user_data = True + common.OPTIONS.spl_downgrade = True + metadata = self.GetLegacyOtaMetadata(target_info, source_info) + # Reset spl_downgrade so other tests are unaffected + common.OPTIONS.spl_downgrade = False + + self.assertDictEqual( + { + 'ota-downgrade': 'yes', + 'ota-type': 'AB', + 'ota-required-cache': '0', + 'ota-wipe': 'yes', + 'post-build': 'build-fingerprint-target', + 'post-build-incremental': 'build-version-incremental-target', + 'post-sdk-level': '27', + 'post-security-patch-level': '2017-12-01', + 'post-timestamp': '1500000000', + 'pre-device': 'product-device', + 'pre-build': 'build-fingerprint-source', + 'pre-build-incremental': 'build-version-incremental-source', + 'spl-downgrade': 'yes', + }, + metadata) + + post_build = GetPackageMetadata(target_info, source_info).postcondition + self.assertEqual('vendor', post_build.partition_state[0].partition_name) + self.assertEqual('12345678', post_build.partition_state[0].version) + + pre_build = GetPackageMetadata(target_info, source_info).precondition + self.assertEqual('vendor', pre_build.partition_state[0].partition_name) + self.assertEqual('87654321', pre_build.partition_state[0].version) + + @test_utils.SkipIfExternalToolsUnavailable() def test_GetTargetFilesZipForSecondaryImages(self): input_file = construct_target_files(secondary=True) diff --git a/tools/releasetools/test_sign_apex.py b/tools/releasetools/test_sign_apex.py index 7723de7a91..8470f202c5 100644 --- a/tools/releasetools/test_sign_apex.py +++ b/tools/releasetools/test_sign_apex.py @@ -59,21 +59,6 @@ class SignApexTest(test_utils.ReleaseToolsTestCase): self.assertTrue(os.path.exists(signed_test_apex)) @test_utils.SkipIfExternalToolsUnavailable() - def test_SignSepolicyApex(self): - test_apex = os.path.join(self.testdata_dir, 'sepolicy.apex') - payload_key = os.path.join(self.testdata_dir, 'testkey_RSA4096.key') - container_key = os.path.join(self.testdata_dir, 'testkey') - apk_keys = {'SEPolicy-33.zip': os.path.join(self.testdata_dir, 'testkey')} - signed_test_apex = sign_apex.SignApexFile( - 'avbtool', - test_apex, - payload_key, - container_key, - False, - None) - self.assertTrue(os.path.exists(signed_test_apex)) - - @test_utils.SkipIfExternalToolsUnavailable() def test_SignCompressedApexFile(self): apex = os.path.join(test_utils.get_current_dir(), 'com.android.apex.compressed.v1.capex') payload_key = os.path.join(self.testdata_dir, 'testkey_RSA4096.key') diff --git a/tools/releasetools/test_sign_target_files_apks.py b/tools/releasetools/test_sign_target_files_apks.py index 0cd7dac184..7ac1cff0bb 100644 --- a/tools/releasetools/test_sign_target_files_apks.py +++ b/tools/releasetools/test_sign_target_files_apks.py @@ -22,8 +22,8 @@ import zipfile import common import test_utils from sign_target_files_apks import ( - CheckApkAndApexKeysAvailable, EditTags, GetApkFileInfo, ReadApexKeysInfo, - ReplaceCerts, ReplaceGkiSigningKey, RewriteAvbProps, RewriteProps, + CheckApkAndApexKeysAvailable, EditTags, GetApkFileInfo, ParseAvbInfo, + ReadApexKeysInfo, ReplaceCerts, RewriteAvbProps, RewriteProps, WriteOtacerts) @@ -537,51 +537,85 @@ name="apex.apexd_test_different_app.apex" public_key="system/apex/apexd/apexd_te 'build/make/target/product/security/testkey', None), }, keys_info) - def test_ReplaceGkiSigningKey(self): - common.OPTIONS.gki_signing_key = 'release_gki_key' - common.OPTIONS.gki_signing_algorithm = 'release_gki_algorithm' - common.OPTIONS.gki_signing_extra_args = 'release_gki_signature_extra_args' + def test_ParseAvbInfo(self): + avb_info_string = """ + Footer version: 1.0 + Image size: 9999999 bytes + Original image size: 8888888 bytes + VBMeta offset: 7777777 + VBMeta size: 1111 bytes + -- + Minimum libavb version: 1.0 + Header Block: 222 bytes + Authentication Block: 333 bytes + Auxiliary Block: 888 bytes + Public key (sha1): abababababababababababababababababababab + Algorithm: SHA256_RSA2048 + Rollback Index: 0 + Flags: 0 + Rollback Index Location: 0 + Release String: 'avbtool 1.3.0' + Descriptors: + Hashtree descriptor: + Version of dm-verity: 1 + Image Size: 8888888 bytes + Tree Offset: 8888888 + Tree Size: 44444 bytes + Data Block Size: 4444 bytes + Hash Block Size: 4444 bytes + FEC num roots: 0 + FEC offset: 0 + FEC size: 0 bytes + Hash Algorithm: sha1 + Partition Name: partition-name + Salt: cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd + Root Digest: efefefefefefefefefefefefefefefefefef + Flags: 0 + Prop: prop.key -> 'prop.value' + """ - misc_info = { - 'gki_signing_key_path': 'default_gki_key', - 'gki_signing_algorithm': 'default_gki_algorithm', - 'gki_signing_signature_args': 'default_gki_signature_args', - } - expected_dict = { - 'gki_signing_key_path': 'release_gki_key', - 'gki_signing_algorithm': 'release_gki_algorithm', - 'gki_signing_signature_args': 'release_gki_signature_extra_args', - } - ReplaceGkiSigningKey(misc_info) - self.assertDictEqual(expected_dict, misc_info) - - def test_ReplaceGkiSigningKey_MissingSigningAlgorithm(self): - common.OPTIONS.gki_signing_key = 'release_gki_key' - common.OPTIONS.gki_signing_algorithm = None - common.OPTIONS.gki_signing_extra_args = 'release_gki_signature_extra_args' - - misc_info = { - 'gki_signing_key_path': 'default_gki_key', - 'gki_signing_algorithm': 'default_gki_algorithm', - 'gki_signing_signature_args': 'default_gki_signature_args', - } - self.assertRaises(ValueError, ReplaceGkiSigningKey, misc_info) - - def test_ReplaceGkiSigningKey_MissingSigningKeyNop(self): - common.OPTIONS.gki_signing_key = None - common.OPTIONS.gki_signing_algorithm = 'release_gki_algorithm' - common.OPTIONS.gki_signing_extra_args = 'release_gki_signature_extra_args' - - # No change to misc_info if common.OPTIONS.gki_signing_key is missing. - misc_info = { - 'gki_signing_key_path': 'default_gki_key', - 'gki_signing_algorithm': 'default_gki_algorithm', - 'gki_signing_signature_args': 'default_gki_signature_args', - } - expected_dict = { - 'gki_signing_key_path': 'default_gki_key', - 'gki_signing_algorithm': 'default_gki_algorithm', - 'gki_signing_signature_args': 'default_gki_signature_args', - } - ReplaceGkiSigningKey(misc_info) - self.assertDictEqual(expected_dict, misc_info) + self.assertEqual( + { + 'Footer version': '1.0', + 'Image size': '9999999 bytes', + 'Original image size': '8888888 bytes', + 'VBMeta offset': '7777777', + 'VBMeta size': '1111 bytes', + 'Minimum libavb version': '1.0', + 'Header Block': '222 bytes', + 'Authentication Block': '333 bytes', + 'Auxiliary Block': '888 bytes', + 'Public key (sha1)': 'abababababababababababababababababababab', + 'Algorithm': 'SHA256_RSA2048', + 'Rollback Index': '0', + 'Flags': '0', + 'Rollback Index Location': '0', + 'Release String': "'avbtool 1.3.0'", + 'Descriptors': [ + { + 'Hashtree descriptor': { + 'Version of dm-verity': '1', + 'Image Size': '8888888 bytes', + 'Tree Offset': '8888888', + 'Tree Size': '44444 bytes', + 'Data Block Size': '4444 bytes', + 'Hash Block Size': '4444 bytes', + 'FEC num roots': '0', + 'FEC offset': '0', + 'FEC size': '0 bytes', + 'Hash Algorithm': 'sha1', + 'Partition Name': 'partition-name', + 'Salt': 'cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd', + 'Root Digest': 'efefefefefefefefefefefefefefefefefef', + 'Flags': '0', + } + }, + { + 'Prop': { + 'prop.key': 'prop.value', + } + }, + ], + }, + ParseAvbInfo(avb_info_string), + )
\ No newline at end of file diff --git a/tools/releasetools/test_validate_target_files.py b/tools/releasetools/test_validate_target_files.py index 48b563d7b0..4d4b9e51e7 100644 --- a/tools/releasetools/test_validate_target_files.py +++ b/tools/releasetools/test_validate_target_files.py @@ -156,7 +156,6 @@ class ValidateTargetFilesTest(test_utils.ReleaseToolsTestCase): verity_key_mincrypt) info_dict = { - 'system_root_image' : 'true', 'verity' : 'true', } options = { diff --git a/tools/releasetools/testdata/sepolicy.apex b/tools/releasetools/testdata/sepolicy.apex Binary files differdeleted file mode 100644 index 2c646cdfd8..0000000000 --- a/tools/releasetools/testdata/sepolicy.apex +++ /dev/null diff --git a/tools/releasetools/validate_target_files.py b/tools/releasetools/validate_target_files.py index beb9e75dfd..88fd8929e5 100755 --- a/tools/releasetools/validate_target_files.py +++ b/tools/releasetools/validate_target_files.py @@ -132,7 +132,7 @@ def ValidateFileConsistency(input_zip, input_tmp, info_dict): return # Verify IMAGES/system.img if applicable. - # Some targets, e.g., gki_arm64, gki_x86_64, etc., are system.img-less. + # Some targets are system.img-less. if 'IMAGES/system.img' in input_zip.namelist(): CheckAllFiles('system') @@ -361,18 +361,15 @@ def ValidateVerifiedBootImages(input_tmp, info_dict, options): "Mismatching mincrypt verity key files" logging.info('Verified the content of /verity_key') - # For devices with a separate ramdisk (i.e. non-system-as-root), there must - # be a copy in ramdisk. - if info_dict.get("system_root_image") != "true": - verity_key_ramdisk = os.path.join( - input_tmp, 'BOOT', 'RAMDISK', 'verity_key') - assert os.path.exists( - verity_key_ramdisk), 'Missing verity_key in ramdisk' + verity_key_ramdisk = os.path.join( + input_tmp, 'BOOT', 'RAMDISK', 'verity_key') + assert os.path.exists( + verity_key_ramdisk), 'Missing verity_key in ramdisk' - assert filecmp.cmp( - verity_key_mincrypt, verity_key_ramdisk, shallow=False), \ - 'Mismatching verity_key files in root and ramdisk' - logging.info('Verified the content of /verity_key in ramdisk') + assert filecmp.cmp( + verity_key_mincrypt, verity_key_ramdisk, shallow=False), \ + 'Mismatching verity_key files in root and ramdisk' + logging.info('Verified the content of /verity_key in ramdisk') # Then verify the verity signed system/vendor/product images, against the # verity pubkey in mincrypt format. @@ -430,7 +427,8 @@ def ValidateVerifiedBootImages(input_tmp, info_dict, options): key_file = options.get(key_name, info_dict[key_name]) chained_partition_arg = common.GetAvbChainedPartitionArg( partition, info_dict, key_file) - cmd.extend(['--expected_chain_partition', chained_partition_arg]) + cmd.extend(['--expected_chain_partition', + chained_partition_arg.to_string()]) # Handle the boot image with a non-default name, e.g. boot-5.4.img boot_images = info_dict.get("boot_images") diff --git a/tools/releasetools/verity_utils.py b/tools/releasetools/verity_utils.py index 755241d282..7caeed48af 100644 --- a/tools/releasetools/verity_utils.py +++ b/tools/releasetools/verity_utils.py @@ -31,6 +31,7 @@ import sys import common import sparse_img from rangelib import RangeSet +from hashlib import sha256 logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ FIXED_SALT = "aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7" MAX_VBMETA_SIZE = 64 * 1024 MAX_FOOTER_SIZE = 4096 + class BuildVerityImageError(Exception): """An Exception raised during verity image building.""" @@ -64,6 +66,11 @@ def CreateVerityImageBuilder(prop_dict): # partition_size could be None at this point, if using dynamic partitions. if partition_size: partition_size = int(partition_size) + # Set up the salt (based on fingerprint) that will be used when adding AVB + # hash / hashtree footers. + salt = prop_dict.get("avb_salt") + if salt is None: + salt = sha256(prop_dict.get("fingerprint", "").encode()).hexdigest() # Verified Boot 2.0 if (prop_dict.get("avb_hash_enable") == "true" or @@ -81,7 +88,7 @@ def CreateVerityImageBuilder(prop_dict): prop_dict["avb_avbtool"], key_path, algorithm, - prop_dict.get("avb_salt"), + salt, prop_dict["avb_add_hash_footer_args"]) # Image uses hashtree footer. @@ -92,7 +99,7 @@ def CreateVerityImageBuilder(prop_dict): prop_dict["avb_avbtool"], key_path, algorithm, - prop_dict.get("avb_salt"), + salt, prop_dict["avb_add_hashtree_footer_args"]) return None @@ -141,11 +148,7 @@ class VerifiedBootVersion2VerityImageBuilder(VerityImageBuilder): self.footer_type = footer_type self.avbtool = avbtool self.algorithm = algorithm - self.key_path = key_path - if key_path and not os.path.exists(key_path) and OPTIONS.search_path: - new_key_path = os.path.join(OPTIONS.search_path, key_path) - if os.path.exists(new_key_path): - self.key_path = new_key_path + self.key_path = common.ResolveAVBSigningPathArgs(key_path) self.salt = salt self.signing_args = signing_args @@ -283,7 +286,7 @@ class VerifiedBootVersion2VerityImageBuilder(VerityImageBuilder): def CreateCustomImageBuilder(info_dict, partition_name, partition_size, - key_path, algorithm, signing_args): + key_path, algorithm, signing_args): builder = None if info_dict.get("avb_enable") == "true": builder = VerifiedBootVersion2VerityImageBuilder( diff --git a/tools/sbom/Android.bp b/tools/sbom/Android.bp new file mode 100644 index 0000000000..2b2b573234 --- /dev/null +++ b/tools/sbom/Android.bp @@ -0,0 +1,94 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +python_binary_host { + name: "generate-sbom", + srcs: [ + "generate-sbom.py", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, + libs: [ + "metadata_file_proto_py", + "libprotobuf-python", + "sbom_lib", + ], +} + +python_library_host { + name: "sbom_lib", + srcs: [ + "sbom_data.py", + "sbom_writers.py", + ], +} + +python_test_host { + name: "sbom_writers_test", + main: "sbom_writers_test.py", + srcs: [ + "sbom_writers_test.py", + ], + data: [ + "testdata/*", + ], + libs: [ + "sbom_lib", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, + test_suites: ["general-tests"], +} + +python_test_host { + name: "sbom_data_test", + main: "sbom_data_test.py", + srcs: [ + "sbom_data_test.py", + ], + libs: [ + "sbom_lib", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, + test_suites: ["general-tests"], +} + +python_binary_host { + name: "generate-sbom-framework_res", + srcs: [ + "generate-sbom-framework_res.py", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, + libs: [ + "sbom_lib", + ], +}
\ No newline at end of file diff --git a/tools/sbom/generate-sbom-framework_res.py b/tools/sbom/generate-sbom-framework_res.py new file mode 100644 index 0000000000..d0d232d635 --- /dev/null +++ b/tools/sbom/generate-sbom-framework_res.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import hashlib +import json +import sbom_data +import sbom_writers + +''' +This script generates SBOM of framework_res.jar of layoutlib shipped with Android Studio. + +The generated SBOM contains some placeholders which should be substituted by release_layoutlib.sh. +The placeholders include: document name, document namespace, organization, created timestamp and +the SHA1 checksum of framework_res.jar. +''' + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='store_true', default=False, + help='Print more information.') + parser.add_argument('--output_file', required=True, + help='The generated SBOM file in SPDX format.') + parser.add_argument('--layoutlib_sbom', required=True, + help='The file path of the SBOM of layoutlib.') + + return parser.parse_args() + + +def main(): + global args + args = get_args() + + doc = sbom_data.Document(name='<name>', + namespace='<namespace>', + creators=['Organization: <organization>'], + created='<created>') + + filename = 'data/framework_res.jar' + file_id = f'SPDXRef-{sbom_data.encode_for_spdxid(filename)}' + file = sbom_data.File(id=file_id, name=filename, checksum='SHA1: <checksum>') + + package_name = 'framework_res' + package_id = f'SPDXRef-PREBUILT-{sbom_data.encode_for_spdxid(package_name)}' + package = sbom_data.Package(id=package_id, name=package_name, version='<package_version>', + download_location=sbom_data.VALUE_NONE, + supplier='Organization: <organization>', + files_analyzed=True, + verification_code='<package_verification_code>') + package.file_ids.append(file_id) + + doc.packages.append(package) + doc.files.append(file) + doc.describes = package_id + + with open(args.layoutlib_sbom, 'r', encoding='utf-8') as f: + layoutlib_sbom = json.load(f) + + with open(args.layoutlib_sbom, 'rb') as f: + sha1 = hashlib.file_digest(f, 'sha1') + + layoutlib_sbom_namespace = layoutlib_sbom[sbom_writers.PropNames.DOCUMENT_NAMESPACE] + external_doc_ref = 'DocumentRef-layoutlib' + doc.external_refs = [ + sbom_data.DocumentExternalReference(external_doc_ref, layoutlib_sbom_namespace, + f'SHA1: {sha1.hexdigest()}')] + + resource_file_spdxids = [] + for file in layoutlib_sbom[sbom_writers.PropNames.FILES]: + if file[sbom_writers.PropNames.FILE_NAME].startswith('data/res/'): + resource_file_spdxids.append(file[sbom_writers.PropNames.SPDXID]) + + doc.relationships = [ + sbom_data.Relationship(package_id, sbom_data.RelationshipType.CONTAINS, file_id) + ] + for spdxid in resource_file_spdxids: + doc.relationships.append( + sbom_data.Relationship(file_id, sbom_data.RelationshipType.GENERATED_FROM, + f'{external_doc_ref}:{spdxid}')) + + # write sbom file + with open(args.output_file, 'w', encoding='utf-8') as f: + sbom_writers.JSONWriter.write(doc, f) + + +if __name__ == '__main__': + main() diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py new file mode 100755 index 0000000000..72f896b42c --- /dev/null +++ b/tools/sbom/generate-sbom.py @@ -0,0 +1,638 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Generate the SBOM of the current target product in SPDX format. +Usage example: + generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \ + --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \ + --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \ + --product_mfr=Google +""" + +import argparse +import csv +import datetime +import google.protobuf.text_format as text_format +import hashlib +import os +import metadata_file_pb2 +import sbom_data +import sbom_writers + + +# Package type +PKG_SOURCE = 'SOURCE' +PKG_UPSTREAM = 'UPSTREAM' +PKG_PREBUILT = 'PREBUILT' + +# Security tag +NVD_CPE23 = 'NVD-CPE2.3:' + +# Report +ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:' +ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:' +ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:' +ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:' +ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-exist installed files:' +INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:' + +SOONG_PREBUILT_MODULE_TYPES = [ + 'android_app_import', + 'android_library_import', + 'cc_prebuilt_binary', + 'cc_prebuilt_library', + 'cc_prebuilt_library_headers', + 'cc_prebuilt_library_shared', + 'cc_prebuilt_library_static', + 'cc_prebuilt_object', + 'dex_import', + 'java_import', + 'java_sdk_library_import', + 'java_system_modules_import', + 'libclang_rt_prebuilt_library_static', + 'libclang_rt_prebuilt_library_shared', + 'llvm_prebuilt_library_static', + 'ndk_prebuilt_object', + 'ndk_prebuilt_shared_stl', + 'nkd_prebuilt_static_stl', + 'prebuilt_apex', + 'prebuilt_bootclasspath_fragment', + 'prebuilt_dsp', + 'prebuilt_firmware', + 'prebuilt_kernel_modules', + 'prebuilt_rfsa', + 'prebuilt_root', + 'rust_prebuilt_dylib', + 'rust_prebuilt_library', + 'rust_prebuilt_rlib', + 'vndk_prebuilt_shared', +] + +THIRD_PARTY_IDENTIFIER_TYPES = [ + # Types defined in metadata_file.proto + 'Git', + 'SVN', + 'Hg', + 'Darcs', + 'VCS', + 'Archive', + 'PrebuiltByAlphabet', + 'LocalSource', + 'Other', + # OSV ecosystems defined at https://ossf.github.io/osv-schema/#affectedpackage-field. + 'Go', + 'npm', + 'OSS-Fuzz', + 'PyPI', + 'RubyGems', + 'crates.io', + 'Hackage', + 'GHC', + 'Packagist', + 'Maven', + 'NuGet', + 'Linux', + 'Debian', + 'Alpine', + 'Hex', + 'Android', + 'GitHub Actions', + 'Pub', + 'ConanCenter', + 'Rocky Linux', + 'AlmaLinux', + 'Bitnami', + 'Photon OS', + 'CRAN', + 'Bioconductor', + 'SwiftURL' +] + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') + parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.') + parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.') + parser.add_argument('--build_version', required=True, help='The build version.') + parser.add_argument('--product_mfr', required=True, help='The product manufacturer.') + parser.add_argument('--module_name', help='The module name. If specified, the generated SBOM is for the module.') + parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format') + parser.add_argument('--unbundled_apk', action='store_true', default=False, help='Generate SBOM for unbundled APKs') + parser.add_argument('--unbundled_apex', action='store_true', default=False, help='Generate SBOM for unbundled APEXs') + + return parser.parse_args() + + +def log(*info): + if args.verbose: + for i in info: + print(i) + + +def new_package_id(package_name, type): + return f'SPDXRef-{type}-{sbom_data.encode_for_spdxid(package_name)}' + + +def new_file_id(file_path): + return f'SPDXRef-{sbom_data.encode_for_spdxid(file_path)}' + + +def checksum(file_path): + h = hashlib.sha1() + if os.path.islink(file_path): + h.update(os.readlink(file_path).encode('utf-8')) + else: + with open(file_path, 'rb') as f: + h.update(f.read()) + return f'SHA1: {h.hexdigest()}' + + +def is_soong_prebuilt_module(file_metadata): + return (file_metadata['soong_module_type'] and + file_metadata['soong_module_type'] in SOONG_PREBUILT_MODULE_TYPES) + + +def is_source_package(file_metadata): + module_path = file_metadata['module_path'] + return module_path.startswith('external/') and not is_prebuilt_package(file_metadata) + + +def is_prebuilt_package(file_metadata): + module_path = file_metadata['module_path'] + if module_path: + return (module_path.startswith('prebuilts/') or + is_soong_prebuilt_module(file_metadata) or + file_metadata['is_prebuilt_make_module']) + + kernel_module_copy_files = file_metadata['kernel_module_copy_files'] + if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'): + return True + + return False + + +def get_source_package_info(file_metadata, metadata_file_path): + """Return source package info exists in its METADATA file, currently including name, security tag + and external SBOM reference. + + See go/android-spdx and go/android-sbom-gen for more details. + """ + if not metadata_file_path: + return file_metadata['module_path'], [] + + metadata_proto = metadata_file_protos[metadata_file_path] + external_refs = [] + for tag in metadata_proto.third_party.security.tag: + if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()): + external_refs.append( + sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe23Type, + locator=tag.removeprefix(NVD_CPE23))) + elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()): + external_refs.append( + sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe22Type, + locator=tag.removeprefix(NVD_CPE23))) + + if metadata_proto.name: + return metadata_proto.name, external_refs + else: + return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name + + +def get_prebuilt_package_name(file_metadata, metadata_file_path): + """Return name of a prebuilt package, which can be from the METADATA file, metadata file path, + module path or kernel module's source path if the installed file is a kernel module. + + See go/android-spdx and go/android-sbom-gen for more details. + """ + name = None + if metadata_file_path: + metadata_proto = metadata_file_protos[metadata_file_path] + if metadata_proto.name: + name = metadata_proto.name + else: + name = metadata_file_path + elif file_metadata['module_path']: + name = file_metadata['module_path'] + elif file_metadata['kernel_module_copy_files']: + src_path = file_metadata['kernel_module_copy_files'].split(':')[0] + name = os.path.dirname(src_path) + + return name.removeprefix('prebuilts/').replace('/', '-') + + +def get_metadata_file_path(file_metadata): + """Search for METADATA file of a package and return its path.""" + metadata_path = '' + if file_metadata['module_path']: + metadata_path = file_metadata['module_path'] + elif file_metadata['kernel_module_copy_files']: + metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) + + while metadata_path and not os.path.exists(metadata_path + '/METADATA'): + metadata_path = os.path.dirname(metadata_path) + + return metadata_path + + +def get_package_version(metadata_file_path): + """Return a package's version in its METADATA file.""" + if not metadata_file_path: + return None + metadata_proto = metadata_file_protos[metadata_file_path] + return metadata_proto.third_party.version + + +def get_package_homepage(metadata_file_path): + """Return a package's homepage URL in its METADATA file.""" + if not metadata_file_path: + return None + metadata_proto = metadata_file_protos[metadata_file_path] + if metadata_proto.third_party.homepage: + return metadata_proto.third_party.homepage + for url in metadata_proto.third_party.url: + if url.type == metadata_file_pb2.URL.Type.HOMEPAGE: + return url.value + + return None + + +def get_package_download_location(metadata_file_path): + """Return a package's code repository URL in its METADATA file.""" + if not metadata_file_path: + return None + metadata_proto = metadata_file_protos[metadata_file_path] + if metadata_proto.third_party.url: + urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type) + if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE: + return urls[0].value + elif len(urls) > 1: + return urls[1].value + + return None + + +def get_sbom_fragments(installed_file_metadata, metadata_file_path): + """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT + package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its + METADATA file. + + See go/android-spdx and go/android-sbom-gen for more details. + """ + external_doc_ref = None + packages = [] + relationships = [] + + # Info from METADATA file + homepage = get_package_homepage(metadata_file_path) + version = get_package_version(metadata_file_path) + download_location = get_package_download_location(metadata_file_path) + + if is_source_package(installed_file_metadata): + # Source fork packages + name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path) + source_package_id = new_package_id(name, PKG_SOURCE) + source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version, + download_location=sbom_data.VALUE_NONE, + supplier='Organization: ' + args.product_mfr, + external_refs=external_refs) + + upstream_package_id = new_package_id(name, PKG_UPSTREAM) + upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version, + supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION, + download_location=download_location) + packages += [source_package, upstream_package] + relationships.append(sbom_data.Relationship(id1=source_package_id, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=upstream_package_id)) + elif is_prebuilt_package(installed_file_metadata): + # Prebuilt fork packages + name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path) + prebuilt_package_id = new_package_id(name, PKG_PREBUILT) + prebuilt_package = sbom_data.Package(id=prebuilt_package_id, + name=name, + download_location=sbom_data.VALUE_NONE, + version=version if version else args.build_version, + supplier='Organization: ' + args.product_mfr) + + upstream_package_id = new_package_id(name, PKG_UPSTREAM) + upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version = version, + supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION, + download_location=download_location) + packages += [prebuilt_package, upstream_package] + relationships.append(sbom_data.Relationship(id1=prebuilt_package_id, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=upstream_package_id)) + + if metadata_file_path: + metadata_proto = metadata_file_protos[metadata_file_path] + if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref': + sbom_url = metadata_proto.third_party.sbom_ref.url + sbom_checksum = metadata_proto.third_party.sbom_ref.checksum + upstream_element_id = metadata_proto.third_party.sbom_ref.element_id + if sbom_url and sbom_checksum and upstream_element_id: + doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{sbom_data.encode_for_spdxid(name)}' + external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id, + uri=sbom_url, + checksum=sbom_checksum) + relationships.append( + sbom_data.Relationship(id1=upstream_package_id, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=doc_ref_id + ':' + upstream_element_id)) + + return external_doc_ref, packages, relationships + + +def save_report(report_file_path, report): + with open(report_file_path, 'w', encoding='utf-8') as report_file: + for type, issues in report.items(): + report_file.write(type + '\n') + for issue in issues: + report_file.write('\t' + issue + '\n') + report_file.write('\n') + + +# Validate the metadata generated by Make for installed files and report if there is no metadata. +def installed_file_has_metadata(installed_file_metadata, report): + installed_file = installed_file_metadata['installed_file'] + module_path = installed_file_metadata['module_path'] + product_copy_files = installed_file_metadata['product_copy_files'] + kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] + is_platform_generated = installed_file_metadata['is_platform_generated'] + + if (not module_path and + not product_copy_files and + not kernel_module_copy_files and + not is_platform_generated and + not installed_file.endswith('.fsv_meta')): + report[ISSUE_NO_METADATA].append(installed_file) + return False + + return True + + +# Validate identifiers in a package's METADATA. +# 1) Only known identifier type is allowed +# 2) Only one identifier's primary_source can be true +def validate_package_metadata(metadata_file_path, package_metadata): + primary_source_found = False + for identifier in package_metadata.third_party.identifier: + if identifier.type not in THIRD_PARTY_IDENTIFIER_TYPES: + sys.exit(f'Unknown value of third_party.identifier.type in {metadata_file_path}/METADATA: {identifier.type}.') + if primary_source_found and identifier.primary_source: + sys.exit( + f'Field "primary_source" is set to true in multiple third_party.identifier in {metadata_file_path}/METADATA.') + primary_source_found = identifier.primary_source + + +def report_metadata_file(metadata_file_path, installed_file_metadata, report): + if metadata_file_path: + report[INFO_METADATA_FOUND_FOR_PACKAGE].append( + 'installed_file: {}, module_path: {}, METADATA file: {}'.format( + installed_file_metadata['installed_file'], + installed_file_metadata['module_path'], + metadata_file_path + '/METADATA')) + + package_metadata = metadata_file_pb2.Metadata() + with open(metadata_file_path + '/METADATA', 'rt') as f: + text_format.Parse(f.read(), package_metadata) + + validate_package_metadata(metadata_file_path, package_metadata) + + if not metadata_file_path in metadata_file_protos: + metadata_file_protos[metadata_file_path] = package_metadata + if not package_metadata.name: + report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"') + + if not package_metadata.third_party.version: + report[ISSUE_METADATA_FILE_INCOMPLETE].append( + f'{metadata_file_path}/METADATA does not has "third_party.version"') + + for tag in package_metadata.third_party.security.tag: + if not tag.startswith(NVD_CPE23): + report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append( + f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA') + else: + report[ISSUE_NO_METADATA_FILE].append( + "installed_file: {}, module_path: {}".format( + installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) + + +def generate_sbom_for_unbundled_apk(): + with open(args.metadata, newline='') as sbom_metadata_file: + reader = csv.DictReader(sbom_metadata_file) + doc = sbom_data.Document(name=args.build_version, + namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', + creators=['Organization: ' + args.product_mfr]) + for installed_file_metadata in reader: + installed_file = installed_file_metadata['installed_file'] + if args.output_file != installed_file_metadata['build_output_path'] + '.spdx.json': + continue + + module_path = installed_file_metadata['module_path'] + package_id = new_package_id(module_path, PKG_PREBUILT) + package = sbom_data.Package(id=package_id, + name=module_path, + version=args.build_version, + supplier='Organization: ' + args.product_mfr) + file_id = new_file_id(installed_file) + file = sbom_data.File(id=file_id, + name=installed_file, + checksum=checksum(installed_file_metadata['build_output_path'])) + relationship = sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=package_id) + doc.add_package(package) + doc.files.append(file) + doc.describes = file_id + doc.add_relationship(relationship) + doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + break + + with open(args.output_file, 'w', encoding='utf-8') as file: + sbom_writers.JSONWriter.write(doc, file) + fragment_file = args.output_file.removesuffix('.spdx.json') + '-fragment.spdx' + with open(fragment_file, 'w', encoding='utf-8') as file: + sbom_writers.TagValueWriter.write(doc, file, fragment=True) + + +def main(): + global args + args = get_args() + log('Args:', vars(args)) + + if args.unbundled_apk: + generate_sbom_for_unbundled_apk() + return + + global metadata_file_protos + metadata_file_protos = {} + + product_package_id = sbom_data.SPDXID_PRODUCT + product_package_name = sbom_data.PACKAGE_NAME_PRODUCT + if args.module_name: + # Build SBOM of a module so use the module name instead. + product_package_id = f'SPDXRef-{sbom_data.encode_for_spdxid(args.module_name)}' + product_package_name = args.module_name + product_package = sbom_data.Package(id=product_package_id, + name=product_package_name, + download_location=sbom_data.VALUE_NONE, + version=args.build_version, + supplier='Organization: ' + args.product_mfr, + files_analyzed=True) + doc_name = args.build_version + if args.module_name: + doc_name = f'{args.build_version}/{args.module_name}' + doc = sbom_data.Document(name=doc_name, + namespace=f'https://www.google.com/sbom/spdx/android/{doc_name}', + creators=['Organization: ' + args.product_mfr], + describes=product_package_id) + if not args.unbundled_apex: + doc.packages.append(product_package) + + doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, + name=sbom_data.PACKAGE_NAME_PLATFORM, + download_location=sbom_data.VALUE_NONE, + version=args.build_version, + supplier='Organization: ' + args.product_mfr)) + + # Report on some issues and information + report = { + ISSUE_NO_METADATA: [], + ISSUE_NO_METADATA_FILE: [], + ISSUE_METADATA_FILE_INCOMPLETE: [], + ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], + ISSUE_INSTALLED_FILE_NOT_EXIST: [], + INFO_METADATA_FOUND_FOR_PACKAGE: [], + } + + # Scan the metadata in CSV file and create the corresponding package and file records in SPDX + with open(args.metadata, newline='') as sbom_metadata_file: + reader = csv.DictReader(sbom_metadata_file) + for installed_file_metadata in reader: + installed_file = installed_file_metadata['installed_file'] + module_path = installed_file_metadata['module_path'] + product_copy_files = installed_file_metadata['product_copy_files'] + kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] + build_output_path = installed_file_metadata['build_output_path'] + is_static_lib = installed_file_metadata['is_static_lib'] + + if not installed_file_has_metadata(installed_file_metadata, report): + continue + if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)): + # Ignore non-existing static library files for now since they are not shipped on devices. + report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file) + continue + + file_id = new_file_id(installed_file) + # TODO(b/285453664): Soong should report the information of statically linked libraries to Make. + # This happens when a different sanitized version of static libraries is used in linking. + # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be + # located correctly because Soong doesn't report the information to Make. + sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' # SHA1 of empty string + if os.path.islink(build_output_path) or os.path.isfile(build_output_path): + sha1 = checksum(build_output_path) + doc.files.append(sbom_data.File(id=file_id, + name=installed_file, + checksum=sha1)) + + if not is_static_lib: + if not args.unbundled_apex: + product_package.file_ids.append(file_id) + elif len(doc.files) > 1: + doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id)) + + if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): + metadata_file_path = get_metadata_file_path(installed_file_metadata) + report_metadata_file(metadata_file_path, installed_file_metadata, report) + + # File from source fork packages or prebuilt fork packages + external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path) + if len(pkgs) > 0: + if external_doc_ref: + doc.add_external_ref(external_doc_ref) + for p in pkgs: + doc.add_package(p) + for rel in rels: + doc.add_relationship(rel) + fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=fork_package_id)) + elif module_path or installed_file_metadata['is_platform_generated']: + # File from PLATFORM package + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + elif product_copy_files: + # Format of product_copy_files: <source path>:<dest path> + src_path = product_copy_files.split(':')[0] + # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device, + # so process them as files from PLATFORM package + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + elif installed_file.endswith('.fsv_meta'): + # See build/make/core/Makefile:2988 + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + elif kernel_module_copy_files.startswith('ANDROID-GEN'): + # For the four files generated for _dlkm, _ramdisk partitions + # See build/make/core/Makefile:323 + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + + # Process static libraries and whole static libraries the installed file links to + static_libs = installed_file_metadata['static_libraries'] + whole_static_libs = installed_file_metadata['whole_static_libraries'] + all_static_libs = (static_libs + ' ' + whole_static_libs).strip() + if all_static_libs: + for lib in all_static_libs.split(' '): + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.STATIC_LINK, + id2=new_file_id(lib + '.a'))) + + if args.unbundled_apex: + doc.describes = doc.files[0].id + + # Save SBOM records to output file + doc.generate_packages_verification_code() + doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + prefix = args.output_file + if prefix.endswith('.spdx'): + prefix = prefix.removesuffix('.spdx') + elif prefix.endswith('.spdx.json'): + prefix = prefix.removesuffix('.spdx.json') + + output_file = prefix + '.spdx' + if args.unbundled_apex: + output_file = prefix + '-fragment.spdx' + with open(output_file, 'w', encoding="utf-8") as file: + sbom_writers.TagValueWriter.write(doc, file, fragment=args.unbundled_apex) + if args.json: + with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file: + sbom_writers.JSONWriter.write(doc, file) + + save_report(prefix + '-gen-report.txt', report) + + +if __name__ == '__main__': + main() diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py new file mode 100644 index 0000000000..b5ac8a59c3 --- /dev/null +++ b/tools/sbom/sbom_data.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Define data classes that model SBOMs defined by SPDX. The data classes could be +written out to different formats (tagvalue, JSON, etc) of SPDX with corresponding +writer utilities. + +Rrefer to SPDX 2.3 spec: https://spdx.github.io/spdx-spec/v2.3/ and go/android-spdx for details of +fields in each data class. +""" + +from dataclasses import dataclass, field +from typing import List +import hashlib + +SPDXID_DOC = 'SPDXRef-DOCUMENT' +SPDXID_PRODUCT = 'SPDXRef-PRODUCT' +SPDXID_PLATFORM = 'SPDXRef-PLATFORM' + +PACKAGE_NAME_PRODUCT = 'PRODUCT' +PACKAGE_NAME_PLATFORM = 'PLATFORM' + +VALUE_NOASSERTION = 'NOASSERTION' +VALUE_NONE = 'NONE' + + +class PackageExternalRefCategory: + SECURITY = 'SECURITY' + PACKAGE_MANAGER = 'PACKAGE-MANAGER' + PERSISTENT_ID = 'PERSISTENT-ID' + OTHER = 'OTHER' + + +class PackageExternalRefType: + cpe22Type = 'cpe22Type' + cpe23Type = 'cpe23Type' + + +@dataclass +class PackageExternalRef: + category: PackageExternalRefCategory + type: PackageExternalRefType + locator: str + + +@dataclass +class Package: + name: str + id: str + version: str = None + supplier: str = None + download_location: str = None + files_analyzed: bool = False + verification_code: str = None + file_ids: List[str] = field(default_factory=list) + external_refs: List[PackageExternalRef] = field(default_factory=list) + + +@dataclass +class File: + id: str + name: str + checksum: str + + +class RelationshipType: + DESCRIBES = 'DESCRIBES' + VARIANT_OF = 'VARIANT_OF' + GENERATED_FROM = 'GENERATED_FROM' + CONTAINS = 'CONTAINS' + STATIC_LINK = 'STATIC_LINK' + + +@dataclass +class Relationship: + id1: str + relationship: RelationshipType + id2: str + + +@dataclass +class DocumentExternalReference: + id: str + uri: str + checksum: str + + +@dataclass +class Document: + name: str + namespace: str + id: str = SPDXID_DOC + describes: str = SPDXID_PRODUCT + creators: List[str] = field(default_factory=list) + created: str = None + external_refs: List[DocumentExternalReference] = field(default_factory=list) + packages: List[Package] = field(default_factory=list) + files: List[File] = field(default_factory=list) + relationships: List[Relationship] = field(default_factory=list) + + def add_external_ref(self, external_ref): + if not any(external_ref.uri == ref.uri for ref in self.external_refs): + self.external_refs.append(external_ref) + + def add_package(self, package): + if not any(package.id == p.id for p in self.packages): + self.packages.append(package) + + def add_relationship(self, rel): + if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship + for r in self.relationships): + self.relationships.append(rel) + + def generate_packages_verification_code(self): + for package in self.packages: + if not package.file_ids: + continue + + checksums = [] + for file in self.files: + if file.id in package.file_ids: + checksums.append(file.checksum.split(': ')[1]) + checksums.sort() + h = hashlib.sha1() + h.update(''.join(checksums).encode(encoding='utf-8')) + package.verification_code = h.hexdigest() + +def encode_for_spdxid(s): + """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-""" + result = '' + for c in s: + if c.isalnum() or c in '.-': + result += c + elif c in '_@/': + result += '-' + else: + result += '0x' + c.encode('utf-8').hex() + + return result.lstrip('-')
\ No newline at end of file diff --git a/tools/sbom/sbom_data_test.py b/tools/sbom/sbom_data_test.py new file mode 100644 index 0000000000..69bc9d2d29 --- /dev/null +++ b/tools/sbom/sbom_data_test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import unittest +import sbom_data + +BUILD_FINGER_PRINT = 'build_finger_print' +SUPPLIER_GOOGLE = 'Organization: Google' +SUPPLIER_UPSTREAM = 'Organization: upstream' + +SPDXID_PREBUILT_PACKAGE1 = 'SPDXRef-PREBUILT-package1' +SPDXID_SOURCE_PACKAGE1 = 'SPDXRef-SOURCE-package1' +SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1' + +SPDXID_FILE1 = 'SPDXRef-file1' +SPDXID_FILE2 = 'SPDXRef-file2' +SPDXID_FILE3 = 'SPDXRef-file3' +SPDXID_FILE4 = 'SPDXRef-file4' + + +class SBOMDataTest(unittest.TestCase): + + def setUp(self): + # SBOM of a product + self.sbom_doc = sbom_data.Document(name='test doc', + namespace='http://www.google.com/sbom/spdx/android', + creators=[SUPPLIER_GOOGLE], + created='2023-03-31T22:17:58Z', + describes=sbom_data.SPDXID_PRODUCT) + self.sbom_doc.add_external_ref( + sbom_data.DocumentExternalReference(id='DocumentRef-external_doc_ref', + uri='external_doc_uri', + checksum='SHA1: 1234567890')) + self.sbom_doc.add_package( + sbom_data.Package(id=sbom_data.SPDXID_PRODUCT, + name=sbom_data.PACKAGE_NAME_PRODUCT, + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + files_analyzed=True, + verification_code='', + file_ids=[SPDXID_FILE1, SPDXID_FILE2, SPDXID_FILE3, SPDXID_FILE4])) + + self.sbom_doc.add_package( + sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, + name=sbom_data.PACKAGE_NAME_PLATFORM, + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE1, + name='Prebuilt package1', + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1, + name='Source package1', + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + external_refs=[sbom_data.PackageExternalRef( + category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe22Type, + locator='cpe:/a:jsoncpp_project:jsoncpp:1.9.4')] + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_UPSTREAM_PACKAGE1, + name='Upstream package1', + supplier=SUPPLIER_UPSTREAM, + version='1.1', + )) + + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_SOURCE_PACKAGE1, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=SPDXID_UPSTREAM_PACKAGE1)) + + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', + checksum='SHA1: 356a192b7913b04c54574d18c28d46e6395428ab')) # sha1 hash of 1 + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', + checksum='SHA1: da4b9237bacccdf19c0760cab7aec4a8359010b0')) # sha1 hash of 2 + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', + checksum='SHA1: 77de68daecd823babbb58edb1c8e14d7106e83bb')) # sha1 hash of 3 + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE4, name='file4.a', + checksum='SHA1: 1b6453892473a467d07372d45eb05abc2031647a')) # sha1 of 4 + + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE2, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_PREBUILT_PACKAGE1)) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE3, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_SOURCE_PACKAGE1 + )) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.STATIC_LINK, + id2=SPDXID_FILE4 + )) + + def test_package_verification_code(self): + checksums = [] + for file in self.sbom_doc.files: + checksums.append(file.checksum.split(': ')[1]) + checksums.sort() + h = hashlib.sha1() + h.update(''.join(checksums).encode(encoding='utf-8')) + expected_package_verification_code = h.hexdigest() + + self.sbom_doc.generate_packages_verification_code() + self.assertEqual(expected_package_verification_code, self.sbom_doc.packages[0].verification_code) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py new file mode 100644 index 0000000000..1cb864db75 --- /dev/null +++ b/tools/sbom/sbom_writers.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Serialize objects defined in package sbom_data to SPDX format: tagvalue, JSON. +""" + +import json +import sbom_data + +SPDX_VER = 'SPDX-2.3' +DATA_LIC = 'CC0-1.0' + + +class Tags: + # Common + SPDXID = 'SPDXID' + SPDX_VERSION = 'SPDXVersion' + DATA_LICENSE = 'DataLicense' + DOCUMENT_NAME = 'DocumentName' + DOCUMENT_NAMESPACE = 'DocumentNamespace' + CREATED = 'Created' + CREATOR = 'Creator' + EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef' + + # Package + PACKAGE_NAME = 'PackageName' + PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation' + PACKAGE_VERSION = 'PackageVersion' + PACKAGE_SUPPLIER = 'PackageSupplier' + FILES_ANALYZED = 'FilesAnalyzed' + PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode' + PACKAGE_EXTERNAL_REF = 'ExternalRef' + # Package license + PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded' + PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles' + PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared' + PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments' + + # File + FILE_NAME = 'FileName' + FILE_CHECKSUM = 'FileChecksum' + # File license + FILE_LICENSE_CONCLUDED = 'LicenseConcluded' + FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile' + FILE_LICENSE_COMMENTS = 'LicenseComments' + FILE_COPYRIGHT_TEXT = 'FileCopyrightText' + FILE_NOTICE = 'FileNotice' + FILE_ATTRIBUTION_TEXT = 'FileAttributionText' + + # Relationship + RELATIONSHIP = 'Relationship' + + +class TagValueWriter: + @staticmethod + def marshal_doc_headers(sbom_doc): + headers = [ + f'{Tags.SPDX_VERSION}: {SPDX_VER}', + f'{Tags.DATA_LICENSE}: {DATA_LIC}', + f'{Tags.SPDXID}: {sbom_doc.id}', + f'{Tags.DOCUMENT_NAME}: {sbom_doc.name}', + f'{Tags.DOCUMENT_NAMESPACE}: {sbom_doc.namespace}', + ] + for creator in sbom_doc.creators: + headers.append(f'{Tags.CREATOR}: {creator}') + headers.append(f'{Tags.CREATED}: {sbom_doc.created}') + for doc_ref in sbom_doc.external_refs: + headers.append( + f'{Tags.EXTERNAL_DOCUMENT_REF}: {doc_ref.id} {doc_ref.uri} {doc_ref.checksum}') + headers.append('') + return headers + + @staticmethod + def marshal_package(sbom_doc, package, fragment): + download_location = sbom_data.VALUE_NOASSERTION + if package.download_location: + download_location = package.download_location + tagvalues = [ + f'{Tags.PACKAGE_NAME}: {package.name}', + f'{Tags.SPDXID}: {package.id}', + f'{Tags.PACKAGE_DOWNLOAD_LOCATION}: {download_location}', + f'{Tags.FILES_ANALYZED}: {str(package.files_analyzed).lower()}', + ] + if package.version: + tagvalues.append(f'{Tags.PACKAGE_VERSION}: {package.version}') + if package.supplier: + tagvalues.append(f'{Tags.PACKAGE_SUPPLIER}: {package.supplier}') + if package.verification_code: + tagvalues.append(f'{Tags.PACKAGE_VERIFICATION_CODE}: {package.verification_code}') + if package.external_refs: + for external_ref in package.external_refs: + tagvalues.append( + f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}') + + tagvalues.append('') + + if package.id == sbom_doc.describes and not fragment: + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') + tagvalues.append('') + + for file in sbom_doc.files: + if file.id in package.file_ids: + tagvalues += TagValueWriter.marshal_file(file) + + return tagvalues + + @staticmethod + def marshal_packages(sbom_doc, fragment): + tagvalues = [] + marshaled_relationships = [] + i = 0 + packages = sbom_doc.packages + while i < len(packages): + if (i + 1 < len(packages) + and packages[i].id.startswith('SPDXRef-SOURCE-') + and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')): + # Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other + # in SBOMs in tagvalue format. + tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment) + tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment) + rel = next((r for r in sbom_doc.relationships if + r.id1 == packages[i].id and + r.id2 == packages[i + 1].id and + r.relationship == sbom_data.RelationshipType.VARIANT_OF), None) + if rel: + marshaled_relationships.append(rel) + tagvalues.append(TagValueWriter.marshal_relationship(rel)) + tagvalues.append('') + + i += 2 + else: + tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment) + i += 1 + + return tagvalues, marshaled_relationships + + @staticmethod + def marshal_file(file): + tagvalues = [ + f'{Tags.FILE_NAME}: {file.name}', + f'{Tags.SPDXID}: {file.id}', + f'{Tags.FILE_CHECKSUM}: {file.checksum}', + '', + ] + + return tagvalues + + @staticmethod + def marshal_files(sbom_doc, fragment): + tagvalues = [] + files_in_packages = [] + for package in sbom_doc.packages: + files_in_packages += package.file_ids + for file in sbom_doc.files: + if file.id in files_in_packages: + continue + tagvalues += TagValueWriter.marshal_file(file) + if file.id == sbom_doc.describes and not fragment: + # Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable. + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') + tagvalues.append('') + return tagvalues + + @staticmethod + def marshal_relationship(rel): + return f'{Tags.RELATIONSHIP}: {rel.id1} {rel.relationship} {rel.id2}' + + @staticmethod + def marshal_relationships(sbom_doc, marshaled_rels): + tagvalues = [] + sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.id2 + r.id1) + for rel in sorted_rels: + if any(r.id1 == rel.id1 and r.id2 == rel.id2 and r.relationship == rel.relationship + for r in marshaled_rels): + continue + tagvalues.append(TagValueWriter.marshal_relationship(rel)) + tagvalues.append('') + return tagvalues + + @staticmethod + def write(sbom_doc, file, fragment=False): + content = [] + if not fragment: + content += TagValueWriter.marshal_doc_headers(sbom_doc) + content += TagValueWriter.marshal_files(sbom_doc, fragment) + tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment) + content += tagvalues + content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships) + file.write('\n'.join(content)) + + +class PropNames: + # Common + SPDXID = 'SPDXID' + SPDX_VERSION = 'spdxVersion' + DATA_LICENSE = 'dataLicense' + NAME = 'name' + DOCUMENT_NAMESPACE = 'documentNamespace' + CREATION_INFO = 'creationInfo' + CREATORS = 'creators' + CREATED = 'created' + EXTERNAL_DOCUMENT_REF = 'externalDocumentRefs' + DOCUMENT_DESCRIBES = 'documentDescribes' + EXTERNAL_DOCUMENT_ID = 'externalDocumentId' + EXTERNAL_DOCUMENT_URI = 'spdxDocument' + EXTERNAL_DOCUMENT_CHECKSUM = 'checksum' + ALGORITHM = 'algorithm' + CHECKSUM_VALUE = 'checksumValue' + + # Package + PACKAGES = 'packages' + PACKAGE_DOWNLOAD_LOCATION = 'downloadLocation' + PACKAGE_VERSION = 'versionInfo' + PACKAGE_SUPPLIER = 'supplier' + FILES_ANALYZED = 'filesAnalyzed' + PACKAGE_VERIFICATION_CODE = 'packageVerificationCode' + PACKAGE_VERIFICATION_CODE_VALUE = 'packageVerificationCodeValue' + PACKAGE_EXTERNAL_REFS = 'externalRefs' + PACKAGE_EXTERNAL_REF_CATEGORY = 'referenceCategory' + PACKAGE_EXTERNAL_REF_TYPE = 'referenceType' + PACKAGE_EXTERNAL_REF_LOCATOR = 'referenceLocator' + PACKAGE_HAS_FILES = 'hasFiles' + + # File + FILES = 'files' + FILE_NAME = 'fileName' + FILE_CHECKSUMS = 'checksums' + + # Relationship + RELATIONSHIPS = 'relationships' + REL_ELEMENT_ID = 'spdxElementId' + REL_RELATED_ELEMENT_ID = 'relatedSpdxElement' + REL_TYPE = 'relationshipType' + + +class JSONWriter: + @staticmethod + def marshal_doc_headers(sbom_doc): + headers = { + PropNames.SPDX_VERSION: SPDX_VER, + PropNames.DATA_LICENSE: DATA_LIC, + PropNames.SPDXID: sbom_doc.id, + PropNames.NAME: sbom_doc.name, + PropNames.DOCUMENT_NAMESPACE: sbom_doc.namespace, + PropNames.CREATION_INFO: {} + } + creators = [creator for creator in sbom_doc.creators] + headers[PropNames.CREATION_INFO][PropNames.CREATORS] = creators + headers[PropNames.CREATION_INFO][PropNames.CREATED] = sbom_doc.created + external_refs = [] + for doc_ref in sbom_doc.external_refs: + checksum = doc_ref.checksum.split(': ') + external_refs.append({ + PropNames.EXTERNAL_DOCUMENT_ID: f'{doc_ref.id}', + PropNames.EXTERNAL_DOCUMENT_URI: doc_ref.uri, + PropNames.EXTERNAL_DOCUMENT_CHECKSUM: { + PropNames.ALGORITHM: checksum[0], + PropNames.CHECKSUM_VALUE: checksum[1] + } + }) + if external_refs: + headers[PropNames.EXTERNAL_DOCUMENT_REF] = external_refs + headers[PropNames.DOCUMENT_DESCRIBES] = [sbom_doc.describes] + + return headers + + @staticmethod + def marshal_packages(sbom_doc): + packages = [] + for p in sbom_doc.packages: + package = { + PropNames.NAME: p.name, + PropNames.SPDXID: p.id, + PropNames.PACKAGE_DOWNLOAD_LOCATION: p.download_location if p.download_location else sbom_data.VALUE_NOASSERTION, + PropNames.FILES_ANALYZED: p.files_analyzed + } + if p.version: + package[PropNames.PACKAGE_VERSION] = p.version + if p.supplier: + package[PropNames.PACKAGE_SUPPLIER] = p.supplier + if p.verification_code: + package[PropNames.PACKAGE_VERIFICATION_CODE] = { + PropNames.PACKAGE_VERIFICATION_CODE_VALUE: p.verification_code + } + if p.external_refs: + package[PropNames.PACKAGE_EXTERNAL_REFS] = [] + for ref in p.external_refs: + ext_ref = { + PropNames.PACKAGE_EXTERNAL_REF_CATEGORY: ref.category, + PropNames.PACKAGE_EXTERNAL_REF_TYPE: ref.type, + PropNames.PACKAGE_EXTERNAL_REF_LOCATOR: ref.locator, + } + package[PropNames.PACKAGE_EXTERNAL_REFS].append(ext_ref) + if p.file_ids: + package[PropNames.PACKAGE_HAS_FILES] = [] + for file_id in p.file_ids: + package[PropNames.PACKAGE_HAS_FILES].append(file_id) + + packages.append(package) + + return {PropNames.PACKAGES: packages} + + @staticmethod + def marshal_files(sbom_doc): + files = [] + for f in sbom_doc.files: + file = { + PropNames.FILE_NAME: f.name, + PropNames.SPDXID: f.id + } + checksum = f.checksum.split(': ') + file[PropNames.FILE_CHECKSUMS] = [{ + PropNames.ALGORITHM: checksum[0], + PropNames.CHECKSUM_VALUE: checksum[1], + }] + files.append(file) + return {PropNames.FILES: files} + + @staticmethod + def marshal_relationships(sbom_doc): + relationships = [] + sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.relationship + r.id2 + r.id1) + for r in sorted_rels: + rel = { + PropNames.REL_ELEMENT_ID: r.id1, + PropNames.REL_RELATED_ELEMENT_ID: r.id2, + PropNames.REL_TYPE: r.relationship, + } + relationships.append(rel) + + return {PropNames.RELATIONSHIPS: relationships} + + @staticmethod + def write(sbom_doc, file): + doc = {} + doc.update(JSONWriter.marshal_doc_headers(sbom_doc)) + doc.update(JSONWriter.marshal_packages(sbom_doc)) + doc.update(JSONWriter.marshal_files(sbom_doc)) + doc.update(JSONWriter.marshal_relationships(sbom_doc)) + file.write(json.dumps(doc, indent=4)) diff --git a/tools/sbom/sbom_writers_test.py b/tools/sbom/sbom_writers_test.py new file mode 100644 index 0000000000..cf85e014b7 --- /dev/null +++ b/tools/sbom/sbom_writers_test.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import pathlib +import unittest +import sbom_data +import sbom_writers + +BUILD_FINGER_PRINT = 'build_finger_print' +SUPPLIER_GOOGLE = 'Organization: Google' +SUPPLIER_UPSTREAM = 'Organization: upstream' + +SPDXID_PREBUILT_PACKAGE1 = 'SPDXRef-PREBUILT-package1' +SPDXID_SOURCE_PACKAGE1 = 'SPDXRef-SOURCE-package1' +SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1' + +SPDXID_FILE1 = 'SPDXRef-file1' +SPDXID_FILE2 = 'SPDXRef-file2' +SPDXID_FILE3 = 'SPDXRef-file3' +SPDXID_FILE4 = 'SPDXRef-file4' + + +class SBOMWritersTest(unittest.TestCase): + + def setUp(self): + # SBOM of a product + self.sbom_doc = sbom_data.Document(name='test doc', + namespace='http://www.google.com/sbom/spdx/android', + creators=[SUPPLIER_GOOGLE], + created='2023-03-31T22:17:58Z', + describes=sbom_data.SPDXID_PRODUCT) + self.sbom_doc.add_external_ref( + sbom_data.DocumentExternalReference(id='DocumentRef-external_doc_ref', + uri='external_doc_uri', + checksum='SHA1: 1234567890')) + self.sbom_doc.add_package( + sbom_data.Package(id=sbom_data.SPDXID_PRODUCT, + name=sbom_data.PACKAGE_NAME_PRODUCT, + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + files_analyzed=True, + verification_code='123456', + file_ids=[SPDXID_FILE1, SPDXID_FILE2, SPDXID_FILE3])) + + self.sbom_doc.add_package( + sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, + name=sbom_data.PACKAGE_NAME_PLATFORM, + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE1, + name='Prebuilt package1', + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1, + name='Source package1', + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + external_refs=[sbom_data.PackageExternalRef( + category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe22Type, + locator='cpe:/a:jsoncpp_project:jsoncpp:1.9.4')] + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_UPSTREAM_PACKAGE1, + name='Upstream package1', + supplier=SUPPLIER_UPSTREAM, + version='1.1', + )) + + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_SOURCE_PACKAGE1, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=SPDXID_UPSTREAM_PACKAGE1)) + + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', checksum='SHA1: 11111')) + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222')) + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333')) + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444')) + + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE2, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_PREBUILT_PACKAGE1)) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE3, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_SOURCE_PACKAGE1 + )) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.STATIC_LINK, + id2=SPDXID_FILE4 + )) + + # SBOM fragment of a APK + self.unbundled_sbom_doc = sbom_data.Document(name='test doc', + namespace='http://www.google.com/sbom/spdx/android', + creators=[SUPPLIER_GOOGLE], + created='2023-03-31T22:17:58Z', + describes=SPDXID_FILE1) + + self.unbundled_sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE1, name='/bin/file1.apk', checksum='SHA1: 11111')) + self.unbundled_sbom_doc.add_package( + sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1, + name='Unbundled apk package', + download_location=sbom_data.VALUE_NONE, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT)) + self.unbundled_sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_SOURCE_PACKAGE1)) + + def test_tagvalue_writer(self): + with io.StringIO() as output: + sbom_writers.TagValueWriter.write(self.sbom_doc, output) + expected_output = pathlib.Path('testdata/expected_tagvalue_sbom.spdx').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + def test_tagvalue_writer_doc_describes_file(self): + with io.StringIO() as output: + self.sbom_doc.describes = SPDXID_FILE4 + sbom_writers.TagValueWriter.write(self.sbom_doc, output) + expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_doc_describes_file.spdx').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + def test_tagvalue_writer_unbundled(self): + with io.StringIO() as output: + sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True) + expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_unbundled.spdx').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + def test_json_writer(self): + with io.StringIO() as output: + sbom_writers.JSONWriter.write(self.sbom_doc, output) + expected_output = pathlib.Path('testdata/expected_json_sbom.spdx.json').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tools/sbom/testdata/expected_json_sbom.spdx.json b/tools/sbom/testdata/expected_json_sbom.spdx.json new file mode 100644 index 0000000000..53936c5c9d --- /dev/null +++ b/tools/sbom/testdata/expected_json_sbom.spdx.json @@ -0,0 +1,152 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "test doc", + "documentNamespace": "http://www.google.com/sbom/spdx/android", + "creationInfo": { + "creators": [ + "Organization: Google" + ], + "created": "2023-03-31T22:17:58Z" + }, + "externalDocumentRefs": [ + { + "externalDocumentId": "DocumentRef-external_doc_ref", + "spdxDocument": "external_doc_uri", + "checksum": { + "algorithm": "SHA1", + "checksumValue": "1234567890" + } + } + ], + "documentDescribes": [ + "SPDXRef-PRODUCT" + ], + "packages": [ + { + "name": "PRODUCT", + "SPDXID": "SPDXRef-PRODUCT", + "downloadLocation": "NONE", + "filesAnalyzed": true, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google", + "packageVerificationCode": { + "packageVerificationCodeValue": "123456" + }, + "hasFiles": [ + "SPDXRef-file1", + "SPDXRef-file2", + "SPDXRef-file3" + ] + }, + { + "name": "PLATFORM", + "SPDXID": "SPDXRef-PLATFORM", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google" + }, + { + "name": "Prebuilt package1", + "SPDXID": "SPDXRef-PREBUILT-package1", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google" + }, + { + "name": "Source package1", + "SPDXID": "SPDXRef-SOURCE-package1", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceType": "cpe22Type", + "referenceLocator": "cpe:/a:jsoncpp_project:jsoncpp:1.9.4" + } + ] + }, + { + "name": "Upstream package1", + "SPDXID": "SPDXRef-UPSTREAM-package1", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "versionInfo": "1.1", + "supplier": "Organization: upstream" + } + ], + "files": [ + { + "fileName": "/bin/file1", + "SPDXID": "SPDXRef-file1", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "11111" + } + ] + }, + { + "fileName": "/bin/file2", + "SPDXID": "SPDXRef-file2", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "22222" + } + ] + }, + { + "fileName": "/bin/file3", + "SPDXID": "SPDXRef-file3", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "33333" + } + ] + }, + { + "fileName": "file4.a", + "SPDXID": "SPDXRef-file4", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "44444" + } + ] + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-file1", + "relatedSpdxElement": "SPDXRef-PLATFORM", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-file2", + "relatedSpdxElement": "SPDXRef-PREBUILT-package1", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-file3", + "relatedSpdxElement": "SPDXRef-SOURCE-package1", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-file1", + "relatedSpdxElement": "SPDXRef-file4", + "relationshipType": "STATIC_LINK" + }, + { + "spdxElementId": "SPDXRef-SOURCE-package1", + "relatedSpdxElement": "SPDXRef-UPSTREAM-package1", + "relationshipType": "VARIANT_OF" + } + ] +}
\ No newline at end of file diff --git a/tools/sbom/testdata/expected_tagvalue_sbom.spdx b/tools/sbom/testdata/expected_tagvalue_sbom.spdx new file mode 100644 index 0000000000..e6fd17e9c3 --- /dev/null +++ b/tools/sbom/testdata/expected_tagvalue_sbom.spdx @@ -0,0 +1,70 @@ +SPDXVersion: SPDX-2.3 +DataLicense: CC0-1.0 +SPDXID: SPDXRef-DOCUMENT +DocumentName: test doc +DocumentNamespace: http://www.google.com/sbom/spdx/android +Creator: Organization: Google +Created: 2023-03-31T22:17:58Z +ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890 + +FileName: file4.a +SPDXID: SPDXRef-file4 +FileChecksum: SHA1: 44444 + +PackageName: PRODUCT +SPDXID: SPDXRef-PRODUCT +PackageDownloadLocation: NONE +FilesAnalyzed: true +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +PackageVerificationCode: 123456 + +Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-PRODUCT + +FileName: /bin/file1 +SPDXID: SPDXRef-file1 +FileChecksum: SHA1: 11111 + +FileName: /bin/file2 +SPDXID: SPDXRef-file2 +FileChecksum: SHA1: 22222 + +FileName: /bin/file3 +SPDXID: SPDXRef-file3 +FileChecksum: SHA1: 33333 + +PackageName: PLATFORM +SPDXID: SPDXRef-PLATFORM +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Prebuilt package1 +SPDXID: SPDXRef-PREBUILT-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Source package1 +SPDXID: SPDXRef-SOURCE-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4 + +PackageName: Upstream package1 +SPDXID: SPDXRef-UPSTREAM-package1 +PackageDownloadLocation: NOASSERTION +FilesAnalyzed: false +PackageVersion: 1.1 +PackageSupplier: Organization: upstream + +Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1 + +Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM +Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1 +Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1 +Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4 diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx new file mode 100644 index 0000000000..428d7e3ddb --- /dev/null +++ b/tools/sbom/testdata/expected_tagvalue_sbom_doc_describes_file.spdx @@ -0,0 +1,70 @@ +SPDXVersion: SPDX-2.3 +DataLicense: CC0-1.0 +SPDXID: SPDXRef-DOCUMENT +DocumentName: test doc +DocumentNamespace: http://www.google.com/sbom/spdx/android +Creator: Organization: Google +Created: 2023-03-31T22:17:58Z +ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890 + +FileName: file4.a +SPDXID: SPDXRef-file4 +FileChecksum: SHA1: 44444 + +Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-file4 + +PackageName: PRODUCT +SPDXID: SPDXRef-PRODUCT +PackageDownloadLocation: NONE +FilesAnalyzed: true +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +PackageVerificationCode: 123456 + +FileName: /bin/file1 +SPDXID: SPDXRef-file1 +FileChecksum: SHA1: 11111 + +FileName: /bin/file2 +SPDXID: SPDXRef-file2 +FileChecksum: SHA1: 22222 + +FileName: /bin/file3 +SPDXID: SPDXRef-file3 +FileChecksum: SHA1: 33333 + +PackageName: PLATFORM +SPDXID: SPDXRef-PLATFORM +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Prebuilt package1 +SPDXID: SPDXRef-PREBUILT-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Source package1 +SPDXID: SPDXRef-SOURCE-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4 + +PackageName: Upstream package1 +SPDXID: SPDXRef-UPSTREAM-package1 +PackageDownloadLocation: NOASSERTION +FilesAnalyzed: false +PackageVersion: 1.1 +PackageSupplier: Organization: upstream + +Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1 + +Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM +Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1 +Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1 +Relationship: SPDXRef-file1 STATIC_LINK SPDXRef-file4 diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx new file mode 100644 index 0000000000..a00c291ad7 --- /dev/null +++ b/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx @@ -0,0 +1,12 @@ +FileName: /bin/file1.apk +SPDXID: SPDXRef-file1 +FileChecksum: SHA1: 11111 + +PackageName: Unbundled apk package +SPDXID: SPDXRef-SOURCE-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-SOURCE-package1 diff --git a/tools/signapk/src/com/android/signapk/SignApk.java b/tools/signapk/src/com/android/signapk/SignApk.java index 25c53d328c..6b2341bc80 100644 --- a/tools/signapk/src/com/android/signapk/SignApk.java +++ b/tools/signapk/src/com/android/signapk/SignApk.java @@ -687,7 +687,7 @@ class SignApk { if (entryName.endsWith(".so")) { // Align .so contents to memory page boundary to enable memory-mapped // execution. - return 4096; + return 16384; } else { return defaultAlignment; } @@ -986,15 +986,17 @@ class SignApk { } private static class ZipSections { - ByteBuffer beforeCentralDir; + DataSource beforeCentralDir; + + // The following fields are still valid after closing the backing DataSource. + long beforeCentralDirSize; ByteBuffer centralDir; ByteBuffer eocd; } - private static ZipSections findMainZipSections(ByteBuffer apk) + private static ZipSections findMainZipSections(DataSource apk) throws IOException, ZipFormatException { - apk.slice(); - ApkUtils.ZipSections sections = ApkUtils.findZipSections(DataSources.asDataSource(apk)); + ApkUtils.ZipSections sections = ApkUtils.findZipSections(apk); long centralDirStartOffset = sections.getZipCentralDirectoryOffset(); long centralDirSizeBytes = sections.getZipCentralDirectorySizeBytes(); long centralDirEndOffset = centralDirStartOffset + centralDirSizeBytes; @@ -1005,25 +1007,20 @@ class SignApk { + ". CD end: " + centralDirEndOffset + ", EoCD start: " + eocdStartOffset); } - apk.position(0); - apk.limit((int) centralDirStartOffset); - ByteBuffer beforeCentralDir = apk.slice(); - apk.position((int) centralDirStartOffset); - apk.limit((int) centralDirEndOffset); - ByteBuffer centralDir = apk.slice(); + ZipSections result = new ZipSections(); - apk.position((int) eocdStartOffset); - apk.limit(apk.capacity()); - ByteBuffer eocd = apk.slice(); + result.beforeCentralDir = apk.slice(0, centralDirStartOffset); + result.beforeCentralDirSize = result.beforeCentralDir.size(); - apk.position(0); - apk.limit(apk.capacity()); + long centralDirSize = centralDirEndOffset - centralDirStartOffset; + if (centralDirSize >= Integer.MAX_VALUE) throw new IndexOutOfBoundsException(); + result.centralDir = apk.getByteBuffer(centralDirStartOffset, (int)centralDirSize); + + long eocdSize = apk.size() - eocdStartOffset; + if (eocdSize >= Integer.MAX_VALUE) throw new IndexOutOfBoundsException(); + result.eocd = apk.getByteBuffer(eocdStartOffset, (int)eocdSize); - ZipSections result = new ZipSections(); - result.beforeCentralDir = beforeCentralDir; - result.centralDir = centralDir; - result.eocd = eocd; return result; } @@ -1278,11 +1275,8 @@ class SignApk { // signatures) apkSigner.inputApkSigningBlock(null); - // Build the output APK in memory, by copying input APK's ZIP entries across - // and then signing the output APK. - ByteArrayOutputStream v1SignedApkBuf = new ByteArrayOutputStream(); CountingOutputStream outputJarCounter = - new CountingOutputStream(v1SignedApkBuf); + new CountingOutputStream(outputFile); JarOutputStream outputJar = new JarOutputStream(outputJarCounter); // Use maximum compression for compressed entries because the APK lives forever // on the system partition. @@ -1295,24 +1289,31 @@ class SignApk { addV1Signature(apkSigner, addV1SignatureRequest, outputJar, timestamp); addV1SignatureRequest.done(); } + + // close output and switch to input mode outputJar.close(); - ByteBuffer v1SignedApk = ByteBuffer.wrap(v1SignedApkBuf.toByteArray()); - v1SignedApkBuf.reset(); - ByteBuffer[] outputChunks = new ByteBuffer[] {v1SignedApk}; + outputJar = null; + outputJarCounter = null; + outputFile = null; + RandomAccessFile v1SignedApk = new RandomAccessFile(outputFilename, "r"); - ZipSections zipSections = findMainZipSections(v1SignedApk); + ZipSections zipSections = findMainZipSections(DataSources.asDataSource( + v1SignedApk)); ByteBuffer eocd = ByteBuffer.allocate(zipSections.eocd.remaining()); eocd.put(zipSections.eocd); eocd.flip(); eocd.order(ByteOrder.LITTLE_ENDIAN); + + ByteBuffer[] outputChunks = new ByteBuffer[] {}; + // This loop is supposed to be iterated twice at most. // The second pass is to align the file size after amending EOCD comments // with assumption that re-generated signing block would be the same size. while (true) { ApkSignerEngine.OutputApkSigningBlockRequest2 addV2SignatureRequest = apkSigner.outputZipSections2( - DataSources.asDataSource(zipSections.beforeCentralDir), + zipSections.beforeCentralDir, DataSources.asDataSource(zipSections.centralDir), DataSources.asDataSource(eocd)); if (addV2SignatureRequest == null) break; @@ -1330,11 +1331,10 @@ class SignApk { modifiedEocd.order(ByteOrder.LITTLE_ENDIAN); ApkUtils.setZipEocdCentralDirectoryOffset( modifiedEocd, - zipSections.beforeCentralDir.remaining() + padding + + zipSections.beforeCentralDir.size() + padding + apkSigningBlock.length); outputChunks = new ByteBuffer[] { - zipSections.beforeCentralDir, ByteBuffer.allocate(padding), ByteBuffer.wrap(apkSigningBlock), zipSections.centralDir, @@ -1348,7 +1348,7 @@ class SignApk { // Calculate the file size eocd = modifiedEocd; - int fileSize = 0; + long fileSize = zipSections.beforeCentralDirSize; for (ByteBuffer buf : outputChunks) { fileSize += buf.remaining(); } @@ -1357,7 +1357,7 @@ class SignApk { break; } // Pad EOCD comment to align the file size. - int commentLen = alignment - fileSize % alignment; + int commentLen = alignment - (int)(fileSize % alignment); modifiedEocd = ByteBuffer.allocate(eocd.remaining() + commentLen); modifiedEocd.put(eocd); modifiedEocd.rewind(); @@ -1368,6 +1368,12 @@ class SignApk { eocd = modifiedEocd; } + // close input and switch back to output mode + v1SignedApk.close(); + v1SignedApk = null; + outputFile = new FileOutputStream(outputFilename, true); + outputFile.getChannel().truncate(zipSections.beforeCentralDirSize); + // This assumes outputChunks are array-backed. To avoid this assumption, the // code could be rewritten to use FileChannel. for (ByteBuffer outputChunk : outputChunks) { diff --git a/tools/test_post_process_props.py b/tools/test_post_process_props.py index 236f9edfdf..2addefcb6f 100644 --- a/tools/test_post_process_props.py +++ b/tools/test_post_process_props.py @@ -255,23 +255,17 @@ class PropListTestcase(unittest.TestCase): stderr_redirect = io.StringIO() with contextlib.redirect_stderr(stderr_redirect): props = PropList("hello") - props.put("ro.board.first_api_level","25") - - # ro.board.first_api_level must be less than or equal to the sdk version - self.assertFalse(validate_grf_props(props, 20)) - self.assertTrue(validate_grf_props(props, 26)) - self.assertTrue(validate_grf_props(props, 35)) + props.put("ro.board.first_api_level","202504") + props.put("ro.build.version.codename", "REL") # manually set ro.board.api_level to an invalid value - props.put("ro.board.api_level","20") - self.assertFalse(validate_grf_props(props, 26)) + props.put("ro.board.api_level","202404") + self.assertFalse(validate_grf_props(props)) props.get_all_props()[-1].make_as_comment() # manually set ro.board.api_level to a valid value - props.put("ro.board.api_level","26") - self.assertTrue(validate_grf_props(props, 26)) - # ro.board.api_level must be less than or equal to the sdk version - self.assertFalse(validate_grf_props(props, 25)) + props.put("ro.board.api_level","202504") + self.assertTrue(validate_grf_props(props)) if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/tools/tool_event_logger/Android.bp b/tools/tool_event_logger/Android.bp new file mode 100644 index 0000000000..7a1d2aaa71 --- /dev/null +++ b/tools/tool_event_logger/Android.bp @@ -0,0 +1,67 @@ +// Copyright 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Set of error prone rules to ensure code quality +// PackageLocation check requires the androidCompatible=false otherwise it does not do anything. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], + default_team: "trendy_team_adte", +} + +python_library_host { + name: "tool_event_proto", + srcs: [ + "proto/tool_event.proto", + ], + proto: { + canonical_path_from_root: false, + }, +} + +python_binary_host { + name: "tool_event_logger", + pkg_path: "tool_event_logger", + srcs: [ + "tool_event_logger.py", + ], + libs: [ + "asuite_cc_client", + "tool_event_proto", + ], + main: "tool_event_logger.py", +} + +python_test_host { + name: "tool_event_logger_test", + main: "tool_event_logger_test.py", + pkg_path: "tool_event_logger", + srcs: [ + "tool_event_logger.py", + "tool_event_logger_test.py", + ], + test_options: { + unit_test: true, + }, + libs: [ + "asuite_cc_client", + "tool_event_proto", + ], + version: { + py3: { + embedded_launcher: true, + enabled: true, + }, + }, +} diff --git a/tools/tool_event_logger/OWNERS b/tools/tool_event_logger/OWNERS new file mode 100644 index 0000000000..b692c9edf3 --- /dev/null +++ b/tools/tool_event_logger/OWNERS @@ -0,0 +1,4 @@ +include platform/tools/asuite:/OWNERS + +zhuoyao@google.com +hzalek@google.com
\ No newline at end of file diff --git a/tools/tool_event_logger/proto/tool_event.proto b/tools/tool_event_logger/proto/tool_event.proto new file mode 100644 index 0000000000..61e28a25e7 --- /dev/null +++ b/tools/tool_event_logger/proto/tool_event.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package tools.asuite.tool_event_logger; + +message ToolEvent { + // Occurs immediately upon execution of the tool. + message InvocationStarted { + string command_args = 1; + string cwd = 2; + string os = 3; + } + + // Occurs when tool exits for any reason. + message InvocationStopped { + int32 exit_code = 2; + string exit_log = 3; + } + + // ------------------------ + // FIELDS FOR ToolEvent + // ------------------------ + // Random string generated to identify the invocation. + string invocation_id = 1; + // Internal user name. + string user_name = 2; + // The root of Android source. + string source_root = 3; + // Name of the tool used. + string tool_tag = 6; + + oneof event { + InvocationStarted invocation_started = 4; + InvocationStopped invocation_stopped = 5; + } +} diff --git a/tools/tool_event_logger/tool_event_logger.py b/tools/tool_event_logger/tool_event_logger.py new file mode 100644 index 0000000000..65a9696011 --- /dev/null +++ b/tools/tool_event_logger/tool_event_logger.py @@ -0,0 +1,229 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import datetime +import getpass +import logging +import os +import platform +import sys +import tempfile +import uuid + +from atest.metrics import clearcut_client +from atest.proto import clientanalytics_pb2 +from proto import tool_event_pb2 + +LOG_SOURCE = 2395 + + +class ToolEventLogger: + """Logs tool events to Sawmill through Clearcut.""" + + def __init__( + self, + tool_tag: str, + invocation_id: str, + user_name: str, + source_root: str, + platform_version: str, + python_version: str, + client: clearcut_client.Clearcut, + ): + self.tool_tag = tool_tag + self.invocation_id = invocation_id + self.user_name = user_name + self.source_root = source_root + self.platform_version = platform_version + self.python_version = python_version + self._clearcut_client = client + + @classmethod + def create(cls, tool_tag: str): + return ToolEventLogger( + tool_tag=tool_tag, + invocation_id=str(uuid.uuid4()), + user_name=getpass.getuser(), + source_root=os.environ.get('ANDROID_BUILD_TOP', ''), + platform_version=platform.platform(), + python_version=platform.python_version(), + client=clearcut_client.Clearcut(LOG_SOURCE), + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.flush() + + def log_invocation_started(self, event_time: datetime, command_args: str): + """Creates an event log with invocation started info.""" + event = self._create_tool_event() + event.invocation_started.CopyFrom( + tool_event_pb2.ToolEvent.InvocationStarted( + command_args=command_args, + os=f'{self.platform_version}:{self.python_version}', + ) + ) + + logging.debug('Log invocation_started: %s', event) + self._log_clearcut_event(event, event_time) + + def log_invocation_stopped( + self, + event_time: datetime, + exit_code: int, + exit_log: str, + ): + """Creates an event log with invocation stopped info.""" + event = self._create_tool_event() + event.invocation_stopped.CopyFrom( + tool_event_pb2.ToolEvent.InvocationStopped( + exit_code=exit_code, + exit_log=exit_log, + ) + ) + + logging.debug('Log invocation_stopped: %s', event) + self._log_clearcut_event(event, event_time) + + def flush(self): + """Sends all batched events to Clearcut.""" + logging.debug('Sending events to Clearcut.') + self._clearcut_client.flush_events() + + def _create_tool_event(self): + return tool_event_pb2.ToolEvent( + tool_tag=self.tool_tag, + invocation_id=self.invocation_id, + user_name=self.user_name, + source_root=self.source_root, + ) + + def _log_clearcut_event( + self, tool_event: tool_event_pb2.ToolEvent, event_time: datetime + ): + log_event = clientanalytics_pb2.LogEvent( + event_time_ms=int(event_time.timestamp() * 1000), + source_extension=tool_event.SerializeToString(), + ) + self._clearcut_client.log(log_event) + + +class ArgumentParserWithLogging(argparse.ArgumentParser): + + def error(self, message): + logging.error('Failed to parse args with error: %s', message) + super().error(message) + + +def create_arg_parser(): + """Creates an instance of the default ToolEventLogger arg parser.""" + + parser = ArgumentParserWithLogging( + description='Build and upload logs for Android dev tools', + add_help=True, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + '--tool_tag', + type=str, + required=True, + help='Name of the tool.', + ) + + parser.add_argument( + '--start_timestamp', + type=lambda ts: datetime.datetime.fromtimestamp(float(ts)), + required=True, + help=( + 'Timestamp when the tool starts. The timestamp should have the format' + '%s.%N which represents the seconds elapses since epoch.' + ), + ) + + parser.add_argument( + '--end_timestamp', + type=lambda ts: datetime.datetime.fromtimestamp(float(ts)), + required=True, + help=( + 'Timestamp when the tool exits. The timestamp should have the format' + '%s.%N which represents the seconds elapses since epoch.' + ), + ) + + parser.add_argument( + '--tool_args', + type=str, + help='Parameters that are passed to the tool.', + ) + + parser.add_argument( + '--exit_code', + type=int, + required=True, + help='Tool exit code.', + ) + + parser.add_argument( + '--exit_log', + type=str, + help='Logs when tool exits.', + ) + + parser.add_argument( + '--dry_run', + action='store_true', + help='Dry run the tool event logger if set.', + ) + + return parser + + +def configure_logging(): + root_logging_dir = tempfile.mkdtemp(prefix='tool_event_logger_') + + log_fmt = '%(asctime)s %(filename)s:%(lineno)s:%(levelname)s: %(message)s' + date_fmt = '%Y-%m-%d %H:%M:%S' + _, log_path = tempfile.mkstemp(dir=root_logging_dir, suffix='.log') + + logging.basicConfig( + filename=log_path, level=logging.DEBUG, format=log_fmt, datefmt=date_fmt + ) + + +def main(argv: list[str]): + args = create_arg_parser().parse_args(argv[1:]) + + if args.dry_run: + logging.debug('This is a dry run.') + return + + try: + with ToolEventLogger.create(args.tool_tag) as logger: + logger.log_invocation_started(args.start_timestamp, args.tool_args) + logger.log_invocation_stopped( + args.end_timestamp, args.exit_code, args.exit_log + ) + except Exception as e: + logging.error('Log failed with unexpected error: %s', e) + raise + + +if __name__ == '__main__': + configure_logging() + main(sys.argv) diff --git a/tools/tool_event_logger/tool_event_logger_test.py b/tools/tool_event_logger/tool_event_logger_test.py new file mode 100644 index 0000000000..34b6c357cc --- /dev/null +++ b/tools/tool_event_logger/tool_event_logger_test.py @@ -0,0 +1,209 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittests for ToolEventLogger.""" + +import datetime +import logging +import unittest +from unittest import mock + +from atest.metrics import clearcut_client +from proto import tool_event_pb2 +from tool_event_logger import tool_event_logger + +TEST_INVOCATION_ID = 'test_invocation_id' +TEST_USER_NAME = 'test_user' +TEST_TOOL_TAG = 'test_tool' +TEST_SOURCE_ROOT = 'test_source_root' +TEST_PLATFORM_VERSION = 'test_platform_version' +TEST_PYTHON_VERSION = 'test_python_version' +TEST_EVENT_TIMESTAMP = datetime.datetime.now() + + +class ToolEventLoggerTest(unittest.TestCase): + + def setUp(self): + super().setUp() + self.clearcut_client = FakeClearcutClient() + self.logger = tool_event_logger.ToolEventLogger( + TEST_TOOL_TAG, + TEST_INVOCATION_ID, + TEST_USER_NAME, + TEST_SOURCE_ROOT, + TEST_PLATFORM_VERSION, + TEST_PYTHON_VERSION, + client=self.clearcut_client, + ) + + def test_log_event_timestamp(self): + with self.logger: + self.logger.log_invocation_started( + datetime.datetime.fromtimestamp(100.101), 'test_command' + ) + + self.assertEqual( + self.clearcut_client.get_last_sent_event().event_time_ms, 100101 + ) + + def test_log_event_basic_information(self): + with self.logger: + self.logger.log_invocation_started(TEST_EVENT_TIMESTAMP, 'test_command') + + sent_event = self.clearcut_client.get_last_sent_event() + log_event = tool_event_pb2.ToolEvent.FromString(sent_event.source_extension) + self.assertEqual(log_event.invocation_id, TEST_INVOCATION_ID) + self.assertEqual(log_event.user_name, TEST_USER_NAME) + self.assertEqual(log_event.tool_tag, TEST_TOOL_TAG) + self.assertEqual(log_event.source_root, TEST_SOURCE_ROOT) + + def test_log_invocation_started(self): + expected_invocation_started = tool_event_pb2.ToolEvent.InvocationStarted( + command_args='test_command', + os=TEST_PLATFORM_VERSION + ':' + TEST_PYTHON_VERSION, + ) + + with self.logger: + self.logger.log_invocation_started(TEST_EVENT_TIMESTAMP, 'test_command') + + self.assertEqual(self.clearcut_client.get_number_of_sent_events(), 1) + sent_event = self.clearcut_client.get_last_sent_event() + self.assertEqual( + expected_invocation_started, + tool_event_pb2.ToolEvent.FromString( + sent_event.source_extension + ).invocation_started, + ) + + def test_log_invocation_stopped(self): + expected_invocation_stopped = tool_event_pb2.ToolEvent.InvocationStopped( + exit_code=0, + exit_log='exit_log', + ) + + with self.logger: + self.logger.log_invocation_stopped(TEST_EVENT_TIMESTAMP, 0, 'exit_log') + + self.assertEqual(self.clearcut_client.get_number_of_sent_events(), 1) + sent_event = self.clearcut_client.get_last_sent_event() + self.assertEqual( + expected_invocation_stopped, + tool_event_pb2.ToolEvent.FromString( + sent_event.source_extension + ).invocation_stopped, + ) + + def test_log_multiple_events(self): + with self.logger: + self.logger.log_invocation_started(TEST_EVENT_TIMESTAMP, 'test_command') + self.logger.log_invocation_stopped(TEST_EVENT_TIMESTAMP, 0, 'exit_log') + + self.assertEqual(self.clearcut_client.get_number_of_sent_events(), 2) + + +class MainTest(unittest.TestCase): + + REQUIRED_ARGS = [ + '', + '--tool_tag', + 'test_tool', + '--start_timestamp', + '1', + '--end_timestamp', + '2', + '--exit_code', + '0', + ] + + def test_log_and_exit_with_missing_required_args(self): + with self.assertLogs() as logs: + with self.assertRaises(SystemExit) as ex: + tool_event_logger.main(['', '--tool_tag', 'test_tool']) + + with self.subTest('Verify exception code'): + self.assertEqual(ex.exception.code, 2) + + with self.subTest('Verify log messages'): + self.assertIn( + 'the following arguments are required', + '\n'.join(logs.output), + ) + + def test_log_and_exit_with_invalid_args(self): + with self.assertLogs() as logs: + with self.assertRaises(SystemExit) as ex: + tool_event_logger.main(['', '--start_timestamp', 'test']) + + with self.subTest('Verify exception code'): + self.assertEqual(ex.exception.code, 2) + + with self.subTest('Verify log messages'): + self.assertIn( + '--start_timestamp: invalid', + '\n'.join(logs.output), + ) + + def test_log_and_exit_with_dry_run(self): + with self.assertLogs(level=logging.DEBUG) as logs: + tool_event_logger.main(self.REQUIRED_ARGS + ['--dry_run']) + + with self.subTest('Verify log messages'): + self.assertIn('dry run', '\n'.join(logs.output)) + + @mock.patch.object(clearcut_client, 'Clearcut') + def test_log_and_exit_with_unexpected_exception(self, mock_cc): + mock_cc.return_value = FakeClearcutClient(raise_log_exception=True) + + with self.assertLogs() as logs: + with self.assertRaises(Exception) as ex: + tool_event_logger.main(self.REQUIRED_ARGS) + + with self.subTest('Verify log messages'): + self.assertIn('unexpected error', '\n'.join(logs.output)) + + @mock.patch.object(clearcut_client, 'Clearcut') + def test_success(self, mock_cc): + mock_clear_cut_client = FakeClearcutClient() + mock_cc.return_value = mock_clear_cut_client + + tool_event_logger.main(self.REQUIRED_ARGS) + + self.assertEqual(mock_clear_cut_client.get_number_of_sent_events(), 2) + + +class FakeClearcutClient: + + def __init__(self, raise_log_exception=False): + self.pending_log_events = [] + self.sent_log_events = [] + self.raise_log_exception = raise_log_exception + + def log(self, log_event): + if self.raise_log_exception: + raise Exception('unknown exception') + self.pending_log_events.append(log_event) + + def flush_events(self): + self.sent_log_events.extend(self.pending_log_events) + self.pending_log_events.clear() + + def get_number_of_sent_events(self): + return len(self.sent_log_events) + + def get_last_sent_event(self): + return self.sent_log_events[-1] + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/whichgit b/tools/whichgit index b0bf2e42f8..55c8c6fb5a 100755 --- a/tools/whichgit +++ b/tools/whichgit @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import itertools import os import subprocess import sys @@ -10,15 +11,34 @@ def get_build_var(var): check=True, capture_output=True, text=True).stdout.strip() +def get_all_modules(): + product_out = subprocess.run(["build/soong/soong_ui.bash", "--dumpvar-mode", "--abs", "PRODUCT_OUT"], + check=True, capture_output=True, text=True).stdout.strip() + result = subprocess.run(["cat", product_out + "/all_modules.txt"], check=True, capture_output=True, text=True) + return result.stdout.strip().split("\n") + + +def batched(iterable, n): + # introduced in itertools 3.12, could delete once that's universally available + if n < 1: + raise ValueError('n must be at least one') + it = iter(iterable) + while batch := tuple(itertools.islice(it, n)): + yield batch + + def get_sources(modules): - result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f", - "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja", - "-t", "inputs", "-d", ] + modules, - stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True) - if result.returncode != 0: - sys.stderr.write(result.stdout) - sys.exit(1) - return set([f for f in result.stdout.split("\n") if not f.startswith("out/")]) + sources = set() + for module_group in batched(modules, 40_000): + result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f", + "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja", + "-t", "inputs", "-d", ] + list(module_group), + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True) + if result.returncode != 0: + sys.stderr.write(result.stdout) + sys.exit(1) + sources.update(set([f for f in result.stdout.split("\n") if not f.startswith("out/")])) + return sources def m_nothing(): @@ -50,57 +70,76 @@ def get_referenced_projects(git_dirs, files): referenced_dirs.add(d) prev_dir = d break - return [d[0:-1] for d in referenced_dirs] + return referenced_dirs def main(argv): # Argument parsing ap = argparse.ArgumentParser(description="List the required git projects for the given modules") ap.add_argument("--products", nargs="*", - help="The TARGET_PRODUCT to check. If not provided just uses whatever has" - + " already been built") + help="One or more TARGET_PRODUCT to check, or \"*\" for all. If not provided" + + "just uses whatever has already been built") ap.add_argument("--variants", nargs="*", help="The TARGET_BUILD_VARIANTS to check. If not provided just uses whatever has" + " already been built, or eng if --products is supplied") ap.add_argument("--modules", nargs="*", - help="The build modules to check, or droid it not supplied") + help="The build modules to check, or \"*\" for all, or droid if not supplied") ap.add_argument("--why", nargs="*", help="Also print the input files used in these projects, or \"*\" for all") + ap.add_argument("--unused", help="List the unused git projects for the given modules rather than" + + "the used ones. Ignores --why", action="store_true") args = ap.parse_args(argv[1:]) modules = args.modules if args.modules else ["droid"] + match args.products: + case ["*"]: + products = get_build_var("all_named_products").split(" ") + case _: + products = args.products + # Get the list of sources for all of the requested build combos - if not args.products and not args.variants: + if not products and not args.variants: + m_nothing() + if args.modules == ["*"]: + modules = get_all_modules() sources = get_sources(modules) else: - if not args.products: + if not products: sys.stderr.write("Error: --products must be supplied if --variants is supplied") sys.exit(1) sources = set() build_num = 1 - for product in args.products: + for product in products: os.environ["TARGET_PRODUCT"] = product variants = args.variants if args.variants else ["user", "userdebug", "eng"] for variant in variants: - sys.stderr.write(f"Analyzing build {build_num} of {len(args.products)*len(variants)}\r") + sys.stderr.write(f"Analyzing build {build_num} of {len(products)*len(variants)}\r") os.environ["TARGET_BUILD_VARIANT"] = variant m_nothing() + if args.modules == ["*"]: + modules = get_all_modules() sources.update(get_sources(modules)) build_num += 1 sys.stderr.write("\n\n") sources = sorted(sources) - # Print the list of git directories that has one or more of the sources in it - for project in sorted(get_referenced_projects(get_git_dirs(), sources)): - print(project) - if args.why: - if "*" in args.why or project in args.why: - prefix = project + "/" - for f in sources: - if f.startswith(prefix): - print(" " + f) + if args.unused: + # Print the list of git directories that don't contain sources + used_git_dirs = set(get_git_dirs()) + for project in sorted(used_git_dirs.difference(set(get_referenced_projects(used_git_dirs, sources)))): + print(project[0:-1]) + else: + # Print the list of git directories that has one or more of the sources in it + for project in sorted(get_referenced_projects(get_git_dirs(), sources)): + print(project[0:-1]) + if args.why: + if "*" in args.why or project[0:-1] in args.why: + prefix = project + for f in sources: + if f.startswith(prefix): + print(" " + f) if __name__ == "__main__": diff --git a/tools/zipalign/Android.bp b/tools/zipalign/Android.bp index 0e1d58ed74..8be7e25f22 100644 --- a/tools/zipalign/Android.bp +++ b/tools/zipalign/Android.bp @@ -70,6 +70,7 @@ cc_test_host { "libgmock", ], data: [ + "tests/data/apkWithUncompressedSharedLibs.zip", "tests/data/archiveWithOneDirectoryEntry.zip", "tests/data/diffOrders.zip", "tests/data/holes.zip", diff --git a/tools/zipalign/ZipAlign.cpp b/tools/zipalign/ZipAlign.cpp index 23840e3945..f32f90b130 100644 --- a/tools/zipalign/ZipAlign.cpp +++ b/tools/zipalign/ZipAlign.cpp @@ -17,6 +17,7 @@ #include "ZipFile.h" #include <stdio.h> +#include <string.h> #include <stdlib.h> #include <unistd.h> @@ -36,17 +37,14 @@ static bool isDirectory(ZipEntry* entry) { } static int getAlignment(bool pageAlignSharedLibs, int defaultAlignment, - ZipEntry* pEntry) { - - static const int kPageAlignment = 4096; - + ZipEntry* pEntry, int pageSize) { if (!pageAlignSharedLibs) { return defaultAlignment; } const char* ext = strrchr(pEntry->getFileName(), '.'); if (ext && strcmp(ext, ".so") == 0) { - return kPageAlignment; + return pageSize; } return defaultAlignment; @@ -56,7 +54,7 @@ static int getAlignment(bool pageAlignSharedLibs, int defaultAlignment, * Copy all entries from "pZin" to "pZout", aligning as needed. */ static int copyAndAlign(ZipFile* pZin, ZipFile* pZout, int alignment, bool zopfli, - bool pageAlignSharedLibs) + bool pageAlignSharedLibs, int pageSize) { int numEntries = pZin->getNumEntries(); ZipEntry* pEntry; @@ -84,7 +82,8 @@ static int copyAndAlign(ZipFile* pZin, ZipFile* pZout, int alignment, bool zopfl status = pZout->add(pZin, pEntry, padding, &pNewEntry); } } else { - const int alignTo = getAlignment(pageAlignSharedLibs, alignment, pEntry); + const int alignTo = getAlignment(pageAlignSharedLibs, alignment, pEntry, + pageSize); //printf("--- %s: orig at %ld(+%d) len=%ld, adding pad=%d\n", // pEntry->getFileName(), (long) pEntry->getFileOffset(), @@ -107,7 +106,7 @@ static int copyAndAlign(ZipFile* pZin, ZipFile* pZout, int alignment, bool zopfl * output file exists and "force" wasn't specified. */ int process(const char* inFileName, const char* outFileName, - int alignment, bool force, bool zopfli, bool pageAlignSharedLibs) + int alignment, bool force, bool zopfli, bool pageAlignSharedLibs, int pageSize) { ZipFile zin, zout; @@ -127,7 +126,7 @@ int process(const char* inFileName, const char* outFileName, } if (zin.open(inFileName, ZipFile::kOpenReadOnly) != OK) { - fprintf(stderr, "Unable to open '%s' as zip archive\n", inFileName); + fprintf(stderr, "Unable to open '%s' as zip archive: %s\n", inFileName, strerror(errno)); return 1; } if (zout.open(outFileName, @@ -138,7 +137,8 @@ int process(const char* inFileName, const char* outFileName, return 1; } - int result = copyAndAlign(&zin, &zout, alignment, zopfli, pageAlignSharedLibs); + int result = copyAndAlign(&zin, &zout, alignment, zopfli, pageAlignSharedLibs, + pageSize); if (result != 0) { printf("zipalign: failed rewriting '%s' to '%s'\n", inFileName, outFileName); @@ -150,7 +150,7 @@ int process(const char* inFileName, const char* outFileName, * Verify the alignment of a zip archive. */ int verify(const char* fileName, int alignment, bool verbose, - bool pageAlignSharedLibs) + bool pageAlignSharedLibs, int pageSize) { ZipFile zipFile; bool foundBad = false; @@ -181,7 +181,8 @@ int verify(const char* fileName, int alignment, bool verbose, continue; } else { off_t offset = pEntry->getFileOffset(); - const int alignTo = getAlignment(pageAlignSharedLibs, alignment, pEntry); + const int alignTo = getAlignment(pageAlignSharedLibs, alignment, pEntry, + pageSize); if ((offset % alignTo) != 0) { if (verbose) { printf("%8jd %s (BAD - %jd)\n", diff --git a/tools/zipalign/ZipAlignMain.cpp b/tools/zipalign/ZipAlignMain.cpp index 53fc8d4f62..2f24403a1a 100644 --- a/tools/zipalign/ZipAlignMain.cpp +++ b/tools/zipalign/ZipAlignMain.cpp @@ -34,15 +34,18 @@ void usage(void) fprintf(stderr, "Zip alignment utility\n"); fprintf(stderr, "Copyright (C) 2009 The Android Open Source Project\n\n"); fprintf(stderr, - "Usage: zipalign [-f] [-p] [-v] [-z] <align> infile.zip outfile.zip\n" - " zipalign -c [-p] [-v] <align> infile.zip\n\n" ); + "Usage: zipalign [-f] [-p] [-P <pagesize_kb>] [-v] [-z] <align> infile.zip outfile.zip\n" + " zipalign -c [-p] [-P <pagesize_kb>] [-v] <align> infile.zip\n\n" ); fprintf(stderr, " <align>: alignment in bytes, e.g. '4' provides 32-bit alignment\n"); fprintf(stderr, " -c: check alignment only (does not modify file)\n"); fprintf(stderr, " -f: overwrite existing outfile.zip\n"); - fprintf(stderr, " -p: page-align uncompressed .so files\n"); + fprintf(stderr, " -p: 4kb page-align uncompressed .so files\n"); fprintf(stderr, " -v: verbose output\n"); fprintf(stderr, " -z: recompress using Zopfli\n"); + fprintf(stderr, " -P <pagesize_kb>: Align uncompressed .so files to the specified\n"); + fprintf(stderr, " page size. Valid values for <pagesize_kb> are 4, 16\n"); + fprintf(stderr, " and 64. '-P' cannot be used in combination with '-p'.\n"); } @@ -57,12 +60,16 @@ int main(int argc, char* const argv[]) bool verbose = false; bool zopfli = false; bool pageAlignSharedLibs = false; + int pageSize = 4096; + bool legacyPageAlignmentFlag = false; // -p + bool pageAlignmentFlag = false; // -P <pagesize_kb> int result = 1; int alignment; char* endp; int opt; - while ((opt = getopt(argc, argv, "fcpvz")) != -1) { + + while ((opt = getopt(argc, argv, "fcpvzP:")) != -1) { switch (opt) { case 'c': check = true; @@ -77,7 +84,29 @@ int main(int argc, char* const argv[]) zopfli = true; break; case 'p': + legacyPageAlignmentFlag = true; pageAlignSharedLibs = true; + pageSize = 4096; + break; + case 'P': + pageAlignmentFlag = true; + pageAlignSharedLibs = true; + + if (!optarg) { + fprintf(stderr, "ERROR: -P requires an argument\n"); + wantUsage = true; + goto bail; + } + + pageSize = atoi(optarg); + if (pageSize != 4 && pageSize != 16 && pageSize != 64) { + fprintf(stderr, "ERROR: Invalid argument for -P: %s\n", optarg); + wantUsage = true; + goto bail; + } + + pageSize *= 1024; // Convert from kB to bytes. + break; default: fprintf(stderr, "ERROR: unknown flag -%c\n", opt); @@ -86,6 +115,13 @@ int main(int argc, char* const argv[]) } } + if (legacyPageAlignmentFlag && pageAlignmentFlag) { + fprintf(stderr, "ERROR: Invalid options: '-P <pagesize_kb>' and '-p'" + "cannot be used in combination.\n"); + wantUsage = true; + goto bail; + } + if (!((check && (argc - optind) == 2) || (!check && (argc - optind) == 3))) { wantUsage = true; goto bail; @@ -100,14 +136,15 @@ int main(int argc, char* const argv[]) if (check) { /* check existing archive for correct alignment */ - result = verify(argv[optind + 1], alignment, verbose, pageAlignSharedLibs); + result = verify(argv[optind + 1], alignment, verbose, pageAlignSharedLibs, pageSize); } else { /* create the new archive */ - result = process(argv[optind + 1], argv[optind + 2], alignment, force, zopfli, pageAlignSharedLibs); + result = process(argv[optind + 1], argv[optind + 2], alignment, force, zopfli, + pageAlignSharedLibs, pageSize); /* trust, but verify */ if (result == 0) { - result = verify(argv[optind + 2], alignment, verbose, pageAlignSharedLibs); + result = verify(argv[optind + 2], alignment, verbose, pageAlignSharedLibs, pageSize); } } diff --git a/tools/zipalign/ZipEntry.cpp b/tools/zipalign/ZipEntry.cpp index 689999ee18..0355e56238 100644 --- a/tools/zipalign/ZipEntry.cpp +++ b/tools/zipalign/ZipEntry.cpp @@ -18,6 +18,8 @@ // Access to entries in a Zip archive. // +#define _POSIX_THREAD_SAFE_FUNCTIONS // For mingw localtime_r(). + #define LOG_TAG "zip" #include "ZipEntry.h" @@ -354,31 +356,29 @@ time_t ZipEntry::getModWhen(void) const */ void ZipEntry::setModWhen(time_t when) { -#if !defined(_WIN32) - struct tm tmResult; -#endif - time_t even; - uint16_t zdate, ztime; - - struct tm* ptm; - /* round up to an even number of seconds */ - even = (when & 1) ? (when + 1) : when; + time_t even = (when & 1) ? (when + 1) : when; /* expand */ -#if !defined(_WIN32) - ptm = localtime_r(&even, &tmResult); -#else - ptm = localtime(&even); -#endif - - int year; - year = ptm->tm_year; - if (year < 80) - year = 80; - - zdate = (year - 80) << 9 | (ptm->tm_mon+1) << 5 | ptm->tm_mday; - ztime = ptm->tm_hour << 11 | ptm->tm_min << 5 | ptm->tm_sec >> 1; + struct tm tmResult; + struct tm* ptm = localtime_r(&even, &tmResult); + + // The earliest valid time for ZIP file entries is 1980-01-01. See: + // https://users.cs.jmu.edu/buchhofp/forensics/formats/pkzip.html. + // Set any time before 1980 to 1980-01-01. + if (ptm->tm_year < 80) { + ptm->tm_year = 80; + ptm->tm_mon = 0; + ptm->tm_mday = 1; + ptm->tm_hour = 0; + ptm->tm_min = 0; + ptm->tm_sec = 0; + } + + uint16_t zdate = static_cast<uint16_t>( + (ptm->tm_year - 80) << 9 | (ptm->tm_mon + 1) << 5 | ptm->tm_mday); + uint16_t ztime = static_cast<uint16_t>( + ptm->tm_hour << 11 | ptm->tm_min << 5 | ptm->tm_sec >> 1); mCDE.mLastModFileTime = mLFH.mLastModFileTime = ztime; mCDE.mLastModFileDate = mLFH.mLastModFileDate = zdate; diff --git a/tools/zipalign/include/ZipAlign.h b/tools/zipalign/include/ZipAlign.h index ab3608682c..85dda1400a 100644 --- a/tools/zipalign/include/ZipAlign.h +++ b/tools/zipalign/include/ZipAlign.h @@ -25,24 +25,28 @@ namespace android { * - force : Overwrite output if it exists, fail otherwise. * - zopfli : Recompress compressed entries with more efficient algorithm. * Copy compressed entries as-is, and unaligned, otherwise. - * - pageAlignSharedLibs: Align .so files to 4096 and other files to + * - pageAlignSharedLibs: Align .so files to @pageSize and other files to * alignTo, or all files to alignTo if false.. + * - pageSize: Specifies the page size of the target device. This is used + * to correctly page-align shared libraries. * * Returns 0 on success. */ int process(const char* input, const char* output, int alignTo, bool force, - bool zopfli, bool pageAlignSharedLibs); + bool zopfli, bool pageAlignSharedLibs, int pageSize); /* * Verify the alignment of a zip archive. * - alignTo: Alignment (in bytes) for uncompressed entries. - * - pageAlignSharedLibs: Align .so files to 4096 and other files to + * - pageAlignSharedLibs: Align .so files to @pageSize and other files to * alignTo, or all files to alignTo if false.. + * - pageSize: Specifies the page size of the target device. This is used + * to correctly page-align shared libraries. * * Returns 0 on success. */ int verify(const char* fileName, int alignTo, bool verbose, - bool pageAlignSharedLibs); + bool pageAlignSharedLibs, int pageSize); } // namespace android diff --git a/tools/zipalign/tests/data/apkWithUncompressedSharedLibs.zip b/tools/zipalign/tests/data/apkWithUncompressedSharedLibs.zip Binary files differnew file mode 100644 index 0000000000..930e3b5010 --- /dev/null +++ b/tools/zipalign/tests/data/apkWithUncompressedSharedLibs.zip diff --git a/tools/zipalign/tests/src/align_test.cpp b/tools/zipalign/tests/src/align_test.cpp index a8433fad47..07ad7ccda2 100644 --- a/tools/zipalign/tests/src/align_test.cpp +++ b/tools/zipalign/tests/src/align_test.cpp @@ -48,11 +48,12 @@ static std::string GetTempPath(const std::string& filename) { TEST(Align, Unaligned) { const std::string src = GetTestPath("unaligned.zip"); const std::string dst = GetTempPath("unaligned_out.zip"); + int pageSize = 4096; - int processed = process(src.c_str(), dst.c_str(), 4, true, false, 4096); + int processed = process(src.c_str(), dst.c_str(), 4, true, false, false, pageSize); ASSERT_EQ(0, processed); - int verified = verify(dst.c_str(), 4, true, false); + int verified = verify(dst.c_str(), 4, true, false, pageSize); ASSERT_EQ(0, verified); } @@ -60,18 +61,19 @@ TEST(Align, DoubleAligment) { const std::string src = GetTestPath("unaligned.zip"); const std::string tmp = GetTempPath("da_aligned.zip"); const std::string dst = GetTempPath("da_d_aligner.zip"); + int pageSize = 4096; - int processed = process(src.c_str(), tmp.c_str(), 4, true, false, 4096); + int processed = process(src.c_str(), tmp.c_str(), 4, true, false, false, pageSize); ASSERT_EQ(0, processed); - int verified = verify(tmp.c_str(), 4, true, false); + int verified = verify(tmp.c_str(), 4, true, false, pageSize); ASSERT_EQ(0, verified); // Align the result of the previous run. Essentially double aligning. - processed = process(tmp.c_str(), dst.c_str(), 4, true, false, 4096); + processed = process(tmp.c_str(), dst.c_str(), 4, true, false, false, pageSize); ASSERT_EQ(0, processed); - verified = verify(dst.c_str(), 4, true, false); + verified = verify(dst.c_str(), 4, true, false, pageSize); ASSERT_EQ(0, verified); // Nothing should have changed between tmp and dst. @@ -90,11 +92,12 @@ TEST(Align, DoubleAligment) { TEST(Align, Holes) { const std::string src = GetTestPath("holes.zip"); const std::string dst = GetTempPath("holes_out.zip"); + int pageSize = 4096; - int processed = process(src.c_str(), dst.c_str(), 4, true, false, 4096); + int processed = process(src.c_str(), dst.c_str(), 4, true, false, true, pageSize); ASSERT_EQ(0, processed); - int verified = verify(dst.c_str(), 4, false, true); + int verified = verify(dst.c_str(), 4, false, true, pageSize); ASSERT_EQ(0, verified); } @@ -102,28 +105,85 @@ TEST(Align, Holes) { TEST(Align, DifferenteOrders) { const std::string src = GetTestPath("diffOrders.zip"); const std::string dst = GetTempPath("diffOrders_out.zip"); + int pageSize = 4096; - int processed = process(src.c_str(), dst.c_str(), 4, true, false, 4096); + int processed = process(src.c_str(), dst.c_str(), 4, true, false, true, pageSize); ASSERT_EQ(0, processed); - int verified = verify(dst.c_str(), 4, false, true); + int verified = verify(dst.c_str(), 4, false, true, pageSize); ASSERT_EQ(0, verified); } TEST(Align, DirectoryEntryDoNotRequireAlignment) { const std::string src = GetTestPath("archiveWithOneDirectoryEntry.zip"); - int verified = verify(src.c_str(), 4, false, true); + int pageSize = 4096; + int verified = verify(src.c_str(), 4, false, true, pageSize); ASSERT_EQ(0, verified); } TEST(Align, DirectoryEntry) { const std::string src = GetTestPath("archiveWithOneDirectoryEntry.zip"); const std::string dst = GetTempPath("archiveWithOneDirectoryEntry_out.zip"); + int pageSize = 4096; - int processed = process(src.c_str(), dst.c_str(), 4, true, false, 4096); + int processed = process(src.c_str(), dst.c_str(), 4, true, false, true, pageSize); ASSERT_EQ(0, processed); ASSERT_EQ(true, sameContent(src, dst)); - int verified = verify(dst.c_str(), 4, false, true); + int verified = verify(dst.c_str(), 4, false, true, pageSize); + ASSERT_EQ(0, verified); +} + +class UncompressedSharedLibsTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { + src = GetTestPath("apkWithUncompressedSharedLibs.zip"); + dst = GetTempPath("apkWithUncompressedSharedLibs_out.zip"); + } + + static std::string src; + static std::string dst; +}; + +std::string UncompressedSharedLibsTest::src; +std::string UncompressedSharedLibsTest::dst; + +TEST_F(UncompressedSharedLibsTest, Unaligned) { + int pageSize = 4096; + + int processed = process(src.c_str(), dst.c_str(), 4, true, false, false, pageSize); + ASSERT_EQ(0, processed); + + int verified = verify(dst.c_str(), 4, true, true, pageSize); + ASSERT_NE(0, verified); // .so's not page-aligned +} + +TEST_F(UncompressedSharedLibsTest, AlignedPageSize4kB) { + int pageSize = 4096; + + int processed = process(src.c_str(), dst.c_str(), 4, true, false, true, pageSize); + ASSERT_EQ(0, processed); + + int verified = verify(dst.c_str(), 4, true, true, pageSize); + ASSERT_EQ(0, verified); +} + +TEST_F(UncompressedSharedLibsTest, AlignedPageSize16kB) { + int pageSize = 16384; + + int processed = process(src.c_str(), dst.c_str(), 4, true, false, true, pageSize); + ASSERT_EQ(0, processed); + + int verified = verify(dst.c_str(), 4, true, true, pageSize); + ASSERT_EQ(0, verified); +} + +TEST_F(UncompressedSharedLibsTest, AlignedPageSize64kB) { + int pageSize = 65536; + + int processed = process(src.c_str(), dst.c_str(), 4, true, false, true, pageSize); + ASSERT_EQ(0, processed); + + int verified = verify(dst.c_str(), 4, true, true, pageSize); ASSERT_EQ(0, verified); } |