diff options
| -rw-r--r-- | init/init_ddk.py | 117 | ||||
| -rw-r--r-- | init/init_ddk_test.py | 63 | ||||
| -rw-r--r-- | kleaf/tests/integration_test/ddk_workspace_test/extra_setup.py | 30 |
3 files changed, 140 insertions, 70 deletions
diff --git a/init/init_ddk.py b/init/init_ddk.py index a93306b..998dd02 100644 --- a/init/init_ddk.py +++ b/init/init_ddk.py @@ -27,7 +27,6 @@ import subprocess import sys import tarfile import tempfile -import textwrap import urllib.parse _TOOLS_BAZEL = "tools/bazel" @@ -124,8 +123,9 @@ class KleafProjectSetter: def _try_rel_workspace(self, path: pathlib.Path): """Tries to convert |path| to be relative to ddk_workspace.""" if not self.ddk_workspace: - raise KleafProjectSetterError("ERROR: _try_rel_workspace called " - "without --ddk_workspace set!") + raise KleafProjectSetterError( + "ERROR: _try_rel_workspace called without --ddk_workspace set!" + ) try: return path.relative_to(self.ddk_workspace) except ValueError: @@ -137,6 +137,31 @@ class KleafProjectSetter: ) return path + def _get_local_path_overrides(self): + """Naive algorithm to extract local_path_override()'s from local @kleaf.""" + path_attr_prefix = 'path = "' + section = [] + overrides = [] + module_bazel = self.kleaf_repo / _MODULE_BAZEL_FILE + # Modify path so it is relative to the current DDK workspace. + kleaf_repo = self._try_rel_workspace(self.kleaf_repo) + with open(module_bazel, "r", encoding="utf-8") as src: + for line in src: + if line.startswith("local_path_override("): + section.append(line) + continue + if not section: + continue + elif line.lstrip().startswith(path_attr_prefix): + line = line.strip().removeprefix(path_attr_prefix) + line = line.removesuffix('",') + line = f' path = "{kleaf_repo / line}",\n' + section.append(line) + if line.strip() == ")": + overrides.append("".join(section)) + section.clear() + return "".join(overrides) + def _generate_module_bazel(self): """Configures the dependencies for the DDK workspace.""" if not self.ddk_workspace: @@ -147,6 +172,12 @@ class KleafProjectSetter: module_bazel_content += _KLEAF_DEPENDENCY_TEMPLATE.format( kleaf_repo_relative=self._try_rel_workspace(self.kleaf_repo), ) + module_bazel_content += self._get_local_path_overrides() + # b/338440785 Due to an issue in Bazel, rules_cc seems to be + # implicitly added in a fallback WORKSPACE.bzlmod file, hence + # forcing an empty one here. + workspace_bzlmod = self.ddk_workspace / "WORKSPACE.bzlmod" + workspace_bzlmod.touch(exist_ok=True) if self.prebuilts_dir: module_bazel_content += "\n" module_bazel_content += _LOCAL_PREBUILTS_CONTENT_TEMPLATE.format( @@ -169,19 +200,25 @@ class KleafProjectSetter: if not kleaf_repo.is_absolute(): kleaf_repo = pathlib.Path("%workspace%") / kleaf_repo + bazelrc_content = [] + bazelrc_content.append(( + "common" + f" --registry=file://{kleaf_repo}/external/bazelbuild-bazel-central-registry" + )) + # Explicitly disable internet usage. + bazelrc_content.append("common --config=no_internet") + self._update_file( bazelrc, - textwrap.dedent(f"""\ - common --config=internet - common --registry=file://{kleaf_repo}/external/bazelbuild-bazel-central-registry - """), + "\n".join(bazelrc_content), ) def _get_url(self, remote_filename: str) -> str | None: """Returns a valid url when it can be formed with target and id.""" if not self.url_fmt: - raise KleafProjectSetterError("ERROR: _get_url called without " - "url_fmt set!") + raise KleafProjectSetterError( + "ERROR: _get_url called without url_fmt set!" + ) url = self.url_fmt.format( build_id=self.build_id, build_target=self.build_target, @@ -223,7 +260,8 @@ class KleafProjectSetter: url = self._get_url(remote_filename) if not url: raise KleafProjectSetterError( - f"ERROR: Unable to download {remote_filename}: can't infer URL") + f"ERROR: Unable to download {remote_filename}: can't infer URL" + ) # Workaround: Rely on host keychain to download files. # This is needed otheriwese downloads fail when running this script # using the hermetic Python toolchain. @@ -242,7 +280,8 @@ class KleafProjectSetter: """Infers the list of files to be downloaded using download_configs.json.""" if not self.prebuilts_dir: raise KleafProjectSetterError( - "ERROR: _infer_download_list called without --prebuilts_dir!") + "ERROR: _infer_download_list called without --prebuilts_dir!" + ) download_configs = self.prebuilts_dir / "download_configs.json" with open(download_configs, "w+", encoding="utf-8") as config: self._download("download_configs.json", pathlib.Path(config.name)) @@ -252,7 +291,8 @@ class KleafProjectSetter: """Downloads prebuilts from a given build_id when provided.""" if not self.prebuilts_dir: raise KleafProjectSetterError( - "ERROR: _download_prebuilts called without --prebuilts_dir!") + "ERROR: _download_prebuilts called without --prebuilts_dir!" + ) logging.info("Downloading prebuilts into %s", self.prebuilts_dir) files_dict = self._infer_download_list() with concurrent.futures.ThreadPoolExecutor() as executor: @@ -282,7 +322,7 @@ class KleafProjectSetter: self._populate_kleaf_repo_extra_files() def _handle_prebuilts(self) -> None: - if not self.ddk_workspace or not self.prebuilts_dir: + if not self.prebuilts_dir: return self.prebuilts_dir.mkdir(parents=True, exist_ok=True) if self._can_download_artifacts(): @@ -295,45 +335,56 @@ class KleafProjectSetter: # --local assumes the kernel source tree is complete. return if not self.kleaf_repo: - logging.info("Skipped populating --kleaf_repo because " - "it is unspecified") + logging.info( + "Skipped populating --kleaf_repo because it is unspecified" + ) return if not self.prebuilts_dir: logging.info( - "No prebuilts specified, skip populating %s", self.kleaf_repo) + "No prebuilts specified, skip populating %s", self.kleaf_repo + ) return self._extract_headers_archive(self.prebuilts_dir, self.kleaf_repo) build_config_constants = self.prebuilts_dir / "build.config.constants" if not build_config_constants.is_file(): - logging.warning("%s is not a file, skip copying", - build_config_constants) + logging.warning( + "%s is not a file, skip copying", build_config_constants + ) return - shutil.copy(build_config_constants, - self.kleaf_repo / "common/build.config.constants") + shutil.copy( + build_config_constants, + self.kleaf_repo / "common/build.config.constants", + ) if not (self.kleaf_repo / "common/BUILD.bazel").is_file(): (self.kleaf_repo / "common/BUILD.bazel").write_text("") @staticmethod - def _extract_headers_archive(prebuilts_dir: pathlib.Path, - kleaf_repo: pathlib.Path): + def _extract_headers_archive( + prebuilts_dir: pathlib.Path, kleaf_repo: pathlib.Path + ): """Extracts DDK headers archive from prebuilts_dir into kleaf_repo""" # TODO: This should be target-specific. The name of the output is # currently (2024-05-16) defined by common/BUILD.bazel, but it may # change in the future. - header_archives = list(prebuilts_dir.glob( - "*_ddk_headers_archive.tar.gz")) + header_archives = list( + prebuilts_dir.glob("*_ddk_headers_archive.tar.gz") + ) if not header_archives: - logging.warning("No _ddk_headers_archive.tar.gz found in %s, " - "skipping header extraction.", - prebuilts_dir) + logging.warning( + "No _ddk_headers_archive.tar.gz found in %s, " + "skipping header extraction.", + prebuilts_dir, + ) return if len(header_archives) > 1: raise KleafProjectSetterError( - f"Multiple _ddk_headers_archive.tar.gz found in " - f"{prebuilts_dir}: {header_archives}") - logging.info("Extracting header archive %s to %s", - header_archives[0], kleaf_repo) + "Multiple _ddk_headers_archive.tar.gz found in " + f"{prebuilts_dir}: {header_archives}" + ) + logging.info( + "Extracting header archive %s to %s", header_archives[0], kleaf_repo + ) with tarfile.open(header_archives[0]) as tar: tar.extractall(kleaf_repo) @@ -406,7 +457,9 @@ if __name__ == "__main__": args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - + # Validate pre-condition. + if args.local and not args.kleaf_repo: + parser.error("--local requires --kleaf_repo.") try: KleafProjectSetter(**vars(args)).run() except KleafProjectSetterError as e: diff --git a/init/init_ddk_test.py b/init/init_ddk_test.py index db6773f..77fa3ba 100644 --- a/init/init_ddk_test.py +++ b/init/init_ddk_test.py @@ -177,7 +177,6 @@ class KleafProjectSetterTest(parameterized.TestCase): """Tests prebuilts setup is correct for relative and non-relative to workspace dirs.""" with tempfile.TemporaryDirectory() as tmp: ddk_workspace = pathlib.Path(tmp) / "ddk_workspace" - # Verify the right local_artifact_path is set for prebuilts # in a relative to workspace directory. prebuilts_dir_rel = ddk_workspace / "prebuilts_dir" @@ -226,13 +225,15 @@ class KleafProjectSetterTest(parameterized.TestCase): prebuilts_dir = ddk_workspace / "prebuilts_dir" download_configs = ddk_workspace / "download_configs.json" download_configs.parent.mkdir(parents=True, exist_ok=True) - download_configs.write_text(json.dumps({ - "non-existent-file": { - "target_suffix": "non-existent-file", - "mandatory": False, - "remote_filename_fmt": "non-existent-file", - } - })) + download_configs.write_text( + json.dumps({ + "non-existent-file": { + "target_suffix": "non-existent-file", + "mandatory": False, + "remote_filename_fmt": "non-existent-file", + } + }) + ) with open(download_configs, "r", encoding="utf-8"): url_fmt = f"file://{str(download_configs.parent)}/{{filename}}" init_ddk.KleafProjectSetter( @@ -245,6 +246,52 @@ class KleafProjectSetterTest(parameterized.TestCase): url_fmt=url_fmt, ).run() + @parameterized.named_parameters( + # (Name, MODULE.bazel in @kleaf, expectation) + ("Empty", "", ""), + ( + "Dependencies", + """ +local_path_override( + module_name = "abseil-py", + path = "external/python/absl-py", +) +local_path_override( + module_name = "apple_support", + path = "external/bazelbuild-apple_support", +) + """, + """local_path_override( + module_name = "abseil-py", + path = "kleaf_repo/external/python/absl-py", +) +local_path_override( + module_name = "apple_support", + path = "kleaf_repo/external/bazelbuild-apple_support", +)\n""", + ), + ) + def test_local_path_overrides_extraction( + self, current_content, wanted_content + ): + """Tests extraction of local path overrides works correctly.""" + with tempfile.TemporaryDirectory() as tmp: + ddk_workspace = pathlib.Path(tmp) / "ddk_workspace" + kleaf_repo = ddk_workspace / "kleaf_repo" + kleaf_repo.mkdir(parents=True, exist_ok=True) + kleaf_repo_module_bazel = kleaf_repo / init_ddk._MODULE_BAZEL_FILE + kleaf_repo_module_bazel.write_text(current_content) + got_content = init_ddk.KleafProjectSetter( + build_id=None, + build_target=None, + ddk_workspace=ddk_workspace, + kleaf_repo=kleaf_repo, + local=True, + prebuilts_dir=None, + url_fmt=None, + )._get_local_path_overrides() + self.assertEqual(got_content, wanted_content) + # This could be run as: tools/bazel test //build/kernel:init_ddk_test --test_output=all if __name__ == "__main__": diff --git a/kleaf/tests/integration_test/ddk_workspace_test/extra_setup.py b/kleaf/tests/integration_test/ddk_workspace_test/extra_setup.py index bfc38d9..ca9bbac 100644 --- a/kleaf/tests/integration_test/ddk_workspace_test/extra_setup.py +++ b/kleaf/tests/integration_test/ddk_workspace_test/extra_setup.py @@ -45,36 +45,6 @@ class DdkExtraSetup: bazel_dep(name = "bazel_skylib") """), file=out_file) - # Copy local_path_override() from @kleaf because we do not - # have Internet on CI. - # TODO(b/338439996): Use offline flag in init_ddk.py instead. - with (self.ddk_workspace / self.kleaf_repo_rel / - "MODULE.bazel").open() as src: - self._copy_local_path_override(src, out_file) - - def _copy_local_path_override(self, src, dst): - """Naive algorithm to parse src and copy local_path_override() to dst""" - section = [] - path_attr_prefix = 'path = "' - - # Modify path so it is relative to the current DDK workspace. - # TODO(b/338439996): Use offline flag in init_ddk.py instead. - for line in src: - if line.startswith("local_path_override("): - section.append(line) - continue - if section: - if line.lstrip().startswith(path_attr_prefix): - line = line.strip() - line = line.removeprefix( - path_attr_prefix).removesuffix('",') - line = f' path = "{self.kleaf_repo_rel / line}",\n' - section.append(line) - - if line.strip() == ")": - print("".join(section), file=dst) - section.clear() - def run(self): self._generate_device_bazelrc() self._generate_module_bazel() |