diff options
author | Pierre Labatut <plabatut@google.com> | 2024-04-22 14:31:43 +0200 |
---|---|---|
committer | Pierre Labatut <plabatut@google.com> | 2024-04-22 17:21:21 +0200 |
commit | b2a44887e9e52f43aae0f255b78fd8527571baaa (patch) | |
tree | 7a29299af8bd7886efa9d048e568b87a30427aad | |
parent | 05a58b98d9539869ef47036cd8dfc722ec2ab477 (diff) | |
download | cuttlefish_vmm-b2a44887e9e52f43aae0f255b78fd8527571baaa.tar.gz |
Add support for aarch64 builds
* Requires clients to install qemu-user-static.
* Requires setting multiarch/qemu-user-static on podman.
* Uses non-sysroot based build for aarch64.
* Adds flags to select architecture, defaults to x86_64.
* Based on exploratory change from plabatut@.
Bug: 331635690
Test: Verified container build of x86_64, aarch64.
Test: Verified launch of resulting QEMU binary on ARM64 host.
Test: Verified qemu/scripts/check.sh.
Change-Id: I770210e5145e95f468ab0dfff72dff5e9e94910f
-rw-r--r-- | qemu/manifest.xml | 2 | ||||
-rw-r--r-- | qemu/scripts/genrepo.py | 6 | ||||
-rwxr-xr-x | qemu/scripts/rebuild.py | 251 | ||||
-rwxr-xr-x | qemu/scripts/rebuild_in_container.sh | 41 |
4 files changed, 242 insertions, 58 deletions
diff --git a/qemu/manifest.xml b/qemu/manifest.xml index 602cdcf..55c465b 100644 --- a/qemu/manifest.xml +++ b/qemu/manifest.xml @@ -65,4 +65,4 @@ <project path="qemu/third_party/rust/crates/zerocopy" name="platform/external/rust/crates/zerocopy" revision="738d78f6696da049502fc4436a70ca8799e1569e" /> <project path="qemu/third_party/rust/crates/zerocopy-derive" name="platform/external/rust/crates/zerocopy-derive" revision="e31ff4fde30f2bef09ae8f20b93d57b9040c7c1c" /> <project path="qemu/third_party/zlib" name="platform/external/zlib" revision="81774276a9cbf47177a1b7555bb6e3ec73bdcd25" /> -</manifest>
\ No newline at end of file +</manifest> diff --git a/qemu/scripts/genrepo.py b/qemu/scripts/genrepo.py index 0884c0c..004a0fd 100644 --- a/qemu/scripts/genrepo.py +++ b/qemu/scripts/genrepo.py @@ -287,6 +287,12 @@ def GenerateRepoManifest(projects, out): ET.SubElement(elem, "linkfile", src=".", dest=linkat) tree = ET.ElementTree(manifest) + tree.getroot().insert( + 0, + ET.Comment( + "DO NOT EDIT. This file is generated by " + os.path.basename(__file__) + ), + ) ET.indent(tree) tree.write(out, encoding="unicode") diff --git a/qemu/scripts/rebuild.py b/qemu/scripts/rebuild.py index df4d714..54e972a 100755 --- a/qemu/scripts/rebuild.py +++ b/qemu/scripts/rebuild.py @@ -15,6 +15,7 @@ """A script to rebuild QEMU from scratch on Linux.""" import argparse +from enum import Enum import os from pathlib import Path import shlex @@ -28,6 +29,11 @@ from typing import List from typing import Sequence +class Architecture(Enum): + AARCH64 = "aarch64" + X86_64 = "x86_64" + + def copy_file(src: Path, dst: Path) -> None: log(" COPY_FILE %s --> %s" % (src, dst)) os.makedirs(dst.parent, exist_ok=True) @@ -39,36 +45,69 @@ def log(msg: str) -> None: def create_dev_environment( - build_dir: Path, install_dir: Path, prebuilts_dir: Path, clang_dir: Path + build_dir: Path, + install_dir: Path, + prebuilts_dir: Path, + clang_dir: Path, + target_arch: Architecture, ) -> Dict[str, str]: sysroot = str(build_dir / "sysroot") binprefix = "%s/" % (clang_dir / "bin") env = os.environ.copy() path = env["PATH"] ld_library_path = env.get("LD_LIBRARY_PATH", "") - env.update({ - "CC": f"{binprefix}clang --sysroot={sysroot}", - "CXX": f"{binprefix}clang++ --sysroot={sysroot} -stdlib=libc++", - # FIXME: this file does not exist. - "LD": f"{binprefix}llvm-ld --sysroot={sysroot}", - "AR": f"{binprefix}llvm-ar", - "NM": f"{binprefix}llvm-nm", - "PKG_CONFIG_PATH": ":".join([ - f"{install_dir}/usr/lib/x86_64-linux-gnu/pkgconfig", - f"{install_dir}/usr/lib/pkgconfig", - f"{sysroot}/usr/lib/pkgconfig", - ]), - "PATH": f"{install_dir}/usr/bin:{path}", - "LD_LIBRARY_PATH": ":".join([ - f"{install_dir}/usr/lib/x86_64-linux-gnu", - f"{install_dir}/usr/lib", - f"{clang_dir}/lib:{ld_library_path}", - ]), - # Required to ensure that configure scripts that do not rely - # on pkg-config find their dependencies properly. - "CFLAGS": f"-I{install_dir}/usr/include", - "LDFLAGS": f"-Wl,-L{install_dir}/usr/lib", - }) + + if target_arch == Architecture.AARCH64: + cargo_path = Path(env["HOME"]) / ".cargo" / "bin" + env.update({ + "CC": f"clang", + "CXX": f"clang++ -stdlib=libc++", + # FIXME: this file does not exist. + "LD": f"llvm-ld", + "AR": f"llvm-ar", + "NM": f"llvm-nm", + "PKG_CONFIG_PATH": ":".join([ + f"{install_dir}/usr/lib/aarch64-linux-gnu/pkgconfig", + f"{install_dir}/usr/lib/pkgconfig", + ]), + "PATH": f"{cargo_path}:{install_dir}/usr/bin:{path}", + "LD_LIBRARY_PATH": ":".join([ + f"{install_dir}/usr/lib/aarch64-linux-gnu", + f"{install_dir}/usr/lib", + f"{ld_library_path}", + ]), + # Required to ensure that configure scripts that do not rely + # on pkg-config find their dependencies properly. + "CFLAGS": f"-I{install_dir}/usr/include", + "CXXFLAGS": f"-I{install_dir}/usr/include", + "LDFLAGS": f"-Wl,-L{install_dir}/usr/lib", + }) + else: + env.update({ + "CC": f"{binprefix}clang --sysroot={sysroot}", + "CXX": f"{binprefix}clang++ --sysroot={sysroot} -stdlib=libc++", + # FIXME: this file does not exist. + "LD": f"{binprefix}llvm-ld --sysroot={sysroot}", + "AR": f"{binprefix}llvm-ar", + "NM": f"{binprefix}llvm-nm", + "PKG_CONFIG_PATH": ":".join([ + f"{install_dir}/usr/lib/x86_64-linux-gnu/pkgconfig", + f"{install_dir}/usr/lib/pkgconfig", + f"{install_dir}/usr/lib64/pkgconfig", + f"{sysroot}/usr/lib/pkgconfig", + ]), + "PATH": f"{install_dir}/usr/bin:{path}", + "LD_LIBRARY_PATH": ":".join([ + f"{install_dir}/usr/lib/x86_64-linux-gnu", + f"{install_dir}/usr/lib", + f"{clang_dir}/lib:{ld_library_path}", + ]), + # Required to ensure that configure scripts that do not rely + # on pkg-config find their dependencies properly. + "CFLAGS": f"-I{install_dir}/usr/include", + "CXXFLAGS": f"-I{install_dir}/usr/include", + "LDFLAGS": f"-Wl,-L{install_dir}/usr/lib", + }) return env @@ -161,7 +200,8 @@ class BuildConfig(object): or running commands. """ - def __init__(self, build_dir: Path, top_dir: Path): + def __init__(self, build_dir: Path, top_dir: Path, target_arch: Architecture): + self._target_arch = target_arch self._build_dir = build_dir self._sysroot_dir = build_dir / "sysroot" self._install_dir = build_dir / "dest-install" @@ -173,6 +213,7 @@ class BuildConfig(object): self._install_dir, self._prebuilts_dir, self._clang_dir, + self._target_arch, ) # By default, run commands directly. Subclasses can override @@ -185,6 +226,10 @@ class BuildConfig(object): self._env[varname] = "ccache " + self._env[varname] @property + def target_arch(self) -> Architecture: + return self._target_arch + + @property def build_dir(self) -> Path: return self._build_dir @@ -350,6 +395,10 @@ project = Project() @project.task([]) def build_task_for_sysroot(build: BuildConfig): + if build.target_arch == Architecture.AARCH64: + # We don't build the sysroot for AARCH64 or its dependencies. + return + # populate_sysroot(build.build_dir / 'sysroot', build.prebuilts_dir), dst_sysroot = build.build_dir / "sysroot" dst_sysroot_lib_dir = dst_sysroot / "usr" / "lib" @@ -380,6 +429,10 @@ def build_task_for_sysroot(build: BuildConfig): @project.task([build_task_for_sysroot]) def build_task_for_ninja(build: BuildConfig): + if build.target_arch == Architecture.AARCH64: + # We use apt to install ninja. + return + build.copy_file( build.prebuilts_dir / "ninja" / "ninja", build.install_dir / "usr" / "bin" / "ninja", @@ -388,6 +441,10 @@ def build_task_for_ninja(build: BuildConfig): @project.task([]) def build_task_for_python(build: BuildConfig): + if build.target_arch == Architecture.AARCH64: + # We use apt to install python3.10. + return + src_python_dir = build.third_party_dir / "python" dst_python_dir = build.install_dir / "usr" for d in ("bin", "lib", "share"): @@ -401,11 +458,13 @@ def build_task_for_meson(build: BuildConfig): meson_packager = ( build.third_party_dir / "meson" / "packaging" / "create_zipapp.py" ) + usr_bin = build.install_dir / "usr" / "bin" + build.run(["mkdir", "-p", usr_bin], env=None) build.run([ "python3", "-S", meson_packager, - "--outfile=%s" % (build.install_dir / "usr" / "bin" / "meson"), + "--outfile=%s" % (usr_bin / "meson"), "--interpreter", "/usr/bin/env python3", build.third_party_dir / "meson", @@ -414,7 +473,14 @@ def build_task_for_meson(build: BuildConfig): @project.task([]) def build_task_for_rust(build: BuildConfig): - log("Install prebuilt rust.") + if build.target_arch == Architecture.AARCH64: + # We use rustup to install rust, but we need the following libraries + # for the aarch64 portable build. + dst_dir = build.install_dir / "usr" / "lib64" + for lib in ["libc++.so.1", "libc++abi.so.1", "libunwind.so.1"]: + build.copy_file(Path(f"/lib/aarch64-linux-gnu/{lib}"), dst_dir / lib) + return + src_rust_dir = build.prebuilts_dir / "rust" / "linux-x86" / "1.73.0" dst_rust_dir = build.install_dir / "usr" for d in ("bin", "lib", "lib64", "share"): @@ -430,6 +496,10 @@ def build_task_for_rust(build: BuildConfig): @project.task([build_task_for_sysroot]) def build_task_for_make(build: BuildConfig) -> None: + if build.target_arch == Architecture.AARCH64: + # We use apt to install make. + return + build.copy_file( build.prebuilts_dir / "build-tools" / "linux-x86" / "bin" / "make", build.install_dir / "usr" / "bin" / "make", @@ -438,7 +508,10 @@ def build_task_for_make(build: BuildConfig) -> None: @project.task([]) def build_task_for_cmake(build: BuildConfig): - log("Install Cmake prebuilt.") + if build.target_arch == Architecture.AARCH64: + # We use apt to install cmake. + return + build.copy_file( build.prebuilts_dir / "cmake" / "bin" / "cmake", build.install_dir / "usr" / "bin" / "cmake", @@ -806,8 +879,12 @@ def build_task_for_libdrm(build: BuildConfig): ) -@project.task([]) +@project.task([build_task_for_sysroot]) def build_task_for_egl(build: BuildConfig): + if build.target_arch == Architecture.AARCH64: + # We use apt to install the EGL headers. + return + build.copy_dir( build.third_party_dir / "egl" / "api" / "KHR", build.sysroot_dir / "usr" / "include" / "KHR", @@ -862,23 +939,35 @@ def build_task_for_gfxstream(build: BuildConfig): def build_task_for_rutabaga(build: BuildConfig): out_dir = build.make_subdir(Path("rutabaga")) cmd_args = [ - build.install_dir / "usr/bin/cargo", + "cargo", "build", "--offline", "--features=gfxstream", "--release", ] - env = { - "CARGO_TARGET_DIR": str(out_dir), - "GFXSTREAM_PATH": str(build.build_dir / "gfxstream" / "host"), - "PATH": f"{build.install_dir}/usr/bin:{os.environ['PATH']}", - "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": ( - f"{build.clang_dir}/bin/clang" - ), - "RUSTFLAGS": ( - f"-Clink-arg=--sysroot={build.sysroot_dir} -Clink-arg=-Wl,-rpath,$ORIGIN" - ), - } + if build.target_arch == Architecture.AARCH64: + cargo_path = Path(os.environ["HOME"]) / ".cargo" / "bin" + env = { + "CARGO_TARGET_DIR": str(out_dir), + "GFXSTREAM_PATH": str(build.build_dir / "gfxstream" / "host"), + "PATH": ( + f"{cargo_path}:{build.install_dir}/usr/bin:{os.environ['PATH']}" + ), + "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "clang", + "RUSTFLAGS": f"-Clink-arg=-Wl,-rpath,$ORIGIN", + } + else: + env = { + "CARGO_TARGET_DIR": str(out_dir), + "GFXSTREAM_PATH": str(build.build_dir / "gfxstream" / "host"), + "PATH": f"{build.install_dir}/usr/bin:{os.environ['PATH']}", + "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": ( + f"{build.clang_dir}/bin/clang" + ), + "RUSTFLAGS": ( + f"-Clink-arg=--sysroot={build.sysroot_dir} -Clink-arg=-Wl,-rpath,$ORIGIN" + ), + } rutabaga_src_dir = build.third_party_dir / "crosvm" / "rutabaga_gfx" / "ffi" build.run(cmd_args, rutabaga_src_dir, env) @@ -889,7 +978,7 @@ def build_task_for_rutabaga(build: BuildConfig): build.run( ["ln", "-sf", "librutabaga_gfx_ffi.so", "librutabaga_gfx_ffi.so.0"], build.install_dir / "usr" / "lib", - env={} + env={}, ) build.copy_file( out_dir / "release" / "rutabaga_gfx_ffi.pc", @@ -910,22 +999,24 @@ def build_task_for_libgbm(build: BuildConfig): # gbm is part of mesa which is a large project. # The dependency is taken fron the system. build.copy_file( - "/usr/lib/x86_64-linux-gnu/libgbm.so.1", + Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/libgbm.so.1"), build.install_dir / "usr/lib/libgbm.so.1", ) build.copy_file( - "/usr/lib/x86_64-linux-gnu/libgbm.so", + Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/libgbm.so"), build.install_dir / "usr/lib/libgbm.so", ) build.copy_file( - "/usr/lib/x86_64-linux-gnu/libgbm.so.1.0.0", + Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/libgbm.so.1.0.0"), build.install_dir / "usr/lib/libgbm.so.1.0.0", ) build.copy_file( - "/usr/lib/x86_64-linux-gnu/pkgconfig/gbm.pc", + Path(f"/usr/lib/{build.target_arch.value}-linux-gnu/pkgconfig/gbm.pc"), build.install_dir / "usr/lib/pkgconfig/gbm.pc", ) - build.copy_file("/usr/include/gbm.h", build.install_dir / "usr/include/gbm.h") + build.copy_file( + Path("/usr/include/gbm.h"), build.install_dir / "usr/include/gbm.h" + ) @project.task([ @@ -993,11 +1084,38 @@ def build_task_for_virglrenderer(build: BuildConfig): @project.task([ + build_task_for_meson, + build_task_for_ninja, +]) +def build_task_for_dtc(build: BuildConfig): + src_dir = build.third_party_dir / "dtc" + build_dir = build.make_subdir(Path("dtc")) + build.run( + [ + "meson", + "setup", + "--prefix=%s/usr" % build.install_dir, + "--libdir=%s/usr/lib" % build.install_dir, + # Option taken from qemu/meson.build subproject dtc. + "-Dtools=false", + "-Dyaml=disabled", + "-Dpython=disabled", + "-Ddefault_library=static", + build_dir, + src_dir, + ], + ) + + build.run(["ninja", "install"], build_dir) + + +@project.task([ build_task_for_make, build_task_for_libslirp, build_task_for_glib, build_task_for_pixman, build_task_for_zlib, + build_task_for_dtc, build_task_for_pkg_config, build_task_for_rutabaga, build_task_for_gfxstream, @@ -1011,12 +1129,33 @@ def build_task_for_qemu(build: BuildConfig): ] src_dir = build.third_party_dir / "qemu" build_dir = build.make_subdir(Path("qemu")) + + # Copy third-pary projects to qemu/subprojects directory so that meson does + # not try to checkout those sources with git. + for name in ("keycodemapdb", "berkeley-softfloat-3", "berkeley-testfloat-3"): + subproject = src_dir / "subprojects" / name + build.run(["rm", "-rf", src_dir / "subprojects" / name], env={}) + build.run( + ["cp", "-r", build.third_party_dir / name, src_dir / "subprojects"], + env={}, + ) + # Add project files to the sources. + packagefiles = src_dir / "subprojects" / "packagefiles" / name + for package_file in packagefiles.glob("*"): + build.run( + ["cp", package_file, subproject], + env={}, + ) + cmd_args: List[str | Path] = [ src_dir.resolve() / "configure", "--prefix=/usr", "--target-list=%s" % ",".join(target_list), "--disable-plugins", "--enable-virglrenderer", + # Disable source checkout of missing dependencies, so that + # missing project chan be found early . + "--disable-download", # Cuttlefish is packaged in host archives that are assembled in # `$ANDROID_BUILD_TOP/out/host/linux-x86`. # Binaries are in `./bin` and resources are in `./usr/share` which is @@ -1056,11 +1195,18 @@ def build_task_for_qemu_portable(build: BuildConfig): "dest-install/usr/lib/librutabaga_gfx_ffi.so.0", "dest-install/usr/lib64/libc++.so.1", ] + if build.target_arch == Architecture.AARCH64: + files += [ + "dest-install/usr/lib64/libc++abi.so.1", + "dest-install/usr/lib64/libunwind.so.1", + ] + # Meson install directory depends on the system and differs between podman and # the developer's workstation. Probe the file system to pick the right location. either_or = [ - "dest-install/usr/lib/x86_64-linux-gnu/libgfxstream_backend.so.0", + f"dest-install/usr/lib/{build.target_arch}-linux-gnu/libgfxstream_backend.so.0", "dest-install/usr/lib/libgfxstream_backend.so.0", + "dest-install/usr/lib64/libgfxstream_backend.so.0", ] try: files.append( @@ -1105,6 +1251,13 @@ def build_task_for_qemu_test(build: BuildConfig): def main() -> int: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--arch", + type=Architecture, + choices=list(Architecture), + default=Architecture.X86_64, + help="Target architecture.", + ) parser.add_argument("--build-dir", required=True, help="Build directory.") parser.add_argument("--ccache", action="store_true", help="Enable ccache.") parser.add_argument( @@ -1124,7 +1277,7 @@ def main() -> int: build_dir = Path(args.build_dir) top_dir = Path(os.path.dirname(__file__)).parent - build_config = BuildConfig(build_dir, top_dir) + build_config = BuildConfig(build_dir, top_dir, args.arch) if args.ccache: build_config.enable_ccache() diff --git a/qemu/scripts/rebuild_in_container.sh b/qemu/scripts/rebuild_in_container.sh index a381736..66acd3d 100755 --- a/qemu/scripts/rebuild_in_container.sh +++ b/qemu/scripts/rebuild_in_container.sh @@ -2,10 +2,15 @@ # Starts an isolated build in a container set -e +ARCH="x86_64" FROM_EXISTING_SOURCES=0 while [[ $# -gt 0 ]]; do case $1 in + --arch) + ARCH="$2" + shift 2 + ;; --from_existing_sources) FROM_EXISTING_SOURCES=1 shift @@ -13,7 +18,7 @@ while [[ $# -gt 0 ]]; do *) echo "Build QEMU from sources in a container." echo - echo "usage: $0 [--from_existing_sources]" + echo "usage: $0 [--from_existing_sources] [--arch x86_64|aarch64]" echo echo "Options:" echo " --from_existing_sources: Do not checkout sources with repo," @@ -37,18 +42,34 @@ if [ "$FROM_EXISTING_SOURCES" -eq 0 ]; then echo "Check out sources with repo at: ${SRC_DIR}" rm -rf "${SRC_DIR}" mkdir -p "${SRC_DIR}" + cd "${SRC_DIR}" repo init --manifest-url "${GIT_ROOT}" \ --manifest-name=qemu/manifest.xml repo sync else - echo "Reuse existing source checkout at: ${SRC_DIR}" readonly SRC_DIR="$GIT_ROOT" + echo "Reuse existing source checkout at: ${SRC_DIR}" +fi + +readonly COMMON_APT_PACKAGES="autoconf libgbm-dev libtool texinfo" +readonly AARCH64_APT_PACKAGES="clang cmake curl libc++-dev libc++abi-dev \ + libegl-dev libgcc-12-dev libxml2-utils llvm make ninja-build \ + patchelf pkg-config python3 python3-distutils python3-venv" + +APT_PACKAGES="${COMMON_APT_PACKAGES}" +RUSTUP_COMMAND=":" +PYTHON_BIN="/src/qemu/third_party/python/bin/python3" +if [[ "$ARCH" == "aarch64" ]]; then + APT_PACKAGES="${APT_PACKAGES} ${AARCH64_APT_PACKAGES}" + RUSTUP_COMMAND="curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -q -y" + PYTHON_BIN="python3" fi readonly COMMAND="apt-get update && \ -apt-get -qy install autoconf libtool texinfo libgbm-dev && \ -/src/qemu/third_party/python/bin/python3 /src/qemu/scripts/rebuild.py --build-dir /out" +apt-get -qy install ${APT_PACKAGES} && \ +${RUSTUP_COMMAND} && \ +${PYTHON_BIN} /src/qemu/scripts/rebuild.py --arch ${ARCH} --build-dir /out" # Note: `/src` is mounted with a file overlay so that cargo can write # `third_party/crossvm/rutabaga_gfx/ffi/Cargo.lock` file. We didn't find @@ -59,16 +80,20 @@ podman run --name qemu-build \ --pids-limit=-1 \ --volume "${SRC_DIR}:/src:O" \ --volume "${WORK_DIR}:/out" \ - docker.io/debian:11-slim \ + --arch ${ARCH} \ + docker.io/debian:12-slim \ bash -c "${COMMAND}" +# Tip: Use `--interactive`, `--tty` and +# `bash -c "${COMMAND};$SHELL"` for interactive debug. + if [ "$FROM_EXISTING_SOURCES" -eq 0 ]; then - readonly DEST="${GIT_ROOT}/qemu/x86_64-linux-gnu" - echo "Overwrite prebuild at: ${DEST}" + readonly DEST="${GIT_ROOT}/qemu/${ARCH}-linux-gnu" + echo "Overwrite prebuilt at: ${DEST}" rm -rf "${DEST}/*" tar -xvf "${WORK_DIR}/qemu-portable.tar.gz" -C "${DEST}" fi echo "Binary available at: ${WORK_DIR}/qemu-portable/bin" -echo "Done."
\ No newline at end of file +echo "Done." |