diff options
Diffstat (limited to 'tools/wheelmaker.py')
| -rw-r--r-- | tools/wheelmaker.py | 436 |
1 files changed, 436 insertions, 0 deletions
diff --git a/tools/wheelmaker.py b/tools/wheelmaker.py new file mode 100644 index 0000000..63b833f --- /dev/null +++ b/tools/wheelmaker.py @@ -0,0 +1,436 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import collections +import hashlib +import os +import re +import sys +import zipfile +from pathlib import Path + + +def commonpath(path1, path2): + ret = [] + for a, b in zip(path1.split(os.path.sep), path2.split(os.path.sep)): + if a != b: + break + ret.append(a) + return os.path.sep.join(ret) + + +def escape_filename_segment(segment): + """Escapes a filename segment per https://www.python.org/dev/peps/pep-0427/#escaping-and-unicode""" + return re.sub(r"[^\w\d.]+", "_", segment, re.UNICODE) + + +class WheelMaker(object): + def __init__( + self, + name, + version, + build_tag, + python_tag, + abi, + platform, + outfile=None, + strip_path_prefixes=None, + ): + self._name = name + self._version = version + self._build_tag = build_tag + self._python_tag = python_tag + self._abi = abi + self._platform = platform + self._outfile = outfile + self._strip_path_prefixes = ( + strip_path_prefixes if strip_path_prefixes is not None else [] + ) + + self._distinfo_dir = ( + escape_filename_segment(self._name) + + "-" + + escape_filename_segment(self._version) + + ".dist-info/" + ) + self._zipfile = None + # Entries for the RECORD file as (filename, hash, size) tuples. + self._record = [] + + def __enter__(self): + self._zipfile = zipfile.ZipFile( + self.filename(), mode="w", compression=zipfile.ZIP_DEFLATED + ) + return self + + def __exit__(self, type, value, traceback): + self._zipfile.close() + self._zipfile = None + + def wheelname(self) -> str: + components = [self._name, self._version] + if self._build_tag: + components.append(self._build_tag) + components += [self._python_tag, self._abi, self._platform] + return "-".join(components) + ".whl" + + def filename(self) -> str: + if self._outfile: + return self._outfile + return self.wheelname() + + def disttags(self): + return ["-".join([self._python_tag, self._abi, self._platform])] + + def distinfo_path(self, basename): + return self._distinfo_dir + basename + + def _serialize_digest(self, hash): + # https://www.python.org/dev/peps/pep-0376/#record + # "base64.urlsafe_b64encode(digest) with trailing = removed" + digest = base64.urlsafe_b64encode(hash.digest()) + digest = b"sha256=" + digest.rstrip(b"=") + return digest + + def add_string(self, filename, contents): + """Add given 'contents' as filename to the distribution.""" + if sys.version_info[0] > 2 and isinstance(contents, str): + contents = contents.encode("utf-8", "surrogateescape") + self._zipfile.writestr(filename, contents) + hash = hashlib.sha256() + hash.update(contents) + self._add_to_record(filename, self._serialize_digest(hash), len(contents)) + + def add_file(self, package_filename, real_filename): + """Add given file to the distribution.""" + + def arcname_from(name): + # Always use unix path separators. + normalized_arcname = name.replace(os.path.sep, "/") + # Don't manipulate names filenames in the .distinfo directory. + if normalized_arcname.startswith(self._distinfo_dir): + return normalized_arcname + for prefix in self._strip_path_prefixes: + if normalized_arcname.startswith(prefix): + return normalized_arcname[len(prefix) :] + + return normalized_arcname + + if os.path.isdir(real_filename): + directory_contents = os.listdir(real_filename) + for file_ in directory_contents: + self.add_file( + "{}/{}".format(package_filename, file_), + "{}/{}".format(real_filename, file_), + ) + return + + arcname = arcname_from(package_filename) + + self._zipfile.write(real_filename, arcname=arcname) + # Find the hash and length + hash = hashlib.sha256() + size = 0 + with open(real_filename, "rb") as f: + while True: + block = f.read(2**20) + if not block: + break + hash.update(block) + size += len(block) + self._add_to_record(arcname, self._serialize_digest(hash), size) + + def add_wheelfile(self): + """Write WHEEL file to the distribution""" + # TODO(pstradomski): Support non-purelib wheels. + wheel_contents = """\ +Wheel-Version: 1.0 +Generator: bazel-wheelmaker 1.0 +Root-Is-Purelib: {} +""".format( + "true" if self._platform == "any" else "false" + ) + for tag in self.disttags(): + wheel_contents += "Tag: %s\n" % tag + self.add_string(self.distinfo_path("WHEEL"), wheel_contents) + + def add_metadata(self, metadata, name, description, version): + """Write METADATA file to the distribution.""" + # https://www.python.org/dev/peps/pep-0566/ + # https://packaging.python.org/specifications/core-metadata/ + metadata = re.sub("^Name: .*$", "Name: %s" % name, metadata, flags=re.MULTILINE) + metadata += "Version: %s\n\n" % version + # setuptools seems to insert UNKNOWN as description when none is + # provided. + metadata += description if description else "UNKNOWN" + metadata += "\n" + self.add_string(self.distinfo_path("METADATA"), metadata) + + def add_recordfile(self): + """Write RECORD file to the distribution.""" + record_path = self.distinfo_path("RECORD") + entries = self._record + [(record_path, b"", b"")] + entries.sort() + contents = b"" + for filename, digest, size in entries: + if sys.version_info[0] > 2 and isinstance(filename, str): + filename = filename.lstrip("/").encode("utf-8", "surrogateescape") + contents += b"%s,%s,%s\n" % (filename, digest, size) + self.add_string(record_path, contents) + + def _add_to_record(self, filename, hash, size): + size = str(size).encode("ascii") + self._record.append((filename, hash, size)) + + +def get_files_to_package(input_files): + """Find files to be added to the distribution. + + input_files: list of pairs (package_path, real_path) + """ + files = {} + for package_path, real_path in input_files: + files[package_path] = real_path + return files + + +def resolve_argument_stamp( + argument: str, volatile_status_stamp: Path, stable_status_stamp: Path +) -> str: + """Resolve workspace status stamps format strings found in the argument string + + Args: + argument (str): The raw argument represenation for the wheel (may include stamp variables) + volatile_status_stamp (Path): The path to a volatile workspace status file + stable_status_stamp (Path): The path to a stable workspace status file + + Returns: + str: A resolved argument string + """ + lines = ( + volatile_status_stamp.read_text().splitlines() + + stable_status_stamp.read_text().splitlines() + ) + for line in lines: + if not line: + continue + key, value = line.split(" ", maxsplit=1) + stamp = "{" + key + "}" + argument = argument.replace(stamp, value) + + return argument + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Builds a python wheel") + metadata_group = parser.add_argument_group("Wheel name, version and platform") + metadata_group.add_argument( + "--name", required=True, type=str, help="Name of the distribution" + ) + metadata_group.add_argument( + "--version", required=True, type=str, help="Version of the distribution" + ) + metadata_group.add_argument( + "--build_tag", + type=str, + default="", + help="Optional build tag for the distribution", + ) + metadata_group.add_argument( + "--python_tag", + type=str, + default="py3", + help="Python version, e.g. 'py2' or 'py3'", + ) + metadata_group.add_argument("--abi", type=str, default="none") + metadata_group.add_argument( + "--platform", type=str, default="any", help="Target platform. " + ) + + output_group = parser.add_argument_group("Output file location") + output_group.add_argument( + "--out", type=str, default=None, help="Override name of ouptut file" + ) + output_group.add_argument( + "--name_file", + type=Path, + help="A file where the canonical name of the " "wheel will be written", + ) + + output_group.add_argument( + "--strip_path_prefix", + type=str, + action="append", + default=[], + help="Path prefix to be stripped from input package files' path. " + "Can be supplied multiple times. Evaluated in order.", + ) + + wheel_group = parser.add_argument_group("Wheel metadata") + wheel_group.add_argument( + "--metadata_file", + type=Path, + help="Contents of the METADATA file (before appending contents of " + "--description_file)", + ) + wheel_group.add_argument( + "--description_file", help="Path to the file with package description" + ) + wheel_group.add_argument( + "--description_content_type", help="Content type of the package description" + ) + wheel_group.add_argument( + "--entry_points_file", + help="Path to a correctly-formatted entry_points.txt file", + ) + + contents_group = parser.add_argument_group("Wheel contents") + contents_group.add_argument( + "--input_file", + action="append", + help="'package_path;real_path' pairs listing " + "files to be included in the wheel. " + "Can be supplied multiple times.", + ) + contents_group.add_argument( + "--input_file_list", + action="append", + help="A file that has all the input files defined as a list to avoid " + "the long command", + ) + contents_group.add_argument( + "--extra_distinfo_file", + action="append", + help="'filename;real_path' pairs listing extra files to include in" + "dist-info directory. Can be supplied multiple times.", + ) + + build_group = parser.add_argument_group("Building requirements") + build_group.add_argument( + "--volatile_status_file", + type=Path, + help="Pass in the stamp info file for stamping", + ) + build_group.add_argument( + "--stable_status_file", + type=Path, + help="Pass in the stamp info file for stamping", + ) + + return parser.parse_args(sys.argv[1:]) + + +def main() -> None: + arguments = parse_args() + + if arguments.input_file: + input_files = [i.split(";") for i in arguments.input_file] + else: + input_files = [] + + if arguments.extra_distinfo_file: + extra_distinfo_file = [i.split(";") for i in arguments.extra_distinfo_file] + else: + extra_distinfo_file = [] + + if arguments.input_file_list: + for input_file in arguments.input_file_list: + with open(input_file) as _file: + input_file_list = _file.read().splitlines() + for _input_file in input_file_list: + input_files.append(_input_file.split(";")) + + all_files = get_files_to_package(input_files) + # Sort the files for reproducible order in the archive. + all_files = sorted(all_files.items()) + + strip_prefixes = [p for p in arguments.strip_path_prefix] + + if arguments.volatile_status_file and arguments.stable_status_file: + name = resolve_argument_stamp( + arguments.name, + arguments.volatile_status_file, + arguments.stable_status_file, + ) + else: + name = arguments.name + + if arguments.volatile_status_file and arguments.stable_status_file: + version = resolve_argument_stamp( + arguments.version, + arguments.volatile_status_file, + arguments.stable_status_file, + ) + else: + version = arguments.version + + with WheelMaker( + name=name, + version=version, + build_tag=arguments.build_tag, + python_tag=arguments.python_tag, + abi=arguments.abi, + platform=arguments.platform, + outfile=arguments.out, + strip_path_prefixes=strip_prefixes, + ) as maker: + for package_filename, real_filename in all_files: + maker.add_file(package_filename, real_filename) + maker.add_wheelfile() + + description = None + if arguments.description_file: + if sys.version_info[0] == 2: + with open(arguments.description_file, "rt") as description_file: + description = description_file.read() + else: + with open( + arguments.description_file, "rt", encoding="utf-8" + ) as description_file: + description = description_file.read() + + metadata = None + if sys.version_info[0] == 2: + with open(arguments.metadata_file, "rt") as metadata_file: + metadata = metadata_file.read() + else: + with open(arguments.metadata_file, "rt", encoding="utf-8") as metadata_file: + metadata = metadata_file.read() + + maker.add_metadata( + metadata=metadata, name=name, description=description, version=version + ) + + if arguments.entry_points_file: + maker.add_file( + maker.distinfo_path("entry_points.txt"), arguments.entry_points_file + ) + + # Sort the files for reproducible order in the archive. + for filename, real_path in sorted(extra_distinfo_file): + maker.add_file(maker.distinfo_path(filename), real_path) + + maker.add_recordfile() + + # Since stamping may otherwise change the target name of the + # wheel, the canonical name (with stamps resolved) is written + # to a file so consumers of the wheel can easily determine + # the correct name. + arguments.name_file.write_text(maker.wheelname()) + + +if __name__ == "__main__": + main() |