diff options
Diffstat (limited to 'pkg/private/tar/build_tar.py')
-rw-r--r-- | pkg/private/tar/build_tar.py | 459 |
1 files changed, 0 insertions, 459 deletions
diff --git a/pkg/private/tar/build_tar.py b/pkg/private/tar/build_tar.py deleted file mode 100644 index 08a3a06..0000000 --- a/pkg/private/tar/build_tar.py +++ /dev/null @@ -1,459 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This tool build tar files from a list of inputs.""" - -import argparse -import os -import tarfile -import tempfile - -from pkg.private import archive -from pkg.private import helpers -from pkg.private import build_info -from pkg.private import manifest -from pkg.private.tar import tar_writer - - -def normpath(path): - """Normalize a path to the format we need it. - - os.path.normpath changes / to \ on windows, but tarfile needs / style paths. - - Args: - path: (str) path to normalize. - """ - return os.path.normpath(path).replace(os.path.sep, '/') - - -class TarFile(object): - """A class to generates a TAR file.""" - - class DebError(Exception): - pass - - def __init__(self, output, directory, compression, compressor, default_mtime): - # Directory prefix on all output paths - d = directory.strip('/') - self.directory = (d + '/') if d else None - self.output = output - self.compression = compression - self.compressor = compressor - self.default_mtime = default_mtime - - def __enter__(self): - self.tarfile = tar_writer.TarFileWriter( - self.output, - self.compression, - self.compressor, - default_mtime=self.default_mtime) - return self - - def __exit__(self, t, v, traceback): - self.tarfile.close() - - def normalize_path(self, path: str) -> str: - dest = normpath(path) - # paths should not have a leading ./ - if dest.startswith('./'): - dest = dest[2:] - # No path should ever come in with slashs on either end, but protect - # against that anyway. - dest = dest.strip('/') - # This prevents a potential problem for users with both a prefix_dir and - # symlinks that also repeat the prefix_dir. The old behavior was that we - # would get just the symlink path. Now we are prefixing with the prefix, - # so you get the file in the wrong place. - # We silently de-dup that. If people come up with a real use case for - # the /a/b/a/b/rest... output we can start an issue and come up with a - # solution at that time. - if self.directory and not dest.startswith(self.directory): - dest = self.directory + dest - return dest - - def add_file(self, f, destfile, mode=None, ids=None, names=None): - """Add a file to the tar file. - - Args: - f: the file to add to the layer - destfile: the name of the file in the layer - mode: (int) force to set the specified mode, by default the value from the - source is taken. - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. `f` will be - copied to `self.directory/destfile` in the layer. - """ - dest = self.normalize_path(destfile) - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - mode = 0o755 if os.access(f, os.X_OK) else 0o644 - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - self.tarfile.add_file( - dest, - file_content=f, - mode=mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_empty_file(self, - destfile, - mode=None, - ids=None, - names=None, - kind=tarfile.REGTYPE): - """Add a file to the tar file. - - Args: - destfile: the name of the file in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. - kind: type of the file. tarfile.DIRTYPE for directory. An empty file - will be created as `destfile` in the layer. - """ - dest = destfile.lstrip('/') # Remove leading slashes - # If mode is unspecified, assume read only - if mode is None: - mode = 0o644 - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - dest = normpath(dest) - self.tarfile.add_file( - dest, - content='' if kind == tarfile.REGTYPE else None, - kind=kind, - mode=mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_empty_dir(self, destpath, mode=None, ids=None, names=None): - """Add a directory to the tar file. - - Args: - destpath: the name of the directory in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. An empty - file will be created as `destfile` in the layer. - """ - self.add_empty_file( - destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE) - - def add_tar(self, tar): - """Merge a tar file into the destination tar file. - - All files presents in that tar will be added to the output file - under self.directory/path. No user name nor group name will be - added to the output. - - Args: - tar: the tar file to add - """ - self.tarfile.add_tar(tar, numeric=True, prefix=self.directory) - - def add_link(self, symlink, destination, mode=None, ids=None, names=None): - """Add a symbolic link pointing to `destination`. - - Args: - symlink: the name of the symbolic link to add. - destination: where the symbolic link point to. - mode: (int) force to set the specified posix mode (e.g. 0o755). The - default is derived from the source - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. An empty - file will be created as `destfile` in the layer. - """ - dest = self.normalize_path(symlink) - self.tarfile.add_file( - dest, - tarfile.SYMTYPE, - link=destination, - mode = mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_deb(self, deb): - """Extract a debian package in the output tar. - - All files presents in that debian package will be added to the - output tar under the same paths. No user name nor group names will - be added to the output. - - Args: - deb: the tar file to add - - Raises: - DebError: if the format of the deb archive is incorrect. - """ - with archive.SimpleArReader(deb) as arfile: - current = next(arfile) - while current and not current.filename.startswith('data.'): - current = next(arfile) - if not current: - raise self.DebError(deb + ' does not contains a data file!') - tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1]) - with open(tmpfile[1], 'wb') as f: - f.write(current.data) - self.add_tar(tmpfile[1]) - os.remove(tmpfile[1]) - - def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None): - """Add a tree artifact to the tar file. - - Args: - tree_top: the top of the tree to add - destpath: the path under which to place the files - mode: (int) force to set the specified posix mode (e.g. 0o755). The - default is derived from the source - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. `f` will be - copied to `self.directory/destfile` in the layer. - """ - # We expect /-style paths. - tree_top = normpath(tree_top) - - dest = destpath.strip('/') # redundant, dests should never have / here - if self.directory and self.directory != '/': - dest = self.directory.lstrip('/') + '/' + dest - - # Again, we expect /-style paths. - dest = normpath(dest) - # normpath may be ".", and dest paths should not start with "./" - dest = '' if dest == '.' else dest + '/' - - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - - to_write = {} - for root, dirs, files in os.walk(tree_top): - # While `tree_top` uses '/' as a path separator, results returned by - # `os.walk` and `os.path.join` on Windows may not. - root = normpath(root) - - dirs = sorted(dirs) - rel_path_from_top = root[len(tree_top):].lstrip('/') - if rel_path_from_top: - dest_dir = dest + rel_path_from_top + '/' - else: - dest_dir = dest - for dir in dirs: - to_write[dest_dir + dir] = None - for file in sorted(files): - content_path = os.path.abspath(os.path.join(root, file)) - if os.name == "nt": - # "To specify an extended-length path, use the `\\?\` prefix. For - # example, `\\?\D:\very long path`."[1] - # - # [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation - to_write[dest_dir + file] = "\\\\?\\" + content_path - else: - to_write[dest_dir + file] = content_path - - for path in sorted(to_write.keys()): - content_path = to_write[path] - if not content_path: - # This is an intermediate directory. Bazel has no API to specify modes - # for this, so the least surprising thing we can do is make it the - # canonical rwxr-xr-x - self.add_empty_file( - path, - mode=0o755, - ids=ids, - names=names, - kind=tarfile.DIRTYPE) - else: - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - f_mode = 0o755 if os.access(content_path, os.X_OK) else 0o644 - else: - f_mode = mode - self.tarfile.add_file( - path, - file_content=content_path, - mode=f_mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_manifest_entry(self, entry, file_attributes): - # Use the pkg_tar mode/owner remapping as a fallback - non_abs_path = entry.dest.strip('/') - if file_attributes: - attrs = file_attributes(non_abs_path) - else: - attrs = {} - # But any attributes from the manifest have higher precedence - if entry.mode is not None and entry.mode != '': - attrs['mode'] = int(entry.mode, 8) - if entry.user: - if entry.group: - attrs['names'] = (entry.user, entry.group) - else: - # Use group that legacy tar process would assign - attrs['names'] = (entry.user, attrs.get('names')[1]) - if entry.uid is not None: - if entry.gid is not None: - attrs['ids'] = (entry.uid, entry.gid) - else: - attrs['ids'] = (entry.uid, entry.uid) - if entry.type == manifest.ENTRY_IS_LINK: - self.add_link(entry.dest, entry.src, **attrs) - elif entry.type == manifest.ENTRY_IS_DIR: - self.add_empty_dir(self.normalize_path(entry.dest), **attrs) - elif entry.type == manifest.ENTRY_IS_TREE: - self.add_tree(entry.src, entry.dest, **attrs) - elif entry.type == manifest.ENTRY_IS_EMPTY_FILE: - self.add_empty_file(self.normalize_path(entry.dest), **attrs) - else: - self.add_file(entry.src, entry.dest, **attrs) - - -def main(): - parser = argparse.ArgumentParser( - description='Helper for building tar packages', - fromfile_prefix_chars='@') - parser.add_argument('--output', required=True, - help='The output file, mandatory.') - parser.add_argument('--manifest', - help='manifest of contents to add to the layer.') - parser.add_argument('--mode', - help='Force the mode on the added files (in octal).') - parser.add_argument( - '--mtime', - help='Set mtime on tar file entries. May be an integer or the' - ' value "portable", to get the value 2000-01-01, which is' - ' is usable with non *nix OSes.') - parser.add_argument('--tar', action='append', - help='A tar file to add to the layer') - parser.add_argument('--deb', action='append', - help='A debian package to add to the layer') - parser.add_argument( - '--directory', - help='Directory in which to store the file inside the layer') - - compression = parser.add_mutually_exclusive_group() - compression.add_argument('--compression', - help='Compression (`gz` or `bz2`), default is none.') - compression.add_argument('--compressor', - help='Compressor program and arguments, ' - 'e.g. `pigz -p 4`') - - parser.add_argument( - '--modes', action='append', - help='Specific mode to apply to specific file (from the file argument),' - ' e.g., path/to/file=0455.') - parser.add_argument( - '--owners', action='append', - help='Specify the numeric owners of individual files, ' - 'e.g. path/to/file=0.0.') - parser.add_argument( - '--owner', default='0.0', - help='Specify the numeric default owner of all files,' - ' e.g., 0.0') - parser.add_argument( - '--owner_name', - help='Specify the owner name of all files, e.g. root.root.') - parser.add_argument( - '--owner_names', action='append', - help='Specify the owner names of individual files, e.g. ' - 'path/to/file=root.root.') - parser.add_argument('--stamp_from', default='', - help='File to find BUILD_STAMP in') - options = parser.parse_args() - - # Parse modes arguments - default_mode = None - if options.mode: - # Convert from octal - default_mode = int(options.mode, 8) - - mode_map = {} - if options.modes: - for filemode in options.modes: - (f, mode) = helpers.SplitNameValuePairAtSeparator(filemode, '=') - if f[0] == '/': - f = f[1:] - mode_map[f] = int(mode, 8) - - default_ownername = ('', '') - if options.owner_name: - default_ownername = options.owner_name.split('.', 1) - names_map = {} - if options.owner_names: - for file_owner in options.owner_names: - (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=') - (user, group) = owner.split('.', 1) - if f[0] == '/': - f = f[1:] - names_map[f] = (user, group) - - default_ids = options.owner.split('.', 1) - default_ids = (int(default_ids[0]), int(default_ids[1])) - ids_map = {} - if options.owners: - for file_owner in options.owners: - (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=') - (user, group) = owner.split('.', 1) - if f[0] == '/': - f = f[1:] - ids_map[f] = (int(user), int(group)) - - default_mtime = options.mtime - if options.stamp_from: - default_mtime = build_info.get_timestamp(options.stamp_from) - - # Add objects to the tar file - with TarFile( - options.output, - directory = helpers.GetFlagValue(options.directory), - compression = options.compression, - compressor = options.compressor, - default_mtime=default_mtime) as output: - - def file_attributes(filename): - if filename.startswith('/'): - filename = filename[1:] - return { - 'mode': mode_map.get(filename, default_mode), - 'ids': ids_map.get(filename, default_ids), - 'names': names_map.get(filename, default_ownername), - } - - if options.manifest: - with open(options.manifest, 'r') as manifest_fp: - manifest_entries = manifest.read_entries_from(manifest_fp) - for entry in manifest_entries: - output.add_manifest_entry(entry, file_attributes) - - for tar in options.tar or []: - output.add_tar(tar) - for deb in options.deb or []: - output.add_deb(deb) - - -if __name__ == '__main__': - main() |