aboutsummaryrefslogtreecommitdiff
path: root/pkg/private/tar/build_tar.py
diff options
context:
space:
mode:
Diffstat (limited to 'pkg/private/tar/build_tar.py')
-rw-r--r--pkg/private/tar/build_tar.py459
1 files changed, 0 insertions, 459 deletions
diff --git a/pkg/private/tar/build_tar.py b/pkg/private/tar/build_tar.py
deleted file mode 100644
index 08a3a06..0000000
--- a/pkg/private/tar/build_tar.py
+++ /dev/null
@@ -1,459 +0,0 @@
-# Copyright 2015 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""This tool build tar files from a list of inputs."""
-
-import argparse
-import os
-import tarfile
-import tempfile
-
-from pkg.private import archive
-from pkg.private import helpers
-from pkg.private import build_info
-from pkg.private import manifest
-from pkg.private.tar import tar_writer
-
-
-def normpath(path):
- """Normalize a path to the format we need it.
-
- os.path.normpath changes / to \ on windows, but tarfile needs / style paths.
-
- Args:
- path: (str) path to normalize.
- """
- return os.path.normpath(path).replace(os.path.sep, '/')
-
-
-class TarFile(object):
- """A class to generates a TAR file."""
-
- class DebError(Exception):
- pass
-
- def __init__(self, output, directory, compression, compressor, default_mtime):
- # Directory prefix on all output paths
- d = directory.strip('/')
- self.directory = (d + '/') if d else None
- self.output = output
- self.compression = compression
- self.compressor = compressor
- self.default_mtime = default_mtime
-
- def __enter__(self):
- self.tarfile = tar_writer.TarFileWriter(
- self.output,
- self.compression,
- self.compressor,
- default_mtime=self.default_mtime)
- return self
-
- def __exit__(self, t, v, traceback):
- self.tarfile.close()
-
- def normalize_path(self, path: str) -> str:
- dest = normpath(path)
- # paths should not have a leading ./
- if dest.startswith('./'):
- dest = dest[2:]
- # No path should ever come in with slashs on either end, but protect
- # against that anyway.
- dest = dest.strip('/')
- # This prevents a potential problem for users with both a prefix_dir and
- # symlinks that also repeat the prefix_dir. The old behavior was that we
- # would get just the symlink path. Now we are prefixing with the prefix,
- # so you get the file in the wrong place.
- # We silently de-dup that. If people come up with a real use case for
- # the /a/b/a/b/rest... output we can start an issue and come up with a
- # solution at that time.
- if self.directory and not dest.startswith(self.directory):
- dest = self.directory + dest
- return dest
-
- def add_file(self, f, destfile, mode=None, ids=None, names=None):
- """Add a file to the tar file.
-
- Args:
- f: the file to add to the layer
- destfile: the name of the file in the layer
- mode: (int) force to set the specified mode, by default the value from the
- source is taken.
- ids: (uid, gid) for the file to set ownership
- names: (username, groupname) for the file to set ownership. `f` will be
- copied to `self.directory/destfile` in the layer.
- """
- dest = self.normalize_path(destfile)
- # If mode is unspecified, derive the mode from the file's mode.
- if mode is None:
- mode = 0o755 if os.access(f, os.X_OK) else 0o644
- if ids is None:
- ids = (0, 0)
- if names is None:
- names = ('', '')
- self.tarfile.add_file(
- dest,
- file_content=f,
- mode=mode,
- uid=ids[0],
- gid=ids[1],
- uname=names[0],
- gname=names[1])
-
- def add_empty_file(self,
- destfile,
- mode=None,
- ids=None,
- names=None,
- kind=tarfile.REGTYPE):
- """Add a file to the tar file.
-
- Args:
- destfile: the name of the file in the layer
- mode: force to set the specified mode, defaults to 644
- ids: (uid, gid) for the file to set ownership
- names: (username, groupname) for the file to set ownership.
- kind: type of the file. tarfile.DIRTYPE for directory. An empty file
- will be created as `destfile` in the layer.
- """
- dest = destfile.lstrip('/') # Remove leading slashes
- # If mode is unspecified, assume read only
- if mode is None:
- mode = 0o644
- if ids is None:
- ids = (0, 0)
- if names is None:
- names = ('', '')
- dest = normpath(dest)
- self.tarfile.add_file(
- dest,
- content='' if kind == tarfile.REGTYPE else None,
- kind=kind,
- mode=mode,
- uid=ids[0],
- gid=ids[1],
- uname=names[0],
- gname=names[1])
-
- def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
- """Add a directory to the tar file.
-
- Args:
- destpath: the name of the directory in the layer
- mode: force to set the specified mode, defaults to 644
- ids: (uid, gid) for the file to set ownership
- names: (username, groupname) for the file to set ownership. An empty
- file will be created as `destfile` in the layer.
- """
- self.add_empty_file(
- destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE)
-
- def add_tar(self, tar):
- """Merge a tar file into the destination tar file.
-
- All files presents in that tar will be added to the output file
- under self.directory/path. No user name nor group name will be
- added to the output.
-
- Args:
- tar: the tar file to add
- """
- self.tarfile.add_tar(tar, numeric=True, prefix=self.directory)
-
- def add_link(self, symlink, destination, mode=None, ids=None, names=None):
- """Add a symbolic link pointing to `destination`.
-
- Args:
- symlink: the name of the symbolic link to add.
- destination: where the symbolic link point to.
- mode: (int) force to set the specified posix mode (e.g. 0o755). The
- default is derived from the source
- ids: (uid, gid) for the file to set ownership
- names: (username, groupname) for the file to set ownership. An empty
- file will be created as `destfile` in the layer.
- """
- dest = self.normalize_path(symlink)
- self.tarfile.add_file(
- dest,
- tarfile.SYMTYPE,
- link=destination,
- mode = mode,
- uid=ids[0],
- gid=ids[1],
- uname=names[0],
- gname=names[1])
-
- def add_deb(self, deb):
- """Extract a debian package in the output tar.
-
- All files presents in that debian package will be added to the
- output tar under the same paths. No user name nor group names will
- be added to the output.
-
- Args:
- deb: the tar file to add
-
- Raises:
- DebError: if the format of the deb archive is incorrect.
- """
- with archive.SimpleArReader(deb) as arfile:
- current = next(arfile)
- while current and not current.filename.startswith('data.'):
- current = next(arfile)
- if not current:
- raise self.DebError(deb + ' does not contains a data file!')
- tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1])
- with open(tmpfile[1], 'wb') as f:
- f.write(current.data)
- self.add_tar(tmpfile[1])
- os.remove(tmpfile[1])
-
- def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
- """Add a tree artifact to the tar file.
-
- Args:
- tree_top: the top of the tree to add
- destpath: the path under which to place the files
- mode: (int) force to set the specified posix mode (e.g. 0o755). The
- default is derived from the source
- ids: (uid, gid) for the file to set ownership
- names: (username, groupname) for the file to set ownership. `f` will be
- copied to `self.directory/destfile` in the layer.
- """
- # We expect /-style paths.
- tree_top = normpath(tree_top)
-
- dest = destpath.strip('/') # redundant, dests should never have / here
- if self.directory and self.directory != '/':
- dest = self.directory.lstrip('/') + '/' + dest
-
- # Again, we expect /-style paths.
- dest = normpath(dest)
- # normpath may be ".", and dest paths should not start with "./"
- dest = '' if dest == '.' else dest + '/'
-
- if ids is None:
- ids = (0, 0)
- if names is None:
- names = ('', '')
-
- to_write = {}
- for root, dirs, files in os.walk(tree_top):
- # While `tree_top` uses '/' as a path separator, results returned by
- # `os.walk` and `os.path.join` on Windows may not.
- root = normpath(root)
-
- dirs = sorted(dirs)
- rel_path_from_top = root[len(tree_top):].lstrip('/')
- if rel_path_from_top:
- dest_dir = dest + rel_path_from_top + '/'
- else:
- dest_dir = dest
- for dir in dirs:
- to_write[dest_dir + dir] = None
- for file in sorted(files):
- content_path = os.path.abspath(os.path.join(root, file))
- if os.name == "nt":
- # "To specify an extended-length path, use the `\\?\` prefix. For
- # example, `\\?\D:\very long path`."[1]
- #
- # [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
- to_write[dest_dir + file] = "\\\\?\\" + content_path
- else:
- to_write[dest_dir + file] = content_path
-
- for path in sorted(to_write.keys()):
- content_path = to_write[path]
- if not content_path:
- # This is an intermediate directory. Bazel has no API to specify modes
- # for this, so the least surprising thing we can do is make it the
- # canonical rwxr-xr-x
- self.add_empty_file(
- path,
- mode=0o755,
- ids=ids,
- names=names,
- kind=tarfile.DIRTYPE)
- else:
- # If mode is unspecified, derive the mode from the file's mode.
- if mode is None:
- f_mode = 0o755 if os.access(content_path, os.X_OK) else 0o644
- else:
- f_mode = mode
- self.tarfile.add_file(
- path,
- file_content=content_path,
- mode=f_mode,
- uid=ids[0],
- gid=ids[1],
- uname=names[0],
- gname=names[1])
-
- def add_manifest_entry(self, entry, file_attributes):
- # Use the pkg_tar mode/owner remapping as a fallback
- non_abs_path = entry.dest.strip('/')
- if file_attributes:
- attrs = file_attributes(non_abs_path)
- else:
- attrs = {}
- # But any attributes from the manifest have higher precedence
- if entry.mode is not None and entry.mode != '':
- attrs['mode'] = int(entry.mode, 8)
- if entry.user:
- if entry.group:
- attrs['names'] = (entry.user, entry.group)
- else:
- # Use group that legacy tar process would assign
- attrs['names'] = (entry.user, attrs.get('names')[1])
- if entry.uid is not None:
- if entry.gid is not None:
- attrs['ids'] = (entry.uid, entry.gid)
- else:
- attrs['ids'] = (entry.uid, entry.uid)
- if entry.type == manifest.ENTRY_IS_LINK:
- self.add_link(entry.dest, entry.src, **attrs)
- elif entry.type == manifest.ENTRY_IS_DIR:
- self.add_empty_dir(self.normalize_path(entry.dest), **attrs)
- elif entry.type == manifest.ENTRY_IS_TREE:
- self.add_tree(entry.src, entry.dest, **attrs)
- elif entry.type == manifest.ENTRY_IS_EMPTY_FILE:
- self.add_empty_file(self.normalize_path(entry.dest), **attrs)
- else:
- self.add_file(entry.src, entry.dest, **attrs)
-
-
-def main():
- parser = argparse.ArgumentParser(
- description='Helper for building tar packages',
- fromfile_prefix_chars='@')
- parser.add_argument('--output', required=True,
- help='The output file, mandatory.')
- parser.add_argument('--manifest',
- help='manifest of contents to add to the layer.')
- parser.add_argument('--mode',
- help='Force the mode on the added files (in octal).')
- parser.add_argument(
- '--mtime',
- help='Set mtime on tar file entries. May be an integer or the'
- ' value "portable", to get the value 2000-01-01, which is'
- ' is usable with non *nix OSes.')
- parser.add_argument('--tar', action='append',
- help='A tar file to add to the layer')
- parser.add_argument('--deb', action='append',
- help='A debian package to add to the layer')
- parser.add_argument(
- '--directory',
- help='Directory in which to store the file inside the layer')
-
- compression = parser.add_mutually_exclusive_group()
- compression.add_argument('--compression',
- help='Compression (`gz` or `bz2`), default is none.')
- compression.add_argument('--compressor',
- help='Compressor program and arguments, '
- 'e.g. `pigz -p 4`')
-
- parser.add_argument(
- '--modes', action='append',
- help='Specific mode to apply to specific file (from the file argument),'
- ' e.g., path/to/file=0455.')
- parser.add_argument(
- '--owners', action='append',
- help='Specify the numeric owners of individual files, '
- 'e.g. path/to/file=0.0.')
- parser.add_argument(
- '--owner', default='0.0',
- help='Specify the numeric default owner of all files,'
- ' e.g., 0.0')
- parser.add_argument(
- '--owner_name',
- help='Specify the owner name of all files, e.g. root.root.')
- parser.add_argument(
- '--owner_names', action='append',
- help='Specify the owner names of individual files, e.g. '
- 'path/to/file=root.root.')
- parser.add_argument('--stamp_from', default='',
- help='File to find BUILD_STAMP in')
- options = parser.parse_args()
-
- # Parse modes arguments
- default_mode = None
- if options.mode:
- # Convert from octal
- default_mode = int(options.mode, 8)
-
- mode_map = {}
- if options.modes:
- for filemode in options.modes:
- (f, mode) = helpers.SplitNameValuePairAtSeparator(filemode, '=')
- if f[0] == '/':
- f = f[1:]
- mode_map[f] = int(mode, 8)
-
- default_ownername = ('', '')
- if options.owner_name:
- default_ownername = options.owner_name.split('.', 1)
- names_map = {}
- if options.owner_names:
- for file_owner in options.owner_names:
- (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=')
- (user, group) = owner.split('.', 1)
- if f[0] == '/':
- f = f[1:]
- names_map[f] = (user, group)
-
- default_ids = options.owner.split('.', 1)
- default_ids = (int(default_ids[0]), int(default_ids[1]))
- ids_map = {}
- if options.owners:
- for file_owner in options.owners:
- (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=')
- (user, group) = owner.split('.', 1)
- if f[0] == '/':
- f = f[1:]
- ids_map[f] = (int(user), int(group))
-
- default_mtime = options.mtime
- if options.stamp_from:
- default_mtime = build_info.get_timestamp(options.stamp_from)
-
- # Add objects to the tar file
- with TarFile(
- options.output,
- directory = helpers.GetFlagValue(options.directory),
- compression = options.compression,
- compressor = options.compressor,
- default_mtime=default_mtime) as output:
-
- def file_attributes(filename):
- if filename.startswith('/'):
- filename = filename[1:]
- return {
- 'mode': mode_map.get(filename, default_mode),
- 'ids': ids_map.get(filename, default_ids),
- 'names': names_map.get(filename, default_ownername),
- }
-
- if options.manifest:
- with open(options.manifest, 'r') as manifest_fp:
- manifest_entries = manifest.read_entries_from(manifest_fp)
- for entry in manifest_entries:
- output.add_manifest_entry(entry, file_attributes)
-
- for tar in options.tar or []:
- output.add_tar(tar)
- for deb in options.deb or []:
- output.add_deb(deb)
-
-
-if __name__ == '__main__':
- main()