Skip to content

Commit

Permalink
Add tar xattr support
Browse files Browse the repository at this point in the history
  • Loading branch information
@bozaro
bozaro committed Jun 5, 2023
1 parent ec4ad13 commit 027f82d
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 15 deletions.
76 changes: 64 additions & 12 deletions pkg/private/tar/build_tar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""This tool build tar files from a list of inputs."""

import argparse
import base64
import os
import tarfile
import tempfile
Expand All @@ -36,27 +37,47 @@ def normpath(path):
return os.path.normpath(path).replace(os.path.sep, '/')


def parse_xattr(xattr_list):
xattrs = {}
if xattr_list:
for item in xattr_list:
idx = item.index("=")
if idx < 0:
raise ValueError("Unexpected xattr item format {} (want key=value)".format(item))
key = item[:idx]
raw = item[idx+1:]
if raw.startswith("0x"):
xattrs[key] = bytes.fromhex(raw[2:]).decode('latin-1')
elif raw.startswith('"') and raw.endswith('"') and len(raw) >= 2:
xattrs[key] = raw[1:-1]
else:
xattrs[key] = base64.b64decode(raw).decode('latin-1')
return xattrs


class TarFile(object):
"""A class to generates a TAR file."""

class DebError(Exception):
pass

def __init__(self, output, directory, compression, compressor, default_mtime):
def __init__(self, output, directory, compression, compressor, default_mtime, use_xattr):
# Directory prefix on all output paths
d = directory.strip('/')
self.directory = (d + '/') if d else None
self.output = output
self.compression = compression
self.compressor = compressor
self.default_mtime = default_mtime
self.use_xattr = use_xattr

def __enter__(self):
self.tarfile = tar_writer.TarFileWriter(
self.output,
self.compression,
self.compressor,
default_mtime=self.default_mtime)
default_mtime=self.default_mtime,
use_xattr=self.use_xattr)
return self

def __exit__(self, t, v, traceback):
Expand All @@ -74,7 +95,7 @@ def normalize_path(self, path: str) -> str:
dest = self.directory + dest
return dest

def add_file(self, f, destfile, mode=None, ids=None, names=None):
def add_file(self, f, destfile, mode=None, ids=None, names=None, xattr=None):
"""Add a file to the tar file.
Args:
Expand All @@ -85,6 +106,7 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. `f` will be
copied to `self.directory/destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
dest = self.normalize_path(destfile)
# If mode is unspecified, derive the mode from the file's mode.
Expand All @@ -101,14 +123,16 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
uid=ids[0],
gid=ids[1],
uname=names[0],
gname=names[1])
gname=names[1],
xattr=xattr)

def add_empty_file(self,
destfile,
mode=None,
ids=None,
names=None,
kind=tarfile.REGTYPE):
kind=tarfile.REGTYPE,
xattr=None):
"""Add a file to the tar file.
Args:
Expand All @@ -118,6 +142,7 @@ def add_empty_file(self,
names: (username, groupname) for the file to set ownership.
kind: type of the file. tarfile.DIRTYPE for directory. An empty file
will be created as `destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
dest = destfile.lstrip('/') # Remove leading slashes
# If mode is unspecified, assume read only
Expand All @@ -136,9 +161,10 @@ def add_empty_file(self,
uid=ids[0],
gid=ids[1],
uname=names[0],
gname=names[1])
gname=names[1],
xattr=xattr)

def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
def add_empty_dir(self, destpath, mode=None, ids=None, names=None, xattr=None):
"""Add a directory to the tar file.
Args:
Expand All @@ -147,9 +173,10 @@ def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. An empty
file will be created as `destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
self.add_empty_file(
destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE)
destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE, xattr=xattr)

def add_tar(self, tar):
"""Merge a tar file into the destination tar file.
Expand All @@ -163,7 +190,7 @@ def add_tar(self, tar):
"""
self.tarfile.add_tar(tar, numeric=True, prefix=self.directory)

def add_link(self, symlink, destination, mode=None, ids=None, names=None):
def add_link(self, symlink, destination, mode=None, ids=None, names=None, xattr=None):
"""Add a symbolic link pointing to `destination`.
Args:
Expand All @@ -174,6 +201,7 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. An empty
file will be created as `destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
dest = self.normalize_path(symlink)
self.tarfile.add_file(
Expand All @@ -184,7 +212,8 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
uid=ids[0],
gid=ids[1],
uname=names[0],
gname=names[1])
gname=names[1],
xattr=xattr)

def add_deb(self, deb):
"""Extract a debian package in the output tar.
Expand All @@ -211,7 +240,7 @@ def add_deb(self, deb):
self.add_tar(tmpfile[1])
os.remove(tmpfile[1])

def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None, xattr=None):
"""Add a tree artifact to the tar file.
Args:
Expand All @@ -222,6 +251,7 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. `f` will be
copied to `self.directory/destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
# We expect /-style paths.
tree_top = normpath(tree_top)
Expand Down Expand Up @@ -374,6 +404,14 @@ def main():
'--owner_names', action='append',
help='Specify the owner names of individual files, e.g. '
'path/to/file=root.root.')
parser.add_argument(
'--xattr', action='append',
help='Specify the xattr of all files, e.g. '
'security.capability=0x0100000200000001000000000000000000000000')
parser.add_argument(
'--file_xattr', action='append',
help='Specify the xattr of individual files, e.g. '
'path/to/file=security.capability=0x0100000200000001000000000000000000000000')
parser.add_argument('--stamp_from', default='',
help='File to find BUILD_STAMP in')
options = parser.parse_args()
Expand Down Expand Up @@ -404,6 +442,18 @@ def main():
f = f[1:]
names_map[f] = (user, group)

default_xattr = parse_xattr(options.xattr)
xattr_map = {}
if options.file_xattr:
xattr_by_file = {}
for file_xattr in options.file_xattr:
(f, xattr) = helpers.SplitNameValuePairAtSeparator(file_xattr, '=')
xattrs = xattr_by_file.get(f, [])
xattrs.append(xattr)
xattr_by_file[f] = xattrs
for f in xattr_by_file:
xattr_map[f] = parse_xattr(xattr_by_file[f])

default_ids = options.owner.split('.', 1)
default_ids = (int(default_ids[0]), int(default_ids[1]))
ids_map = {}
Expand All @@ -425,7 +475,8 @@ def main():
directory = helpers.GetFlagValue(options.directory),
compression = options.compression,
compressor = options.compressor,
default_mtime=default_mtime) as output:
default_mtime=default_mtime,
use_xattr=bool(xattr_map or default_xattr)) as output:

def file_attributes(filename):
if filename.startswith('/'):
Expand All @@ -434,6 +485,7 @@ def file_attributes(filename):
'mode': mode_map.get(filename, default_mode),
'ids': ids_map.get(filename, default_ids),
'names': names_map.get(filename, default_ownername),
'xattr': {**xattr_map.get(filename, {}), **default_xattr}
}

if options.manifest:
Expand Down
10 changes: 10 additions & 0 deletions pkg/private/tar/tar.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,14 @@ def _pkg_tar_impl(ctx):
"--owner_names",
"%s=%s" % (_quote(key), ctx.attr.ownernames[key]),
)
if ctx.attr.xattr:
for item in ctx.attr.xattr:
args.add("--xattr", item)
if ctx.attr.xattrs:
for file in ctx.attr.xattrs:
xattr = ctx.attr.xattrs[file]
for item in xattr:
args.add("--file_xattr", "%s=%s" % (_quote(file), item))
for empty_file in ctx.attr.empty_files:
add_empty_file(content_map, empty_file, ctx.label)
for empty_dir in ctx.attr.empty_dirs or []:
Expand Down Expand Up @@ -264,6 +272,8 @@ pkg_tar_impl = rule(
"ownername": attr.string(default = "."),
"owners": attr.string_dict(),
"ownernames": attr.string_dict(),
"xattr": attr.string_list(),
"xattrs": attr.string_list_dict(),
"extension": attr.string(default = "tar"),
"symlinks": attr.string_dict(),
"empty_files": attr.string_list(),
Expand Down
17 changes: 14 additions & 3 deletions pkg/private/tar/tar_writer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ def __init__(self,
compression='',
compressor='',
default_mtime=None,
preserve_tar_mtimes=True):
preserve_tar_mtimes=True,
use_xattr=False):
"""TarFileWriter wraps tarfile.open().
Args:
Expand All @@ -60,6 +61,7 @@ def __init__(self,
preserve_tar_mtimes: if true, keep file mtimes from input tar file.
"""
self.preserve_mtime = preserve_tar_mtimes
self.use_xattr = use_xattr
if default_mtime is None:
self.default_mtime = 0
elif default_mtime == 'portable':
Expand Down Expand Up @@ -98,7 +100,7 @@ def __init__(self,
self.name = name

self.tar = tarfile.open(name=name, mode=mode, fileobj=self.fileobj,
format=tarfile.GNU_FORMAT)
format=tarfile.PAX_FORMAT if use_xattr else tarfile.GNU_FORMAT)
self.members = set()
self.directories = set()
# Preseed the added directory list with things we should not add. If we
Expand Down Expand Up @@ -191,7 +193,8 @@ def add_file(self,
uname='',
gname='',
mtime=None,
mode=None):
mode=None,
xattr=None):
"""Add a file to the current tar.
Args:
Expand Down Expand Up @@ -234,6 +237,14 @@ def add_file(self,
tarinfo.mode = mode
if link:
tarinfo.linkname = link
if xattr:
if not self.use_xattr:
raise self.Error('This tar file was created without `use_xattr` flag but try to create file with xattr: {}, {}'.
format(name, xattr))
pax_headers = {}
for key in xattr:
pax_headers["SCHILY.xattr." + key] = xattr[key]
tarinfo.pax_headers = pax_headers
if content:
content_bytes = content.encode('utf-8')
tarinfo.size = len(content_bytes)
Expand Down

0 comments on commit 027f82d

Please sign in to comment.