Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add tar xattr support #707

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add tar xattr support #707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 64 additions & 12 deletions pkg/private/tar/build_tar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""This tool build tar files from a list of inputs."""

import argparse
import base64
import os
import tarfile
import tempfile
Expand All @@ -36,27 +37,47 @@ def normpath(path):
return os.path.normpath(path).replace(os.path.sep, '/')


def parse_xattr(xattr_list):
xattrs = {}
if xattr_list:
for item in xattr_list:
idx = item.index("=")
if idx < 0:
raise ValueError("Unexpected xattr item format {} (want key=value)".format(item))
key = item[:idx]
raw = item[idx+1:]
if raw.startswith("0x"):
xattrs[key] = bytes.fromhex(raw[2:]).decode('latin-1')
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You probably want to use .decode('utf-8', "surrogateescape") here as certain xattrs will be re-encoded incorrectly otherwise. On re-encoding the tarfile module uses surrogateescape to restore any values that don't cleanly decode to UTF-8.

elif raw.startswith('"') and raw.endswith('"') and len(raw) >= 2:
xattrs[key] = raw[1:-1]
else:
xattrs[key] = base64.b64decode(raw).decode('latin-1')
return xattrs


class TarFile(object):
"""A class to generates a TAR file."""

class DebError(Exception):
pass

def __init__(self, output, directory, compression, compressor, default_mtime):
def __init__(self, output, directory, compression, compressor, default_mtime, use_xattr):
# Directory prefix on all output paths
d = directory.strip('/')
self.directory = (d + '/') if d else None
self.output = output
self.compression = compression
self.compressor = compressor
self.default_mtime = default_mtime
self.use_xattr = use_xattr

def __enter__(self):
self.tarfile = tar_writer.TarFileWriter(
self.output,
self.compression,
self.compressor,
default_mtime=self.default_mtime)
default_mtime=self.default_mtime,
use_xattr=self.use_xattr)
return self

def __exit__(self, t, v, traceback):
Expand All @@ -74,7 +95,7 @@ def normalize_path(self, path: str) -> str:
dest = self.directory + dest
return dest

def add_file(self, f, destfile, mode=None, ids=None, names=None):
def add_file(self, f, destfile, mode=None, ids=None, names=None, xattr=None):
"""Add a file to the tar file.

Args:
Expand All @@ -85,6 +106,7 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. `f` will be
copied to `self.directory/destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
dest = self.normalize_path(destfile)
# If mode is unspecified, derive the mode from the file's mode.
Expand All @@ -101,14 +123,16 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
uid=ids[0],
gid=ids[1],
uname=names[0],
gname=names[1])
gname=names[1],
xattr=xattr)

def add_empty_file(self,
destfile,
mode=None,
ids=None,
names=None,
kind=tarfile.REGTYPE):
kind=tarfile.REGTYPE,
xattr=None):
"""Add a file to the tar file.

Args:
Expand All @@ -118,6 +142,7 @@ def add_empty_file(self,
names: (username, groupname) for the file to set ownership.
kind: type of the file. tarfile.DIRTYPE for directory. An empty file
will be created as `destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
dest = destfile.lstrip('/') # Remove leading slashes
# If mode is unspecified, assume read only
Expand All @@ -136,9 +161,10 @@ def add_empty_file(self,
uid=ids[0],
gid=ids[1],
uname=names[0],
gname=names[1])
gname=names[1],
xattr=xattr)

def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
def add_empty_dir(self, destpath, mode=None, ids=None, names=None, xattr=None):
"""Add a directory to the tar file.

Args:
Expand All @@ -147,9 +173,10 @@ def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. An empty
file will be created as `destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
self.add_empty_file(
destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE)
destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE, xattr=xattr)

def add_tar(self, tar):
"""Merge a tar file into the destination tar file.
Expand All @@ -163,7 +190,7 @@ def add_tar(self, tar):
"""
self.tarfile.add_tar(tar, numeric=True, prefix=self.directory)

def add_link(self, symlink, destination, mode=None, ids=None, names=None):
def add_link(self, symlink, destination, mode=None, ids=None, names=None, xattr=None):
"""Add a symbolic link pointing to `destination`.

Args:
Expand All @@ -174,6 +201,7 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. An empty
file will be created as `destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
dest = self.normalize_path(symlink)
self.tarfile.add_file(
Expand All @@ -184,7 +212,8 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
uid=ids[0],
gid=ids[1],
uname=names[0],
gname=names[1])
gname=names[1],
xattr=xattr)

def add_deb(self, deb):
"""Extract a debian package in the output tar.
Expand All @@ -211,7 +240,7 @@ def add_deb(self, deb):
self.add_tar(tmpfile[1])
os.remove(tmpfile[1])

def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None, xattr=None):
"""Add a tree artifact to the tar file.

Args:
Expand All @@ -222,6 +251,7 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
ids: (uid, gid) for the file to set ownership
names: (username, groupname) for the file to set ownership. `f` will be
copied to `self.directory/destfile` in the layer.
xattr: (strings) xattr list in getfattr-like output style.
"""
# We expect /-style paths.
tree_top = normpath(tree_top)
Expand Down Expand Up @@ -374,6 +404,14 @@ def main():
'--owner_names', action='append',
help='Specify the owner names of individual files, e.g. '
'path/to/file=root.root.')
parser.add_argument(
'--xattr', action='append',
help='Specify the xattr of all files, e.g. '
'security.capability=0x0100000200000001000000000000000000000000')
parser.add_argument(
'--file_xattr', action='append',
help='Specify the xattr of individual files, e.g. '
'path/to/file=security.capability=0x0100000200000001000000000000000000000000')
parser.add_argument('--stamp_from', default='',
help='File to find BUILD_STAMP in')
options = parser.parse_args()
Expand Down Expand Up @@ -404,6 +442,18 @@ def main():
f = f[1:]
names_map[f] = (user, group)

default_xattr = parse_xattr(options.xattr)
xattr_map = {}
if options.file_xattr:
xattr_by_file = {}
for file_xattr in options.file_xattr:
(f, xattr) = helpers.SplitNameValuePairAtSeparator(file_xattr, '=')
xattrs = xattr_by_file.get(f, [])
xattrs.append(xattr)
xattr_by_file[f] = xattrs
for f in xattr_by_file:
xattr_map[f] = parse_xattr(xattr_by_file[f])

default_ids = options.owner.split('.', 1)
default_ids = (int(default_ids[0]), int(default_ids[1]))
ids_map = {}
Expand All @@ -425,7 +475,8 @@ def main():
directory = helpers.GetFlagValue(options.directory),
compression = options.compression,
compressor = options.compressor,
default_mtime=default_mtime) as output:
default_mtime=default_mtime,
use_xattr=bool(xattr_map or default_xattr)) as output:

def file_attributes(filename):
if filename.startswith('/'):
Expand All @@ -434,6 +485,7 @@ def file_attributes(filename):
'mode': mode_map.get(filename, default_mode),
'ids': ids_map.get(filename, default_ids),
'names': names_map.get(filename, default_ownername),
'xattr': {**xattr_map.get(filename, {}), **default_xattr}
}

if options.manifest:
Expand Down
10 changes: 10 additions & 0 deletions pkg/private/tar/tar.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,14 @@ def _pkg_tar_impl(ctx):
"--owner_names",
"%s=%s" % (_quote(key), ctx.attr.ownernames[key]),
)
if ctx.attr.xattr:
for item in ctx.attr.xattr:
args.add("--xattr", item)
if ctx.attr.xattrs:
for file in ctx.attr.xattrs:
xattr = ctx.attr.xattrs[file]
for item in xattr:
args.add("--file_xattr", "%s=%s" % (_quote(file), item))
for empty_file in ctx.attr.empty_files:
add_empty_file(content_map, empty_file, ctx.label)
for empty_dir in ctx.attr.empty_dirs or []:
Expand Down Expand Up @@ -264,6 +272,8 @@ pkg_tar_impl = rule(
"ownername": attr.string(default = "."),
"owners": attr.string_dict(),
"ownernames": attr.string_dict(),
"xattr": attr.string_list(),
"xattrs": attr.string_list_dict(),
"extension": attr.string(default = "tar"),
"symlinks": attr.string_dict(),
"empty_files": attr.string_list(),
Expand Down
17 changes: 14 additions & 3 deletions pkg/private/tar/tar_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ def __init__(self,
compression='',
compressor='',
default_mtime=None,
preserve_tar_mtimes=True):
preserve_tar_mtimes=True,
use_xattr=False):
"""TarFileWriter wraps tarfile.open().

Args:
Expand All @@ -60,6 +61,7 @@ def __init__(self,
preserve_tar_mtimes: if true, keep file mtimes from input tar file.
"""
self.preserve_mtime = preserve_tar_mtimes
self.use_xattr = use_xattr
if default_mtime is None:
self.default_mtime = 0
elif default_mtime == 'portable':
Expand Down Expand Up @@ -98,7 +100,7 @@ def __init__(self,
self.name = name

self.tar = tarfile.open(name=name, mode=mode, fileobj=self.fileobj,
format=tarfile.GNU_FORMAT)
format=tarfile.PAX_FORMAT if use_xattr else tarfile.GNU_FORMAT)
self.members = set()
self.directories = set()
# Preseed the added directory list with things we should not add. If we
Expand Down Expand Up @@ -191,7 +193,8 @@ def add_file(self,
uname='',
gname='',
mtime=None,
mode=None):
mode=None,
xattr=None):
"""Add a file to the current tar.

Args:
Expand Down Expand Up @@ -234,6 +237,14 @@ def add_file(self,
tarinfo.mode = mode
if link:
tarinfo.linkname = link
if xattr:
if not self.use_xattr:
raise self.Error('This tar file was created without `use_xattr` flag but try to create file with xattr: {}, {}'.
format(name, xattr))
pax_headers = {}
for key in xattr:
pax_headers["SCHILY.xattr." + key] = xattr[key]
tarinfo.pax_headers = pax_headers
if content:
content_bytes = content.encode('utf-8')
tarinfo.size = len(content_bytes)
Expand Down
10 changes: 10 additions & 0 deletions tests/tar/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,16 @@ pkg_tar(
ownername = "titi.tata",
ownernames = {"etc/nsswitch.conf": "tata.titi"},
owners = {"etc/nsswitch.conf": "24.42"},
xattr = [
"security.capability=0x0100000200000001000000000000000000000000", # setcap cap_sys_resource+ep tool && getfattr -d -e hex -m - tool
],
xattrs = {
"etc/nsswitch.conf": [
"user.foo=\"foo\"", # String
"user.bar=0x626172", # Hex
"user.baz=YmF6", # Base64
],
},
package_dir = "/",
strip_prefix = ".",
symlinks = {"usr/bin/java": "/path/to/bin/java"},
Expand Down