Skip to content

Support for zstd compression #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jklukas merged 1 commit into from
Sep 29, 2025
Merged

Support for zstd compression #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion minidock/container_data.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,22 @@ def _magic_path(ctx, f, output_layer):

def __container_data_impl(
ctx):
layer = ctx.actions.declare_file("%s.tgz" % ctx.attr.name)
# Determine file extension based on compression type
compression = ctx.attr.compression
if compression == "gz" or compression == "tgz":
ext = ".tgz"
elif compression == "bz2" or compression == "bzip2":
ext = ".tar.bz2"
elif compression == "xz":
ext = ".tar.xz"
elif compression == "lzma":
ext = ".tar.lzma"
elif compression == "zstd":
ext = ".tar.zst"
else:
ext = ".tar"

layer = ctx.actions.declare_file("%s%s" % (ctx.attr.name, ext))

files = ctx.files.files
args = ctx.actions.args()
Expand All @@ -97,6 +112,8 @@ def __container_data_impl(
ctx.actions.write(manifest_file, json.encode(manifest))
args.add(manifest_file, format = "--manifest=%s")
args.add(ctx.attr.gzip_compression_level, format = "--gzip_compression_level=%s")
args.add(ctx.attr.zstd_compression_level, format = "--zstd_compression_level=%s")
args.add(compression, format = "--compression=%s")
args.add(ctx.attr.mtime, format = "--mtime=%s")

ctx.actions.run(
Expand Down Expand Up @@ -152,6 +169,14 @@ container_data = rule(
default=9,
doc = "The gzip compression level to use when making .tar.gz outputs, use low effort for already compressed things like jars"
),
"zstd_compression_level": attr.int(
default=3,
doc = "The zstd compression level to use when making .tar.zst outputs (1-22)"
),
"compression": attr.string(
default="gz",
doc = "Compression format: gz, bz2, xz, lzma, zstd, or '' for no compression"
),
"files": attr.label_list(
allow_files = True,
doc = """File to add to the layer.
Expand Down
59 changes: 52 additions & 7 deletions minidock/container_data_tools/build_tar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,16 @@ def __init__(self,
name,
compression='',
gzip_compression_level=9,
zstd_compression_level=3,
root_directory='./',
default_mtime=None,
preserve_tar_mtimes=True):
"""TarFileWriter wraps tarfile.open().
Args:
name: the tar file name.
compression: compression type: bzip2, bz2, gz, tgz, xz, lzma.
compression: compression type: bzip2, bz2, gz, tgz, xz, lzma, zstd.
gzip_compression_level: compression level for gzip (1-9).
zstd_compression_level: compression level for zstd (1-22).
root_directory: virtual root to prepend to elements in the archive.
default_mtime: default mtime to use for elements in the archive.
May be an integer or the value 'portable' to use the date
Expand All @@ -66,17 +69,21 @@ def __init__(self,
self.gz = compression in ['tgz', 'gz']
# Support xz compression through xz... until we can use Py3
self.xz = compression in ['xz', 'lzma']
# Support zstd compression through zstd command line tool
self.zstd = compression == 'zstd'
self.zstd_compression_level = zstd_compression_level
self.name = name
self.root_directory = root_directory.rstrip('/')
self.preserve_mtime = preserve_tar_mtimes
if default_mtime is None:
self.default_mtime = 0
elif default_mtime == 'portable':
self.default_mtime = PORTABLE_MTIME
self.default_mtime = 946684800 # January 1, 2000, 00:00:00 UTC
else:
self.default_mtime = int(default_mtime)

self.fileobj = None
self._raw_file = None
if self.gz:
# The Tarfile class doesn't allow us to specify gzip's mtime attribute.
# Instead, we manually re-implement gzopen from tarfile.py and set mtime.
Expand Down Expand Up @@ -287,7 +294,9 @@ def add_tar(self,
compression = 'bz2'
elif compression == 'lzma':
compression = 'xz'
elif compression not in ['gz', 'bz2', 'xz']:
elif compression == 'zst':
compression = 'zstd'
elif compression not in ['gz', 'bz2', 'xz', 'zstd']:
compression = ''
if compression == 'xz':
# Python 2 does not support lzma, our py3 support is terrible so let's
Expand All @@ -306,6 +315,20 @@ def add_tar(self,
f = io.BytesIO(p.stdout.read())
p.wait()
intar = tarfile.open(fileobj=f, mode='r:')
elif compression == 'zstd':
# Handle zstd compression through zstd command line tool
# Note that we buffer the file in memory and it can have an important
# memory footprint but it's probably fine as we don't use them for really
# large files.
if subprocess.call('which zstd', shell=True, stdout=subprocess.PIPE):
raise self.Error('Cannot handle .zstd compression: '
'zstd command not found.')
p = subprocess.Popen('zstd -dc %s' % tar,
shell=True,
stdout=subprocess.PIPE)
f = io.BytesIO(p.stdout.read())
p.wait()
intar = tarfile.open(fileobj=f, mode='r:')
else:
if compression in ['gz', 'bz2']:
# prevent performance issues due to accidentally-introduced seeks
Expand Down Expand Up @@ -378,6 +401,19 @@ def close(self):
# Close the gzip file object if necessary.
if self.fileobj:
self.fileobj.close()

if self.zstd:
# Support zstd compression through zstd command line tool
# Following same pattern as xz to maintain no-external-dependencies principle
if subprocess.call('which zstd', shell=True, stdout=subprocess.PIPE):
raise self.Error('Cannot handle .zstd compression: '
'zstd command not found.')
subprocess.call(
'mv {0} {0}.d && zstd -z -{1} {0}.d && mv {0}.d.zst {0}'.format(
self.name, self.zstd_compression_level),
shell=True,
stdout=subprocess.PIPE)

if self.xz:
# Support xz compression through xz... until we can use Py3
if subprocess.call('which xz', shell=True, stdout=subprocess.PIPE):
Expand Down Expand Up @@ -410,11 +446,12 @@ def parse_pkg_name(metadata, filename):

def __init__(self, output, directory, root_directory,
default_mtime, enable_mtime_preservation,
force_posixpath, gzip_compression_level):
force_posixpath, gzip_compression_level, zstd_compression_level=3, compression='gz'):
self.directory = directory
self.output = output
self.compression = "gz"
self.compression = compression
self.gzip_compression_level = gzip_compression_level
self.zstd_compression_level = zstd_compression_level
self.root_directory = root_directory
self.default_mtime = default_mtime
self.enable_mtime_preservation = enable_mtime_preservation
Expand All @@ -425,6 +462,7 @@ def __enter__(self):
self.output,
self.compression,
self.gzip_compression_level,
self.zstd_compression_level,
self.root_directory,
self.default_mtime,
self.enable_mtime_preservation,
Expand Down Expand Up @@ -678,7 +716,8 @@ def main(FLAGS):
with TarFile(FLAGS.output, FLAGS.directory,
FLAGS.root_directory, FLAGS.mtime,
FLAGS.enable_mtime_preservation,
FLAGS.force_posixpath, FLAGS.gzip_compression_level) as output:
FLAGS.force_posixpath, FLAGS.gzip_compression_level,
FLAGS.zstd_compression_level, FLAGS.compression) as output:
def file_attributes(filename):
if filename.startswith('/'):
filename = filename[1:]
Expand Down Expand Up @@ -751,7 +790,7 @@ def file_attributes(filename):

def validate_link(l):
if not all([value.find(':') > 0 for value in l]):
raise argparse.ArgumentTypeError(msg)
raise argparse.ArgumentTypeError("Link must be in format 'source:target'")
return l

parser.add_argument('--link', type=validate_link, default=[], action='append',
Expand Down Expand Up @@ -789,6 +828,12 @@ def validate_link(l):
parser.add_argument('--gzip_compression_level', type=int, default=9,
help='Set the gzip compression level to use.')

parser.add_argument('--zstd_compression_level', type=int, default=3,
help='Set the zstd compression level to use (1-22).')

parser.add_argument('--compression', type=str, default='gz',
help='Set the compression type: gz, bz2, xz, lzma, zstd, or empty string for no compression.')

main(parser.parse_args())