Stream decompressed data for gzip

.gitignore

@@ -2,3 +2,7 @@ bazel-*
 .ijwb
 .idea/
 
+# Python
+__pycache__/
+*.pyc
+*.pyo

minidock/container_data_tools/build_tar.py

@@ -30,7 +30,7 @@
 import re
 import tarfile
 import tempfile
-
+import shutil
 
 
 
@@ -405,24 +405,67 @@ def close(self):
     if self.zstd:
       # Support zstd compression through zstd command line tool
       # Following same pattern as xz to maintain no-external-dependencies principle
-      if subprocess.call('which zstd', shell=True, stdout=subprocess.PIPE):
+      if subprocess.call(['which', 'zstd'], stdout=subprocess.PIPE, stderr=subprocess.PIPE):
         raise self.Error('Cannot handle .zstd compression: '
                          'zstd command not found.')
-      subprocess.call(
-          'mv {0} {0}.d && zstd -z -{1} {0}.d && mv {0}.d.zst {0}'.format(
-              self.name, self.zstd_compression_level),
-          shell=True,
-          stdout=subprocess.PIPE)
+
+      # Hybrid approach: use optimized method for files < 500MB, original for larger files
+      file_size = os.path.getsize(self.name)
+      threshold = 500 * 1024 * 1024  # 500MB
+
+      if file_size < threshold:
+        # Optimized: let zstd operate directly on file (better for small/medium files)
+        temp_name = self.name + '.tmp'
+        try:
+          subprocess.run(
+              ['zstd', '-z', f'-{self.zstd_compression_level}', self.name, '-o', temp_name],
+              check=True,
+              stdout=subprocess.PIPE,
+              stderr=subprocess.PIPE
+          )
+          os.replace(temp_name, self.name)
+        finally:
+          if os.path.exists(temp_name):
+            os.remove(temp_name)
+      else:
+        # Original approach for very large files (better performance for 500MB+)
+        subprocess.call(
+            'mv {0} {0}.d && zstd -z -{1} {0}.d && mv {0}.d.zst {0}'.format(
+                self.name, self.zstd_compression_level),
+            shell=True,
+            stdout=subprocess.PIPE)
 
     if self.xz:
       # Support xz compression through xz... until we can use Py3
-      if subprocess.call('which xz', shell=True, stdout=subprocess.PIPE):
+      if subprocess.call(['which', 'xz'], stdout=subprocess.PIPE, stderr=subprocess.PIPE):
         raise self.Error('Cannot handle .xz and .lzma compression: '
                          'xz not found.')
-      subprocess.call(
-          'mv {0} {0}.d && xz -z {0}.d && mv {0}.d.xz {0}'.format(self.name),
-          shell=True,
-          stdout=subprocess.PIPE)
+
+      # Hybrid approach: use optimized method for files < 500MB, original for larger files
+      file_size = os.path.getsize(self.name)
+      threshold = 500 * 1024 * 1024  # 500MB
+
+      if file_size < threshold:
+        # Optimized: let xz operate directly on file (better for small/medium files)
+        temp_name = self.name + '.tmp'
+        try:
+          with open(temp_name, 'wb') as outfile:
+            subprocess.run(
+                ['xz', '-z', '-c', self.name],
+                stdout=outfile,
+                check=True,
+                stderr=subprocess.PIPE
+            )
+          os.replace(temp_name, self.name)
+        finally:
+          if os.path.exists(temp_name):
+            os.remove(temp_name)
+      else:
+        # Original approach for very large files (better performance for 500MB+)
+        subprocess.call(
+            'mv {0} {0}.d && xz -z {0}.d && mv {0}.d.xz {0}'.format(self.name),
+            shell=True,
+            stdout=subprocess.PIPE)
 
 
 
@@ -605,17 +648,21 @@ def add_link(self, symlink, destination):
   def write_temp_file(self, data, suffix='tar', mode='wb'):
     # deb(5) states members may optionally be compressed with gzip or xz
     if suffix.endswith('.gz'):
-      with gzip.GzipFile(fileobj=io.BytesIO(data)) as f:
-        data = f.read()
-      suffix = suffix[:-3]
-
-    (_, tmpfile) = tempfile.mkstemp(suffix=suffix)
-    try:
-      with open(tmpfile, mode=mode) as f:
-        f.write(data)
-      yield tmpfile
-    finally:
-      os.remove(tmpfile)
+        suffix = suffix[:-3]
+        (_, tmpfile) = tempfile.mkstemp(suffix=suffix)
+        with gzip.GzipFile(fileobj=io.BytesIO(data)) as gz, open(tmpfile, mode=mode) as out:
+            shutil.copyfileobj(gz, out)
+        yield tmpfile
+        os.remove(tmpfile)
+    else:
+        # Non-compressed case: write data directly to temp file
+        (_, tmpfile) = tempfile.mkstemp(suffix=suffix)
+        try:
+            with open(tmpfile, mode=mode) as f:
+                f.write(data)
+            yield tmpfile
+        finally:
+            os.remove(tmpfile)
 
   def add_pkg_metadata(self, metadata_tar, deb):
     try: