Version 0.5.0 (#65)

* add --uncompress gzip hashing, verify fallback, and CI tests
* improve ctrl-c shutdown handling during worker cleanup
* make CI run pytest and normalize macOS path assertions
* bumps version to 0.5

Author: rsgalloway

.github/workflows/tests.yml

@@ -0,0 +1,34 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.11"
+          - "3.12"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: python -m pip install -e . pytest
+
+      - name: Run tests
+        run: python -m pytest tests -q

CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## 0.5.0 - 2026-04-04
+
+### Added
+
+- Added `--uncompress` support for hashing the decompressed contents of `.gz` files.
+- Added gzip-aware verification fallback so manifest entries can be checked against matching `.gz` files.
+- Added basic GitHub Actions test coverage across multiple operating systems and Python versions.
+- Added unit coverage for gzip hashing, verification, and shutdown helper behavior.
+
+### Changed
+
+- Manifest output now uses the uncompressed filename when `--uncompress` is enabled.
+- `--uncompress` bypasses the hash cache so compressed-byte and decompressed-content hashes are not mixed.
+- Improved shutdown handling so `Ctrl+C` exits more cleanly during worker cleanup and progress-thread teardown.
+
+### Fixed
+
+- Fixed SQLite cache merge retries to clean up attached temp databases between attempts.
+- Fixed the lock-failure retry test so it still exercises the intended failure path.

README.md

@@ -79,6 +79,18 @@ available) or regenerated hash values if mtimes are missing or different:
 $ hashio --verify hash.json
 ```
 
+To hash the decompressed contents of `.gz` files instead of the archive bytes,
+use `--uncompress`. In this mode, manifest entries are written using the
+uncompressed filename:
+
+```bash
+$ hashio sample.txt.gz -o hash.json --uncompress
+$ hashio --verify hash.json --uncompress
+```
+
+Note: `--uncompress` currently supports `.gz` files only, and bypasses the hash
+cache so compressed-byte hashes are not mixed with decompressed-content hashes.
+
 #### Portability
 
 To make a portable hash file, use `-or` to make the paths relative to the

lib/hashio/__init__.py

@@ -35,4 +35,4 @@
 
 __author__ = "ryan@rsgalloway.com"
 __prog__ = "hashio"
-__version__ = "0.4.10"
+__version__ = "0.5.0"

lib/hashio/cache.py

@@ -88,8 +88,9 @@ def wrapper(*args, **kwargs):
                     else:
                         logger.warning("sqlite3.OperationalError: %s", str(e))
                         raise
+            db_path = getattr(args[0], "db_path", "<unknown>")
             raise RuntimeError(
-                f"{fn.__name__} failed after {retries} retries with error: {last_err} ({args[0].db_path})"
+                f"{fn.__name__} failed after {retries} retries with error: {last_err} ({db_path})"
             )
 
         return wrapper
@@ -304,22 +305,30 @@ def merge(self, path: str):
 
         :param path: The path to the other SQLite database file.
         """
-        self.conn.execute("ATTACH DATABASE ? AS tempdb", (path,))
-        # do not insert IDs
-        self.conn.executescript(
+        attached = False
+        try:
+            self.conn.execute("ATTACH DATABASE ? AS tempdb", (path,))
+            attached = True
+            # do not insert IDs
+            self.conn.executescript(
+                """
+                INSERT OR IGNORE INTO files (path, mtime, algo, hash, size, inode)
+                SELECT path, mtime, algo, hash, size, inode FROM tempdb.files;
+
+                INSERT OR IGNORE INTO snapshots (name, created_at, path)
+                SELECT name, created_at, path FROM tempdb.snapshots;
+
+                INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
+                SELECT snapshot_id, file_id FROM tempdb.snapshot_files;
             """
-            INSERT OR IGNORE INTO files (path, mtime, algo, hash, size, inode)
-            SELECT path, mtime, algo, hash, size, inode FROM tempdb.files;
-
-            INSERT OR IGNORE INTO snapshots (name, created_at, path)
-            SELECT name, created_at, path FROM tempdb.snapshots;
-
-            INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
-            SELECT snapshot_id, file_id FROM tempdb.snapshot_files;
-        """
-        )
-        self.conn.commit()
-        self.conn.execute("DETACH DATABASE tempdb")
+            )
+            self.conn.commit()
+        finally:
+            if attached:
+                try:
+                    self.conn.execute("DETACH DATABASE tempdb")
+                except sqlite3.Error:
+                    pass
 
     def query(
         self, pattern: str, algo: Optional[str] = None, since: Optional[str] = None

lib/hashio/cli.py

@@ -48,6 +48,24 @@
 from hashio.worker import HashWorker
 
 
+def safe_join_thread(thread: threading.Thread, timeout: float = None):
+    """Join a thread while suppressing KeyboardInterrupt during shutdown."""
+    try:
+        thread.join(timeout=timeout)
+    except KeyboardInterrupt:
+        return False
+    return not thread.is_alive()
+
+
+def stop_workers(workers):
+    """Stop workers while suppressing repeated interrupts during cleanup."""
+    for worker in workers:
+        try:
+            worker.stop()
+        except KeyboardInterrupt:
+            continue
+
+
 def format_result(row):
     """Format a row from the cache query into a human-readable string."""
     # unpack row data
@@ -162,6 +180,11 @@ def parse_args():
         nargs="*",
         help="verify checksums from a previously created hash file",
     )
+    parser.add_argument(
+        "--uncompress",
+        action="store_true",
+        help="hash and verify the decompressed contents of .gz files",
+    )
 
     # mutually exclusive group for cache operations
     cache_group = parser.add_argument_group("cache")
@@ -351,11 +374,13 @@ def is_under_root(path, root):
                 print(f"file not found: {config.CACHE_FILENAME}")
                 return 0
             for algo, value, miss in verify_checksums(
-                config.CACHE_FILENAME, start=args.start
+                config.CACHE_FILENAME, start=args.start, uncompress=args.uncompress
             ):
                 print("{0} {1}".format(algo, miss))
         elif len(args.verify) == 1:
-            for algo, value, miss in verify_checksums(args.verify[0], start=args.start):
+            for algo, value, miss in verify_checksums(
+                args.verify[0], start=args.start, uncompress=args.uncompress
+            ):
                 print("{0} {1}".format(algo, miss))
         elif len(args.verify) == 2:
             source = args.verify[0]
@@ -392,6 +417,7 @@ def is_under_root(path, root):
                 snapshot=args_dict["snapshot"],
                 force=args_dict["force"],
                 verbose=verbose,
+                uncompress=args_dict["uncompress"],
             )
             workers.append(worker)
 
@@ -410,18 +436,18 @@ def is_under_root(path, root):
         while any(t.is_alive() for t in worker_threads):
             # wait for all workers to finish
             for t in worker_threads:
-                t.join(timeout=0.2)
+                safe_join_thread(t, timeout=0.2)
 
             # now wait for progress threads to exit
             for t in progress_threads:
-                t.join()
+                safe_join_thread(t, timeout=0.2)
 
     except KeyboardInterrupt:
-        for worker in workers:
-            worker.stop()
+        stop_workers(workers)
         for t in progress_threads:
-            t.join(timeout=0.2)
+            safe_join_thread(t, timeout=0.2)
         print("stopping...")
+        return 130
 
     finally:
         if len(paths) > 1 and not (args.verbose or args.summarize):

lib/hashio/encoder.py

@@ -37,12 +37,12 @@
 import os
 import xxhash
 import zlib
-from typing import List, Tuple
+from typing import List, Tuple, Union
 
 from hashio import config
 from hashio.exporter import CacheExporter
 from hashio.logger import logger
-from hashio.utils import read_file, walk
+from hashio.utils import is_gzip_path, read_file, read_file_uncompressed, walk
 
 
 def bytes_to_long(data: bytes):
@@ -321,21 +321,35 @@ def checksum_data(data: bytes, encoder: Encoder, buffer_size: int = config.BUF_S
     return value
 
 
-def checksum_file(path: str, encoder: Encoder, buffer_size: int = config.BUF_SIZE):
+def checksum_file(
+    path: str,
+    encoder: Encoder,
+    buffer_size: int = config.BUF_SIZE,
+    uncompress: bool = False,
+    with_size: bool = False,
+) -> Union[str, Tuple[str, int]]:
     """Creates a checksum for a given filepath and encoder. Note: resets
     encoder, existing data will be lost.
 
     >>> checksum_file("example.txt", MD5Encoder())
 
     :param path: the path to the filepath being hashed
     :param encoder: instance of Encoder subclass
-    :return: hexdigest of the checksum
+    :param buffer_size: size of each read chunk in bytes
+    :param uncompress: if True, hash decompressed contents for supported files
+    :param with_size: if True, also return the total bytes hashed
+    :return: checksum hex digest, or ``(checksum, size)`` when ``with_size`` is True
     """
     encoder.reset()
-    for data in read_file(path, buffer_size=buffer_size):
+    size = 0
+    reader = read_file_uncompressed if uncompress and is_gzip_path(path) else read_file
+    for data in reader(path, buffer_size=buffer_size):
+        size += len(data)
         encoder.update(data)
     value = encoder.hexdigest()
     encoder.reset()
+    if with_size:
+        return value, size
     return value
 
 
@@ -372,7 +386,11 @@ def checksum_text(data: str, encoder: Encoder):
 
 
 def checksum_path(
-    path: str, encoder: Encoder, filetype: str = "a", use_cache: bool = True
+    path: str,
+    encoder: Encoder,
+    filetype: str = "a",
+    use_cache: bool = True,
+    uncompress: bool = False,
 ):
     """Returns a checksum of for a given path, encoder and filetype.
 
@@ -384,12 +402,12 @@ def checksum_path(
     :param use_cache: cache results to filesystem
     :return: hexdigest of the checksum
     """
-    if use_cache:
+    if use_cache and not uncompress:
         cached_value = CacheExporter.find(path, encoder.name)
         if cached_value:
             return cached_value
     if os.path.isfile(path) and filetype in ("a", "f"):
-        return checksum_file(path, encoder)
+        return checksum_file(path, encoder, uncompress=uncompress)
     elif os.path.isdir(path) and filetype in ("a", "d"):
         return checksum_folder(path, encoder)
 
@@ -400,6 +418,7 @@ def checksum_gen(
     filetype: str = "f",
     recursive: bool = True,
     use_cache: bool = True,
+    uncompress: bool = False,
 ):
     """Checksum generator that yields tuple of (filepath, value).
 
@@ -415,12 +434,12 @@ def checksum_gen(
     """
     if recursive:
         for subpath in walk(path, filetype):
-            value = checksum_path(subpath, encoder, filetype, use_cache)
+            value = checksum_path(subpath, encoder, filetype, use_cache, uncompress)
             if value:
                 yield (subpath, value)
 
     else:
-        value = checksum_path(path, encoder, filetype, use_cache)
+        value = checksum_path(path, encoder, filetype, use_cache, uncompress)
         if value:
             yield (path, value)
 
@@ -629,7 +648,7 @@ def dedupe_caches(target: str, source: str, algo: str = config.DEFAULT_ALGO):
     return [(t, s) for t, s in dedupe_cache_gen(target, source, algo=algo)]
 
 
-def verify_checksums(path: str, start: str = None):
+def verify_checksums(path: str, start: str = None, uncompress: bool = False):
     """Generator that yields a data tuple for hash misses in a previously
     generated output file. Compares mtimes in the output file with the
     filesystem.
@@ -653,29 +672,36 @@ def verify_checksums(path: str, start: str = None):
 
     for filename, metadata in data.items():
         filepath = os.path.join(root, filename)
+        hash_path = filepath
+
+        if uncompress and not os.path.exists(hash_path):
+            gzip_path = f"{filepath}.gz"
+            if os.path.exists(gzip_path):
+                hash_path = gzip_path
 
         # iterate over all the hash algos...
         for algo in ENCODER_MAP.keys():
             if algo not in metadata.keys():
                 continue
 
             # check if file exists and compare mtimes
-            if not os.path.exists(filepath):
+            if not os.path.exists(hash_path):
                 logger.warning("missing: %s", filepath)
                 continue
             # if mtimes match, skip
-            elif metadata.get("mtime") == os.stat(filepath).st_mtime:
+            elif metadata.get("mtime") == os.stat(hash_path).st_mtime:
                 continue
 
             # if mtimes don't match, re-hash the file
-            logger.debug("mtime miss on %s", filepath)
+            logger.debug("mtime miss on %s", hash_path)
             old_value = metadata.get(algo)
             encoder = ENCODER_MAP.get(algo)()
-            new_value = checksum_path(filepath, encoder)
+            do_uncompress = uncompress and is_gzip_path(hash_path)
+            new_value = checksum_path(hash_path, encoder, uncompress=do_uncompress)
 
             # hash values don't match, file must have changed
             if (new_value and old_value) and (new_value != old_value):
-                logger.debug("hash miss on %s %s", algo, filepath)
+                logger.debug("hash miss on %s %s", algo, hash_path)
                 yield (algo, new_value, filepath)