Add layered zip indexes

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x,1.22.x,1.23.x]
+        go-version: [1.23.x,1.24.x,1.25.x]
         os: [ubuntu-latest]
     steps:
       - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}
@@ -30,6 +30,16 @@ jobs:
         env:
           GO111MODULE: on
         run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.60.1
-          $(go env GOPATH)/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
           go test -race ./...
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with:
+          go-version: stable
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.1.6

.github/workflows/vulncheck.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [ 1.22.x ]
+        go-version: [ 1.25.x ]
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3

.golangci.yml

@@ -1,27 +1,40 @@
-linters-settings:
-  staticcheck:
-    checks:
-      - all
-      - '-SA6002' # disable the rule SA6002, slices are fine.
-  misspell:
-    locale: US
-
+version: "2"
 linters:
-  disable-all: true
+  default: none
   enable:
-    - typecheck
-    - goimports
-    - misspell
     - govet
     - ineffassign
-    - gosimple
+    - misspell
     - staticcheck
     - unused
-
-issues:
-  exclude-use-default: false
-  exclude:
-      - should have a package comment
-      - comment on exported method
-      - should have comment or be unexported
-      - error strings should not be capitalized or end with punctuation or a newline
+  settings:
+    misspell:
+      locale: US
+    staticcheck:
+      checks:
+        - all
+        - -SA6002
+  exclusions:
+    generated: lax
+    rules:
+      - path: (.+)\.go$
+        text: should have a package comment
+      - path: (.+)\.go$
+        text: comment on exported method
+      - path: (.+)\.go$
+        text: should have comment or be unexported
+      - path: (.+)\.go$
+        text: error strings should not be capitalized or end with punctuation or a newline
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

README.md

@@ -81,17 +81,123 @@ Similar to [stdlib zip](https://golang.org/pkg/archive/zip/), not all methods/fl
 
 For expert users, `(*File).OpenRaw` allows access to the compressed data.
 
+
 ## Compression Methods
 
 By default, zipindex keeps files stored uncompressed or deflate compressed.
 This covers the most commonly seen compression methods.
 
-Furthermore, files compressed with [zstandard](https://facebook.github.io/zstd/) 
+Furthermore, files compressed with [zstandard](https://facebook.github.io/zstd/)
 as method 93 will be preserved and can be read back.
 
-Use [`RegisterDecompressor`](https://pkg.go.dev/github.com/minio/zipindex#RegisterDecompressor) 
+Use [`RegisterDecompressor`](https://pkg.go.dev/github.com/minio/zipindex#RegisterDecompressor)
 to register non-standard decompressors.
 
+# Layered Indexes
+
+The `LayeredIndex[T]` type provides a way to combine multiple zip indexes into a single searchable entity. This is useful when you need to overlay multiple archives or apply incremental updates without rebuilding the entire index.
+
+## Key Features
+
+- **Generic type parameter**: Each layer is associated with a comparable reference type `T` (e.g., version number, timestamp, file path)
+- **Override semantics**: Files in newer layers override files with the same path in older layers
+- **Delete layers**: Special layers that remove files from previous layers
+- **Efficient lookups**: Find files across all layers with proper precedence
+
+## Basic Usage
+
+```go
+// Create a new layered index with string references
+layered := zipindex.NewLayeredIndex[string]()
+
+// Add base layer
+baseFiles, _ := zipindex.ReadFile("base.zip")
+err := layered.AddLayer(baseFiles, "v1.0")
+
+// Add update layer (overrides files from base)
+updateFiles, _ := zipindex.ReadFile("update.zip")
+err = layered.AddLayer(updateFiles, "v1.1")
+
+// Add a delete layer (removes specified files)
+deleteFiles := zipindex.Files{{Name: "obsolete.txt"}}
+err = layered.AddDeleteLayer(deleteFiles, "cleanup")
+
+// Find a file across all layers
+file, found := layered.Find("readme.txt")
+if found {
+    // file.File contains the file info
+    // file.LayerRef contains the layer reference (e.g., "v1.1")
+}
+
+// Iterate over all files
+for ref, file := range layered.FilesIter() {
+    fmt.Printf("File %s from layer %v\n", file.Name, ref)
+}
+
+// Merge all layers into a single index, this will lose the reference information
+merged := layered.ToSingleIndex()
+serialized, _ := merged.Serialize()
+```
+
+## API Reference
+
+### Creation and Layer Management
+- `NewLayeredIndex[T]()` - Create a new empty layered index
+- `AddLayer(files, ref)` - Add a layer (returns error if ref already exists)
+- `AddDeleteLayer(files, ref)` - Add a delete layer to remove files from previous layers
+- `RemoveLayer(index)` - Remove layer by index
+- `RemoveLayerByRef(ref)` - Remove all layers with the given reference
+- `Clear()` - Remove all layers
+
+### File Access
+- `Find(name)` - Find a file across all layers, returns `(*FileWithRef[T], bool)`
+- `FindInLayer(name, ref)` - Find a file in a specific layer only
+- `FilesIter()` - Iterator that yields `(T, File)` pairs on merged indexes
+- `Files()` - Get all files as `[]FileWithRef[T]` after applying layer operations
+- `HasFile(name)` - Check if a file exists
+
+### Layer Information
+- `LayerCount()` - Number of layers
+- `GetLayerRef(index)` - Get reference for a layer
+- `FileCount()` - Total unique files after applying operations
+- `IsEmpty()` - True if no files remain after applying all operations
+
+### Conversion
+- `ToSingleIndex()` - Merge all layers into a single `Files` collection
+
+### Serialization
+- `SerializeLayered(RefSerializer[T])` - Serialize the layered index preserving all layers
+- `DeserializeLayered[T](data, RefSerializer[T])` - Reconstruct a layered index from serialized data
+
+The serialization requires providing a `RefSerializer[T]` with functions to convert your reference type to/from bytes:
+
+```go
+// Example for string references
+stringSerializer := RefSerializer[string]{
+    Marshal: func(s string) ([]byte, error) {
+        return []byte(s), nil
+    },
+    Unmarshal: func(b []byte) (string, error) {
+        return string(b), nil
+    },
+}
+
+// Serialize
+data, err := layered.SerializeLayered(stringSerializer)
+
+// Deserialize
+layered2, err := DeserializeLayered(data, stringSerializer)
+```
+
+### Important Notes
+
+1. **Deletion semantics**: Delete layers only remove files that exist in *previous* layers. Files added in subsequent layers are not affected.
+
+2. **Directory handling**: When a file is deleted, empty parent directories are automatically removed. A directory is kept if it still contains any files.
+
+3. **Duplicate references**: The same reference cannot be used twice. Attempting to add a layer with an existing reference returns an error.
+
+4. **Performance**: The layered index maintains files in memory. For large numbers of layers or files, consider merging to a single index periodically.
 
 ## License
 

go.mod

@@ -1,10 +1,10 @@
 module github.com/minio/zipindex
 
-go 1.20
+go 1.23
 
 require (
-	github.com/klauspost/compress v1.17.9
-	github.com/tinylib/msgp v1.2.0
+	github.com/klauspost/compress v1.18.0
+	github.com/tinylib/msgp v1.4.0
 )
 
-require github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
+require github.com/philhofer/fwd v1.2.0 // indirect

go.sum

@@ -1,6 +1,12 @@
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4=
 github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
+github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
+github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
 github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY=
 github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=
+github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8=
+github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o=