Merge pull request #136 from cokelaer/main

Enhance shell modules, tests, and CI configurations

Author: cokelaer

.github/workflows/main.yml

@@ -76,4 +76,4 @@ jobs:
         else
           echo "No parent commit. Skipping git diff."
         fi
-        pytest test.py -vv
+        pytest wrappers/test.py -vv

.github/workflows/pypi.yml

@@ -10,6 +10,9 @@ jobs:
   build-n-publish:
     name: Build and publish to PyPI
     runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
 
     steps:
       - uses: actions/checkout@v4
@@ -27,16 +30,6 @@ jobs:
           rm -rf dist
           python -m build
 
-      - name: Publish to Test PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-          repository-url: https://test.pypi.org/legacy/
-
       - name: Publish to PyPI
         if: startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/shells.yml

@@ -61,8 +61,15 @@ jobs:
           fi
 
           # Keep only shells that actually have a test/ directory
+          # Exclude shells whose container download is too slow for CI
+          EXCLUDED="medaka"
           TESTABLE=""
           for shell in $CHANGED; do
+            tool=$(echo "$shell" | cut -d/ -f1)
+            if echo "$EXCLUDED" | grep -qw "$tool"; then
+              echo "Skipping excluded shell: $shell"
+              continue
+            fi
             if [ -d "${SHELLS_ROOT}/${shell}/test" ]; then
               TESTABLE="${TESTABLE} ${shell}"
             fi

.github/workflows/snippets.yml

@@ -0,0 +1,37 @@
+name: Tests snippets
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - "sequana_wrappers/snippets/**"
+  pull_request:
+    branches-ignore: []
+    paths:
+      - "sequana_wrappers/snippets/**"
+  workflow_dispatch:
+  schedule:
+    # Monthly full run — same cadence as other test suites
+    - cron: "0 0 4 * *"
+
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          pip install sequana_pipetools snakemake pytest
+          pip install -e .
+
+      - name: Run snippet tests
+        run: pytest -v .

.gitignore

@@ -4,3 +4,4 @@ __pycache__
 .cache
 .idea
 .vscode
+CLAUDE.md

MANIFEST.in

@@ -0,0 +1,6 @@
+recursive-exclude * test *
+recursive-exclude * tests *
+exclude Makefile
+exclude Snakefile
+global-exclude Snakefile
+global-exclude Makefile

README.md

@@ -2,12 +2,13 @@
 
 [![Tests wrappers](https://github.com/sequana/sequana-wrappers/actions/workflows/main.yml/badge.svg)](https://github.com/sequana/sequana-wrappers/actions/workflows/main.yml)
 [![Tests shells](https://github.com/sequana/sequana-wrappers/actions/workflows/shells.yml/badge.svg)](https://github.com/sequana/sequana-wrappers/actions/workflows/shells.yml)
+[![Tests snippets](https://github.com/sequana/sequana-wrappers/actions/workflows/snippets.yml/badge.svg)](https://github.com/sequana/sequana-wrappers/actions/workflows/snippets.yml)
 [![Tests](http://joss.theoj.org/papers/10.21105/joss.00352/status.svg)](http://joss.theoj.org/papers/10.21105/joss.00352)
 
 |||
 | --- | --- |
 | Overview | Shell command library and Snakemake wrappers for Sequana pipelines |
-| Status | Production (`wrappers/` — maintenance only) / Active (`shells/`) |
+| Status | Production (`wrappers/` — maintenance only) / Active (`shells/`, `snippets/`) |
 | Issues | Please fill a report on [github/sequana/sequana-wrappers](https://github.com/sequana/sequana/issues) |
 | Python version | Python 3.8+ |
 | Citation | Cokelaer et al, (2017), ‘Sequana’: a Set of Snakemake NGS pipelines, Journal of Open Source Software, 2(16), 352, [doi:10.21105/joss.00352](http://www.doi2bib.org/bib/10.21105%2Fjoss.00352) |
@@ -28,6 +29,21 @@ tool commands to Sequana pipelines:
   inside the container.  This is the active development track and the
   recommended approach for all new Sequana pipelines.
 
+- **`sequana_wrappers/snippets/`** — versioned Python callables for pipeline
+  steps that require Python logic but still benefit from shared, versioned
+  definitions. Used via `run:` blocks (not `shell:`).  See
+  [the snippets section below](#the-snippets-directory--rationale) for details.
+
+  All wrappers available in shells except 3 (require Python imports from the sequana library —
+   not expressible as pure bash):
+  - fastq_stats — uses sequana.FastQC + matplotlib
+  - freebayes_vcf_filter — uses sequana.VCF_freebayes Python class
+  - snpeff_add_locus_in_fasta — uses sequana.SnpEff.add_locus_in_fasta()
+
+  The `rulegraph` rule formerly in wrappers has been migrated to
+  `sequana_wrappers/snippets/rulegraph/` because it requires Python imports from
+  `sequana_pipetools` — it cannot run as a pure bash command inside a container.
+
 ## Quick start — shells (recommended)
 
 Install the package:
@@ -47,6 +63,25 @@ rule minimap2:
     shell:   manager.get_shell("minimap2/align", "v1")
 ```
 
+## Quick start — snippets (Python run blocks)
+
+When a pipeline step requires Python logic (host-side imports, file path
+resolution, etc.) but you still want the code to be shared and versioned, use
+`get_run` with a `run:` block:
+
+```python
+rule rulegraph:
+    input:   "Snakefile"
+    output:  "rulegraph/rulegraph.svg"
+    params:  configname="config.yaml"
+    run:
+        manager.get_run("rulegraph/run", "v1")(snakemake)
+```
+
+The snippet's `execute(input, output, params)` function runs on the **host**
+(where `sequana_pipetools` and other Python dependencies are available) — no
+container is involved.
+
 ## Quick start — wrappers (legacy)
 
 ```bash
@@ -151,23 +186,19 @@ sequana-wrappers/
 ├── wrappers/                        # existing — kept for backward compat
 │   ├── bwa/align/wrapper.py
 │   └── ...
-└── sequana_wrappers/shells/         # Python package — container-first shell library
-    ├── __init__.py
-    ├── bwa/
-    │   ├── __init__.py
-    │   ├── align/
-    │   │   ├── __init__.py
-    │   │   └── v1/
-    │   │       ├── __init__.py
-    │   │       └── cmd.py           # frozen at release v1
-    │   └── build/
-    │       ├── __init__.py
-    │       └── v1/cmd.py
-    ├── bamtools/
-    │   └── stats/
-    │       ├── __init__.py
-    │       └── v1/cmd.py
-    └── ...
+└── sequana_wrappers/
+    ├── __init__.py                  # get_shell() and get_run()
+    ├── shells/                      # container-first shell strings
+    │   ├── bwa/
+    │   │   ├── align/
+    │   │   │   └── v1/cmd.py        # frozen at release v1
+    │   │   └── build/
+    │   │       └── v1/cmd.py
+    │   ├── bamtools/stats/v1/cmd.py
+    │   └── ...
+    └── snippets/                    # host-side Python callables
+        ├── rulegraph/run/v1/code.py
+        └── ...
 ```
 
 ### Versioning convention
@@ -278,6 +309,53 @@ and bump `version` in `pyproject.toml`.  No git tag required — the directory
 
 ---
 
+# The `snippets/` directory — rationale
+
+Some pipeline steps require Python logic that cannot be expressed as a pure
+bash command string.  Examples: generating a rule graph (needs
+`sequana_pipetools.DOTParser`), post-processing VCF files with a custom Python
+class, or running tools that depend on host-side Python libraries.
+
+These steps **cannot use `shell:` + `container:`** (the container has no Python
+runtime; and even if it did, the ABI mismatch problem described above applies).
+They also cannot use `wrapper:` for the same ABI reason.
+
+The solution is a `snippets/` library of versioned Python callables that are
+invoked inside Snakemake `run:` blocks, running entirely on the host where all
+Python dependencies are available:
+
+```
+sequana_wrappers/snippets/<tool>/<command>/<version>/code.py
+```
+
+Each `code.py` exports an `execute(input, output, params)` function.  Versioning
+follows the same convention as shells (`v1`, `v2`, …, `dev`).  `get_run` loads
+the callable by path and version — no git-time fetching, no ABI concerns.
+
+| Property | Wrappers | Shell library | Snippet library |
+|---|---|---|---|
+| Python in container | Required | Not needed | N/A (host-side) |
+| Container needed | Optional | Yes | No |
+| Snakemake directive | `wrapper:` | `shell:` | `run:` |
+| Reusable & versioned | Yes | Yes | **Yes** |
+| Python imports on host | Yes | No | **Yes** |
+
+The repository layout extended with snippets:
+
+```
+sequana-wrappers/
+├── wrappers/                            # legacy — maintenance only
+├── sequana_wrappers/
+│   ├── shells/                          # container-first shell strings
+│   │   ├── bwa/align/v1/cmd.py
+│   │   └── ...
+│   └── snippets/                        # host-side Python callables
+│       ├── rulegraph/run/v1/code.py
+│       └── ...
+```
+
+---
+
 # Notes for developers
 
 ## Overview

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
-requires = ["setuptools>=61"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=2.0.0"]
+build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "sequana_wrappers"
@@ -10,6 +10,14 @@ readme = "README.md"
 license = { text = "BSD-3-Clause" }
 requires-python = ">=3.8"
 
-[tool.setuptools.packages.find]
-where = ["."]
-include = ["sequana_wrappers*"]
+[tool.poetry]
+packages = [{ include = "sequana_wrappers" }]
+exclude = [
+    "sequana_wrappers/shells/**/test",
+    "sequana_wrappers/shells/**/test/**",
+    "sequana_wrappers/shells/**/README.md",
+]
+
+[tool.pytest.ini_options]
+norecursedirs = ["sequana_wrappers/shells", "dist", "build", ".git"]
+

sequana_wrappers/__init__.py

@@ -4,21 +4,13 @@
 def get_shell(tool_path: str, version: str) -> str:
     """Return a shell command string from the sequana_wrappers shell library.
 
-    ``tool_path`` is a slash-separated string encoding the tool and command
-    (and any future sub-commands), e.g. ``"bwa/align"`` or ``"bamtools/stats"``.
-
-    Every version — including ``"dev"`` — maps to a subdirectory of the same
-    name under ``shells/<tool>/<command>/``.  No silent fallback: an explicit
-    error is raised if the requested version does not exist.
-
     :param tool_path: slash-separated tool/command path, e.g. ``"bwa/align"``
     :param version: shell command version, e.g. ``"v1"``, or ``"dev"`` for the
         development (unreleased) version.
 
     Example usage in a pipeline rules file::
 
-        shell: manager.get_shell("bwa/align", "v1")    # pinned — recommended
-        shell: manager.get_shell("bwa/align", "dev")   # development version
+        shell: manager.get_shell("bwa/align", "v1")
     """
     parts = tool_path.split("/")
     module_path = ".".join(["sequana_wrappers", "shells"] + parts + [version, "cmd"])
@@ -31,3 +23,31 @@ def get_shell(tool_path: str, version: str) -> str:
             f"sequana_wrappers/shells/{'/'.join(parts)}/.\n"
             f"Use version='dev' for the development version."
         )
+
+
+def get_run(tool_path: str, version: str):
+    """Return a Python callable from the sequana_wrappers snippets library.
+
+    The returned callable has the signature ``execute(snakemake)`` and is
+    intended for use inside Snakemake ``run:`` blocks.
+
+    :param tool_path: slash-separated tool/command path, e.g. ``"rulegraph/run"``
+    :param version: snippet version, e.g. ``"v1"``, or ``"dev"`` for the
+        development (unreleased) version.
+
+    Example usage in a pipeline rules file::
+
+        run:
+            manager.get_run("rulegraph/run", "v1")(snakemake)
+    """
+    parts = tool_path.split("/")
+    module_path = ".".join(["sequana_wrappers", "snippets"] + parts + [version, "code"])
+    try:
+        return importlib.import_module(module_path).execute
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            f"Snippet version '{version}' not found: '{module_path}'.\n"
+            f"Available versions are the subdirectories under "
+            f"sequana_wrappers/snippets/{'/'.join(parts)}/.\n"
+            f"Use version='dev' for the development version."
+        )

sequana_wrappers/shells/Makefile

@@ -0,0 +1,6 @@
+MAKEFILES := $(shell find . -mindepth 3 -name Makefile)
+
+clean:
+	@for mk in $(MAKEFILES); do \
+	    $(MAKE) -C $$(dirname $$mk) clean; \
+	done