bgunnar5 · bgunnar5 · May 4, 2023 · May 4, 2023 · May 4, 2023 · May 4, 2023
diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml
@@ -9,12 +9,29 @@ jobs:
     if: github.event_name == 'pull_request'
 
     steps:
-    - uses: actions/checkout@v1
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0 # Checkout the whole history, in case the target is way far behind
+
+    - name: Check if target branch has been merged
+      run: |
+        if git merge-base --is-ancestor ${{ github.event.pull_request.base.sha }} ${{ github.sha }}; then
+          echo "Target branch has been merged into the source branch."
+        else
+          echo "Target branch has not been merged into the source branch. Please merge in target first."
+          exit 1
+        fi
 
     - name: Check that CHANGELOG has been updated
       run: |
         # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution.
-        git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!"
+        if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q '^CHANGELOG.md$'; then
+          echo "Thanks for helping keep our CHANGELOG up-to-date!"
+        else
+          echo "Please update the CHANGELOG.md file with notes on your contribution."
+          exit 1
+        fi
 
   Lint:
     runs-on: ubuntu-latest

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,44 @@ All notable changes to Merlin will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [unreleased]
+### Added
+- Merlin manager capability to monitor celery workers.
+
+## [1.12.2b1]
+### Added
+- Conflict handler option to the `dict_deep_merge` function in `utils.py`
+- Ability to add module-specific pytest fixtures
+- Added fixtures specifically for testing status functionality
+- Added tests for reading and writing status files, and status conflict handling
+- Added tests for the `dict_deep_merge` function
+- Pytest-mock as a dependency for the test suite (necessary for using mocks and fixtures in the same test)
+- New github action test to make sure target branch has been merged into the source first, so we know histories are ok
+- Check in the status commands to make sure we're not pulling statuses from nested workspaces
+- Added `setuptools` as a requirement for python 3.12 to recognize the `pkg_resources` library
+- Patch to celery results backend to stop ChordErrors being raised and breaking workflows when a single task fails
+- New step return code `$(MERLIN_RAISE_ERROR)` to force an error to be raised by a task (mainly for testing)
+  - Added description of this to docs
+- New test to ensure a single failed task won't break a workflow
+
+### Changed
+- `merlin info` is cleaner and gives python package info
+- merlin version now prints with every banner message
+- Applying filters for `merlin detailed-status` will now log debug statements instead of warnings
+- Modified the unit tests for the `merlin status` command to use pytest rather than unittest
+- Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test
+- Batch block and workers now allow for variables to be used in node settings
+- Task id is now the path to the directory
+
+### Fixed
+- Bugfix for output of `merlin example openfoam_wf_singularity`
+- A bug with the CHANGELOG detection test when the target branch isn't in the ci runner history
+- Link to Merlin banner in readme
+- Issue with escape sequences in ascii art (caught by python 3.12)
+- Bug where Flux wasn't identifying total number of nodes on an allocation
+  - Not supporting Flux versions below 0.17.0
+
+
 ## [1.12.1]
 ### Added
 - New Priority.RETRY value for the Celery task priorities. This will be the new highest priority.

diff --git a/Makefile b/Makefile
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 [![Pull requests](https://img.shields.io/github/issues-pr/LLNL/merlin)](https://github.com/LLNL/merlin/pulls)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/LLNL/merlin.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/LLNL/merlin/context:python)
 
-![Merlin](https://raw.githubusercontent.com/LLNL/merlin/main/docs/images/merlin.png)
+![Merlin](https://raw.githubusercontent.com/LLNL/merlin/main/docs/assets/images/merlin_banner_white.png)
 
 ## A brief introduction to Merlin
 Merlin is a tool for running machine learning based workflows. The goal of

diff --git a/docs/tutorial/4_run_simulation.md b/docs/tutorial/4_run_simulation.md
@@ -72,8 +72,8 @@ In the `openfoam_wf_singularity` directory you should see the following:
   <figcaption>Fig 3. openfoam_wf Directory Structure</figcaption>
 </figure>
 
-- `openfoam_wf.yaml` -- this spec file is partially blank. You will fill in the gaps as you follow this module's steps.
-- `openfoam_wf_template.yaml` -- this is a complete spec file. You can always reference it as an example.
+- `openfoam_wf_singularity.yaml` -- this spec file is partially blank. You will fill in the gaps as you follow this module's steps.
+- `openfoam_wf_singularity_template.yaml` -- this is a complete spec file. You can always reference it as an example.
 - `scripts` -- This directory contains all the necessary scripts for this module.
     - We'll be exploring these scripts as we go with the tutorial.
 - `requirements.txt` -- this is a text file listing this workflow's python dependencies.
@@ -87,22 +87,22 @@ We are going to build a spec file that produces this DAG:
   <figcaption>Fig 4. OpenFOAM DAG</figcaption>
 </figure>
 
-**To start, open** `openfoam_wf.yaml` **using your favorite text editor.**
+**To start, open** `openfoam_wf_singularity.yaml` **using your favorite text editor.**
 
 It should look something like this:
 
 ???+ abstract "Initial Contents of the Spec"
 
     <!--codeinclude-->
-    [openfoam_wf.yaml](../../merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml)
+    [openfoam_wf_singularity.yaml](../../merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml)
     <!--/codeinclude-->
 
 ### Variables
 
 First we specify some variables to make our life easier. Locate the `env` block in our yaml spec:
 
 <!--codeinclude-->
-[](../../merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml) lines:9-15
+[](../../merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml) lines:9-15
 <!--/codeinclude-->
 
 The `OUTPUT_PATH` variable is set to tell Merlin where you want your output directory to be written. The default is the current working directory.
@@ -254,7 +254,7 @@ nonsimworkers:
 
 ### Putting It All Together
 
-By the end, your `openfoam_wf.yaml` should look like the template version in the same directory:
+By the end, your `openfoam_wf_singularity.yaml` should look like the template version in the same directory:
 
 ???+ abstract "Complete Spec File"
 
@@ -273,21 +273,21 @@ Now that you are done with the Specification file, use the following commands fr
 Create the DAG and send tasks to the server with:
 
 ```bash
-merlin run openfoam_wf.yaml
+merlin run openfoam_wf_singularity.yaml
 ```
 
 Open a new terminal window, then start the workers that will consume the tasks we just queued by using:
 
 ```bash
-merlin run-workers openfoam_wf.yaml
+merlin run-workers openfoam_wf_singularity.yaml
 ```
 
 But wait! We realize that 10 samples is not enough to train a good model. We would like to restart with 100 samples instead of 10 (should take about 6 minutes):
 
 After sending the workers to start on their queues, we need to first stop the workers:
 
 ```bash
-merlin stop-workers --spec openfoam_wf.yaml
+merlin stop-workers --spec openfoam_wf_singularity.yaml
 ```
 
 !!! tip
@@ -297,25 +297,25 @@ merlin stop-workers --spec openfoam_wf.yaml
 We stopped these tasks from running but if we were to run the workflow again (with 100 samples instead of 10), we would continue running the 10 samples first! This is because the queues are still filled with the previous attempt's tasks. This can be seen with:
 
 ```bash
-merlin status openfoam_wf.yaml
+merlin status openfoam_wf_singularity.yaml
 ```
 
 We need to purge these queues first in order to repopulate them with the appropriate tasks. This is where we use the `merlin purge` command:
 
 ```bash
-merlin purge openfoam_wf.yaml
+merlin purge openfoam_wf_singularity.yaml
 ```
 
 Now we are free to repopulate the queues with the 100 samples. In our terminal window that's not designated for our workers, we'll queue up tasks again, this time with 100 samples:
 
 ```bash
-merlin run openfoam_wf.yaml --vars N_SAMPLES=100
+merlin run openfoam_wf_singularity.yaml --vars N_SAMPLES=100
 ```
 
 Then in our window for workers, we'll execute:
 
 ```bash
-merlin run-workers openfoam_wf.yaml
+merlin run-workers openfoam_wf_singularity.yaml
 ```
 
 To see your results, look inside the `learn` output directory. You should see a png that looks like this:

diff --git a/docs/user_guide/variables.md b/docs/user_guide/variables.md
@@ -252,3 +252,4 @@ If necessary, users can raise their own return codes within steps. The table bel
 | <pre>`$(MERLIN_SOFT_FAIL)`</pre> | Mark this step as a failure, note in the warning log but keep executing the workflow. Unknown return codes get translated to soft fails, so that they can be logged. | <pre><code><span>echo "Uh-oh, this sample didn't work"</span></br><span>exit $(MERLIN_SOFT_FAIL)</span></code></pre> |
 | <pre>`$(MERLIN_HARD_FAIL)`</pre> | Something went terribly wrong and we need to stop the whole workflow. Raises a `HardFailException` and stops all workers connected to that step. Workers will stop after a 60 second delay to allow the step to be acknowledged by the server. <div class="admonition note"><p class="admonition-title">Note</p><p>Workers in isolated parts of the workflow not consuming from the bad step will continue. you can stop all workers with `$(MERLIN_STOP_WORKERS)`</p></div> | <pre><code><span>echo "Oh no, we've created skynet! Abort!"</span></br><span>exit $(MERLIN_HARD_FAIL)</span></code></pre> |
 | <pre>`$(MERLIN_STOP_WORKERS)`</pre> | Launch a task to stop all active workers. To allow the current task to finish and acknowledge the results to the server, will happen in 60 seconds. | <pre><code><span># send a signal to all workers to stop</span></br><span>exit $(MERLIN_STOP_WORKERS)</span></code></pre> |
+| <pre>`$(MERLIN_RAISE_ERROR)`</pre> | Purposefully raise a general exception. *This is intended to be used for testing, you'll likely want to use `$(MERLIN_SOFT_FAIL)` instead.* | <pre><code><span># send a signal to raise an exception</span></br><span>exit $(MERLIN_RAISE_ERROR)</span></code></pre> |
diff --git a/merlin/__init__.py b/merlin/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -38,7 +38,7 @@
 import sys
 
 
-__version__ = "1.12.1"
+__version__ = "1.12.2b1"
 VERSION = __version__
 PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "")
 

diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -30,6 +30,9 @@
 
 # pylint: skip-file
 
+from merlin import VERSION
+
+
 """
 Holds ascii art strings.
 """
@@ -93,11 +96,11 @@
 
 
   __  __           _ _       
- |  \/  |         | (_)      
- | \  / | ___ _ __| |_ _ __  
- | |\/| |/ _ \ '__| | | '_ \ 
+ |  \\/  |         | (_)      
+ | \\  / | ___ _ __| |_ _ __  
+ | |\\/| |/ _ \\ '__| | | '_ \\ 
  | |  | |  __/ |  | | | | | |
- |_|  |_|\___|_|  |_|_|_| |_|
+ |_|  |_|\\___|_|  |_|_|_| |_|
 
  Machine Learning for HPC Workflows                                 
 
@@ -127,6 +130,7 @@ def _make_banner():
     for hat_line, name_line in zip(hat_lines, name_lines):
         banner = banner + hat_line + name_line + "\n"
 
+    banner = banner + f"  v. {VERSION}\n"
     return banner
 
 

diff --git a/merlin/celery.py b/merlin/celery.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -36,8 +36,10 @@
 from typing import Dict, Optional, Union
 
 import billiard
+import celery
 import psutil
-from celery import Celery
+from celery import Celery, states
+from celery.backends.redis import RedisBackend  # noqa: F401 ; Needed for celery patch
 from celery.signals import worker_process_init
 
 import merlin.common.security.encrypt_backend_traffic
@@ -50,6 +52,37 @@
 LOG: logging.Logger = logging.getLogger(__name__)
 
 
+def patch_celery():
+    """
+    Patch redis backend so that errors in chords don't break workflows.
+    Celery has error callbacks but they do not work properly on chords that
+    are nested within chains.
+
+    Credit to this function goes to: https://danidee10.github.io/2019/07/09/celery-chords.html
+    """
+
+    def _unpack_chord_result(
+        self,
+        tup,
+        decode,
+        EXCEPTION_STATES=states.EXCEPTION_STATES,
+        PROPAGATE_STATES=states.PROPAGATE_STATES,
+    ):
+        _, tid, state, retval = decode(tup)
+
+        if state in EXCEPTION_STATES:
+            retval = self.exception_to_python(retval)
+        if state in PROPAGATE_STATES:
+            # retval is an Exception
+            retval = f"{retval.__class__.__name__}: {str(retval)}"
+
+        return retval
+
+    celery.backends.redis.RedisBackend._unpack_chord_result = _unpack_chord_result
+
+    return celery
+
+
 # This function has to have specific args/return values for celery so ignore pylint
 def route_for_task(name, args, kwargs, options, task=None, **kw):  # pylint: disable=W0613,R1710
     """
@@ -82,7 +115,7 @@ def route_for_task(name, args, kwargs, options, task=None, **kw):  # pylint: dis
     RESULTS_BACKEND_URI = None
 
 # initialize app with essential properties
-app: Celery = Celery(
+app: Celery = patch_celery().Celery(
     "merlin",
     broker=BROKER_URI,
     backend=RESULTS_BACKEND_URI,

diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -48,3 +48,4 @@ class ReturnCode(IntEnum):
     DRY_OK = 103
     RETRY = 104
     STOP_WORKERS = 105
+    RAISE_ERROR = 106
diff --git a/merlin/common/dumper.py b/merlin/common/dumper.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1
+# This file is part of Merlin, Version: 1.12.2b1
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py
@@ -8,7 +8,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py
@@ -8,7 +8,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.1.
+# This file is part of Merlin, Version: 1.12.2b1.
 #
 # For details, see https://github.com/LLNL/merlin.
 #