add more tests and cleanup testing setup code

Author: TheBoje

.github/workflows/c-cpp.yml

@@ -32,7 +32,7 @@ jobs:
       run: make clean && make -j4 CXX="ccache clang++" debug
 
     - name: run tests  
-      run: pip install -U pytest pytest-cov && make clean && make -j4 CXX="ccache clang++" test
+      run: pip install -U pytest pytest-cov && make clean && make -j4 CXX="ccache clang++" profile
 
     - name: ccache stats
       run: ccache -s

Makefile

@@ -24,8 +24,8 @@ BISONFLAGS=-t -d --defines=${OUT_DIR}/parser.tab.h -Wconflicts-rr -Wcounterexamp
 ifeq ($(MAKECMDGOALS), test)
   CXXFLAGS   += -fprofile-arcs -ftest-coverage -g -fprofile-instr-generate -fcoverage-mapping
   CXXLINKING += -lgcov
-else
-  CXXFLAGS   += -DENABLE_UI
+#else
+#  CXXFLAGS   += -DENABLE_UI
 endif
 
 ifeq ($(MAKECMDGOALS), release)
@@ -61,15 +61,17 @@ release: lsystems
 debug: lsystems
 
 test: lsystems tests/run_tests.py
-	LLVM_PROFILE_FILE="build/lsystems.profraw" pytest tests/run_tests.py
+	@LLVM_PROFILE_FILE="build/lsystems.profraw" pytest tests/run_tests.py
+
+profile: test
 	@llvm-profdata merge -output=build/lsystems.profdata build/lsystems.profraw
 	@llvm-cov show build/lsystems -instr-profile=build/lsystems.profdata -format=html -output-dir=build/coverage_report
 	@llvm-cov report build/lsystems -instr-profile=build/lsystems.profdata
 
 lsystems: ${OUT_DIR}/lsystems ${OBJ_SHADER}
 
 run: lsystems examples/default.lsy
-	${OUT_DIR}/lsystems examples/default.lsy
+	@${OUT_DIR}/lsystems examples/default.lsy
 
 ${OUT_DIR}/lsystems: ${OBJ} ${OUT_SHADERS} ${OUT_DIR}/lex.yy.cpp ${OUT_DIR}/parser.tab.o 
 	@echo -e "${COLOR_LINK}[CLANG]${COLOR_RESET} Linking executable ${COLOR_LINK}${COLOR_BOLD}$(@F)${COLOR_RESET}"

tests/failure/missing_context.expected tests/failure/missing/context.expectedtests/failure/missing_context.expected renamed to tests/failure/missing/context.expected

@@ -5,38 +5,37 @@
 # Path to the executable
 LSYS_EXE = pathlib.Path(__file__).parent.parent / "build" / "lsystems"
 
-def collect_test_cases(folder, expect_success=True):
+def collect_test_cases(folder, expect_success):
     input_dir = pathlib.Path(__file__).parent / folder
     test_cases = []
-    for ls_file in input_dir.glob("*.lsy"):
+    for ls_file in input_dir.rglob("*.lsy"):
         expected_file = ls_file.with_suffix(".expected")
-        test_cases.append((ls_file, expected_file, expect_success))
+        test_name = str(ls_file.relative_to(input_dir))
+        test_cases.append(pytest.param(ls_file, expected_file, expect_success, id=test_name))
     return test_cases
 
-@pytest.mark.parametrize("ls_file, expected_file, expect_success", collect_test_cases("success", expect_success=True))
-def test_success_cases(ls_file, expected_file, expect_success):
+def run_test_case(ls_file, expected_file, expect_success):
     result = subprocess.run(
-        [str(LSYS_EXE), str(pathlib.Path(ls_file))],
+        [str(LSYS_EXE), str(ls_file)],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         timeout=2
     )
     output = result.stdout.decode("utf-8")
-    expected = expected_file.read_text(encoding="utf-8")
+    if expect_success:
+        expected = expected_file.read_text(encoding="utf-8")
+        assert result.returncode == 0, f"{ls_file} failed unexpectedly"
+        assert output == expected, f"{ls_file} output mismatch"
+    else:
+        output += result.stderr.decode("utf-8")
+        expected = expected_file.read_text(encoding="utf-8")
+        assert result.returncode != 0, f"{ls_file} should fail but succeeded"
+        assert expected in output, f"{ls_file} error message mismatch"
 
-    assert result.returncode == 0, f"{ls_file.name} failed unexpectedly"
-    assert output == expected, f"{ls_file.name} output mismatch"
+@pytest.mark.parametrize("ls_file, expected_file, expect_success", collect_test_cases("success", expect_success=True))
+def test_success_cases(ls_file, expected_file, expect_success):
+    run_test_case(ls_file, expected_file, expect_success)
 
 @pytest.mark.parametrize("ls_file, expected_file, expect_success", collect_test_cases("failure", expect_success=False))
 def test_failure_cases(ls_file, expected_file, expect_success):
-    result = subprocess.run(
-        [str(LSYS_EXE), str(pathlib.Path(ls_file))],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        timeout=2
-    )
-    output = result.stdout.decode("utf-8") + result.stderr.decode("utf-8")
-    expected = expected_file.read_text(encoding="utf-8")
-
-    assert result.returncode != 0, f"{ls_file.name} should fail but succeeded"
-    assert expected in output, f"{ls_file.name} error message mismatch"
+    run_test_case(ls_file, expected_file, expect_success)

examples/error/missing_context.lsy tests/failure/missing/context.lsyexamples/error/missing_context.lsy renamed to tests/failure/missing/context.lsy

@@ -0,0 +1,18 @@
+node: a < a > b --> b
+node: b < a > a --> b
+node: b < a > b --> A
+node: * < b > * --> a
+config: 
+	derivation: 5
+	angle factor: 60
+	scale factor: 1
+	axiom: baaaaaaaaab
+	ignore: c
+
+derivation 0/5: baaaaaaaaab
+derivation 1/5: abaaaaaaaba
+derivation 2/5: aabaaaaabaa
+derivation 3/5: ababaaababa
+derivation 4/5: aaAababaAaa
+derivation 5/5: aaAaaAaaAaa
+aaAaaAaaAaa