Dev continue

.github/workflows/IntegrationTest.yml

@@ -62,3 +62,6 @@ jobs:
     - name: Integration Test(custom config)
       run: |
         python -m dingo.run.cli --input_path test/data/test_local_json.json --dataset local -e test --data_format json --column_content prediction --custom_config test/config/config_rule.json --log_level=DEBUG
+    - name: Run unit tests with pytest
+      run: |
+        pytest test/scripts --ignore=test/scripts/data

examples/continue/continue.py

@@ -0,0 +1,43 @@
+from dingo.exec import Executor
+from dingo.io import InputArgs
+
+
+def exec_first():
+    input_data = {
+        "eval_group": "sft",
+        "input_path": "../../test/data/test_local_jsonl.jsonl",
+        "save_data": True,
+        "save_correct": True,
+        "dataset": "local",
+        "data_format": "jsonl",
+        "column_id": "id",
+        "column_content": "content",
+        "end_index": 1
+    }
+
+    input_args = InputArgs(**input_data)
+    executor = Executor.exec_map["local"](input_args)
+    result = executor.execute()
+    print(result)
+
+def exec_second():
+    input_data = {
+        "eval_group": "sft",
+        "input_path": "../../test/data/test_local_jsonl.jsonl",
+        "save_data": True,
+        "save_correct": True,
+        "dataset": "local",
+        "data_format": "jsonl",
+        "column_id": "id",
+        "column_content": "content",
+        "start_index": 1
+    }
+
+    input_args = InputArgs(**input_data)
+    executor = Executor.exec_map["local"](input_args)
+    result = executor.execute()
+    print(result)
+
+if __name__ == '__main__':
+    exec_first()
+    exec_second()

test/scripts/io/input/test_continue.py

@@ -0,0 +1,38 @@
+import json
+import os.path
+
+import pytest
+from dingo.exec import Executor
+from dingo.io import InputArgs
+
+
+class TestContinue:
+    def test_continue_local_jsonl(self):
+        input_data = {
+            "eval_group": "sft",
+            "input_path": "test/data/test_local_jsonl.jsonl",
+            "save_data": True,
+            "save_correct": True,
+            "dataset": "local",
+            "data_format": "jsonl",
+            "column_id": "id",
+            "column_content": "content",
+            "start_index": 1
+        }
+
+        input_args = InputArgs(**input_data)
+        executor = Executor.exec_map["local"](input_args)
+        result = executor.execute().to_dict()
+
+        output_path = result['output_path']
+        p = os.path.join(output_path, 'QUALITY_GOOD', 'Data.jsonl')
+        assert os.path.exists(p)
+
+        id = -1
+        with open(p, 'r', encoding='utf-8') as f:
+            for line in f:
+                j = json.loads(line)
+                print(j)
+                id = j['data_id']
+                break
+        assert id == '1'

test/scripts/test_write.py → test/scripts/io/input/test_write.py

@@ -10,7 +10,7 @@ class TestWrite:
     def test_write_local_jsonl(self):
         input_args = InputArgs(**{
             "eval_group": "qa_standard_v1",
-            "input_path": "../data/test_local_jsonl.jsonl",
+            "input_path": "test/data/test_local_jsonl.jsonl",
             "save_data": True,
             "save_correct": True,
             "dataset": "local",
@@ -19,12 +19,8 @@ def test_write_local_jsonl(self):
             "column_content": "content",
         })
         executor = Executor.exec_map["local"](input_args)
-        result = executor.execute()
+        result = executor.execute().to_dict()
         # print(result)
-        output_path = result[0].output_path
+        output_path = result['output_path']
         assert os.path.exists(output_path)
         shutil.rmtree('outputs')
-
-
-if __name__ == '__main__':
-    pytest.main(["-s", "-q"])