Refactor report

Signed-off-by: Herklos <herklos@drakkar.software>

Author: Herklos

.gitignore

@@ -27,6 +27,8 @@ wheels/
 .installed.cfg
 *.egg
 
+node_modules
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -136,6 +138,7 @@ cython_debug/
 .env
 
 *.zip
+*.db
 
 # tensorboard
 tensorboard_logs

MANIFEST.in

@@ -1,5 +1,7 @@
 recursive-include octobot_script/config *.json *.ini
 recursive-include octobot_script/resources *.js *.css *.html
+recursive-exclude octobot_script/resources/report/node_modules *
+recursive-exclude octobot_script/resources/report/src *
 
 include README.md
 include LICENSE

biome.json

@@ -0,0 +1,42 @@
+{
+  "$schema": "https://biomejs.dev/schemas/2.3.8/schema.json",
+  "assist": { "actions": { "source": { "organizeImports": "on" } } },
+  "files": {
+    "includes": [
+      "**",
+      "!**/dist/**/*",
+      "!**/node_modules/**/*",
+      "!**/src/components/ui/**/*"
+    ]
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true,
+      "suspicious": {
+        "noExplicitAny": "off",
+        "noArrayIndexKey": "off"
+      },
+      "style": {
+        "noNonNullAssertion": "off",
+        "noParameterAssign": "error",
+        "useSelfClosingElements": "error",
+        "noUselessElse": "error"
+      }
+    }
+  },
+  "formatter": {
+    "indentStyle": "space"
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "double",
+      "semicolons": "asNeeded"
+    }
+  },
+  "css": {
+    "parser": {
+      "tailwindDirectives": true
+    }
+  }
+}

components.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "new-york",
+  "rsc": false,
+  "tsx": true,
+  "tailwind": {
+    "config": "",
+    "css": "octobot_script/resources/report/src/index.css",
+    "baseColor": "gray",
+    "cssVariables": true,
+    "prefix": ""
+  },
+  "iconLibrary": "lucide",
+  "rtl": false,
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "registries": {}
+}

octobot_script/internal/octobot_mocks.py

@@ -35,6 +35,13 @@ def get_tentacles_config():
         commons_constants.CONFIG_TENTACLES_FILE
     )
     tentacles_setup_config = octobot_tentacles_manager_api.get_tentacles_setup_config(ref_tentacles_config_path)
+    if not tentacles_setup_config.is_successfully_loaded:
+        # reference config not available (tentacles not yet installed via CLI),
+        # populate from currently imported tentacles
+        octobot_tentacles_manager_api.fill_with_installed_tentacles(
+            tentacles_setup_config,
+            tentacles_folder=get_imported_tentacles_path()
+        )
     # activate OctoBot-Script required tentacles
     _force_tentacles_config_activation(tentacles_setup_config)
     return tentacles_setup_config

octobot_script/model/backtest_plot.py

@@ -13,12 +13,12 @@
 #
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot-Script. If not, see <https://www.gnu.org/licenses/>.
+import html
+import os
+import re
+import shutil
 import time
-import webbrowser
-import jinja2
 import json
-import http.server
-import socketserver
 
 import octobot_commons.constants as commons_constants
 import octobot_commons.display as display
@@ -27,17 +27,22 @@
 import octobot.api as octobot_api
 import octobot_script.resources as resources
 import octobot_script.internal.backtester_trading_mode as backtester_trading_mode
+from octobot_script.model.backtest_report_server import BacktestReportServer
 
 
 class BacktestPlot:
     DEFAULT_REPORT_NAME = "report.html"
-    DEFAULT_TEMPLATE = "default_report_template.html"
-    JINJA_ENVIRONMENT = jinja2.Environment(loader=jinja2.FileSystemLoader(
-        resources.get_report_resource_path(None)
-    ))
+    ADVANCED_TEMPLATE = os.path.join("dist", "index.html")
+    DEFAULT_TEMPLATE = ADVANCED_TEMPLATE
+    REPORT_DATA_FILENAME = "report_data.json"
+    REPORT_META_FILENAME = "report_meta.json"
+    REPORT_BUNDLE_FILENAME = "report.json"
+    HISTORY_DIR = "backtesting"
+    HISTORY_TIMESTAMP_FORMAT = "%Y%m%d_%H%M%S"
     GENERATED_TIME_FORMAT = "%Y-%m-%d at %H:%M:%S"
     SERVER_PORT = 5555
     SERVER_HOST = "localhost"
+    SERVE_TIMEOUT = 300  # seconds — keep server alive for history browsing
 
     def __init__(self, backtest_result, run_db_identifier, report_file=None):
         self.backtest_result = backtest_result
@@ -46,49 +51,127 @@ def __init__(self, backtest_result, run_db_identifier, report_file=None):
         self.backtesting_analysis_settings = self.default_backtesting_analysis_settings()
 
     async def fill(self, template_file=None):
-        template = self.JINJA_ENVIRONMENT.get_template(template_file or self.DEFAULT_TEMPLATE)
+        template_name = template_file or self.DEFAULT_TEMPLATE
         template_data = await self._get_template_data()
-        with open(self.report_file, "w") as report:
-            report.write(template.render(template_data))
+        report_dir = os.path.dirname(os.path.abspath(self.report_file))
+        shutil.copy2(resources.get_report_resource_path(template_name), self.report_file)
+        meta = template_data["meta"]
+        with open(os.path.join(report_dir, self.REPORT_DATA_FILENAME), "w", encoding="utf-8") as f:
+            f.write(template_data["full_data"])
+        with open(os.path.join(report_dir, self.REPORT_META_FILENAME), "w", encoding="utf-8") as f:
+            json.dump(meta, f)
+        bundle = {"meta": meta, "data": json.loads(template_data["full_data"])}
+        with open(os.path.join(report_dir, self.REPORT_BUNDLE_FILENAME), "w", encoding="utf-8") as f:
+            json.dump(bundle, f)
+        self._save_history_entry(report_dir, template_data["full_data"], meta, bundle)
+
+    def _save_history_entry(self, report_dir, data_str, meta, bundle):
+        ts = time.strftime(self.HISTORY_TIMESTAMP_FORMAT)
+        run_dir = os.path.join(report_dir, self.HISTORY_DIR, ts)
+        os.makedirs(run_dir, exist_ok=True)
+        with open(os.path.join(run_dir, self.REPORT_DATA_FILENAME), "w", encoding="utf-8") as f:
+            f.write(data_str)
+        with open(os.path.join(run_dir, self.REPORT_META_FILENAME), "w", encoding="utf-8") as f:
+            json.dump(meta, f)
+        with open(os.path.join(run_dir, self.REPORT_BUNDLE_FILENAME), "w", encoding="utf-8") as f:
+            json.dump(bundle, f)
 
     def show(self):
-        backtest_plot_instance = self
-        print(f"Report in {self.report_file}")
+        report_dir = os.path.dirname(os.path.abspath(self.report_file))
+        report_name = os.path.basename(self.report_file)
+        runs_root_dir = os.path.dirname(report_dir)
+        print(f"Report: {self.report_file}")
+        server = BacktestReportServer(
+            report_file=self.report_file,
+            report_dir=report_dir,
+            report_name=report_name,
+            runs_root_dir=runs_root_dir,
+            server_host=self.SERVER_HOST,
+            server_port=self.SERVER_PORT,
+            serve_timeout=self.SERVE_TIMEOUT,
+            history_dir=self.HISTORY_DIR,
+            data_filename=self.REPORT_DATA_FILENAME,
+            meta_filename=self.REPORT_META_FILENAME,
+            bundle_filename=self.REPORT_BUNDLE_FILENAME,
+        )
+        server.serve()
 
-        class ReportRequestHandler(http.server.SimpleHTTPRequestHandler):
-            def log_request(self, *_, **__):
-                # do not log requests
-                pass
+    async def _get_template_data(self):
+        full_data, symbols, time_frames, exchanges = await self._get_full_data()
+        summary = self._extract_summary_metrics(full_data)
+        return {
+            "full_data": full_data,
+            "meta": {
+                "title": f"{', '.join(symbols)}",
+                "creation_time": timestamp_util.convert_timestamp_to_datetime(
+                    time.time(), self.GENERATED_TIME_FORMAT, local_timezone=True
+                ),
+                "strategy_config": self.backtest_result.strategy_config,
+                "symbols": symbols,
+                "time_frames": time_frames,
+                "exchanges": exchanges,
+                "summary": summary,
+            },
+        }
 
-            def do_GET(self):
-                self.send_response(http.HTTPStatus.OK)
-                self.send_header("Content-type", "text/html")
-                self.end_headers()
+    @staticmethod
+    def _extract_summary_metrics(full_data):
+        try:
+            parsed = json.loads(full_data)
+            sub_elements = parsed.get("data", {}).get("sub_elements", [])
+            details = next(
+                (
+                    element for element in sub_elements
+                    if element.get("name") == "backtesting-details" and element.get("type") == "value"
+                ),
+                None
+            )
+            values = details.get("data", {}).get("elements", []) if details else []
+            metrics = {}
+            for element in values:
+                title = element.get("title")
+                value = element.get("value")
+                if title and value is not None:
+                    metrics[str(title)] = str(value)
+                metric_html = element.get("html")
+                if metric_html:
+                    metrics.update(BacktestPlot._extract_metrics_from_html(metric_html))
 
-                with open(backtest_plot_instance.report_file, "rb") as report:
-                    self.wfile.write(report.read())
+            def _find_metric(candidates):
+                for key, value in metrics.items():
+                    lowered = key.lower()
+                    if any(candidate in lowered for candidate in candidates):
+                        return value
+                return None
 
-        try:
-            with socketserver.TCPServer(("", self.SERVER_PORT), ReportRequestHandler) as httpd:
-                webbrowser.open(f"http://{self.SERVER_HOST}:{self.SERVER_PORT}")
-                httpd.handle_request()
+            return {
+                "profitability": _find_metric(("usdt gains", "profitability", "profit", "roi", "return")),
+                "portfolio": _find_metric(("end portfolio usdt value", "end portfolio", "portfolio")),
+                "metrics": metrics,
+            }
         except Exception:
-            webbrowser.open(self.report_file)
+            return {"profitability": None, "portfolio": None, "metrics": {}}
 
-    async def _get_template_data(self):
-        full_data, symbols, time_frames, exchanges = await self._get_full_data()
-        return {
-            "FULL_DATA": full_data,
-            "title": f"{', '.join(symbols)}",
-            "top_title": f"{', '.join(symbols)} on {', '.join(time_frames)} from "
-                         f"{', '.join([e.capitalize() for e in exchanges])}",
-            "creation_time": timestamp_util.convert_timestamp_to_datetime(
-                time.time(), self.GENERATED_TIME_FORMAT, local_timezone=True
-            ),
-            "middle_title": "Portfolio value",
-            "bottom_title": "Details",
-            "strategy_config": self.backtest_result.strategy_config
-        }
+    @staticmethod
+    def _extract_metrics_from_html(metric_html):
+        metrics = {}
+        matches = re.findall(
+            r'backtesting-run-container-values-label[^>]*>\s*(.*?)\s*</div>\s*'
+            r'<div[^>]*backtesting-run-container-values-value[^>]*>\s*(.*?)\s*</div>',
+            metric_html,
+            flags=re.IGNORECASE | re.DOTALL
+        )
+        for raw_label, raw_value in matches:
+            label = BacktestPlot._html_to_text(raw_label)
+            value = BacktestPlot._html_to_text(raw_value)
+            if label and value:
+                metrics[label] = value
+        return metrics
+
+    @staticmethod
+    def _html_to_text(content):
+        no_tags = re.sub(r"<[^>]+>", " ", content or "")
+        return re.sub(r"\s+", " ", html.unescape(no_tags)).strip()
 
     async def _get_full_data(self):
         # tentacles not available during first install