Allow to use profileID and service feeds

Signed-off-by: Herklos <herklos@drakkar.software>

Author: Herklos

example.py

@@ -1,30 +1,38 @@
 import asyncio
-import tulipy    # Can be any TA library.
+import tulipy  # Can be any TA library.
 import octobot_script as obs
+from octobot_script.api.ploting import generate_and_show_report
 
 
-async def rsi_test():
+async def example():
+    async def initialize(ctx):
+        # Compute entries only once per backtest.
+        closes = await obs.Close(ctx, max_history=True)
+        times = await obs.Time(ctx, max_history=True, use_close_time=True)
+        rsi_v = tulipy.rsi(closes, period=ctx.tentacle.trading_config["period"])
+        delta = len(closes) - len(rsi_v)
+        # Populate entries with timestamps of candles where RSI is
+        # bellow the "rsi_value_buy_threshold" configuration.
+        run_data["entries"] = {
+            times[index + delta]
+            for index, rsi_val in enumerate(rsi_v)
+            if rsi_val < ctx.tentacle.trading_config["rsi_value_buy_threshold"]
+        }
+        await obs.plot_indicator(ctx, "RSI", times[delta:], rsi_v, run_data["entries"])
+
     async def strategy(ctx):
-        # Will be called at each candle.
-        if run_data["entries"] is None:
-            # Compute entries only once per backtest.
-            closes = await obs.Close(ctx, max_history=True)
-            times = await obs.Time(ctx, max_history=True, use_close_time=True)
-            rsi_v = tulipy.rsi(closes, period=ctx.tentacle.trading_config["period"])
-            delta = len(closes) - len(rsi_v)
-            # Populate entries with timestamps of candles where RSI is
-            # bellow the "rsi_value_buy_threshold" configuration.
-            run_data["entries"] = {
-                times[index + delta]
-                for index, rsi_val in enumerate(rsi_v)
-                if rsi_val < ctx.tentacle.trading_config["rsi_value_buy_threshold"]
-            }
-            await obs.plot_indicator(ctx, "RSI", times[delta:], rsi_v, run_data["entries"])
+        # Called at each candle.
+        # Uses pre-computed entries times to enter positions when relevant.
+        # Also, instantly set take profits and stop losses.
+        # Position exits could also be set separately.
         if obs.current_live_time(ctx) in run_data["entries"]:
-            # Uses pre-computed entries times to enter positions when relevant.
-            # Also, instantly set take profits and stop losses.
-            # Position exists could also be set separately.
-            await obs.market(ctx, "buy", amount="10%", stop_loss_offset="-15%", take_profit_offset="25%")
+            await obs.market(
+                ctx,
+                "buy",
+                amount="10%",
+                stop_loss_offset="-15%",
+                take_profit_offset="25%",
+            )
 
     # Configuration that will be passed to each run.
     # It will be accessible under "ctx.tentacle.trading_config".
@@ -34,19 +42,23 @@ async def strategy(ctx):
     }
 
     # Read and cache candle data to make subsequent backtesting runs faster.
-    data = await obs.get_data("BTC/USDT", "1d", start_timestamp=1505606400)
+    data = await obs.get_data(
+        "BTC/USDT", "1d", start_timestamp=1505606400, social_services=[]
+    )
     run_data = {
         "entries": None,
     }
     # Run a backtest using the above data, strategy and configuration.
-    res = await obs.run(data, strategy, config)
+    res = await obs.run(
+        data, config, initialize_func=initialize, strategy_func=strategy
+    )
     print(res.describe())
     # Generate and open report including indicators plots
-    await res.plot(show=True)
+    await generate_and_show_report(res)
     # Stop data to release local databases.
     await data.stop()
 
 
 # Call the execution of the script inside "asyncio.run" as
 # OctoBot-Script runs using the python asyncio framework.
-asyncio.run(rsi_test())
+asyncio.run(example())

octobot_script/api/data_fetching.py

@@ -14,44 +14,142 @@
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot-Script. If not, see <https://www.gnu.org/licenses/>.
 
+import datetime
+
 import octobot_backtesting.api as backtesting_api
 import octobot_commons.symbols as commons_symbols
 import octobot_commons.enums as commons_enums
 import octobot_trading.enums as trading_enums
 import octobot_script.internal.octobot_mocks as octobot_mocks
 
 
+def _validate_tentacles_source(tentacles_config, profile_id):
+    if tentacles_config is not None and profile_id is not None:
+        raise ValueError("Only one of tentacles_config or profile_id can be provided.")
+
+
 def _ensure_ms_timestamp(timestamp):
     if timestamp is None:
         return timestamp
     if timestamp < 16737955050:  # Friday 28 May 2500 07:57:30
         return timestamp * 1000
 
 
+def _yesterday_midnight_ms() -> int:
+    """Return today at 00:00:00 UTC (= end of yesterday) in milliseconds.
+    Used as a stable default end_timestamp so data files collected on the
+    same calendar day share an identical end boundary and can be cached."""
+    today_midnight = datetime.datetime.now(datetime.timezone.utc).replace(
+        hour=0, minute=0, second=0, microsecond=0
+    )
+    return int(today_midnight.timestamp() * 1000)
+
+
+def _resolve_end_timestamp_ms(end_timestamp) -> int:
+    if end_timestamp is None:
+        return _yesterday_midnight_ms()
+    return _ensure_ms_timestamp(end_timestamp)
+
+
 async def historical_data(symbol, timeframe, exchange="binance", exchange_type=trading_enums.ExchangeTypes.SPOT.value,
-                          start_timestamp=None, end_timestamp=None):
+                          start_timestamp=None, end_timestamp=None, tentacles_config=None, profile_id=None):
+    _validate_tentacles_source(tentacles_config, profile_id)
     symbols = [symbol]
     time_frames = [commons_enums.TimeFrames(timeframe)]
+    start_timestamp_ms = _ensure_ms_timestamp(start_timestamp)
+    end_timestamp_ms = _resolve_end_timestamp_ms(end_timestamp)
+    existing_file = await backtesting_api.find_matching_data_file(
+        exchange_name=exchange,
+        symbols=symbols,
+        time_frames=time_frames,
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+    )
+    if existing_file:
+        return existing_file
     data_collector_instance = backtesting_api.exchange_historical_data_collector_factory(
         exchange,
         trading_enums.ExchangeTypes(exchange_type),
-        octobot_mocks.get_tentacles_config(),
+        octobot_mocks.get_tentacles_config(tentacles_config, profile_id, activate_strategy_tentacles=False),
         [commons_symbols.parse_symbol(symbol) for symbol in symbols],
         time_frames=time_frames,
-        start_timestamp=_ensure_ms_timestamp(start_timestamp),
-        end_timestamp=_ensure_ms_timestamp(end_timestamp)
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+    )
+    return await backtesting_api.initialize_and_run_data_collector(data_collector_instance)
+
+
+async def social_historical_data(services: list[str], sources: list[str] | None = None,
+                                 symbols: list[str] | None = None, start_timestamp=None, end_timestamp=None,
+                                 tentacles_config=None, profile_id=None):
+    _validate_tentacles_source(tentacles_config, profile_id)
+    start_timestamp_ms = _ensure_ms_timestamp(start_timestamp)
+    end_timestamp_ms = _resolve_end_timestamp_ms(end_timestamp)
+    existing_file = await backtesting_api.find_matching_data_file(
+        services=services,
+        symbols=symbols or [],
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+    )
+    if existing_file:
+        return existing_file
+    data_collector_instance = backtesting_api.social_historical_data_collector_factory(
+        services=services,
+        tentacles_setup_config=octobot_mocks.get_tentacles_config(
+            tentacles_config, profile_id, activate_strategy_tentacles=False
+        ),
+        sources=sources,
+        symbols=[commons_symbols.parse_symbol(symbol) for symbol in symbols] if symbols else None,
+        start_timestamp=start_timestamp_ms,
+        end_timestamp=end_timestamp_ms,
+        config=octobot_mocks.get_config(),
     )
     return await backtesting_api.initialize_and_run_data_collector(data_collector_instance)
 
 
 async def get_data(symbol, time_frame, exchange="binance", exchange_type=trading_enums.ExchangeTypes.SPOT.value,
-                   start_timestamp=None, end_timestamp=None, data_file=None):
-    data = data_file or \
-           await historical_data(symbol, timeframe=time_frame, exchange=exchange, exchange_type=exchange_type,
-                                 start_timestamp=start_timestamp, end_timestamp=end_timestamp)
+                   start_timestamp=None, end_timestamp=None, data_file=None,
+                   social_data_files: list[str] | None = None, social_services: list[str] | None = None,
+                   social_sources: list[str] | None = None, social_symbols: list[str] | None = None,
+                   tentacles_config=None, profile_id=None):
+    _validate_tentacles_source(tentacles_config, profile_id)
+    data_files = [data_file] if data_file else [
+        await historical_data(
+            symbol,
+            timeframe=time_frame,
+            exchange=exchange,
+            exchange_type=exchange_type,
+            start_timestamp=start_timestamp,
+            end_timestamp=end_timestamp,
+            tentacles_config=tentacles_config,
+            profile_id=profile_id,
+        )
+    ]
+
+    if social_data_files is not None:
+        data_files.extend(social_data_files)
+    elif profile_id is not None:
+        social_services = social_services if social_services is not None \
+            else octobot_mocks.get_activated_social_services(
+                tentacles_config, profile_id, requested_sources=social_sources
+            )
+    if social_services:
+        for service in social_services:
+            data_files.append(
+                await social_historical_data(
+                    [service],
+                    sources=social_sources,
+                    symbols=social_symbols,
+                    start_timestamp=start_timestamp,
+                    end_timestamp=end_timestamp,
+                    tentacles_config=tentacles_config,
+                    profile_id=profile_id,
+                )
+            )
+
     return await backtesting_api.create_and_init_backtest_data(
-        [data],
+        data_files,
         octobot_mocks.get_config(),
-        octobot_mocks.get_tentacles_config(),
+        octobot_mocks.get_tentacles_config(tentacles_config, profile_id, activate_strategy_tentacles=False),
         use_accurate_price_time_frame=True
     )

octobot_script/api/execution.py

@@ -18,11 +18,17 @@
 import octobot_script.internal.runners as runners
 
 
-async def run(backtesting_data, update_func, strategy_config,
-              enable_logs=False, enable_storage=True):
+async def run(backtesting_data, strategy_config,
+              enable_logs=False, enable_storage=True,
+              strategy_func=None, initialize_func=None,
+              tentacles_config=None, profile_id=None):
+    if tentacles_config is not None and profile_id is not None:
+        raise ValueError("Only one of tentacles_config or profile_id can be provided.")
     if enable_logs:
         logging_util.load_logging_config()
     return await runners.run(
-        backtesting_data, update_func, strategy_config,
-        enable_logs=enable_logs, enable_storage=enable_storage
+        backtesting_data, strategy_config,
+        enable_logs=enable_logs, enable_storage=enable_storage,
+        strategy_func=strategy_func, initialize_func=initialize_func,
+        tentacles_config=tentacles_config, profile_id=profile_id,
     )

octobot_script/api/ploting.py

@@ -15,15 +15,33 @@
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot-Script. If not, see <https://www.gnu.org/licenses/>.
 
+
 async def plot_indicator(ctx, name, x, y, signals=None):
     # lazy import
     import octobot_script as obs
 
     await obs.plot(ctx, name, x=list(x), y=list(y))
-    value_by_x = {
-        x: y
-        for x, y in zip(x, y)
-    }
+    value_by_x = {x: y for x, y in zip(x, y)}
     if signals:
-        await obs.plot(ctx, "signals", x=list(signals), y=[value_by_x[x] for x in signals], mode="markers")
+        await obs.plot(
+            ctx,
+            "signals",
+            x=list(signals),
+            y=[value_by_x[x] for x in signals],
+            mode="markers",
+        )
+
+
+async def generate_and_show_report(res):
+    """Generate a backtest report and show it.
+
+    Args:
+        res: BacktestResult instance
 
+    Returns:
+        The generated report object
+    """
+    report = await res.plot(show=False)
+    print(f"Report generated at: {report.report_file}")
+    report.show()
+    return report