chore: add xtdb

Author: YaroSpace

.github/workflows/test.yml

@@ -12,6 +12,11 @@ jobs:
     runs-on: ubuntu-latest
 
     services:
+      xtdb:
+        image: ghcr.io/xtdb/xtdb:latest
+        ports:
+          - '5442:5432'
+
       chrome-driver:
         image: erseco/alpine-chromedriver
         ports:

Makefile

@@ -22,6 +22,7 @@ deploy:
 
 	$(call confirm,Push changes?)
 	git push --force
+	sleep 10 && .git/hooks/post-push
 
 lint:
 	clj-kondo --lint src

deps.edn

@@ -6,6 +6,7 @@
         com.github.seancorfield/honeysql        {:mvn/version "2.7.1340"}
         com.github.seancorfield/next.jdbc       {:mvn/version "1.3.1048"}
         org.postgresql/postgresql               {:mvn/version "42.7.7"}
+        com.xtdb/xtdb-api                       {:mvn/version "2.0.0"}
         migratus/migratus                       {:mvn/version "1.6.4"}
         hikari-cp/hikari-cp                     {:mvn/version "3.3.0"}
         clj-http/clj-http                       {:mvn/version "3.13.1"}
@@ -14,18 +15,17 @@
         org.clojure/tools.logging               {:mvn/version "1.3.0"}
         ch.qos.logback/logback-classic          {:mvn/version "1.5.18"}
         nrepl/nrepl                             {:mvn/version "1.3.1"}
-        mvxcvi/puget                            {:mvn/version "1.3.4"}
         etaoin/etaoin                           {:mvn/version "1.1.43"}
         org.clj-commons/slingshot               {:mvn/version "0.13.0"}}
 
  :paths ["src" "resources"]
 
  :aliases {:nrebel
-           {:extra-deps {com.bhauman/rebel-readline-nrepl {:mvn/version "0.1.5"}}
+           {:extra-deps {com.bhauman/rebel-readline-nrepl {:mvn/version "0.1.5"}
+                         mvxcvi/puget                     {:mvn/version "1.3.4"}}
             :exec-fn rebel-readline.nrepl/connect
             :exec-args {:background-print false}
-            :main-opts ["--eval" "(require '[puget.printer :as pp])"
-                        "-m" "rebel-readline.nrepl.main"]
+            ; :main-opts ["--eval" "(alter-var-root #'rebel-readline.clojure.main/syntax-highlight-prn (fn [_] (fn [v] (puget.printer/cprint v))))"]
             :jvm-opts ["--enable-native-access=ALL-UNNAMED"]}
 
            :repl
@@ -41,8 +41,10 @@
                          nubank/matcher-combinators              {:mvn/version "3.9.2"}
                          clj-test-containers/clj-test-containers {:mvn/version "0.7.4"}
                          org.testcontainers/postgresql           {:mvn/version "1.21.3"}
+                         com.xtdb/xtdb-core                      {:mvn/version "2.0.0"}
                          ; djblue/portal                           {:mvn/version "0.59.1"}   ; portal data inspector
                          org.clj-commons/pretty                  {:mvn/version "3.5.0"}
+                         mvxcvi/puget                            {:mvn/version "1.3.4"}
                          dev.weavejester/hashp                   {:mvn/version "0.4.0"}
                          clj-http-fake/clj-http-fake             {:mvn/version "1.0.4"}
                          io.github.noahtheduke/lazytest          {:mvn/version "1.9.1"}}
@@ -55,7 +57,8 @@
                         "-f" "rebel-readline.main/-main"]
             :jvm-opts ["--enable-native-access=ALL-UNNAMED"
                        "-Djdk.attach.allowAttachSelf"]}
-
+                       ; "--add-opens=java.base/java.nio=ALL-UNNAMED" ; for xtdb-local
+                       ; "-Dio.netty.tryReflectionSetAccessible=true"]} ; for xtdb-local
                        ;"-Dio.pedestal.dev-mode=true"]}
 
            :build

dev/user.clj

@@ -1,7 +1,6 @@
 (ns user
   (:require
    [nrepl.core :as nrepl]
-   [nvim-app.utils :as u]
    [clojure.tools.namespace.repl :as repl]
    [rebel-readline.clojure.main :as main]
    [clj-commons.pretty.repl :as pretty-repl]
@@ -12,7 +11,8 @@
    [user.java :refer :all]
    [clojure.test :as test]
    [clojure.tools.logging :as log]
-   [clojure.string :as str])
+   [clojure.string :as str]
+   [clojure.pprint :as pprint])
 
   (:import
    [ch.qos.logback.classic Level]
@@ -101,6 +101,7 @@
       (first (nrepl/message client data)))))
 
 (def cp> puget.printer/cprint)
+; (def pp pprint/pprint)
 
 (defn pn>
   "Print and strip newlines"

docker-compose.yaml

@@ -12,6 +12,16 @@ services:
       - db-data:/var/lib/postgresql/data
     container_name: nvim-app-database
 
+  xtdb:
+    image: ghcr.io/xtdb/xtdb:latest
+    ports:
+      - '5442:5432'
+      - '5080:8080'
+      - '5030:3000'
+    volumes:
+      - xtdb-data:/var/lib/xtdb
+    container_name: nvim-app-xtdb
+
   chromedriver:
     image: erseco/alpine-chromedriver
     deploy:
@@ -53,6 +63,7 @@ services:
 
 volumes:
   db-data:
+  xtdb-data:
   web-public:
 
 networks:

resources/config.edn

@@ -7,9 +7,12 @@
  :db-spec {:jdbc-url #or [#env DATABASE_URL "jdbc:postgresql://localhost:5432/nvim.app"]
            :username #or [#env DATABASE_USER "nvim"]
            :password #or [#env DATABASE_PASSWORD "nvim"]
+           :maximum-pool-size 20
            :connection-init-sql "SET pg_trgm.similarity_threshold = 0.16"
            :logging? #profile {:default false :dev false}}
 
+ :xtdb-spec {:jdbc-url #or [#env XTDB_URL "jdbc:xtdb://localhost:5442/xtdb"]}
+
  :repl    {:port #long #or [#env NVIM_APP_REPL_PORT 7700]
            :enable #or [#env NVIM_APP_REPL_ENABLE false]}
 

src/nvim_app/components/sched.clj

@@ -35,7 +35,7 @@
   (schedule-task scheduler update-repos! (:update-repos-interval-hr config))
 
   (let [interval-hr (:update-previews-interval-hr config)]
-    (schedule-task scheduler (update-previews! interval-hr) interval-hr)))
+    (schedule-task scheduler (update-previews! interval-hr) 4)))
 
 (defn completed-tasks
   ([]

src/nvim_app/components/xtdb.clj

@@ -0,0 +1,41 @@
+(ns nvim-app.components.xtdb
+  (:require
+   [com.stuartsierra.component :as component]
+   [next.jdbc :as jdbc]
+   [xtdb.next.jdbc :as xt]
+   [hikari-cp.core :refer [make-datasource]]
+   [clojure.tools.logging :as log])
+  (:import
+   [com.zaxxer.hikari HikariDataSource]))
+
+(defrecord XtdbComponent [config raw-ds datasource]
+  component/Lifecycle
+
+  (start [this]
+    (log/info (str "Starting XTDB on: " (:jdbc-url config)))
+
+    (if datasource
+      this
+      (try
+        (let [^HikariDataSource ds (make-datasource config)
+              ds-with-opts (jdbc/with-options ds {:builder-fn xt/builder-fn})]
+          (try
+            (assoc this :raw-ds ds :datasource ds-with-opts)
+            (catch Exception e
+              (.close ^HikariDataSource ds)
+              (throw e))))
+
+        (catch Exception e
+          (throw (ex-info "Failed to start XTDB component" {:message (ex-message e)} e))))))
+
+  (stop [this]
+    (log/info "Stopping XTDB Component"
+              (when raw-ds (.close ^HikariDataSource raw-ds))
+              (assoc this :datasource nil))))
+
+(defn new [config]
+  (map->XtdbComponent {:config (:xtdb-spec config)}))
+
+(comment
+  (let [ds (:xtdb-component @nvim-app.state/app-system-atom)]
+    (jdbc/execute! ds ["SELECT * FROM information_schema.tables"])))

src/nvim_app/core.clj

@@ -5,6 +5,7 @@
    [nvim-app.components.app :as app]
    [nvim-app.components.pedestal.core :as pedestal-component]
    [nvim-app.components.database :as database-component]
+   [nvim-app.components.xtdb :as xtdb-component]
    [nvim-app.components.sched :as sched-component]
    [nvim-app.components.repl :as repl-component]
    [nvim-app.utils :refer [ex-format]]
@@ -20,13 +21,15 @@
    (component/system-map
     :repl (repl-component/new config)
     :database-component (database-component/new config)
+    :xtdb-component (xtdb-component/new config)
     :pedestal-component (component/using (pedestal-component/new config)
                                          [:database-component])
     :sched (sched-component/new config)
     :app  (app/new config))
-   (component/system-using {:app [:repl :sched
+   (component/system-using {:app [:database-component
+                                  :xtdb-component
                                   :pedestal-component
-                                  :database-component]})))
+                                  :sched :repl]})))
 
 (defn -main []
   (try

src/nvim_app/db/core.clj

@@ -9,6 +9,9 @@
 (defn get-ds []
   (:database-component @app-system-atom))
 
+(defn get-xtds []
+  (:xtdb-component @app-system-atom))
+
 (defn query!
   ([sql] (query! (get-ds) sql jdbc/execute!))
   ([ds sql] (query! ds sql jdbc/execute!))
@@ -134,6 +137,5 @@
     (reset!!)
     (migration-up!))
   (map #(dissoc % :tsv :topics_tsv) (take 10 (select :repos)))
-  (update! :users {:role "admin"} :where [:id 1])
-  (select :categories)
-  (select-one :app))
+  (update! :users {:role "admin"} :where [:id 0])
+  (jdbc/execute! (get-xtds) ["SELECT * FROM information_schema.tables"]))