greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: frontend tests shouldn't depend on sqlite3 

frontend.test.helper was accidentally coupled to sqlite3 via
logseq.outliner.db-pipeline. Split out sqlite3 dependent fns to
logseq.outliner.cli so it's clear this shouldn't be coupled to frontend
experiment/tanstack-table
Gabriel Horner 2024-06-14 14:37:40 -04:00
parent 56063ed2bf
commit 95db811f55
8 changed files with 73 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
deps/db/script/create_graph.cljs vendored
View File

@ -1,6 +1,6 @@
(ns create-graph
"An example script that creates a DB graph given a sqlite.build EDN file"
(:require [logseq.outliner.db-pipeline :as db-pipeline]
(:require [logseq.outliner.cli :as outliner-cli]
[clojure.string :as string]
[clojure.edn :as edn]
[datascript.core :as d]
@ -27,8 +27,8 @@
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
sqlite-build-edn (merge {:auto-create-ontology? true}
(-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
{:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx sqlite-build-edn)]
conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx sqlite-build-edn)]
(println "Generating" (count (filter :block/name init-tx)) "pages and"
(count (filter :block/content init-tx)) "blocks ...")
(d/transact! conn init-tx)

4
deps/graph-parser/script/db_import.cljs vendored
View File

@ -12,7 +12,7 @@
[babashka.cli :as cli]
[logseq.graph-parser.exporter :as gp-exporter]
[logseq.common.graph :as common-graph]
[logseq.outliner.db-pipeline :as db-pipeline]
[logseq.outliner.cli :as outliner-cli]
[promesa.core :as p]))
(defn- build-graph-files
@ -107,7 +107,7 @@
((juxt node-path/dirname node-path/basename) graph-dir'))
[(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
file-graph' (resolve-path file-graph)
conn (db-pipeline/init-conn dir db-name)
conn (outliner-cli/init-conn dir db-name)
directory? (.isDirectory (fs/statSync file-graph'))]
(p/do!
(if directory?

1
deps/outliner/.carve/config.edn vendored
View File

@ -1,6 +1,7 @@
{:paths ["src"]
:api-namespaces [logseq.outliner.datascript-report
logseq.outliner.pipeline
logseq.outliner.cli
logseq.outliner.core
logseq.outliner.db-pipeline
logseq.outliner.property

48
deps/outliner/src/logseq/outliner/cli.cljs vendored Normal file
View File

@ -0,0 +1,48 @@
(ns ^:node-only logseq.outliner.cli
"Primary ns for outliner CLI fns"
(:require [clojure.string :as string]
[datascript.core :as d]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.db.sqlite.build :as sqlite-build]
[logseq.db.sqlite.db :as sqlite-db]
[logseq.outliner.db-pipeline :as db-pipeline]
["fs" :as fs]
["path" :as node-path]))
(defn- find-on-classpath [classpath rel-path]
(some (fn [dir]
(let [f (node-path/join dir rel-path)]
(when (fs/existsSync f) f)))
(string/split classpath #":")))
(defn- setup-init-data
"Setup initial data same as frontend.handler.repo/create-db"
[conn {:keys [additional-config classpath]}]
(let [config-content
(cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
(do (println "Setting graph's config to empty since no templates/config.edn was found.")
"{}"))
additional-config
;; TODO: Replace with rewrite-clj when it's available
(string/replace-first #"(:file/name-format :triple-lowbar)"
(str "$1 "
(string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
(d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
(defn init-conn
"Create sqlite DB, initialize datascript connection and sync listener and then
transacts initial data. Takes the following options:
* :additional-config - Additional config map to merge into repo config.edn
* :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
that comes with Logseq"
[dir db-name & [opts]]
(fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
;; Same order as frontend.db.conn/start!
(let [conn (sqlite-db/open-db! dir db-name)]
(db-pipeline/add-listener conn)
(setup-init-data conn opts)
conn))
(def build-blocks-tx
"An alias for build-blocks-tx to specify default options for this ns"
sqlite-build/build-blocks-tx)

60
deps/outliner/src/logseq/outliner/db_pipeline.cljs vendored
View File

@ -1,19 +1,13 @@
(ns ^:node-only logseq.outliner.db-pipeline
"This ns provides a datascript listener for DB graphs and helper fns that
build on top of it. The listener adds additional changes that the frontend
also adds per transact. Missing features from frontend.worker.pipeline including:
(ns logseq.outliner.db-pipeline
"This ns provides a datascript listener for DB graphs that is useful for CLIs
and testing (since it doesn't assume a frontend worker exists). The listener adds
additional changes that the frontend also adds per transact. Missing features
from frontend.worker.pipeline including:
* Deleted blocks don't update effected :block/tx-id
* Delete empty property parent"
(:require [clojure.string :as string]
[datascript.core :as d]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.db.sqlite.build :as sqlite-build]
[logseq.db.sqlite.db :as sqlite-db]
(:require [datascript.core :as d]
[logseq.outliner.datascript-report :as ds-report]
[logseq.outliner.pipeline :as outliner-pipeline]
["fs" :as fs]
["path" :as node-path]))
[logseq.outliner.pipeline :as outliner-pipeline]))
(defn- rebuild-block-refs
[{:keys [db-after]} blocks]
@ -45,42 +39,4 @@
"Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
[conn]
(d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
(invoke-hooks conn tx-report))))
(defn- find-on-classpath [classpath rel-path]
(some (fn [dir]
(let [f (node-path/join dir rel-path)]
(when (fs/existsSync f) f)))
(string/split classpath #":")))
(defn- setup-init-data
"Setup initial data same as frontend.handler.repo/create-db"
[conn {:keys [additional-config classpath]}]
(let [config-content
(cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
(do (println "Setting graph's config to empty since no templates/config.edn was found.")
"{}"))
additional-config
;; TODO: Replace with rewrite-clj when it's available
(string/replace-first #"(:file/name-format :triple-lowbar)"
(str "$1 "
(string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
(d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
(defn init-conn
"Create sqlite DB, initialize datascript connection and sync listener and then
transacts initial data. Takes the following options:
* :additional-config - Additional config map to merge into repo config.edn
* :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
that comes with Logseq"
[dir db-name & [opts]]
(fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
;; Same order as frontend.db.conn/start!
(let [conn (sqlite-db/open-db! dir db-name)]
(add-listener conn)
(setup-init-data conn opts)
conn))
(def build-blocks-tx
"An alias for build-blocks-tx to specify default options for this ns"
sqlite-build/build-blocks-tx)
(invoke-hooks conn tx-report))))

6
scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs
View File

@ -1,6 +1,6 @@
(ns logseq.tasks.db-graph.create-graph-with-large-sizes
"Script that generates graphs at large sizes"
(:require [logseq.outliner.db-pipeline :as db-pipeline]
(:require [logseq.outliner.cli :as outliner-cli]
[clojure.string :as string]
[datascript.core :as d]
[babashka.cli :as cli]
@ -66,9 +66,9 @@
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
_ (println "Building tx ...")
{:keys [init-tx]} (db-pipeline/build-blocks-tx (create-init-data options))]
{:keys [init-tx]} (outliner-cli/build-blocks-tx (create-init-data options))]
(println "Built" (count init-tx) "tx," (count (filter :block/original-name init-tx)) "pages and"
(count (filter :block/content init-tx)) "blocks ...")
;; Vary the chunking with page size up to a max to avoid OOM

8
scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs
View File

@ -2,7 +2,7 @@
"Script that generates all the permutations of property types and cardinality.
Also creates a page of queries that exercises most properties
NOTE: This script is also used in CI to confirm graph creation works"
(:require [logseq.outliner.db-pipeline :as db-pipeline]
(:require [logseq.outliner.cli :as outliner-cli]
[logseq.common.util.date-time :as date-time-util]
[logseq.common.util.page-ref :as page-ref]
[logseq.db.frontend.property.type :as db-property-type]
@ -182,9 +182,9 @@
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
:classpath (cp/get-classpath)})
{:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx (create-init-data))
conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
:classpath (cp/get-classpath)})
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx (create-init-data))
existing-names (set (map :v (d/datoms @conn :avet :block/original-name)))
conflicting-names (set/intersection existing-names (set (keep :block/original-name init-tx)))]
(when (seq conflicting-names)

8
scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs
View File

@ -10,7 +10,7 @@
* Some properties are skipped because they are superseded/deprecated or because they have a property
type logseq doesnt' support yet
* schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many"
(:require [logseq.outliner.db-pipeline :as db-pipeline]
(:require [logseq.outliner.cli :as outliner-cli]
[logseq.common.util :as common-util]
[logseq.db.frontend.property :as db-property]
[clojure.string :as string]
@ -396,11 +396,11 @@
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
:classpath (cp/get-classpath)})
conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
:classpath (cp/get-classpath)})
init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
options)
{:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx init-data)]
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)]
(println "Generating" (str (count (filter :block/name init-tx)) " pages with "
(count (:classes init-data)) " classes and "
(count (:properties init-data)) " properties ..."))