greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

enhance: bb task for creating graphs from EDN file 

Converted inferred graph to an EDN file now that this task exists. Also
merge last of tasks.create-graph to relevant ns so that external users
can also create such tasks
experiment/tanstack-table
Gabriel Horner 2024-06-11 10:03:02 -04:00
parent 2d3f152454
commit 0e1ada2ef6
12 changed files with 151 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bb.edn
View File

@ -76,11 +76,17 @@
{:doc "Transact against a DB graph's datascript db"
:task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
dev:db-create
{:doc "Create a DB graph given a sqlite.build EDN file"
:requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq -cp src:../outliner/src script/create_graph.cljs" *command-line-args*)}
dev:db-import
{:doc "Import a file graph to db graph"
:requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq -cp src:../outliner/src:../../scripts/src script/db_import.cljs" *command-line-args*)}
"yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)}
dev:db-datoms
{:doc "Write db's datoms to a file"

38
deps/db/script/create_graph.cljs vendored Normal file
View File

@ -0,0 +1,38 @@
(ns create-graph
"An example script that creates a DB graph given a sqlite.build EDN file"
(:require [logseq.outliner.db-pipeline :as db-pipeline]
[clojure.string :as string]
[clojure.edn :as edn]
[datascript.core :as d]
["path" :as node-path]
["os" :as os]
["fs" :as fs]
[nbb.classpath :as cp]
[nbb.core :as nbb]))
(defn- resolve-path
"If relative path, resolve with $ORIGINAL_PWD"
[path]
(if (node-path/isAbsolute path)
path
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
(defn -main [args]
(when (not= 2 (count args))
(println "Usage: $0 GRAPH-DIR EDN-PATH")
(js/process.exit 1))
(let [[graph-dir edn-path] args
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
sqlite-build-edn (-> (resolve-path edn-path) fs/readFileSync str edn/read-string)
conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
{:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx sqlite-build-edn)]
(println "Generating" (count (filter :block/name init-tx)) "pages and"
(count (filter :block/content init-tx)) "blocks ...")
(d/transact! conn init-tx)
(d/transact! conn block-props-tx)
(println "Created graph" (str db-name "!"))))
(when (= nbb/*file* (:file (meta #'-main)))
(-main *command-line-args*))

21
deps/db/script/create_graph/inferred.edn vendored Normal file
View File

@ -0,0 +1,21 @@
;; Script that generates classes and properties for a demo of inferring properties.
;; To generate this graph:
;; bb dev:db-create inferred deps/db/create_graph/inferred.edn
;;
;; To try the demo in the UI, in any page type:
;; - Good Will Hunting #Movie #Ben-Affleck
;; or
;; - DB 3 #Meeting #Tienson
{:auto-create-ontology? true
:classes {:Movie {:build/schema-properties [:actor :comment]}
:Meeting {:build/schema-properties [:attendee :duration]}}
:properties
{:actor {:block/schema {:type :object :cardinality :many}
:build/schema-classes [:Person]}
:attendee {:block/schema {:type :object :cardinality :many}
:build/schema-classes [:Person]}}
:pages-and-blocks
[{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}}
{:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}}
{:page {:block/original-name "Tienson" :build/tags [:Person]}}
{:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]}

4
deps/graph-parser/script/db_import.cljs vendored
View File

@ -12,7 +12,7 @@
[babashka.cli :as cli]
[logseq.graph-parser.exporter :as gp-exporter]
[logseq.common.graph :as common-graph]
[logseq.tasks.db-graph.create-graph :as create-graph]
[logseq.outliner.db-pipeline :as db-pipeline]
[promesa.core :as p]))
(defn- build-graph-files
@ -107,7 +107,7 @@
((juxt node-path/dirname node-path/basename) graph-dir'))
[(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
file-graph' (resolve-path file-graph)
conn (create-graph/init-conn dir db-name)
conn (db-pipeline/init-conn dir db-name)
directory? (.isDirectory (fs/statSync file-graph'))]
(p/do!
(if directory?

54
deps/outliner/src/logseq/outliner/db_pipeline.cljs vendored
View File

@ -1,12 +1,18 @@
(ns ^:node-only logseq.outliner.db-pipeline
"This ns provides a datascript listener for DB graphs to add additional changes
that the frontend also adds per transact.
Missing features from frontend.worker.pipeline including:
"This ns provides a datascript listener for DB graphs and helper fns that
build on top of it. The listener adds additional changes that the frontend
also adds per transact. Missing features from frontend.worker.pipeline including:
* Deleted blocks don't update effected :block/tx-id
* Delete empty property parent"
(:require [datascript.core :as d]
(:require [clojure.string :as string]
[datascript.core :as d]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.db.sqlite.build :as sqlite-build]
[logseq.db.sqlite.db :as sqlite-db]
[logseq.outliner.datascript-report :as ds-report]
[logseq.outliner.pipeline :as outliner-pipeline]
[logseq.outliner.datascript-report :as ds-report]))
["fs" :as fs]
["path" :as node-path]))
(defn- rebuild-block-refs
@ -40,3 +46,41 @@
[conn]
(d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
(invoke-hooks conn tx-report))))
(defn- find-on-classpath [classpath rel-path]
(some (fn [dir]
(let [f (node-path/join dir rel-path)]
(when (fs/existsSync f) f)))
(string/split classpath #":")))
(defn- setup-init-data
"Setup initial data same as frontend.handler.repo/create-db"
[conn {:keys [additional-config classpath]}]
(let [config-content
(cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
(do (println "Setting graph's config to empty since no templates/config.edn was found.")
"{}"))
additional-config
;; TODO: Replace with rewrite-clj when it's available
(string/replace-first #"(:file/name-format :triple-lowbar)"
(str "$1 "
(string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
(d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
(defn init-conn
"Create sqlite DB, initialize datascript connection and sync listener and then
transacts initial data. Takes the following options:
* :additional-config - Additional config map to merge into repo config.edn
* :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
that comes with Logseq"
[dir db-name & [opts]]
(fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
;; Same order as frontend.db.conn/start!
(let [conn (sqlite-db/open-db! dir db-name)]
(add-listener conn)
(setup-init-data conn opts)
conn))
(def build-blocks-tx
"An alias for build-blocks-tx to specify default options for this ns"
sqlite-build/build-blocks-tx)

14
docs/dev-practices.md
View File

@ -387,6 +387,20 @@ These tasks are specific to database graphs. For these tasks there is a one time
Updated 16 block(s) for graph test-db!
```
* `dev:db-create` - Create a DB graph given a `sqlite.build` EDN file
First in Electron, create the name of the graph you want create e.g. `inferred`.
Then:
```sh
bb dev:db-create inferred deps/db/script/create_graph/inferred.edn
Generating 11 pages and 0 blocks ...
Created graph inferred!
```
Finally, upload this created graph with the dev command: `Replace graph with`
... Switch to the graph and you can use the created graph!
* `dev:db-datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files
```sh

12
scripts/README.md
View File

@ -13,11 +13,11 @@ Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be su
#### Create graph scripts
For database graphs, it is possible to create graphs with the
[logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs)
ns. This ns makes it easy to write scripts that create graphs by supporting a
concise EDN map for graph generation. For example, the
`create_graph_with_properties.cljs` script uses this ns to create a graph with
a variety of properties:
[logseq.outliner.db-pipeline](deps/outliner/src/logseq/outliner/db_pipeline.cljs)
and [logseq.db.sqlite.build](deps/db/src/logseq/db/sqlite/build.cljs). These
namespaces makes it easy to write scripts to create graphs with a concise EDN
map. For example, the `create_graph_with_properties.cljs` script uses this ns to
create a graph with a variety of properties:
```
$ yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs woot
@ -28,7 +28,7 @@ Created graph woot!
This script creates a DB graph with blocks containing several property types for
both single and many cardinality. It also includes queries for most of these
properties. Read the docs in
[logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs)
[logseq.db.sqlite.build](deps/db/src/logseq/db/sqlite/build.cljs)
for specifics on the EDN map.
To create large graphs with varying size:

47
scripts/src/logseq/tasks/db_graph/create_graph.cljs
View File

@ -1,47 +0,0 @@
(ns logseq.tasks.db-graph.create-graph
"This ns provides fns to create a DB graph using EDN. See `init-conn` for
initializing a DB graph with a datascript connection that syncs to a sqlite DB
at the given directory. See `build-blocks-tx` for the EDN format to create a
graph and current limitations"
(:require [logseq.db.sqlite.db :as sqlite-db]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.outliner.db-pipeline :as db-pipeline]
[clojure.string :as string]
[datascript.core :as d]
["fs" :as fs]
["path" :as node-path]
[nbb.classpath :as cp]
[logseq.db.sqlite.build :as sqlite-build]))
(defn- find-on-classpath [rel-path]
(some (fn [dir]
(let [f (node-path/join dir rel-path)]
(when (fs/existsSync f) f)))
(string/split (cp/get-classpath) #":")))
(defn- setup-init-data
"Setup initial data same as frontend.handler.repo/create-db"
[conn additional-config]
(let [config-content
(cond-> (or (some-> (find-on-classpath "templates/config.edn") fs/readFileSync str)
(do (println "Setting graph's config to empty since no templates/config.edn was found.")
"{}"))
additional-config
;; TODO: Replace with rewrite-clj when it's available
(string/replace-first #"(:file/name-format :triple-lowbar)"
(str "$1 "
(string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
(d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
(defn init-conn
"Create sqlite DB, initialize datascript connection and sync listener and then
transacts initial data"
[dir db-name & {:keys [additional-config]}]
(fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
;; Same order as frontend.db.conn/start!
(let [conn (sqlite-db/open-db! dir db-name)]
(db-pipeline/add-listener conn)
(setup-init-data conn additional-config)
conn))
(def build-blocks-tx sqlite-build/build-blocks-tx)

47
scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs
View File

@ -1,47 +0,0 @@
(ns logseq.tasks.db-graph.create-graph-with-inferred-properties
"Script that generates classes and properties for a demo of inferring properties.
To try the demo, in any page type:
- Good Will Hunting #Movie #Ben-Affleck
or
- DB 3 #Meeting #Tienson"
(:require [logseq.tasks.db-graph.create-graph :as create-graph]
[logseq.db.sqlite.build :as sqlite-build]
[clojure.string :as string]
[datascript.core :as d]
["path" :as node-path]
["os" :as os]
[nbb.core :as nbb]))
(defn- create-init-data []
{:auto-create-ontology? true
:classes {:Movie {:build/schema-properties [:actor :comment]}
:Meeting {:build/schema-properties [:attendee :duration]}}
:properties
{:actor {:block/schema {:type :object :cardinality :many}
:build/schema-classes [:Person]}
:attendee {:block/schema {:type :object :cardinality :many}
:build/schema-classes [:Person]}}
:pages-and-blocks
[{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}}
{:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}}
{:page {:block/original-name "Tienson" :build/tags [:Person]}}
{:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]})
(defn -main [args]
(when (not= 1 (count args))
(println "Usage: $0 GRAPH-DIR")
(js/process.exit 1))
(let [graph-dir (first args)
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (create-graph/init-conn dir db-name)
{:keys [init-tx block-props-tx]} (sqlite-build/build-blocks-tx (create-init-data))]
(println "Generating" (count (filter :block/name init-tx)) "pages and"
(count (filter :block/content init-tx)) "blocks ...")
(d/transact! conn init-tx)
(d/transact! conn block-props-tx)
(println "Created graph" (str db-name "!"))))
(when (= nbb/*file* (:file (meta #'-main)))
(-main *command-line-args*))

7
scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs
View File

@ -1,11 +1,12 @@
(ns logseq.tasks.db-graph.create-graph-with-large-sizes
"Script that generates graphs at large sizes"
(:require [logseq.tasks.db-graph.create-graph :as create-graph]
(:require [logseq.outliner.db-pipeline :as db-pipeline]
[clojure.string :as string]
[datascript.core :as d]
[babashka.cli :as cli]
["path" :as node-path]
["os" :as os]
[nbb.classpath :as cp]
[nbb.core :as nbb]))
(def *ids (atom #{}))
@ -65,9 +66,9 @@
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (create-graph/init-conn dir db-name)
conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
_ (println "Building tx ...")
{:keys [init-tx]} (create-graph/build-blocks-tx (create-init-data options))]
{:keys [init-tx]} (db-pipeline/build-blocks-tx (create-init-data options))]
(println "Built" (count init-tx) "tx," (count (filter :block/original-name init-tx)) "pages and"
(count (filter :block/content init-tx)) "blocks ...")
;; Vary the chunking with page size up to a max to avoid OOM

8
scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs
View File

@ -2,7 +2,7 @@
"Script that generates all the permutations of property types and cardinality.
Also creates a page of queries that exercises most properties
NOTE: This script is also used in CI to confirm graph creation works"
(:require [logseq.tasks.db-graph.create-graph :as create-graph]
(:require [logseq.outliner.db-pipeline :as db-pipeline]
[logseq.common.util.date-time :as date-time-util]
[logseq.common.util.page-ref :as page-ref]
[logseq.db.frontend.property.type :as db-property-type]
@ -13,6 +13,7 @@
["path" :as node-path]
["os" :as os]
[babashka.cli :as cli]
[nbb.classpath :as cp]
[nbb.core :as nbb]))
(defn- date-journal-title [date]
@ -181,8 +182,9 @@
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
{:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx (create-init-data))
conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
:classpath (cp/get-classpath)})
{:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx (create-init-data))
existing-names (set (map :v (d/datoms @conn :avet :block/original-name)))
conflicting-names (set/intersection existing-names (set (keep :block/original-name init-tx)))]
(when (seq conflicting-names)

8
scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs
View File

@ -10,7 +10,7 @@
* Some properties are skipped because they are superseded/deprecated or because they have a property
type logseq doesnt' support yet
* schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many"
(:require [logseq.tasks.db-graph.create-graph :as create-graph]
(:require [logseq.outliner.db-pipeline :as db-pipeline]
[logseq.common.util :as common-util]
[logseq.db.frontend.property :as db-property]
[clojure.string :as string]
@ -19,6 +19,7 @@
["path" :as node-path]
["os" :as os]
["fs" :as fs]
[nbb.classpath :as cp]
[nbb.core :as nbb]
[clojure.set :as set]
[clojure.walk :as w]
@ -395,10 +396,11 @@
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
:classpath (cp/get-classpath)})
init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
options)
{:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx init-data)]
{:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx init-data)]
(println "Generating" (str (count (filter :block/name init-tx)) " pages with "
(count (:classes init-data)) " classes and "
(count (:properties init-data)) " properties ..."))