2024-02-24 05:42:31 +00:00
|
|
|
(ns db-import
|
2024-02-26 16:11:01 +00:00
|
|
|
"Imports given file(s) to a db graph. This script is primarily for
|
2024-02-24 05:42:31 +00:00
|
|
|
developing the import feature and for engineers who want to customize
|
|
|
|
the import process"
|
|
|
|
(:require [clojure.string :as string]
|
|
|
|
[datascript.core :as d]
|
|
|
|
["path" :as node-path]
|
|
|
|
["os" :as os]
|
2024-02-27 21:27:26 +00:00
|
|
|
["fs" :as fs]
|
2024-02-26 21:20:03 +00:00
|
|
|
["fs/promises" :as fsp]
|
2024-02-24 05:42:31 +00:00
|
|
|
[nbb.core :as nbb]
|
|
|
|
[babashka.cli :as cli]
|
|
|
|
[logseq.graph-parser.exporter :as gp-exporter]
|
2024-02-26 21:20:03 +00:00
|
|
|
[logseq.common.graph :as common-graph]
|
|
|
|
[logseq.tasks.db-graph.create-graph :as create-graph]
|
|
|
|
[promesa.core :as p]))
|
|
|
|
|
|
|
|
(defn- build-graph-files
|
2024-03-13 20:23:49 +00:00
|
|
|
"Given a file graph directory, return all files including assets and adds relative paths
|
|
|
|
on ::rpath since paths are absolute by default and exporter needs relative paths for
|
|
|
|
some operations"
|
2024-02-27 17:07:02 +00:00
|
|
|
[dir*]
|
2024-02-26 21:20:03 +00:00
|
|
|
(let [dir (node-path/resolve dir*)]
|
|
|
|
(->> (common-graph/get-files dir)
|
2024-03-13 20:23:49 +00:00
|
|
|
(concat (when (fs/existsSync (node-path/join dir* "assets"))
|
|
|
|
(common-graph/readdir (node-path/join dir* "assets"))))
|
|
|
|
(mapv #(hash-map :path %
|
|
|
|
::rpath (node-path/relative dir* %))))))
|
2024-02-24 05:42:31 +00:00
|
|
|
|
2024-02-27 21:27:26 +00:00
|
|
|
(defn- <read-file
|
|
|
|
[file]
|
2024-03-13 20:23:49 +00:00
|
|
|
(p/let [s (fsp/readFile (:path file))]
|
2024-02-27 21:27:26 +00:00
|
|
|
(str s)))
|
|
|
|
|
2024-03-05 22:05:18 +00:00
|
|
|
(defn- <copy-asset-file [file db-graph-dir file-graph-dir]
|
|
|
|
(p/let [parent-dir (node-path/dirname
|
2024-03-13 20:23:49 +00:00
|
|
|
(node-path/join db-graph-dir (node-path/relative file-graph-dir (:path file))))
|
2024-03-05 22:05:18 +00:00
|
|
|
_ (fsp/mkdir parent-dir #js {:recursive true})]
|
2024-03-13 20:23:49 +00:00
|
|
|
(fsp/copyFile (:path file) (node-path/join parent-dir (node-path/basename (:path file))))))
|
2024-03-05 22:05:18 +00:00
|
|
|
|
2024-03-13 20:23:49 +00:00
|
|
|
(defn- import-file-graph-to-db
|
|
|
|
"Import a file graph dir just like UI does. However, unlike the UI the
|
|
|
|
exporter receives file maps containing keys :path and ::rpath since :path
|
|
|
|
are full paths"
|
|
|
|
[file-graph-dir db-graph-dir conn options]
|
|
|
|
(let [*files (build-graph-files file-graph-dir)
|
|
|
|
config-file (first (filter #(string/ends-with? (:path %) "logseq/config.edn") *files))
|
|
|
|
_ (assert config-file "No 'logseq/config.edn' found for file graph dir")
|
|
|
|
options (merge options
|
|
|
|
{;; common options
|
|
|
|
:rpath-key ::rpath
|
|
|
|
:notify-user prn
|
|
|
|
:<read-file <read-file
|
|
|
|
;; :set-ui-state prn
|
|
|
|
;; config file options
|
|
|
|
;; TODO: Add actual default
|
|
|
|
:default-config {}
|
|
|
|
;; asset file options
|
|
|
|
:<copy-asset (fn copy-asset [file]
|
|
|
|
(<copy-asset-file file db-graph-dir file-graph-dir))})]
|
|
|
|
(gp-exporter/export-file-graph conn conn config-file *files options)))
|
2024-02-24 05:42:31 +00:00
|
|
|
|
2024-02-29 20:05:23 +00:00
|
|
|
(defn- resolve-path
|
|
|
|
"If relative path, resolve with $ORIGINAL_PWD"
|
|
|
|
[path]
|
|
|
|
(if (node-path/isAbsolute path)
|
|
|
|
path
|
|
|
|
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
|
|
|
|
|
2024-03-13 20:23:49 +00:00
|
|
|
(defn- import-files-to-db
|
|
|
|
"Import specific doc files for dev purposes"
|
|
|
|
[file conn {:keys [files] :as options}]
|
|
|
|
(let [doc-options (gp-exporter/build-doc-options conn {:macros {}} options)
|
|
|
|
files' (mapv #(hash-map :path %)
|
2024-02-29 20:05:23 +00:00
|
|
|
(into [file] (map resolve-path files)))]
|
2024-03-13 20:23:49 +00:00
|
|
|
(gp-exporter/export-doc-files conn files' <read-file doc-options)))
|
2024-02-27 21:27:26 +00:00
|
|
|
|
2024-02-24 05:42:31 +00:00
|
|
|
(def spec
|
|
|
|
"Options spec"
|
|
|
|
{:help {:alias :h
|
|
|
|
:desc "Print help"}
|
|
|
|
:verbose {:alias :v
|
2024-02-26 21:20:03 +00:00
|
|
|
:desc "Verbose mode"}
|
|
|
|
:tag-classes {:alias :t
|
|
|
|
:coerce []
|
|
|
|
:desc "List of tags to convert to classes"}
|
2024-02-28 22:09:15 +00:00
|
|
|
:files {:alias :f
|
|
|
|
:coerce []
|
|
|
|
:desc "Additional files to import"}
|
2024-02-26 21:20:03 +00:00
|
|
|
:property-classes {:alias :p
|
|
|
|
:coerce []
|
2024-03-29 14:37:09 +00:00
|
|
|
:desc "List of properties whose values convert to classes"}
|
|
|
|
:property-parent-classes
|
|
|
|
{:alias :P
|
|
|
|
:coerce []
|
|
|
|
:desc "List of properties whose values convert to a parent class"}})
|
2024-02-24 05:42:31 +00:00
|
|
|
|
|
|
|
(defn -main [args]
|
|
|
|
(let [[file-graph db-graph-dir] args
|
|
|
|
options (cli/parse-opts args {:spec spec})
|
|
|
|
_ (when (or (< (count args) 2) (:help options))
|
|
|
|
(println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
|
|
|
|
(cli/format-opts {:spec spec})))
|
|
|
|
(js/process.exit 1))
|
|
|
|
[dir db-name] (if (string/includes? db-graph-dir "/")
|
2024-02-29 20:05:23 +00:00
|
|
|
(let [graph-dir' (resolve-path db-graph-dir)]
|
2024-02-24 05:42:31 +00:00
|
|
|
((juxt node-path/dirname node-path/basename) graph-dir'))
|
|
|
|
[(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
|
2024-02-29 20:05:23 +00:00
|
|
|
file-graph' (resolve-path file-graph)
|
2024-02-27 21:27:26 +00:00
|
|
|
conn (create-graph/init-conn dir db-name)
|
|
|
|
directory? (.isDirectory (fs/statSync file-graph'))]
|
2024-02-26 21:20:03 +00:00
|
|
|
(p/do!
|
2024-02-27 21:27:26 +00:00
|
|
|
(if directory?
|
2024-03-05 22:05:18 +00:00
|
|
|
(import-file-graph-to-db file-graph' (node-path/join dir db-name) conn (merge options {:graph-name db-name}))
|
2024-02-27 21:27:26 +00:00
|
|
|
(import-files-to-db file-graph' conn (merge options {:graph-name db-name})))
|
2024-02-26 21:20:03 +00:00
|
|
|
(when (:verbose options) (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
|
|
|
|
(println "Created graph" (str db-name "!")))))
|
2024-02-24 05:42:31 +00:00
|
|
|
|
|
|
|
(when (= nbb/*file* (:file (meta #'-main)))
|
|
|
|
(-main *command-line-args*))
|