greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Split out graph fns and reuse them b/n gp-cli and electron 

Also added test
pull/9318/merge
Gabriel Horner 2023-05-15 09:38:17 -04:00 committed by Tienson Qin
parent 402668bd69
commit 91beda09d5
14 changed files with 164 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
deps/common/nbb.edn vendored Normal file
View File

@ -0,0 +1,4 @@
{:paths ["src"]
:deps
{io.github.nextjournal/nbb-test-runner
{:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

11
deps/common/package.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"name": "@logseq/common",
"version": "1.0.0",
"private": true,
"devDependencies": {
"@logseq/nbb-logseq": "^1.2.173"
},
"scripts": {
"test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"
}
}

73
deps/common/src/logseq/common/graph.cljs vendored Normal file
View File

@ -0,0 +1,73 @@
(ns logseq.common.graph
"This ns provides common fns for a graph directory and only runs in a node environment"
(:require ["fs" :as fs]
["path" :as node-path]
[clojure.string :as string]))
(def ^:private win32?
"Copy of electron.utils/win32? . Too basic to couple the two libraries"
(= (.-platform js/process) "win32"))
(defn- fix-win-path!
"Copy of electron.utils/fix-win-path!. Too basic to couple the two libraries"
[path]
(when (not-empty path)
(if win32?
(string/replace path "\\" "/")
path)))
(defn readdir
"Reads given directory recursively and returns all filenames. Also applies
some graph-specific filtering e.g. symbolic links and files starting with '.'
are removed"
[root-dir]
(->> (tree-seq
(fn [[is-dir _fpath]]
is-dir)
(fn [[_is-dir dir]]
(let [files (fs/readdirSync dir #js {:withFileTypes true})]
(->> files
(remove #(.isSymbolicLink ^js %))
(remove #(string/starts-with? (.-name ^js %) "."))
(map #(do
[(.isDirectory %)
(node-path/join dir (.-name %))])))))
[true root-dir])
(filter (complement first))
(map second)
(map fix-win-path!)
(vec)))
(def ^:private allowed-formats
#{:org :markdown :md :edn :json :js :css :excalidraw :tldr})
(defn- get-ext
[p]
(-> (node-path/extname p)
(subs 1)
keyword))
(defn ignored-path?
"Given a graph directory and path, returns truthy value on whether the path is
ignored. Useful for contexts like reading a graph's directory and file watcher
notifications"
[dir path]
(when (string? path)
(or
(some #(string/starts-with? path (str dir "/" %))
["." ".recycle" "node_modules" "logseq/bak" "version-files"])
(some #(string/includes? path (str "/" % "/"))
["." ".recycle" "node_modules" "logseq/bak" "version-files"])
(some #(string/ends-with? path %)
[".DS_Store" "logseq/graphs-txid.edn"])
;; hidden directory or file
(let [relpath (node-path/relative dir path)]
(or (re-find #"/\.[^.]+" relpath)
(re-find #"^\.[^.]+" relpath))))))
(defn get-files
"Given a graph's root dir, returns a list of all files that it recognizes"
[graph-dir]
(->> (readdir graph-dir)
(remove (partial ignored-path? graph-dir))
(filter #(contains? allowed-formats (get-ext %)))))

35
deps/common/test/logseq/common/graph_test.cljs vendored Normal file
View File

@ -0,0 +1,35 @@
(ns logseq.common.graph-test
(:require [logseq.common.graph :as common-graph]
[cljs.test :refer [deftest is use-fixtures async]]
["fs" :as fs]
["path" :as node-path]))
(use-fixtures
:each
;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging
{:before
#(async done
(if (fs/existsSync "tmp")
(fs/rm "tmp" #js {:recursive true} (fn [err]
(when err (js/console.log err))
(done)))
(done)))})
(defn- create-logseq-graph
"Creates a minimal mock graph"
[dir]
(fs/mkdirSync (node-path/join dir "logseq") #js {:recursive true})
(fs/mkdirSync (node-path/join dir "journals"))
(fs/mkdirSync (node-path/join dir "pages")))
(deftest get-files
(create-logseq-graph "tmp/test-graph")
;; Create files that are recognized
(fs/writeFileSync "tmp/test-graph/pages/foo.md" "")
(fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "")
;; Create files that are ignored
(fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak"))
(fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "")
(fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "")
(is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"]
(common-graph/get-files "tmp/test-graph"))))

15
deps/common/yarn.lock vendored Normal file
View File

@ -0,0 +1,15 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
"@logseq/nbb-logseq@^1.2.173":
version "1.2.173"
resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.2.173.tgz#27a52c350f06ac9c337d73687738f6ea8b2fc3f3"
integrity sha512-ABKPtVnSOiS4Zpk9+UTaGcs5H6EUmRADr9FJ0aEAVpa0WfAyvUbX/NgkQGMe1kKRv3EbIuLwaxfy+txr31OtAg==
dependencies:
import-meta-resolve "^2.1.0"
import-meta-resolve@^2.1.0:
version "2.2.2"
resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-2.2.2.tgz#75237301e72d1f0fbd74dbc6cca9324b164c2cc9"
integrity sha512-f8KcQ1D80V7RnqVm+/lirO9zkOxjGxhaTC1IPrBGd3MEfNgmNG67tSUO9gTi2F3Blr2Az6g1vocaxzkVnWl9MA==

3
deps/graph-parser/deps.edn vendored
View File

@ -3,8 +3,9 @@
;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
{com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
:sha "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
;; local dep
;; local deps
logseq/db {:local/root "../db"}
logseq/common {:local/root "../common"}
;; stubbed in nbb
com.lambdaisland/glogi {:mvn/version "1.1.144"}
;; built in to nbb

2
deps/graph-parser/nbb.edn vendored
View File

@ -2,5 +2,7 @@
:deps
{logseq/db
{:local/root "../db"}
logseq/common
{:local/root "../common"}
io.github.nextjournal/nbb-test-runner
{:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

31
deps/graph-parser/src/logseq/graph_parser/cli.cljs vendored
View File

@ -3,38 +3,25 @@
(:require ["fs" :as fs]
["child_process" :as child-process]
[clojure.edn :as edn]
[clojure.string :as string]
[logseq.common.graph :as common-graph]
[logseq.graph-parser :as graph-parser]
[logseq.graph-parser.config :as gp-config]
[logseq.graph-parser.util :as gp-util]
[logseq.db :as ldb]))
(defn slurp
(defn- slurp
"Return file contents like clojure.core/slurp"
[file]
(str (fs/readFileSync file)))
(defn sh
"Run shell cmd synchronously and print to inherited streams by default. Aims
to be similar to babashka.tasks/shell
TODO: Fail fast when process exits 1"
[cmd opts]
(child-process/spawnSync (first cmd)
(clj->js (rest cmd))
(clj->js (merge {:stdio "inherit"} opts))))
(defn build-graph-files
"Given a git graph directory, returns allowed file paths and their contents in
preparation for parsing"
(defn- build-graph-files
"Given a graph directory, return allowed file paths and their contents in preparation
for parsing"
[dir]
;; -z needed to avoid quoting unusual paths that cause slurp failures.
;; See https://git-scm.com/docs/git-ls-files#_output for more
(let [files (->> (str (.-stdout (sh ["git" "ls-files" "-z"]
{:cwd dir :stdio nil})))
(#(string/split % (re-pattern "\0")))
(map #(hash-map :file/path (str dir "/" %)))
graph-parser/filter-files)]
(mapv #(assoc % :file/content (slurp (:file/path %))) files)))
(->> (common-graph/get-files dir)
(map #(hash-map :file/path %))
graph-parser/filter-files
(mapv #(assoc % :file/content (slurp (:file/path %))))))
(defn- read-config
"Reads repo-specific config from logseq/config.edn"

4
deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs vendored
View File

@ -152,8 +152,8 @@
;; Counts assertions help check for no major regressions. These counts should
;; only increase over time as the docs graph rarely has deletions
(testing "Counts"
(is (= 306 (count files)) "Correct file count")
(is (= 69508 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 304 (count files)) "Correct file count")
(is (= 69502 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 5866
(ffirst

2
deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs vendored
View File

@ -139,7 +139,7 @@ body"
"Heading" 5648,
"Hiccup" 9,
"List" 22,
"Paragraph" 573,
"Paragraph" 571,
"Properties" 87,
"Property_Drawer" 423,
"Quote" 24,

5
src/electron/electron/fs_watcher.cljs
View File

@ -8,7 +8,8 @@
[electron.utils :as utils]
[electron.logger :as logger]
["electron" :refer [app]]
[electron.window :as window]))
[electron.window :as window]
[logseq.common.graph :as common-graph]))
;; TODO: explore different solutions for different platforms
;; 1. https://github.com/Axosoft/nsfw
@ -61,7 +62,7 @@
[dir options]
(let [watcher-opts (clj->js
{:ignored (fn [path]
(utils/ignored-path? dir path))
(common-graph/ignored-path? dir path))
:ignoreInitial true
:ignorePermissionErrors true
:interval polling-interval

40
src/electron/electron/handler.cljs
View File

@ -28,6 +28,7 @@
[electron.state :as state]
[electron.utils :as utils]
[electron.window :as win]
[logseq.common.graph :as common-graph]
[promesa.core :as p]))
(defmulti handle (fn [_window args] (keyword (first args))))
@ -38,28 +39,8 @@
(defmethod handle :mkdir-recur [_window [_ dir]]
(fs/mkdirSync dir #js {:recursive true}))
(defn- readdir
"Read directory recursively, return all filenames"
[root-dir]
(->> (tree-seq
(fn [[is-dir _fpath]]
is-dir)
(fn [[_is-dir dir]]
(let [files (fs/readdirSync dir #js {:withFileTypes true})]
(->> files
(remove #(.isSymbolicLink ^js %))
(remove #(string/starts-with? (.-name ^js %) "."))
(map #(do
[(.isDirectory %)
(.join node-path dir (.-name %))])))))
[true root-dir])
(filter (complement first))
(map second)
(map utils/fix-win-path!)
(vec)))
(defmethod handle :readdir [_window [_ dir]]
(readdir dir))
(common-graph/readdir dir))
(defmethod handle :listdir [_window [_ dir flat?]]
(when (and dir (fs-extra/pathExistsSync dir))
@ -147,21 +128,10 @@
(defmethod handle :stat [_window [_ path]]
(fs/statSync path))
(defonce allowed-formats
#{:org :markdown :md :edn :json :js :css :excalidraw :tldr})
(defn get-ext
[p]
(-> (.extname node-path p)
(subs 1)
keyword))
(defn- get-files
"Returns vec of file-objs"
[path]
(->> (readdir path)
(remove (partial utils/ignored-path? path))
(filter #(contains? allowed-formats (get-ext %)))
(->> (common-graph/get-files path)
(map (fn [path]
(let [stat (fs/statSync path)]
(when-not (.isDirectory stat)
@ -238,10 +208,10 @@
dir))
(defn- get-graphs
"Returns all graph names in the cache directory (strating with `logseq_local_`)"
"Returns all graph names in the cache directory (starting with `logseq_local_`)"
[]
(let [dir (get-graphs-dir)]
(->> (readdir dir)
(->> (common-graph/readdir dir)
(remove #{dir})
(map #(node-path/basename % ".transit"))
(map graph-name->path))))

17
src/electron/electron/utils.cljs
View File

@ -198,23 +198,6 @@
(cfgs/set-item! :settings/agent {:type type :test test})
(cfgs/set-item! :settings/agent {:type type :protocol type :host host :port port :test test})))
(defn ignored-path?
"Ignore given path from file-watcher notification"
[dir path]
(when (string? path)
(or
(some #(string/starts-with? path (str dir "/" %))
["." ".recycle" "node_modules" "logseq/bak" "version-files"])
(some #(string/includes? path (str "/" % "/"))
["." ".recycle" "node_modules" "logseq/bak" "version-files"])
(some #(string/ends-with? path %)
[".DS_Store" "logseq/graphs-txid.edn"])
;; hidden directory or file
(let [relpath (node-path/relative dir path)]
(or (re-find #"/\.[^.]+" relpath)
(re-find #"^\.[^.]+" relpath))))))
(defn should-read-content?
"Skip reading content of file while using file-watcher"
[path]

4
src/test/frontend/handler/repo_conversion_test.cljs
View File

@ -97,8 +97,8 @@
;; Counts assertions help check for no major regressions. These counts should
;; only increase over time as the docs graph rarely has deletions
(testing "Counts"
(is (= 211 (count files)) "Correct file count")
(is (= 42304 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 212 (count files)) "Correct file count")
(is (= 42315 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 3600
(ffirst