diff --git a/deps/common/nbb.edn b/deps/common/nbb.edn new file mode 100644 index 000000000..3774f52e5 --- /dev/null +++ b/deps/common/nbb.edn @@ -0,0 +1,4 @@ +{:paths ["src"] + :deps + {io.github.nextjournal/nbb-test-runner + {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} diff --git a/deps/common/package.json b/deps/common/package.json new file mode 100644 index 000000000..a15d03704 --- /dev/null +++ b/deps/common/package.json @@ -0,0 +1,11 @@ +{ + "name": "@logseq/common", + "version": "1.0.0", + "private": true, + "devDependencies": { + "@logseq/nbb-logseq": "^1.2.173" + }, + "scripts": { + "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner" + } +} diff --git a/deps/common/src/logseq/common/graph.cljs b/deps/common/src/logseq/common/graph.cljs new file mode 100644 index 000000000..cbfddbd41 --- /dev/null +++ b/deps/common/src/logseq/common/graph.cljs @@ -0,0 +1,73 @@ +(ns logseq.common.graph + "This ns provides common fns for a graph directory and only runs in a node environment" + (:require ["fs" :as fs] + ["path" :as node-path] + [clojure.string :as string])) + +(def ^:private win32? + "Copy of electron.utils/win32? . Too basic to couple the two libraries" + (= (.-platform js/process) "win32")) + +(defn- fix-win-path! + "Copy of electron.utils/fix-win-path!. Too basic to couple the two libraries" + [path] + (when (not-empty path) + (if win32? + (string/replace path "\\" "/") + path))) + +(defn readdir + "Reads given directory recursively and returns all filenames. Also applies + some graph-specific filtering e.g. symbolic links and files starting with '.' + are removed" + [root-dir] + (->> (tree-seq + (fn [[is-dir _fpath]] + is-dir) + (fn [[_is-dir dir]] + (let [files (fs/readdirSync dir #js {:withFileTypes true})] + (->> files + (remove #(.isSymbolicLink ^js %)) + (remove #(string/starts-with? (.-name ^js %) ".")) + (map #(do + [(.isDirectory %) + (node-path/join dir (.-name %))]))))) + [true root-dir]) + (filter (complement first)) + (map second) + (map fix-win-path!) + (vec))) + +(def ^:private allowed-formats + #{:org :markdown :md :edn :json :js :css :excalidraw :tldr}) + +(defn- get-ext + [p] + (-> (node-path/extname p) + (subs 1) + keyword)) + +(defn ignored-path? + "Given a graph directory and path, returns truthy value on whether the path is + ignored. Useful for contexts like reading a graph's directory and file watcher + notifications" + [dir path] + (when (string? path) + (or + (some #(string/starts-with? path (str dir "/" %)) + ["." ".recycle" "node_modules" "logseq/bak" "version-files"]) + (some #(string/includes? path (str "/" % "/")) + ["." ".recycle" "node_modules" "logseq/bak" "version-files"]) + (some #(string/ends-with? path %) + [".DS_Store" "logseq/graphs-txid.edn"]) + ;; hidden directory or file + (let [relpath (node-path/relative dir path)] + (or (re-find #"/\.[^.]+" relpath) + (re-find #"^\.[^.]+" relpath)))))) + +(defn get-files + "Given a graph's root dir, returns a list of all files that it recognizes" + [graph-dir] + (->> (readdir graph-dir) + (remove (partial ignored-path? graph-dir)) + (filter #(contains? allowed-formats (get-ext %))))) \ No newline at end of file diff --git a/deps/common/test/logseq/common/graph_test.cljs b/deps/common/test/logseq/common/graph_test.cljs new file mode 100644 index 000000000..48cf766ae --- /dev/null +++ b/deps/common/test/logseq/common/graph_test.cljs @@ -0,0 +1,35 @@ +(ns logseq.common.graph-test + (:require [logseq.common.graph :as common-graph] + [cljs.test :refer [deftest is use-fixtures async]] + ["fs" :as fs] + ["path" :as node-path])) + +(use-fixtures + :each + ;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging + {:before + #(async done + (if (fs/existsSync "tmp") + (fs/rm "tmp" #js {:recursive true} (fn [err] + (when err (js/console.log err)) + (done))) + (done)))}) + +(defn- create-logseq-graph + "Creates a minimal mock graph" + [dir] + (fs/mkdirSync (node-path/join dir "logseq") #js {:recursive true}) + (fs/mkdirSync (node-path/join dir "journals")) + (fs/mkdirSync (node-path/join dir "pages"))) + +(deftest get-files + (create-logseq-graph "tmp/test-graph") + ;; Create files that are recognized + (fs/writeFileSync "tmp/test-graph/pages/foo.md" "") + (fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "") + ;; Create files that are ignored + (fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak")) + (fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "") + (fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "") + (is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"] + (common-graph/get-files "tmp/test-graph")))) \ No newline at end of file diff --git a/deps/common/yarn.lock b/deps/common/yarn.lock new file mode 100644 index 000000000..66b8e38c2 --- /dev/null +++ b/deps/common/yarn.lock @@ -0,0 +1,15 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@logseq/nbb-logseq@^1.2.173": + version "1.2.173" + resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.2.173.tgz#27a52c350f06ac9c337d73687738f6ea8b2fc3f3" + integrity sha512-ABKPtVnSOiS4Zpk9+UTaGcs5H6EUmRADr9FJ0aEAVpa0WfAyvUbX/NgkQGMe1kKRv3EbIuLwaxfy+txr31OtAg== + dependencies: + import-meta-resolve "^2.1.0" + +import-meta-resolve@^2.1.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-2.2.2.tgz#75237301e72d1f0fbd74dbc6cca9324b164c2cc9" + integrity sha512-f8KcQ1D80V7RnqVm+/lirO9zkOxjGxhaTC1IPrBGd3MEfNgmNG67tSUO9gTi2F3Blr2Az6g1vocaxzkVnWl9MA== diff --git a/deps/graph-parser/deps.edn b/deps/graph-parser/deps.edn index c0135314d..c6b65464f 100644 --- a/deps/graph-parser/deps.edn +++ b/deps/graph-parser/deps.edn @@ -3,8 +3,9 @@ ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork :sha "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"} - ;; local dep + ;; local deps logseq/db {:local/root "../db"} + logseq/common {:local/root "../common"} ;; stubbed in nbb com.lambdaisland/glogi {:mvn/version "1.1.144"} ;; built in to nbb diff --git a/deps/graph-parser/nbb.edn b/deps/graph-parser/nbb.edn index 90583c25b..77793d20b 100644 --- a/deps/graph-parser/nbb.edn +++ b/deps/graph-parser/nbb.edn @@ -2,5 +2,7 @@ :deps {logseq/db {:local/root "../db"} + logseq/common + {:local/root "../common"} io.github.nextjournal/nbb-test-runner {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} diff --git a/deps/graph-parser/src/logseq/graph_parser/cli.cljs b/deps/graph-parser/src/logseq/graph_parser/cli.cljs index 79b9aec83..71d8dbd40 100644 --- a/deps/graph-parser/src/logseq/graph_parser/cli.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/cli.cljs @@ -3,38 +3,25 @@ (:require ["fs" :as fs] ["child_process" :as child-process] [clojure.edn :as edn] - [clojure.string :as string] + [logseq.common.graph :as common-graph] [logseq.graph-parser :as graph-parser] [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.util :as gp-util] [logseq.db :as ldb])) -(defn slurp +(defn- slurp "Return file contents like clojure.core/slurp" [file] (str (fs/readFileSync file))) -(defn sh - "Run shell cmd synchronously and print to inherited streams by default. Aims - to be similar to babashka.tasks/shell -TODO: Fail fast when process exits 1" - [cmd opts] - (child-process/spawnSync (first cmd) - (clj->js (rest cmd)) - (clj->js (merge {:stdio "inherit"} opts)))) - -(defn build-graph-files - "Given a git graph directory, returns allowed file paths and their contents in - preparation for parsing" +(defn- build-graph-files + "Given a graph directory, return allowed file paths and their contents in preparation + for parsing" [dir] - ;; -z needed to avoid quoting unusual paths that cause slurp failures. - ;; See https://git-scm.com/docs/git-ls-files#_output for more - (let [files (->> (str (.-stdout (sh ["git" "ls-files" "-z"] - {:cwd dir :stdio nil}))) - (#(string/split % (re-pattern "\0"))) - (map #(hash-map :file/path (str dir "/" %))) - graph-parser/filter-files)] - (mapv #(assoc % :file/content (slurp (:file/path %))) files))) + (->> (common-graph/get-files dir) + (map #(hash-map :file/path %)) + graph-parser/filter-files + (mapv #(assoc % :file/content (slurp (:file/path %)))))) (defn- read-config "Reads repo-specific config from logseq/config.edn" diff --git a/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs b/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs index b10e50df4..6145b5cb5 100644 --- a/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs @@ -152,8 +152,8 @@ ;; Counts assertions help check for no major regressions. These counts should ;; only increase over time as the docs graph rarely has deletions (testing "Counts" - (is (= 306 (count files)) "Correct file count") - (is (= 69508 (count (d/datoms db :eavt))) "Correct datoms count") + (is (= 304 (count files)) "Correct file count") + (is (= 69502 (count (d/datoms db :eavt))) "Correct datoms count") (is (= 5866 (ffirst diff --git a/deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs b/deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs index 2437cab5e..70dac7765 100644 --- a/deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs @@ -139,7 +139,7 @@ body" "Heading" 5648, "Hiccup" 9, "List" 22, - "Paragraph" 573, + "Paragraph" 571, "Properties" 87, "Property_Drawer" 423, "Quote" 24, diff --git a/src/electron/electron/fs_watcher.cljs b/src/electron/electron/fs_watcher.cljs index 2fd09d7cf..d108f4de0 100644 --- a/src/electron/electron/fs_watcher.cljs +++ b/src/electron/electron/fs_watcher.cljs @@ -8,7 +8,8 @@ [electron.utils :as utils] [electron.logger :as logger] ["electron" :refer [app]] - [electron.window :as window])) + [electron.window :as window] + [logseq.common.graph :as common-graph])) ;; TODO: explore different solutions for different platforms ;; 1. https://github.com/Axosoft/nsfw @@ -61,7 +62,7 @@ [dir options] (let [watcher-opts (clj->js {:ignored (fn [path] - (utils/ignored-path? dir path)) + (common-graph/ignored-path? dir path)) :ignoreInitial true :ignorePermissionErrors true :interval polling-interval diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index 4f40cfd7e..312b878e8 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -28,6 +28,7 @@ [electron.state :as state] [electron.utils :as utils] [electron.window :as win] + [logseq.common.graph :as common-graph] [promesa.core :as p])) (defmulti handle (fn [_window args] (keyword (first args)))) @@ -38,28 +39,8 @@ (defmethod handle :mkdir-recur [_window [_ dir]] (fs/mkdirSync dir #js {:recursive true})) -(defn- readdir - "Read directory recursively, return all filenames" - [root-dir] - (->> (tree-seq - (fn [[is-dir _fpath]] - is-dir) - (fn [[_is-dir dir]] - (let [files (fs/readdirSync dir #js {:withFileTypes true})] - (->> files - (remove #(.isSymbolicLink ^js %)) - (remove #(string/starts-with? (.-name ^js %) ".")) - (map #(do - [(.isDirectory %) - (.join node-path dir (.-name %))]))))) - [true root-dir]) - (filter (complement first)) - (map second) - (map utils/fix-win-path!) - (vec))) - (defmethod handle :readdir [_window [_ dir]] - (readdir dir)) + (common-graph/readdir dir)) (defmethod handle :listdir [_window [_ dir flat?]] (when (and dir (fs-extra/pathExistsSync dir)) @@ -147,21 +128,10 @@ (defmethod handle :stat [_window [_ path]] (fs/statSync path)) -(defonce allowed-formats - #{:org :markdown :md :edn :json :js :css :excalidraw :tldr}) - -(defn get-ext - [p] - (-> (.extname node-path p) - (subs 1) - keyword)) - (defn- get-files "Returns vec of file-objs" [path] - (->> (readdir path) - (remove (partial utils/ignored-path? path)) - (filter #(contains? allowed-formats (get-ext %))) + (->> (common-graph/get-files path) (map (fn [path] (let [stat (fs/statSync path)] (when-not (.isDirectory stat) @@ -238,10 +208,10 @@ dir)) (defn- get-graphs - "Returns all graph names in the cache directory (strating with `logseq_local_`)" + "Returns all graph names in the cache directory (starting with `logseq_local_`)" [] (let [dir (get-graphs-dir)] - (->> (readdir dir) + (->> (common-graph/readdir dir) (remove #{dir}) (map #(node-path/basename % ".transit")) (map graph-name->path)))) diff --git a/src/electron/electron/utils.cljs b/src/electron/electron/utils.cljs index 2f7db5bb6..d8e989162 100644 --- a/src/electron/electron/utils.cljs +++ b/src/electron/electron/utils.cljs @@ -198,23 +198,6 @@ (cfgs/set-item! :settings/agent {:type type :test test}) (cfgs/set-item! :settings/agent {:type type :protocol type :host host :port port :test test}))) - -(defn ignored-path? - "Ignore given path from file-watcher notification" - [dir path] - (when (string? path) - (or - (some #(string/starts-with? path (str dir "/" %)) - ["." ".recycle" "node_modules" "logseq/bak" "version-files"]) - (some #(string/includes? path (str "/" % "/")) - ["." ".recycle" "node_modules" "logseq/bak" "version-files"]) - (some #(string/ends-with? path %) - [".DS_Store" "logseq/graphs-txid.edn"]) - ;; hidden directory or file - (let [relpath (node-path/relative dir path)] - (or (re-find #"/\.[^.]+" relpath) - (re-find #"^\.[^.]+" relpath)))))) - (defn should-read-content? "Skip reading content of file while using file-watcher" [path] diff --git a/src/test/frontend/handler/repo_conversion_test.cljs b/src/test/frontend/handler/repo_conversion_test.cljs index 7768ee35c..59a97c642 100644 --- a/src/test/frontend/handler/repo_conversion_test.cljs +++ b/src/test/frontend/handler/repo_conversion_test.cljs @@ -97,8 +97,8 @@ ;; Counts assertions help check for no major regressions. These counts should ;; only increase over time as the docs graph rarely has deletions (testing "Counts" - (is (= 211 (count files)) "Correct file count") - (is (= 42304 (count (d/datoms db :eavt))) "Correct datoms count") + (is (= 212 (count files)) "Correct file count") + (is (= 42315 (count (d/datoms db :eavt))) "Correct datoms count") (is (= 3600 (ffirst