From 261118d4dc1550451483ccad318368aec695451e Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 9 Aug 2024 12:16:20 -0400 Subject: [PATCH] refactor: finish frontend independence from worker and add lint Remaining fixes worker all file related. Fixes LOG-3717 --- .clj-kondo/config.edn | 6 ++-- docs/dev-practices.md | 5 +-- scripts/src/logseq/tasks/dev/lint.clj | 25 ++++++++++--- src/bench/frontend/benchmark_test_runner.cljs | 2 +- .../{worker => common}/file/core.cljs | 35 ++++++++++++++----- .../{worker => common}/file/util.cljs | 12 +++++-- src/main/frontend/handler/export/common.cljs | 8 ++--- src/main/frontend/modules/file/core.cljs | 4 +-- src/main/frontend/util/fs.cljs | 2 +- src/main/frontend/worker/db_worker.cljs | 7 ++-- src/main/frontend/worker/export.cljs | 22 ++---------- src/main/frontend/worker/file.cljs | 10 +++--- .../handler/page/file_based/rename.cljs | 2 +- src/main/frontend/worker/util.cljc | 9 ++--- src/test/frontend/db/name_sanity_test.cljs | 2 +- 15 files changed, 87 insertions(+), 64 deletions(-) rename src/main/frontend/{worker => common}/file/core.cljs (86%) rename src/main/frontend/{worker => common}/file/util.cljs (91%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index fda9e3b78..e21adf049 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -55,6 +55,8 @@ frontend.components.class class-component frontend.components.property property-component frontend.common.date common-date + frontend.common.file.core common-file + frontend.common.file.util wfu frontend.common.missionary-util c.m frontend.common.schema-register sr frontend.common.search-fuzzy fuzzy @@ -134,9 +136,6 @@ frontend.worker.util worker-util frontend.worker.state worker-state frontend.worker.handler.page worker-page - frontend.worker.handler.page.rename worker-page-rename - frontend.worker.handler.file.util wfu - logseq.outliner.batch-tx batch-tx lambdaisland.glogi log logseq.common.config common-config logseq.common.graph common-graph @@ -169,6 +168,7 @@ logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.property gp-property + logseq.outliner.batch-tx batch-tx logseq.outliner.core outliner-core logseq.outliner.op outliner-op logseq.outliner.pipeline outliner-pipeline diff --git a/docs/dev-practices.md b/docs/dev-practices.md index c6aef9140..1f25bc945 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -122,11 +122,12 @@ The main convention is that file and db specific files go under directories name ### Separate Worker from Frontend -The worker and frontend code share common code from deps/ and `frontend.common.*`. However, the worker should never depend on other frontend namespaces as it could pull in libraries like React which cause it to fail hard. Run this linter to ensure worker namespaces are not dependent on other frontend namespaces: +The worker and frontend code share common code from deps/ and `frontend.common.*`. However, the worker should never depend on other frontend namespaces as it could pull in libraries like React which cause it to fail hard. Likewise the frontend should never depend on worker namespaces. Run this linter to ensure worker and frontend namespaces don't require each other: ``` $ bb lint:worker-and-frontend-separate -Success! +Valid worker namespaces! +Valid frontend namespaces! ``` ## Testing diff --git a/scripts/src/logseq/tasks/dev/lint.clj b/scripts/src/logseq/tasks/dev/lint.clj index 0eedb0e5b..4ebd1b585 100644 --- a/scripts/src/logseq/tasks/dev/lint.clj +++ b/scripts/src/logseq/tasks/dev/lint.clj @@ -21,8 +21,7 @@ (println cmd) (shell cmd))) -(defn worker-and-frontend-separate - "Ensures worker is independent of frontend" +(defn- validate-frontend-not-in-worker [] (let [res (shell {:out :string} "git grep -h" "\\[frontend.*:as" "src/main/frontend/worker") @@ -32,8 +31,26 @@ (if (seq req-lines) (do - (println "The following requires should not be in worker namespaces:") + (println "The following frontend requires should not be in worker namespaces:") (println (string/join "\n" req-lines)) (System/exit 1)) - (println "Success!")))) + (println "Valid worker namespaces!")))) + +(defn- validate-worker-not-in-frontend + [] + (let [res (shell {:out :string :continue true} + "grep -r --exclude-dir=worker" "\\[frontend.worker.*:" "src/main/frontend") + req-lines (->> (:out res) string/split-lines)] + (if-not (and (= 1 (:exit res)) (= "" (:out res))) + (do + (println "The following worker requires should not be in frontend namespaces:") + (println (:out res)) + (System/exit 1)) + (println "Valid frontend namespaces!")))) + +(defn worker-and-frontend-separate + "Ensures worker is independent of frontend" + [] + (validate-frontend-not-in-worker) + (validate-worker-not-in-frontend)) diff --git a/src/bench/frontend/benchmark_test_runner.cljs b/src/bench/frontend/benchmark_test_runner.cljs index db7facc5b..7a21f19d8 100644 --- a/src/bench/frontend/benchmark_test_runner.cljs +++ b/src/bench/frontend/benchmark_test_runner.cljs @@ -5,7 +5,7 @@ [clojure.pprint :as pprint] [clojure.test :refer [deftest testing]] [fipp.edn :as fipp] - [frontend.worker.file.util :as wfu])) + [frontend.common.file.util :as wfu])) (def onboarding (edn/read-string (slurped "resources/whiteboard/onboarding.edn"))) diff --git a/src/main/frontend/worker/file/core.cljs b/src/main/frontend/common/file/core.cljs similarity index 86% rename from src/main/frontend/worker/file/core.cljs rename to src/main/frontend/common/file/core.cljs index d35e7a23c..0ffef551e 100644 --- a/src/main/frontend/worker/file/core.cljs +++ b/src/main/frontend/common/file/core.cljs @@ -1,14 +1,16 @@ -(ns frontend.worker.file.core - "Save file to disk. Used by both file and DB graphs" +(ns frontend.common.file.core + "Save file to disk. Used by both file and DB graphs and shared + by worker and frontend namespaces" (:require [clojure.string :as string] - [frontend.worker.file.util :as wfu] + [frontend.common.file.util :as wfu] [logseq.graph-parser.property :as gp-property] [logseq.common.path :as path] [datascript.core :as d] [logseq.db :as ldb] [frontend.common.date :as common-date] - [frontend.worker.util :as worker-util] - [logseq.db.sqlite.util :as sqlite-util])) + [logseq.db.frontend.content :as db-content] + [logseq.db.sqlite.util :as sqlite-util] + [logseq.outliner.tree :as otree])) (defonce *writes (atom {})) (defonce *request-id (atom 0)) @@ -174,10 +176,10 @@ (let [files [[file-path new-content]]] (when (seq files) (let [page-id (:db/id page-block)] - (worker-util/post-message :write-files {:request-id request-id - :page-id page-id - :repo repo - :files files}) + (wfu/post-message :write-files {:request-id request-id + :page-id page-id + :repo repo + :files files}) :sent))))) ;; In e2e tests, "card" page in db has no :file/path (js/console.error "File path from page-block is not valid" page-block tree))] @@ -195,3 +197,18 @@ (if file (ok-handler) (transact-file-tx-if-not-exists! conn page-block ok-handler context))))) + +(defn block->content + "Converts a block including its children (recursively) to plain-text." + [repo db root-block-uuid tree->file-opts context] + (assert (uuid? root-block-uuid)) + (let [init-level (or (:init-level tree->file-opts) + (if (ldb/page? (d/entity db [:block/uuid root-block-uuid])) + 0 + 1)) + blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid))) + (map #(db-content/update-block-content db % (:db/id %)))) + tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))] + (tree->file-content repo db tree + (assoc tree->file-opts :init-level init-level) + context))) diff --git a/src/main/frontend/worker/file/util.cljs b/src/main/frontend/common/file/util.cljs similarity index 91% rename from src/main/frontend/worker/file/util.cljs rename to src/main/frontend/common/file/util.cljs index 8d5cc7a48..6b2b30006 100644 --- a/src/main/frontend/worker/file/util.cljs +++ b/src/main/frontend/common/file/util.cljs @@ -1,7 +1,8 @@ -(ns frontend.worker.file.util - "File name fns" +(ns frontend.common.file.util + "File name fns. Used by worker and frontend namespaces" (:require [clojure.string :as string] - [logseq.common.util :as common-util])) + [logseq.common.util :as common-util] + [logseq.db :as ldb])) ;; Update repo/invalid-graph-name-warning if characters change (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\") @@ -88,3 +89,8 @@ [x] (with-redefs [print-prefix-map print-prefix-map*] (pr-str x))) + +(defn post-message + [type data] + (when (exists? js/self) + (.postMessage js/self (ldb/write-transit-str [type data])))) \ No newline at end of file diff --git a/src/main/frontend/handler/export/common.cljs b/src/main/frontend/handler/export/common.cljs index 77e34c8bc..fa59eb0c7 100644 --- a/src/main/frontend/handler/export/common.cljs +++ b/src/main/frontend/handler/export/common.cljs @@ -13,7 +13,7 @@ [frontend.persist-db.browser :as db-browser] [frontend.state :as state] [frontend.util :as util :refer [concatv mapcatv removev]] - [frontend.worker.export :as worker-export] + [frontend.common.file.core :as common-file] [malli.core :as m] [malli.util :as mu] [promesa.core :as p])) @@ -93,9 +93,9 @@ [page-uuid] (let [repo (state/get-current-repo) db (db/get-db repo)] - (worker-export/block->content repo db page-uuid - nil - {:export-bullet-indentation (state/get-export-bullet-indentation)}))) + (common-file/block->content repo db page-uuid + nil + {:export-bullet-indentation (state/get-export-bullet-indentation)}))) (defn- page-name->ast [page-name] diff --git a/src/main/frontend/modules/file/core.cljs b/src/main/frontend/modules/file/core.cljs index a716122d2..78e99fa47 100644 --- a/src/main/frontend/modules/file/core.cljs +++ b/src/main/frontend/modules/file/core.cljs @@ -1,6 +1,6 @@ (ns frontend.modules.file.core "Convert block trees to content" - (:require [frontend.worker.file.core :as worker-file] + (:require [frontend.common.file.core :as common-file] [frontend.state :as state] [frontend.db :as db])) @@ -11,4 +11,4 @@ (when-let [repo (state/get-current-repo)] (let [db (db/get-db repo) context {:export-bullet-indentation (state/get-export-bullet-indentation)}] - (worker-file/tree->file-content repo db tree opts context)))) + (common-file/tree->file-content repo db tree opts context)))) diff --git a/src/main/frontend/util/fs.cljs b/src/main/frontend/util/fs.cljs index 713ea2c81..37ff6a82b 100644 --- a/src/main/frontend/util/fs.cljs +++ b/src/main/frontend/util/fs.cljs @@ -8,7 +8,7 @@ [frontend.config :as config] [promesa.core :as p] [cljs.reader :as reader] - [frontend.worker.file.util :as wfu])) + [frontend.common.file.util :as wfu])) ;; NOTE: This is not the same ignored-path? as src/electron/electron/utils.cljs. ;; The assets directory is ignored. diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index e10232e8b..92c831b9b 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -8,6 +8,7 @@ [clojure.string :as string] [datascript.core :as d] [datascript.storage :refer [IStorage]] + [frontend.common.file.core :as common-file] [frontend.worker.db-listener :as db-listener] [frontend.worker.db-metadata :as worker-db-metadata] [frontend.worker.db.migrate :as db-migrate] @@ -601,9 +602,9 @@ (assert (common-util/uuid-string? block-uuid-str)) (let [block-uuid (uuid block-uuid-str)] (when-let [conn (worker-state/get-datascript-conn repo)] - (worker-export/block->content repo @conn block-uuid - (ldb/read-transit-str tree->file-opts) - (ldb/read-transit-str context))))) + (common-file/block->content repo @conn block-uuid + (ldb/read-transit-str tree->file-opts) + (ldb/read-transit-str context))))) (get-all-pages [this repo] diff --git a/src/main/frontend/worker/export.cljs b/src/main/frontend/worker/export.cljs index 537e34b94..39cd09444 100644 --- a/src/main/frontend/worker/export.cljs +++ b/src/main/frontend/worker/export.cljs @@ -1,26 +1,10 @@ (ns frontend.worker.export "Export data" (:require [datascript.core :as d] - [frontend.worker.file.core :as worker-file] + [frontend.common.file.core :as common-file] [logseq.db :as ldb] [logseq.graph-parser.property :as gp-property] - [logseq.outliner.tree :as otree] - [logseq.db.frontend.content :as db-content])) - -(defn block->content - "Converts a block including its children (recursively) to plain-text." - [repo db root-block-uuid tree->file-opts context] - (assert (uuid? root-block-uuid)) - (let [init-level (or (:init-level tree->file-opts) - (if (ldb/page? (d/entity db [:block/uuid root-block-uuid])) - 0 - 1)) - blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid))) - (map #(db-content/update-block-content db % (:db/id %)))) - tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))] - (worker-file/tree->file-content repo db tree - (assoc tree->file-opts :init-level init-level) - context))) + [logseq.outliner.tree :as otree])) (defn- safe-keywordize [block] @@ -64,4 +48,4 @@ (map (fn [d] (let [e (d/entity db (:e d))] [(:block/title e) - (block->content repo db (:block/uuid e) {} {})]))))) + (common-file/block->content repo db (:block/uuid e) {} {})]))))) diff --git a/src/main/frontend/worker/file.cljs b/src/main/frontend/worker/file.cljs index f1261a120..6a764e127 100644 --- a/src/main/frontend/worker/file.cljs +++ b/src/main/frontend/worker/file.cljs @@ -3,7 +3,7 @@ (:require [clojure.core.async :as async] [clojure.string :as string] [clojure.set :as set] - [frontend.worker.file.core :as file] + [frontend.common.file.core :as common-file] [logseq.outliner.tree :as otree] [lambdaisland.glogi :as log] [cljs-time.core :as t] @@ -17,9 +17,9 @@ [goog.object :as gobj] [logseq.common.util :as common-util])) -(def *writes file/*writes) -(def dissoc-request! file/dissoc-request!) -(def conj-page-write! file/conj-page-write!) +(def *writes common-file/*writes) +(def dissoc-request! common-file/dissoc-request!) +(def conj-page-write! common-file/conj-page-write!) (defonce file-writes-chan (let [coercer (m/coercer [:catn @@ -81,7 +81,7 @@ (let [tree-or-blocks (if whiteboard? blocks (otree/blocks->vec-tree repo @conn blocks (:db/id page-block)))] (if page-block - (file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context request-id) + (common-file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context request-id) (do (js/console.error (str "can't find page id: " page-db-id)) (dissoc-request! request-id))))))) diff --git a/src/main/frontend/worker/handler/page/file_based/rename.cljs b/src/main/frontend/worker/handler/page/file_based/rename.cljs index 0e8d38b30..e0523c8ee 100644 --- a/src/main/frontend/worker/handler/page/file_based/rename.cljs +++ b/src/main/frontend/worker/handler/page/file_based/rename.cljs @@ -5,7 +5,7 @@ [clojure.string :as string] [clojure.walk :as walk] [logseq.common.util.page-ref :as page-ref] - [frontend.worker.file.util :as wfu] + [frontend.common.file.util :as wfu] [logseq.db :as ldb] [logseq.common.util :as common-util] [logseq.common.config :as common-config] diff --git a/src/main/frontend/worker/util.cljc b/src/main/frontend/worker/util.cljc index 9c7b01096..0d5ebefca 100644 --- a/src/main/frontend/worker/util.cljc +++ b/src/main/frontend/worker/util.cljc @@ -6,8 +6,8 @@ [goog.crypt :as crypt] [goog.crypt.Hmac] [goog.crypt.Sha256] - [logseq.db :as ldb] - [logseq.db.sqlite.common-db :as sqlite-common-db]))) + [logseq.db.sqlite.common-db :as sqlite-common-db] + [frontend.common.file.util :as wfu]))) ;; Copied from https://github.com/tonsky/datascript-todo #?(:clj @@ -23,10 +23,7 @@ #?(:cljs (do - (defn post-message - [type data] - (when (exists? js/self) - (.postMessage js/self (ldb/write-transit-str [type data])))) + (def post-message wfu/post-message) (defn get-pool-name [graph-name] diff --git a/src/test/frontend/db/name_sanity_test.cljs b/src/test/frontend/db/name_sanity_test.cljs index 637d48426..498dabea7 100644 --- a/src/test/frontend/db/name_sanity_test.cljs +++ b/src/test/frontend/db/name_sanity_test.cljs @@ -4,7 +4,7 @@ [logseq.graph-parser.extract :as extract] [frontend.worker.handler.page.file-based.rename :as worker-page-rename] [frontend.util.fs :as fs-util] - [frontend.worker.file.util :as wfu])) + [frontend.common.file.util :as wfu])) (defn- test-page-name "Check if page name can be preserved after escaping"