refactor: finish frontend independence from worker and add lint

Remaining fixes worker all file related. Fixes LOG-3717

.clj-kondo/config.edn

@@ -55,6 +55,8 @@
              frontend.components.class class-component
              frontend.components.property property-component
              frontend.common.date common-date
+             frontend.common.file.core common-file
+             frontend.common.file.util wfu
              frontend.common.missionary-util c.m
              frontend.common.schema-register sr
              frontend.common.search-fuzzy fuzzy
@@ -134,9 +136,6 @@
              frontend.worker.util worker-util
              frontend.worker.state worker-state
              frontend.worker.handler.page worker-page
-             frontend.worker.handler.page.rename worker-page-rename
-             frontend.worker.handler.file.util wfu
-             logseq.outliner.batch-tx batch-tx
              lambdaisland.glogi log
              logseq.common.config common-config
              logseq.common.graph common-graph
@@ -169,6 +168,7 @@
              logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc
              logseq.graph-parser.property gp-property
+             logseq.outliner.batch-tx batch-tx
              logseq.outliner.core outliner-core
              logseq.outliner.op outliner-op
              logseq.outliner.pipeline outliner-pipeline

docs/dev-practices.md

@@ -122,11 +122,12 @@ The main convention is that file and db specific files go under directories name
 
 ### Separate Worker from Frontend
 
-The worker and frontend code share common code from deps/ and `frontend.common.*`. However, the worker should never depend on other frontend namespaces as it could pull in libraries like React which cause it to fail hard. Run this linter to ensure worker namespaces are not dependent on other frontend namespaces:
+The worker and frontend code share common code from deps/ and `frontend.common.*`. However, the worker should never depend on other frontend namespaces as it could pull in libraries like React which cause it to fail hard. Likewise the frontend should never depend on worker namespaces. Run this linter to ensure worker and frontend namespaces don't require each other:
 
 ```
 $ bb lint:worker-and-frontend-separate
-Success!
+Valid worker namespaces!
+Valid frontend namespaces!
 ```
 
 ## Testing

scripts/src/logseq/tasks/dev/lint.clj

@@ -21,8 +21,7 @@
     (println cmd)
     (shell cmd)))
 
-(defn worker-and-frontend-separate
+(defn- validate-frontend-not-in-worker
-  "Ensures worker is independent of frontend"
   []
   (let [res (shell {:out :string}
                    "git grep -h" "\\[frontend.*:as" "src/main/frontend/worker")
@@ -32,8 +31,26 @@
 
     (if (seq req-lines)
       (do
-        (println "The following requires should not be in worker namespaces:")
+        (println "The following frontend requires should not be in worker namespaces:")
         (println (string/join "\n" req-lines))
         (System/exit 1))
-      (println "Success!"))))
+      (println "Valid worker namespaces!"))))
+
+(defn- validate-worker-not-in-frontend
+  []
+  (let [res (shell {:out :string :continue true}
+                   "grep -r --exclude-dir=worker" "\\[frontend.worker.*:" "src/main/frontend")
+        req-lines (->> (:out res) string/split-lines)]
+    (if-not (and (= 1 (:exit res)) (= "" (:out res)))
+      (do
+        (println "The following worker requires should not be in frontend namespaces:")
+        (println (:out res))
+        (System/exit 1))
+      (println "Valid frontend namespaces!"))))
+
+(defn worker-and-frontend-separate
+  "Ensures worker is independent of frontend"
+  []
+  (validate-frontend-not-in-worker)
+  (validate-worker-not-in-frontend))
 

src/bench/frontend/benchmark_test_runner.cljs

@@ -5,7 +5,7 @@
             [clojure.pprint :as pprint]
             [clojure.test :refer [deftest testing]]
             [fipp.edn :as fipp]
-            [frontend.worker.file.util :as wfu]))
+            [frontend.common.file.util :as wfu]))
 
 (def onboarding
   (edn/read-string (slurped "resources/whiteboard/onboarding.edn")))

src/main/frontend/common/file/core.cljs

@@ -1,14 +1,16 @@
-(ns frontend.worker.file.core
+(ns frontend.common.file.core
-  "Save file to disk. Used by both file and DB graphs"
+  "Save file to disk. Used by both file and DB graphs and shared
+   by worker and frontend namespaces"
   (:require [clojure.string :as string]
-            [frontend.worker.file.util :as wfu]
+            [frontend.common.file.util :as wfu]
             [logseq.graph-parser.property :as gp-property]
             [logseq.common.path :as path]
             [datascript.core :as d]
             [logseq.db :as ldb]
             [frontend.common.date :as common-date]
-            [frontend.worker.util :as worker-util]
+            [logseq.db.frontend.content :as db-content]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.outliner.tree :as otree]))
 
 (defonce *writes (atom {}))
 (defonce *request-id (atom 0))
@@ -174,10 +176,10 @@
                      (let [files [[file-path new-content]]]
                        (when (seq files)
                          (let [page-id (:db/id page-block)]
-                           (worker-util/post-message :write-files {:request-id request-id
+                           (wfu/post-message :write-files {:request-id request-id
-                                                                   :page-id page-id
+                                                           :page-id page-id
-                                                                   :repo repo
+                                                           :repo repo
-                                                                   :files files})
+                                                           :files files})
                            :sent)))))
                  ;; In e2e tests, "card" page in db has no :file/path
                  (js/console.error "File path from page-block is not valid" page-block tree))]
@@ -195,3 +197,18 @@
       (if file
         (ok-handler)
         (transact-file-tx-if-not-exists! conn page-block ok-handler context)))))
+
+(defn block->content
+  "Converts a block including its children (recursively) to plain-text."
+  [repo db root-block-uuid tree->file-opts context]
+  (assert (uuid? root-block-uuid))
+  (let [init-level (or (:init-level tree->file-opts)
+                       (if (ldb/page? (d/entity db [:block/uuid root-block-uuid]))
+                         0
+                         1))
+        blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid)))
+                    (map #(db-content/update-block-content db % (:db/id %))))
+        tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
+    (tree->file-content repo db tree
+                        (assoc tree->file-opts :init-level init-level)
+                        context)))

src/main/frontend/common/file/util.cljs

@@ -1,7 +1,8 @@
-(ns frontend.worker.file.util
+(ns frontend.common.file.util
-  "File name fns"
+  "File name fns. Used by worker and frontend namespaces"
   (:require [clojure.string :as string]
-            [logseq.common.util :as common-util]))
+            [logseq.common.util :as common-util]
+            [logseq.db :as ldb]))
 
 ;; Update repo/invalid-graph-name-warning if characters change
 (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\")
@@ -88,3 +89,8 @@
   [x]
   (with-redefs [print-prefix-map print-prefix-map*]
     (pr-str x)))
+
+(defn post-message
+  [type data]
+  (when (exists? js/self)
+    (.postMessage js/self (ldb/write-transit-str [type data]))))

src/main/frontend/handler/export/common.cljs

@@ -13,7 +13,7 @@
             [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.util :as util :refer [concatv mapcatv removev]]
-            [frontend.worker.export :as worker-export]
+            [frontend.common.file.core :as common-file]
             [malli.core :as m]
             [malli.util :as mu]
             [promesa.core :as p]))
@@ -93,9 +93,9 @@
   [page-uuid]
   (let [repo (state/get-current-repo)
         db (db/get-db repo)]
-    (worker-export/block->content repo db page-uuid
+    (common-file/block->content repo db page-uuid
-                                  nil
+                                nil
-                                  {:export-bullet-indentation (state/get-export-bullet-indentation)})))
+                                {:export-bullet-indentation (state/get-export-bullet-indentation)})))
 
 (defn- page-name->ast
   [page-name]

src/main/frontend/modules/file/core.cljs

@@ -1,6 +1,6 @@
 (ns frontend.modules.file.core
   "Convert block trees to content"
-  (:require [frontend.worker.file.core :as worker-file]
+  (:require [frontend.common.file.core :as common-file]
             [frontend.state :as state]
             [frontend.db :as db]))
 
@@ -11,4 +11,4 @@
   (when-let [repo (state/get-current-repo)]
     (let [db (db/get-db repo)
           context {:export-bullet-indentation (state/get-export-bullet-indentation)}]
-      (worker-file/tree->file-content repo db tree opts context))))
+      (common-file/tree->file-content repo db tree opts context))))

src/main/frontend/util/fs.cljs

@@ -8,7 +8,7 @@
             [frontend.config :as config]
             [promesa.core :as p]
             [cljs.reader :as reader]
-            [frontend.worker.file.util :as wfu]))
+            [frontend.common.file.util :as wfu]))
 
 ;; NOTE: This is not the same ignored-path? as src/electron/electron/utils.cljs.
 ;;       The assets directory is ignored.

src/main/frontend/worker/db_worker.cljs

@@ -8,6 +8,7 @@
             [clojure.string :as string]
             [datascript.core :as d]
             [datascript.storage :refer [IStorage]]
+            [frontend.common.file.core :as common-file]
             [frontend.worker.db-listener :as db-listener]
             [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.db.migrate :as db-migrate]
@@ -601,9 +602,9 @@
    (assert (common-util/uuid-string? block-uuid-str))
    (let [block-uuid (uuid block-uuid-str)]
      (when-let [conn (worker-state/get-datascript-conn repo)]
-       (worker-export/block->content repo @conn block-uuid
+       (common-file/block->content repo @conn block-uuid
-                                     (ldb/read-transit-str tree->file-opts)
+                                   (ldb/read-transit-str tree->file-opts)
-                                     (ldb/read-transit-str context)))))
+                                   (ldb/read-transit-str context)))))
 
   (get-all-pages
    [this repo]

src/main/frontend/worker/export.cljs

@@ -1,26 +1,10 @@
 (ns frontend.worker.export
   "Export data"
   (:require [datascript.core :as d]
-            [frontend.worker.file.core :as worker-file]
+            [frontend.common.file.core :as common-file]
             [logseq.db :as ldb]
             [logseq.graph-parser.property :as gp-property]
-            [logseq.outliner.tree :as otree]
+            [logseq.outliner.tree :as otree]))
-            [logseq.db.frontend.content :as db-content]))
-
-(defn block->content
-  "Converts a block including its children (recursively) to plain-text."
-  [repo db root-block-uuid tree->file-opts context]
-  (assert (uuid? root-block-uuid))
-  (let [init-level (or (:init-level tree->file-opts)
-                       (if (ldb/page? (d/entity db [:block/uuid root-block-uuid]))
-                         0
-                         1))
-        blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid)))
-                    (map #(db-content/update-block-content db % (:db/id %))))
-        tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
-    (worker-file/tree->file-content repo db tree
-                                    (assoc tree->file-opts :init-level init-level)
-                                    context)))
 
 (defn- safe-keywordize
   [block]
@@ -64,4 +48,4 @@
        (map (fn [d]
               (let [e (d/entity db (:e d))]
                 [(:block/title e)
-                 (block->content repo db (:block/uuid e) {} {})])))))
+                 (common-file/block->content repo db (:block/uuid e) {} {})])))))

src/main/frontend/worker/file.cljs

@@ -3,7 +3,7 @@
   (:require [clojure.core.async :as async]
             [clojure.string :as string]
             [clojure.set :as set]
-            [frontend.worker.file.core :as file]
+            [frontend.common.file.core :as common-file]
             [logseq.outliner.tree :as otree]
             [lambdaisland.glogi :as log]
             [cljs-time.core :as t]
@@ -17,9 +17,9 @@
             [goog.object :as gobj]
             [logseq.common.util :as common-util]))
 
-(def *writes file/*writes)
+(def *writes common-file/*writes)
-(def dissoc-request! file/dissoc-request!)
+(def dissoc-request! common-file/dissoc-request!)
-(def conj-page-write! file/conj-page-write!)
+(def conj-page-write! common-file/conj-page-write!)
 
 (defonce file-writes-chan
   (let [coercer (m/coercer [:catn
@@ -81,7 +81,7 @@
             (let [tree-or-blocks (if whiteboard? blocks
                                      (otree/blocks->vec-tree repo @conn blocks (:db/id page-block)))]
               (if page-block
-                (file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context request-id)
+                (common-file/save-tree! repo conn page-block tree-or-blocks blocks-just-deleted? context request-id)
                 (do
                   (js/console.error (str "can't find page id: " page-db-id))
                   (dissoc-request! request-id)))))))

src/main/frontend/worker/handler/page/file_based/rename.cljs

@@ -5,7 +5,7 @@
             [clojure.string :as string]
             [clojure.walk :as walk]
             [logseq.common.util.page-ref :as page-ref]
-            [frontend.worker.file.util :as wfu]
+            [frontend.common.file.util :as wfu]
             [logseq.db :as ldb]
             [logseq.common.util :as common-util]
             [logseq.common.config :as common-config]

src/main/frontend/worker/util.cljc

@@ -6,8 +6,8 @@
                      [goog.crypt :as crypt]
                      [goog.crypt.Hmac]
                      [goog.crypt.Sha256]
-                     [logseq.db :as ldb]
+                     [logseq.db.sqlite.common-db :as sqlite-common-db]
-                     [logseq.db.sqlite.common-db :as sqlite-common-db])))
+                     [frontend.common.file.util :as wfu])))
 
 ;; Copied from https://github.com/tonsky/datascript-todo
 #?(:clj
@@ -23,10 +23,7 @@
 
 #?(:cljs
    (do
-     (defn post-message
+     (def post-message wfu/post-message)
-       [type data]
-       (when (exists? js/self)
-         (.postMessage js/self (ldb/write-transit-str [type data]))))
 
      (defn get-pool-name
        [graph-name]

src/test/frontend/db/name_sanity_test.cljs

@@ -4,7 +4,7 @@
             [logseq.graph-parser.extract :as extract]
             [frontend.worker.handler.page.file-based.rename :as worker-page-rename]
             [frontend.util.fs :as fs-util]
-            [frontend.worker.file.util :as wfu]))
+            [frontend.common.file.util :as wfu]))
 
 (defn- test-page-name
   "Check if page name can be preserved after escaping"