Automatically set extract-options correctly

- Removed state paths [:config :text-formats], [:config :img-formats]
  and [:me :settins :date-formatter] as they have been unused since
  their introduction in 2020
- Converted incorrect js/Error refs
- Also merged db.config test helpers

.clj-kondo/config.edn

@@ -30,6 +30,7 @@
              frontend.format.mldoc mldoc
              frontend.format.block block
              frontend.handler.extract extract
+             logseq.graph-parser graph-parser
              logseq.graph-parser.text text
              logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc

src/main/frontend/components/block.cljs

@@ -279,7 +279,7 @@
           (contains? config/audio-formats ext)
           (audio-cp @src)
 
-          (contains? (config/img-formats) ext)
+          (contains? (gp-config/img-formats) ext)
           (resizable-image config title @src metadata full_text true)
 
           (= ext :pdf)

src/main/frontend/components/file.cljs

@@ -5,7 +5,6 @@
             [datascript.core :as d]
             [frontend.components.lazy-editor :as lazy-editor]
             [frontend.components.svg :as svg]
-            [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [frontend.db :as db]
@@ -99,10 +98,10 @@
 
      (cond
        ;; image type
-       (and format (contains? (config/img-formats) format))
+       (and format (contains? (gp-config/img-formats) format))
        [:img {:src path}]
 
-       (and format (contains? (config/text-formats) format))
+       (and format (contains? (gp-config/text-formats) format))
        (when-let [file-content (db/get-file path)]
          (let [content (string/trim file-content)
                mode (util/get-file-ext path)]

src/main/frontend/config.cljs

@@ -5,6 +5,7 @@
             [frontend.util :as util]
             [shadow.resource :as rc]
             [logseq.graph-parser.util :as gp-util]
+            [logseq.graph-parser.config :as gp-config]
             [frontend.mobile.util :as mobile-util]))
 
 (goog-define DEV-RELEASE false)
@@ -63,49 +64,20 @@
     (if dev? path
         (str asset-domain path))))
 
-(defn text-formats
-  []
-  (let [config-formats (some->> (get-in @state/state [:config :text-formats])
-                                (map :keyword)
-                                (set))]
-    (set/union
-     config-formats
-     #{:json :org :md :yml :dat :asciidoc :rst :txt :markdown :adoc :html :js :ts :edn :clj :ml :rb :ex :erl :java :php :c :css
-       :excalidraw})))
-
 (def markup-formats
   #{:org :md :markdown :asciidoc :adoc :rst})
 
-(defn img-formats
-  []
-  (let [config-formats (some->> (get-in @state/state [:config :image-formats])
-                                (map :keyword)
-                                (set))]
-    (set/union
-     config-formats
-     #{:gif :svg :jpeg :ico :png :jpg :bmp :webp})))
-
 (defn doc-formats
   []
-  (let [config-formats (some->> (get-in @state/state [:config :document-formats])
-                                (map :keyword)
-                                (set))]
-    (set/union
-     config-formats
-     #{:doc :docx :xls :xlsx :ppt :pptx :one :pdf :epub})))
+  #{:doc :docx :xls :xlsx :ppt :pptx :one :pdf :epub})
 
 (def audio-formats #{:mp3 :ogg :mpeg :wav :m4a :flac :wma :aac})
 
-(def media-formats (set/union (img-formats) audio-formats))
+(def media-formats (set/union (gp-config/img-formats) audio-formats))
 
 (def html-render-formats
   #{:adoc :asciidoc})
 
-(defn supported-formats
-  []
-  (set/union (text-formats)
-             (img-formats)))
-
 (def mobile?
   (when-not util/node-test?
     (util/safe-re-find #"Mobi" js/navigator.userAgent)))
@@ -114,13 +86,7 @@
 
 (defn get-block-pattern
   [format]
-  (let [format (or format (state/get-preferred-format))
-        format (keyword format)]
-    (case format
-      :org
-      "*"
-
-      "-")))
+  (gp-config/get-block-pattern (or format (state/get-preferred-format))))
 
 (defn get-hr
   [format]

src/main/frontend/format/block.cljs

@@ -6,6 +6,7 @@
             [frontend.db :as db]
             [frontend.format :as format]
             [frontend.state :as state]
+            [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.mldoc :as gp-mldoc]))
 
@@ -15,7 +16,7 @@
   (gp-block/extract-blocks blocks content with-id? format
                            {:user-config (state/get-config)
                             :block-pattern (config/get-block-pattern format)
-                            :supported-formats (config/supported-formats)
+                            :supported-formats (gp-config/supported-formats)
                             :db (db/get-db (state/get-current-repo))
                             :date-formatter (state/get-date-formatter)}))
 

src/main/frontend/handler/file.cljs

@@ -16,6 +16,7 @@
             [lambdaisland.glogi :as log]
             [promesa.core :as p]
             [frontend.mobile.util :as mobile]
+            [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser :as graph-parser]))
 
 ;; TODO: extract all git ops using a channel
@@ -45,11 +46,11 @@
 
 (defn- only-text-formats
   [files]
-  (keep-formats files (config/text-formats)))
+  (keep-formats files (gp-config/text-formats)))
 
 (defn- only-image-formats
   [files]
-  (keep-formats files (config/img-formats)))
+  (keep-formats files (gp-config/img-formats)))
 
 (defn restore-config!
   ([repo-url project-changed-check?]
@@ -142,7 +143,7 @@
                                 :date-formatter (state/get-date-formatter)
                                 :page-name-order (state/page-name-order)
                                 :block-pattern (config/get-block-pattern (gp-util/get-format file))
-                                :supported-formats (config/supported-formats)}})))))
+                                :supported-formats (gp-config/supported-formats)}})))))
 
 ;; TODO: Remove this function in favor of `alter-files`
 (defn alter-file

src/main/frontend/state.cljs

@@ -13,6 +13,7 @@
             [goog.object :as gobj]
             [promesa.core :as p]
             [rum.core :as rum]
+            [logseq.graph-parser.config :as gp-config]
             [frontend.mobile.util :as mobile-util]))
 
 (defonce ^:large-vars/data-var state
@@ -1016,15 +1017,7 @@
 
 (defn get-date-formatter
   []
-  (or
-    (when-let [repo (get-current-repo)]
-      (or
-        (get-in @state [:config repo :journal/page-title-format])
-        ;; for compatibility
-        (get-in @state [:config repo :date-formatter])))
-    ;; TODO:
-    (get-in @state [:me :settings :date-formatter])
-    "MMM do, yyyy"))
+  (gp-config/get-date-formatter (get-config)))
 
 (defn shortcuts []
   (get-in @state [:config (get-current-repo) :shortcuts]))

src/main/logseq/graph_parser.cljs

@@ -14,30 +14,25 @@
                  :file/content content}]
     (d/transact! db [tx-data] {:skip-refresh? true})))
 
-;; TODO: Reuse from frontend.config
-(def supported-formats
-   #{:dat :markdown :bmp :js :png :gif :txt :yml :erl :excalidraw :css :webp :asciidoc :ts :rb :ml :java :c :org :ex :edn :svg :php :rst :json :jpeg :ico :jpg :clj :adoc :html :md})
-
 (defn parse-file
   "Parse file and save parsed data to the given db"
   [db file content {:keys [new? delete-blocks-fn new-graph? extract-options]
                     :or {new? true
                          new-graph? false
-                         delete-blocks-fn (constantly [])
-                         ;; TODO: Reuse these options from state and config
-                         extract-options {:block-pattern "-"
-                                          :date-formatter "MMM do, yyyy"
-                                          :supported-formats supported-formats}}}]
-
+                         delete-blocks-fn (constantly [])}}]
   (db-set-file-content! db file content)
   (let [format (gp-util/get-format file)
         file-content [{:file/path file}]
         tx (if (contains? gp-config/mldoc-support-formats format)
-             (let [[pages blocks]
+             (let [extract-options' (merge {:block-pattern (gp-config/get-block-pattern format)
+                                            :date-formatter "MMM do, yyyy"
+                                            :supported-formats (gp-config/supported-formats)}
+                                           extract-options)
+                   [pages blocks]
                    (extract/extract-blocks-pages
                     file
                     content
-                    (merge extract-options {:db @db}))
+                    (merge extract-options' {:db @db}))
                    delete-blocks (delete-blocks-fn (first pages) file)
                    block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
                    block-refs-ids (->> (mapcat :block/refs blocks)
@@ -59,6 +54,10 @@
     (d/transact! db (gp-util/remove-nils tx) (when new-graph? {:new-graph? true}))))
 
 (defn parse
-  [db files]
-  (doseq [{:file/keys [path content]} files]
-    (parse-file db path content {})))
+  "Main parse fn"
+  ([db files]
+   (parse db files {}))
+  ([db files {:keys [config]}]
+   (let [extract-options {:date-formatter (gp-config/get-date-formatter config)}]
+     (doseq [{:file/keys [path content]} files]
+       (parse-file db path content {:extract-options extract-options})))))

src/main/logseq/graph_parser/cli.cljs

@@ -0,0 +1,20 @@
+(ns logseq.graph-parser.cli
+  "Ns only for use by CLIs as it uses node.js libraries"
+  (:require ["fs" :as fs]
+            [clojure.edn :as edn]
+            [logseq.graph-parser :as graph-parser]))
+
+(defn- read-config
+  "Commandline version of frontend.handler.common/read-config without graceful
+  handling of broken config. Config is assumed to be at $dir/logseq/config.edn "
+  [dir]
+  (if (fs/existsSync (str dir "/logseq/config.edn"))
+    (-> (str dir "/logseq/config.edn") fs/readFileSync str edn/read-string)
+    {}))
+
+(defn parse
+  "Main entry point for parsing"
+  [dir db files]
+  (graph-parser/parse db
+                      files
+                      {:config (read-config dir)}))

src/main/logseq/graph_parser/config.cljs

@@ -1,6 +1,7 @@
 (ns ^:nbb-compatible logseq.graph-parser.config
   "Config that is shared between graph-parser and rest of app"
   (:require [logseq.graph-parser.util :as gp-util]
+            [clojure.set :as set]
             [clojure.string :as string]))
 
 (defonce local-assets-dir "assets")
@@ -22,3 +23,34 @@
 (defn mldoc-support?
   [format]
   (contains? mldoc-support-formats (keyword format)))
+
+(defn text-formats
+  []
+  #{:json :org :md :yml :dat :asciidoc :rst :txt :markdown :adoc :html :js :ts :edn :clj :ml :rb :ex :erl :java :php :c :css
+    :excalidraw})
+
+(defn img-formats
+  []
+  #{:gif :svg :jpeg :ico :png :jpg :bmp :webp})
+
+(defn supported-formats
+  []
+  (set/union (text-formats)
+             (img-formats)))
+
+(defn get-date-formatter
+  [config]
+  (or
+   (:journal/page-title-format config)
+   ;; for compatibility
+   (:date-formatter config)
+   "MMM do, yyyy"))
+
+(defn get-block-pattern
+  [format]
+  (let [format' (keyword format)]
+    (case format'
+      :org
+      "*"
+
+      "-")))

src/main/logseq/graph_parser/date_time_util.cljs

@@ -25,7 +25,7 @@
                           (fn [formatter]
                             (try
                               (tf/parse (tf/formatter formatter) (gp-util/capitalize-all journal-title))
-                              (catch js/Error _e
+                              (catch :default _e
                                 nil)))
                           formatters)
                          (filter some?)

src/main/logseq/graph_parser/extract.cljc

@@ -44,7 +44,6 @@
 (defn- extract-pages-and-blocks
   [format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
   (try
-    #_:clj-kondo/ignore ;;clj-kondo bug
     (let [page (get-page-name file ast page-name-order)
           [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
           blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))

src/main/logseq/graph_parser/util.cljs

@@ -7,9 +7,6 @@
 (defn safe-re-find
   "Copy of frontend.util/safe-re-find. Too basic to couple to main app"
   [pattern s]
-  (when-not (string? s)
-    ;; TODO: sentry
-    (js/console.trace))
   (when (string? s)
     (re-find pattern s)))
 
@@ -65,7 +62,7 @@
        (try
          (js/URL. s)
          true
-         (catch js/Error _e
+         (catch :default _e
            false))))
 
 (defn json->clj

src/test/frontend/db/config.cljs

@@ -1,12 +0,0 @@
-(ns frontend.db.config
-  (:require [frontend.db.conn :as conn]
-            [frontend.state :as state]
-            [frontend.db.persist :as db-persist]))
-
-(defn destroy-db! [] (conn/destroy-all!))
-
-(defn clear-current-repo []
-  (let [current-repo (state/get-current-repo)]
-    (db-persist/delete-graph! current-repo)
-    (destroy-db!)
-    (conn/start! current-repo)))

src/test/frontend/handler/export_test.cljs

@@ -2,7 +2,7 @@
   (:require [cljs.test :refer [async deftest use-fixtures are is]]
             [clojure.edn :as edn]
             [frontend.handler.export :as export]
-            [frontend.db.config :as config]
+            [frontend.test.helper :as test-helper]
             [frontend.handler.repo :as repo-handler]
             [frontend.state :as state]
             [promesa.core :as p]))
@@ -33,10 +33,10 @@
 (use-fixtures :once
   {:before (fn []
              (async done
-                    (config/clear-current-repo)
+                    (test-helper/clear-current-repo)
                     (p/let [_ (import-test-data!)]
                       (done))))
-   :after config/destroy-db!})
+   :after test-helper/destroy-test-db!})
 
 (deftest export-blocks-as-markdown
   (are [expect block-uuid-s]

src/test/frontend/test/helper.cljs

@@ -1,6 +1,8 @@
 (ns frontend.test.helper
   "Common helper fns for tests"
   (:require [frontend.handler.repo :as repo-handler]
+            [frontend.db.persist :as db-persist]
+            [frontend.state :as state]
             [frontend.db.conn :as conn]))
 
 (defonce test-db "test-db")
@@ -13,5 +15,11 @@
   []
   (conn/destroy-all!))
 
+(defn clear-current-repo []
+  (let [current-repo (state/get-current-repo)]
+    (db-persist/delete-graph! current-repo)
+    (destroy-test-db!)
+    (conn/start! current-repo)))
+
 (defn load-test-files [files]
   (repo-handler/parse-files-and-load-to-db! test-db files {:re-render? false}))

src/test/logseq/graph_parser_test.cljs

@@ -36,7 +36,7 @@
   (let [graph-dir "src/test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
         files (docs-graph-helper/build-graph-files graph-dir)
-        conn (gp-db/start-conn) 
+        conn (gp-db/start-conn)
         ; _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false})
         _ (graph-parser/parse conn files)
         db @conn]
@@ -99,7 +99,6 @@
 
       (is (= {:title 98 :id 98
               :updated-at 47 :created-at 47
-              :collapsed 22
               :card-last-score 6 :card-repeats 6 :card-next-schedule 6
               :card-last-interval 6 :card-ease-factor 6 :card-last-reviewed 6
               :alias 6}