mirror of https://github.com/logseq/logseq
Automatically set extract-options correctly
- Removed state paths [:config :text-formats], [:config :img-formats] and [:me :settins :date-formatter] as they have been unused since their introduction in 2020 - Converted incorrect js/Error refs - Also merged db.config test helperspull/5420/head
parent
5bce219ca4
commit
e999ebc149
|
@ -30,6 +30,7 @@
|
|||
frontend.format.mldoc mldoc
|
||||
frontend.format.block block
|
||||
frontend.handler.extract extract
|
||||
logseq.graph-parser graph-parser
|
||||
logseq.graph-parser.text text
|
||||
logseq.graph-parser.block gp-block
|
||||
logseq.graph-parser.mldoc gp-mldoc
|
||||
|
|
|
@ -279,7 +279,7 @@
|
|||
(contains? config/audio-formats ext)
|
||||
(audio-cp @src)
|
||||
|
||||
(contains? (config/img-formats) ext)
|
||||
(contains? (gp-config/img-formats) ext)
|
||||
(resizable-image config title @src metadata full_text true)
|
||||
|
||||
(= ext :pdf)
|
||||
|
|
|
@ -5,7 +5,6 @@
|
|||
[datascript.core :as d]
|
||||
[frontend.components.lazy-editor :as lazy-editor]
|
||||
[frontend.components.svg :as svg]
|
||||
[frontend.config :as config]
|
||||
[frontend.context.i18n :refer [t]]
|
||||
[frontend.date :as date]
|
||||
[frontend.db :as db]
|
||||
|
@ -99,10 +98,10 @@
|
|||
|
||||
(cond
|
||||
;; image type
|
||||
(and format (contains? (config/img-formats) format))
|
||||
(and format (contains? (gp-config/img-formats) format))
|
||||
[:img {:src path}]
|
||||
|
||||
(and format (contains? (config/text-formats) format))
|
||||
(and format (contains? (gp-config/text-formats) format))
|
||||
(when-let [file-content (db/get-file path)]
|
||||
(let [content (string/trim file-content)
|
||||
mode (util/get-file-ext path)]
|
||||
|
|
|
@ -5,6 +5,7 @@
|
|||
[frontend.util :as util]
|
||||
[shadow.resource :as rc]
|
||||
[logseq.graph-parser.util :as gp-util]
|
||||
[logseq.graph-parser.config :as gp-config]
|
||||
[frontend.mobile.util :as mobile-util]))
|
||||
|
||||
(goog-define DEV-RELEASE false)
|
||||
|
@ -63,49 +64,20 @@
|
|||
(if dev? path
|
||||
(str asset-domain path))))
|
||||
|
||||
(defn text-formats
|
||||
[]
|
||||
(let [config-formats (some->> (get-in @state/state [:config :text-formats])
|
||||
(map :keyword)
|
||||
(set))]
|
||||
(set/union
|
||||
config-formats
|
||||
#{:json :org :md :yml :dat :asciidoc :rst :txt :markdown :adoc :html :js :ts :edn :clj :ml :rb :ex :erl :java :php :c :css
|
||||
:excalidraw})))
|
||||
|
||||
(def markup-formats
|
||||
#{:org :md :markdown :asciidoc :adoc :rst})
|
||||
|
||||
(defn img-formats
|
||||
[]
|
||||
(let [config-formats (some->> (get-in @state/state [:config :image-formats])
|
||||
(map :keyword)
|
||||
(set))]
|
||||
(set/union
|
||||
config-formats
|
||||
#{:gif :svg :jpeg :ico :png :jpg :bmp :webp})))
|
||||
|
||||
(defn doc-formats
|
||||
[]
|
||||
(let [config-formats (some->> (get-in @state/state [:config :document-formats])
|
||||
(map :keyword)
|
||||
(set))]
|
||||
(set/union
|
||||
config-formats
|
||||
#{:doc :docx :xls :xlsx :ppt :pptx :one :pdf :epub})))
|
||||
#{:doc :docx :xls :xlsx :ppt :pptx :one :pdf :epub})
|
||||
|
||||
(def audio-formats #{:mp3 :ogg :mpeg :wav :m4a :flac :wma :aac})
|
||||
|
||||
(def media-formats (set/union (img-formats) audio-formats))
|
||||
(def media-formats (set/union (gp-config/img-formats) audio-formats))
|
||||
|
||||
(def html-render-formats
|
||||
#{:adoc :asciidoc})
|
||||
|
||||
(defn supported-formats
|
||||
[]
|
||||
(set/union (text-formats)
|
||||
(img-formats)))
|
||||
|
||||
(def mobile?
|
||||
(when-not util/node-test?
|
||||
(util/safe-re-find #"Mobi" js/navigator.userAgent)))
|
||||
|
@ -114,13 +86,7 @@
|
|||
|
||||
(defn get-block-pattern
|
||||
[format]
|
||||
(let [format (or format (state/get-preferred-format))
|
||||
format (keyword format)]
|
||||
(case format
|
||||
:org
|
||||
"*"
|
||||
|
||||
"-")))
|
||||
(gp-config/get-block-pattern (or format (state/get-preferred-format))))
|
||||
|
||||
(defn get-hr
|
||||
[format]
|
||||
|
|
|
@ -6,6 +6,7 @@
|
|||
[frontend.db :as db]
|
||||
[frontend.format :as format]
|
||||
[frontend.state :as state]
|
||||
[logseq.graph-parser.config :as gp-config]
|
||||
[logseq.graph-parser.property :as gp-property]
|
||||
[logseq.graph-parser.mldoc :as gp-mldoc]))
|
||||
|
||||
|
@ -15,7 +16,7 @@
|
|||
(gp-block/extract-blocks blocks content with-id? format
|
||||
{:user-config (state/get-config)
|
||||
:block-pattern (config/get-block-pattern format)
|
||||
:supported-formats (config/supported-formats)
|
||||
:supported-formats (gp-config/supported-formats)
|
||||
:db (db/get-db (state/get-current-repo))
|
||||
:date-formatter (state/get-date-formatter)}))
|
||||
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
[lambdaisland.glogi :as log]
|
||||
[promesa.core :as p]
|
||||
[frontend.mobile.util :as mobile]
|
||||
[logseq.graph-parser.config :as gp-config]
|
||||
[logseq.graph-parser :as graph-parser]))
|
||||
|
||||
;; TODO: extract all git ops using a channel
|
||||
|
@ -45,11 +46,11 @@
|
|||
|
||||
(defn- only-text-formats
|
||||
[files]
|
||||
(keep-formats files (config/text-formats)))
|
||||
(keep-formats files (gp-config/text-formats)))
|
||||
|
||||
(defn- only-image-formats
|
||||
[files]
|
||||
(keep-formats files (config/img-formats)))
|
||||
(keep-formats files (gp-config/img-formats)))
|
||||
|
||||
(defn restore-config!
|
||||
([repo-url project-changed-check?]
|
||||
|
@ -142,7 +143,7 @@
|
|||
:date-formatter (state/get-date-formatter)
|
||||
:page-name-order (state/page-name-order)
|
||||
:block-pattern (config/get-block-pattern (gp-util/get-format file))
|
||||
:supported-formats (config/supported-formats)}})))))
|
||||
:supported-formats (gp-config/supported-formats)}})))))
|
||||
|
||||
;; TODO: Remove this function in favor of `alter-files`
|
||||
(defn alter-file
|
||||
|
|
|
@ -13,6 +13,7 @@
|
|||
[goog.object :as gobj]
|
||||
[promesa.core :as p]
|
||||
[rum.core :as rum]
|
||||
[logseq.graph-parser.config :as gp-config]
|
||||
[frontend.mobile.util :as mobile-util]))
|
||||
|
||||
(defonce ^:large-vars/data-var state
|
||||
|
@ -1016,15 +1017,7 @@
|
|||
|
||||
(defn get-date-formatter
|
||||
[]
|
||||
(or
|
||||
(when-let [repo (get-current-repo)]
|
||||
(or
|
||||
(get-in @state [:config repo :journal/page-title-format])
|
||||
;; for compatibility
|
||||
(get-in @state [:config repo :date-formatter])))
|
||||
;; TODO:
|
||||
(get-in @state [:me :settings :date-formatter])
|
||||
"MMM do, yyyy"))
|
||||
(gp-config/get-date-formatter (get-config)))
|
||||
|
||||
(defn shortcuts []
|
||||
(get-in @state [:config (get-current-repo) :shortcuts]))
|
||||
|
|
|
@ -14,30 +14,25 @@
|
|||
:file/content content}]
|
||||
(d/transact! db [tx-data] {:skip-refresh? true})))
|
||||
|
||||
;; TODO: Reuse from frontend.config
|
||||
(def supported-formats
|
||||
#{:dat :markdown :bmp :js :png :gif :txt :yml :erl :excalidraw :css :webp :asciidoc :ts :rb :ml :java :c :org :ex :edn :svg :php :rst :json :jpeg :ico :jpg :clj :adoc :html :md})
|
||||
|
||||
(defn parse-file
|
||||
"Parse file and save parsed data to the given db"
|
||||
[db file content {:keys [new? delete-blocks-fn new-graph? extract-options]
|
||||
:or {new? true
|
||||
new-graph? false
|
||||
delete-blocks-fn (constantly [])
|
||||
;; TODO: Reuse these options from state and config
|
||||
extract-options {:block-pattern "-"
|
||||
:date-formatter "MMM do, yyyy"
|
||||
:supported-formats supported-formats}}}]
|
||||
|
||||
delete-blocks-fn (constantly [])}}]
|
||||
(db-set-file-content! db file content)
|
||||
(let [format (gp-util/get-format file)
|
||||
file-content [{:file/path file}]
|
||||
tx (if (contains? gp-config/mldoc-support-formats format)
|
||||
(let [[pages blocks]
|
||||
(let [extract-options' (merge {:block-pattern (gp-config/get-block-pattern format)
|
||||
:date-formatter "MMM do, yyyy"
|
||||
:supported-formats (gp-config/supported-formats)}
|
||||
extract-options)
|
||||
[pages blocks]
|
||||
(extract/extract-blocks-pages
|
||||
file
|
||||
content
|
||||
(merge extract-options {:db @db}))
|
||||
(merge extract-options' {:db @db}))
|
||||
delete-blocks (delete-blocks-fn (first pages) file)
|
||||
block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
|
||||
block-refs-ids (->> (mapcat :block/refs blocks)
|
||||
|
@ -59,6 +54,10 @@
|
|||
(d/transact! db (gp-util/remove-nils tx) (when new-graph? {:new-graph? true}))))
|
||||
|
||||
(defn parse
|
||||
[db files]
|
||||
(doseq [{:file/keys [path content]} files]
|
||||
(parse-file db path content {})))
|
||||
"Main parse fn"
|
||||
([db files]
|
||||
(parse db files {}))
|
||||
([db files {:keys [config]}]
|
||||
(let [extract-options {:date-formatter (gp-config/get-date-formatter config)}]
|
||||
(doseq [{:file/keys [path content]} files]
|
||||
(parse-file db path content {:extract-options extract-options})))))
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
(ns logseq.graph-parser.cli
|
||||
"Ns only for use by CLIs as it uses node.js libraries"
|
||||
(:require ["fs" :as fs]
|
||||
[clojure.edn :as edn]
|
||||
[logseq.graph-parser :as graph-parser]))
|
||||
|
||||
(defn- read-config
|
||||
"Commandline version of frontend.handler.common/read-config without graceful
|
||||
handling of broken config. Config is assumed to be at $dir/logseq/config.edn "
|
||||
[dir]
|
||||
(if (fs/existsSync (str dir "/logseq/config.edn"))
|
||||
(-> (str dir "/logseq/config.edn") fs/readFileSync str edn/read-string)
|
||||
{}))
|
||||
|
||||
(defn parse
|
||||
"Main entry point for parsing"
|
||||
[dir db files]
|
||||
(graph-parser/parse db
|
||||
files
|
||||
{:config (read-config dir)}))
|
|
@ -1,6 +1,7 @@
|
|||
(ns ^:nbb-compatible logseq.graph-parser.config
|
||||
"Config that is shared between graph-parser and rest of app"
|
||||
(:require [logseq.graph-parser.util :as gp-util]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defonce local-assets-dir "assets")
|
||||
|
@ -22,3 +23,34 @@
|
|||
(defn mldoc-support?
|
||||
[format]
|
||||
(contains? mldoc-support-formats (keyword format)))
|
||||
|
||||
(defn text-formats
|
||||
[]
|
||||
#{:json :org :md :yml :dat :asciidoc :rst :txt :markdown :adoc :html :js :ts :edn :clj :ml :rb :ex :erl :java :php :c :css
|
||||
:excalidraw})
|
||||
|
||||
(defn img-formats
|
||||
[]
|
||||
#{:gif :svg :jpeg :ico :png :jpg :bmp :webp})
|
||||
|
||||
(defn supported-formats
|
||||
[]
|
||||
(set/union (text-formats)
|
||||
(img-formats)))
|
||||
|
||||
(defn get-date-formatter
|
||||
[config]
|
||||
(or
|
||||
(:journal/page-title-format config)
|
||||
;; for compatibility
|
||||
(:date-formatter config)
|
||||
"MMM do, yyyy"))
|
||||
|
||||
(defn get-block-pattern
|
||||
[format]
|
||||
(let [format' (keyword format)]
|
||||
(case format'
|
||||
:org
|
||||
"*"
|
||||
|
||||
"-")))
|
||||
|
|
|
@ -25,7 +25,7 @@
|
|||
(fn [formatter]
|
||||
(try
|
||||
(tf/parse (tf/formatter formatter) (gp-util/capitalize-all journal-title))
|
||||
(catch js/Error _e
|
||||
(catch :default _e
|
||||
nil)))
|
||||
formatters)
|
||||
(filter some?)
|
||||
|
|
|
@ -44,7 +44,6 @@
|
|||
(defn- extract-pages-and-blocks
|
||||
[format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
|
||||
(try
|
||||
#_:clj-kondo/ignore ;;clj-kondo bug
|
||||
(let [page (get-page-name file ast page-name-order)
|
||||
[_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
|
||||
blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
|
||||
|
|
|
@ -7,9 +7,6 @@
|
|||
(defn safe-re-find
|
||||
"Copy of frontend.util/safe-re-find. Too basic to couple to main app"
|
||||
[pattern s]
|
||||
(when-not (string? s)
|
||||
;; TODO: sentry
|
||||
(js/console.trace))
|
||||
(when (string? s)
|
||||
(re-find pattern s)))
|
||||
|
||||
|
@ -65,7 +62,7 @@
|
|||
(try
|
||||
(js/URL. s)
|
||||
true
|
||||
(catch js/Error _e
|
||||
(catch :default _e
|
||||
false))))
|
||||
|
||||
(defn json->clj
|
||||
|
|
|
@ -1,12 +0,0 @@
|
|||
(ns frontend.db.config
|
||||
(:require [frontend.db.conn :as conn]
|
||||
[frontend.state :as state]
|
||||
[frontend.db.persist :as db-persist]))
|
||||
|
||||
(defn destroy-db! [] (conn/destroy-all!))
|
||||
|
||||
(defn clear-current-repo []
|
||||
(let [current-repo (state/get-current-repo)]
|
||||
(db-persist/delete-graph! current-repo)
|
||||
(destroy-db!)
|
||||
(conn/start! current-repo)))
|
|
@ -2,7 +2,7 @@
|
|||
(:require [cljs.test :refer [async deftest use-fixtures are is]]
|
||||
[clojure.edn :as edn]
|
||||
[frontend.handler.export :as export]
|
||||
[frontend.db.config :as config]
|
||||
[frontend.test.helper :as test-helper]
|
||||
[frontend.handler.repo :as repo-handler]
|
||||
[frontend.state :as state]
|
||||
[promesa.core :as p]))
|
||||
|
@ -33,10 +33,10 @@
|
|||
(use-fixtures :once
|
||||
{:before (fn []
|
||||
(async done
|
||||
(config/clear-current-repo)
|
||||
(test-helper/clear-current-repo)
|
||||
(p/let [_ (import-test-data!)]
|
||||
(done))))
|
||||
:after config/destroy-db!})
|
||||
:after test-helper/destroy-test-db!})
|
||||
|
||||
(deftest export-blocks-as-markdown
|
||||
(are [expect block-uuid-s]
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
(ns frontend.test.helper
|
||||
"Common helper fns for tests"
|
||||
(:require [frontend.handler.repo :as repo-handler]
|
||||
[frontend.db.persist :as db-persist]
|
||||
[frontend.state :as state]
|
||||
[frontend.db.conn :as conn]))
|
||||
|
||||
(defonce test-db "test-db")
|
||||
|
@ -13,5 +15,11 @@
|
|||
[]
|
||||
(conn/destroy-all!))
|
||||
|
||||
(defn clear-current-repo []
|
||||
(let [current-repo (state/get-current-repo)]
|
||||
(db-persist/delete-graph! current-repo)
|
||||
(destroy-test-db!)
|
||||
(conn/start! current-repo)))
|
||||
|
||||
(defn load-test-files [files]
|
||||
(repo-handler/parse-files-and-load-to-db! test-db files {:re-render? false}))
|
||||
|
|
|
@ -36,7 +36,7 @@
|
|||
(let [graph-dir "src/test/docs"
|
||||
_ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
|
||||
files (docs-graph-helper/build-graph-files graph-dir)
|
||||
conn (gp-db/start-conn)
|
||||
conn (gp-db/start-conn)
|
||||
; _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false})
|
||||
_ (graph-parser/parse conn files)
|
||||
db @conn]
|
||||
|
@ -99,7 +99,6 @@
|
|||
|
||||
(is (= {:title 98 :id 98
|
||||
:updated-at 47 :created-at 47
|
||||
:collapsed 22
|
||||
:card-last-score 6 :card-repeats 6 :card-next-schedule 6
|
||||
:card-last-interval 6 :card-ease-factor 6 :card-last-reviewed 6
|
||||
:alias 6}
|
||||
|
|
Loading…
Reference in New Issue