greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Finish migration block ns to nbb

pull/5299/head
Gabriel Horner 2022-05-12 16:58:26 -04:00
parent 2c725ca5af
commit b1e226745d
15 changed files with 215 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.clj-kondo/config.edn
View File

@ -28,7 +28,8 @@
logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.mldoc gp-mldoc
logseq.graph-parser.util gp-util logseq.graph-parser.util gp-util
logseq.graph-parser.property gp-property logseq.graph-parser.property gp-property
logseq.graph-parser.config gp-config}}} logseq.graph-parser.config gp-config
logseq.graph-parser.date-time-util date-time-util}}}
:hooks {:analyze-call {rum.core/defc hooks.rum/defc :hooks {:analyze-call {rum.core/defc hooks.rum/defc
rum.core/defcs hooks.rum/defcs}} rum.core/defcs hooks.rum/defcs}}

3
src/main/frontend/components/journal.cljs
View File

@ -9,6 +9,7 @@
[frontend.handler.page :as page-handler] [frontend.handler.page :as page-handler]
[frontend.state :as state] [frontend.state :as state]
[logseq.graph-parser.text :as text] [logseq.graph-parser.text :as text]
[logseq.graph-parser.util :as gp-util]
[frontend.ui :as ui] [frontend.ui :as ui]
[frontend.util :as util] [frontend.util :as util]
[goog.object :as gobj] [goog.object :as gobj]
@ -51,7 +52,7 @@
:page)) :page))
(.preventDefault e)))} (.preventDefault e)))}
[:h1.title [:h1.title
(util/capitalize-all title)]] (gp-util/capitalize-all title)]]
(blocks-cp repo page format) (blocks-cp repo page format)

4
src/main/frontend/components/page.cljs
View File

@ -34,7 +34,7 @@
[medley.core :as medley] [medley.core :as medley]
[rum.core :as rum] [rum.core :as rum]
[logseq.graph-parser.util :as gp-util] [logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.block :as gp-block] [frontend.format.block :as block]
[frontend.mobile.util :as mobile-util])) [frontend.mobile.util :as mobile-util]))
(defn- get-page-name (defn- get-page-name
@ -332,7 +332,7 @@
(db/entity repo)) (db/entity repo))
(do (do
(when-not (db/entity repo [:block/name page-name]) (when-not (db/entity repo [:block/name page-name])
(let [m (gp-block/page-name->map path-page-name true)] (let [m (block/page-name->map path-page-name true)]
(db/transact! repo [m]))) (db/transact! repo [m])))
(db/pull [:block/name page-name]))) (db/pull [:block/name page-name])))
{:keys [icon]} (:block/properties page) {:keys [icon]} (:block/properties page)

46
src/main/frontend/date.cljs
View File

@ -5,9 +5,10 @@
[cljs-time.core :as t] [cljs-time.core :as t]
[cljs-time.format :as tf] [cljs-time.format :as tf]
[cljs-time.local :as tl] [cljs-time.local :as tl]
[clojure.string :as string]
[frontend.state :as state] [frontend.state :as state]
[frontend.util :as util] [frontend.util :as util]
[logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.date-time-util :as date-time-util]
[goog.object :as gobj] [goog.object :as gobj]
[lambdaisland.glogi :as log])) [lambdaisland.glogi :as log]))
@ -16,11 +17,6 @@
(when (string? s) (when (string? s)
((gobj/get chrono "parseDate") s))) ((gobj/get chrono "parseDate") s)))
(defn format
[date]
(when-let [formatter-string (state/get-date-formatter)]
(tf/unparse (tf/formatter formatter-string) date)))
(def custom-formatter (tf/formatter "yyyy-MM-dd'T'HH:mm:ssZZ")) (def custom-formatter (tf/formatter "yyyy-MM-dd'T'HH:mm:ssZZ"))
(defn journal-title-formatters (defn journal-title-formatters
@ -55,13 +51,6 @@
"yyyy年MM月dd日"} "yyyy年MM月dd日"}
(state/get-date-formatter))) (state/get-date-formatter)))
;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
(defn safe-journal-title-formatters
[]
(->> [(state/get-date-formatter) "yyyy-MM-dd" "yyyy_MM_dd"]
(remove string/blank?)
distinct))
(defn get-date-time-string (defn get-date-time-string
([] ([]
(get-date-time-string (t/now))) (get-date-time-string (t/now)))
@ -115,7 +104,7 @@
([] ([]
(journal-name (tl/local-now))) (journal-name (tl/local-now)))
([date] ([date]
(format date))) (date-time-util/format date (state/get-date-formatter))))
(defn journal-name-s [s] (defn journal-name-s [s]
(try (try
@ -183,34 +172,19 @@
(defn valid-journal-title? (defn valid-journal-title?
[title] [title]
(and title (and title
(valid? (util/capitalize-all title)))) (valid? (gp-util/capitalize-all title))))
(defn journal-title-> (defn journal-title->
([journal-title then-fn] ([journal-title then-fn]
(journal-title-> journal-title then-fn (safe-journal-title-formatters))) (journal-title-> journal-title then-fn (date-time-util/safe-journal-title-formatters (state/get-date-formatter))))
([journal-title then-fn formatters] ([journal-title then-fn formatters]
(when-not (string/blank? journal-title) (date-time-util/journal-title-> journal-title then-fn formatters)))
(when-let [time (->> (map
(fn [formatter]
(try
(tf/parse (tf/formatter formatter) (util/capitalize-all journal-title))
(catch js/Error _e
nil)))
formatters)
(filter some?)
first)]
(then-fn time)))))
(defn journal-title->int (defn journal-title->int
[journal-title] [journal-title]
(when journal-title (date-time-util/journal-title->int
(let [journal-title (util/capitalize-all journal-title)] journal-title
(journal-title-> journal-title #(util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %)))))) (date-time-util/safe-journal-title-formatters (state/get-date-formatter))))
(defn int->journal-title
[day]
(when day
(format (tf/parse (tf/formatter "yyyyMMdd") (str day)))))
(defn journal-day->ts (defn journal-day->ts
[day] [day]
@ -240,7 +214,7 @@
(defn journal-title->custom-format (defn journal-title->custom-format
[journal-title] [journal-title]
(journal-title-> journal-title format)) (journal-title-> journal-title #(date-time-util/format % (state/get-date-formatter))))
(defn int->local-time-2 (defn int->local-time-2
[n] [n]

19
src/main/frontend/format/block.cljs
View File

@ -9,6 +9,23 @@
[logseq.graph-parser.property :as gp-property] [logseq.graph-parser.property :as gp-property]
[logseq.graph-parser.mldoc :as gp-mldoc])) [logseq.graph-parser.mldoc :as gp-mldoc]))
(defn extract-blocks
"Wrapper around logseq.graph-parser.block/extract-blocks that adds in system state"
[blocks content with-id? format]
(gp-block/extract-blocks blocks content with-id? format
{:user-config (state/get-config)
:block-pattern (config/get-block-pattern format)
:supported-formats (config/supported-formats)
:db (db/get-db (state/get-current-repo))
:date-formatter (state/get-date-formatter)}))
(defn page-name->map
"Wrapper around logseq.graph-parser.block/page-name->map that adds in db"
([original-page-name with-id?]
(page-name->map original-page-name with-id? true))
([original-page-name with-id? with-timestamp?]
(gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter))))
(defn with-parent-and-left (defn with-parent-and-left
[page-id blocks] [page-id blocks]
(loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks) (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
@ -87,7 +104,7 @@
(when-not (string/blank? content) (when-not (string/blank? content)
(let [block (dissoc block :block/pre-block?) (let [block (dissoc block :block/pre-block?)
ast (format/to-edn content format nil) ast (format/to-edn content format nil)
blocks (gp-block/extract-blocks ast content with-id? format) blocks (extract-blocks ast content with-id? format)
new-block (first blocks) new-block (first blocks)
parent-refs (->> (db/get-block-parent (state/get-current-repo) uuid) parent-refs (->> (db/get-block-parent (state/get-current-repo) uuid)
:block/path-refs :block/path-refs

10
src/main/frontend/handler/editor.cljs
View File

@ -266,7 +266,7 @@
(if (and (:block/pre-block? block) (if (and (:block/pre-block? block)
(seq (:block/properties block))) (seq (:block/properties block)))
(let [page-properties (:block/properties block) (let [page-properties (:block/properties block)
str->page (fn [n] (gp-block/page-name->map n true)) str->page (fn [n] (block/page-name->map n true))
refs (->> page-properties refs (->> page-properties
(filter (fn [[_ v]] (coll? v))) (filter (fn [[_ v]] (coll? v)))
(vals) (vals)
@ -670,7 +670,9 @@
(defn properties-block (defn properties-block
[properties format page] [properties format page]
(let [content (property/insert-properties format "" properties) (let [content (property/insert-properties format "" properties)
refs (gp-block/get-page-refs-from-properties properties)] refs (gp-block/get-page-refs-from-properties properties
(db/get-db (state/get-current-repo))
(state/get-date-formatter))]
{:block/pre-block? true {:block/pre-block? true
:block/uuid (db/new-block-id) :block/uuid (db/new-block-id)
:block/properties properties :block/properties properties
@ -1946,7 +1948,7 @@
content* (str (if (= :markdown format) "- " "* ") content* (str (if (= :markdown format) "- " "* ")
(property/insert-properties format content props)) (property/insert-properties format content props))
ast (mldoc/->edn content* (gp-mldoc/default-config format)) ast (mldoc/->edn content* (gp-mldoc/default-config format))
blocks (gp-block/extract-blocks ast content* true format) blocks (block/extract-blocks ast content* true format)
fst-block (first blocks)] fst-block (first blocks)]
(assert fst-block "fst-block shouldn't be nil") (assert fst-block "fst-block shouldn't be nil")
(assoc fst-block :block/level (:block/level block))))))) (assoc fst-block :block/level (:block/level block)))))))
@ -2853,7 +2855,7 @@
[format text] [format text]
(when-let [editing-block (state/get-edit-block)] (when-let [editing-block (state/get-edit-block)]
(let [page-id (:db/id (:block/page editing-block)) (let [page-id (:db/id (:block/page editing-block))
blocks (gp-block/extract-blocks blocks (block/extract-blocks
(mldoc/->edn text (gp-mldoc/default-config format)) text true format) (mldoc/->edn text (gp-mldoc/default-config format)) text true format)
blocks' (block/with-parent-and-left page-id blocks)] blocks' (block/with-parent-and-left page-id blocks)]
(paste-blocks blocks' {})))) (paste-blocks blocks' {}))))

7
src/main/frontend/handler/external.cljs
View File

@ -8,8 +8,9 @@
[clojure.string :as string] [clojure.string :as string]
[frontend.db :as db] [frontend.db :as db]
[frontend.format.mldoc :as mldoc] [frontend.format.mldoc :as mldoc]
[logseq.graph-parser.block :as gp-block] [frontend.format.block :as block]
[logseq.graph-parser.util :as gp-util] [logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.date-time-util :as date-time-util]
[frontend.handler.page :as page] [frontend.handler.page :as page]
[frontend.handler.editor :as editor] [frontend.handler.editor :as editor]
[frontend.util :as util])) [frontend.util :as util]))
@ -50,7 +51,7 @@
(map (map
(fn [title] (fn [title]
(let [day (date/journal-title->int title) (let [day (date/journal-title->int title)
page-name (util/page-name-sanity-lc (date/int->journal-title day))] page-name (util/page-name-sanity-lc (date-time-util/int->journal-title day (state/get-date-formatter)))]
{:block/name page-name {:block/name page-name
:block/journal? true :block/journal? true
:block/journal-day day})) :block/journal-day day}))
@ -74,7 +75,7 @@
(when-let [repo (state/get-current-repo)] (when-let [repo (state/get-current-repo)]
(let [[headers parsed-blocks] (mldoc/opml->edn data) (let [[headers parsed-blocks] (mldoc/opml->edn data)
parsed-blocks (->> parsed-blocks (->>
(gp-block/extract-blocks parsed-blocks "" true :markdown) (block/extract-blocks parsed-blocks "" true :markdown)
(mapv editor/wrap-parse-block)) (mapv editor/wrap-parse-block))
page-name (:title headers)] page-name (:title headers)]
(when (not (db/page-exists? page-name)) (when (not (db/page-exists? page-name))

80
src/main/frontend/handler/extract.cljs
View File

@ -48,8 +48,8 @@
[repo-url format ast properties file content] [repo-url format ast properties file content]
(try (try
(let [page (get-page-name file ast) (let [page (get-page-name file ast)
[_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page) [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page (state/get-date-formatter))
blocks (->> (gp-block/extract-blocks ast content false format) blocks (->> (block/extract-blocks ast content false format)
(block/with-parent-and-left {:block/name page-name})) (block/with-parent-and-left {:block/name page-name}))
ref-pages (atom #{}) ref-pages (atom #{})
ref-tags (atom #{}) ref-tags (atom #{})
@ -66,7 +66,7 @@
:block/page [:block/name page-name] :block/page [:block/name page-name]
:block/refs block-ref-pages :block/refs block-ref-pages
:block/path-refs block-path-ref-pages)))) :block/path-refs block-path-ref-pages))))
blocks) blocks)
page-entity (let [alias (:alias properties) page-entity (let [alias (:alias properties)
alias (if (string? alias) [alias] alias) alias (if (string? alias) [alias] alias)
aliases (and alias aliases (and alias
@ -75,56 +75,56 @@
alias))) alias)))
aliases (->> aliases (->>
(map (map
(fn [alias] (fn [alias]
(let [page-name (util/page-name-sanity-lc alias) (let [page-name (util/page-name-sanity-lc alias)
aliases (distinct aliases (distinct
(conj (conj
(remove #{alias} aliases) (remove #{alias} aliases)
page)) page))
aliases (when (seq aliases) aliases (when (seq aliases)
(map (map
(fn [alias] (fn [alias]
{:block/name (util/page-name-sanity-lc alias)}) {:block/name (util/page-name-sanity-lc alias)})
aliases))] aliases))]
(if (seq aliases) (if (seq aliases)
{:block/name page-name {:block/name page-name
:block/alias aliases} :block/alias aliases}
{:block/name page-name}))) {:block/name page-name})))
aliases) aliases)
(remove nil?))] (remove nil?))]
(cond-> (cond->
(gp-util/remove-nils (gp-util/remove-nils
(assoc (assoc
(gp-block/page-name->map page false) (block/page-name->map page false)
:block/file {:file/path (gp-util/path-normalize file)})) :block/file {:file/path (gp-util/path-normalize file)}))
(seq properties) (seq properties)
(assoc :block/properties properties) (assoc :block/properties properties)
(seq aliases) (seq aliases)
(assoc :block/alias aliases) (assoc :block/alias aliases)
(:tags properties) (:tags properties)
(assoc :block/tags (let [tags (:tags properties) (assoc :block/tags (let [tags (:tags properties)
tags (if (string? tags) [tags] tags) tags (if (string? tags) [tags] tags)
tags (remove string/blank? tags)] tags (remove string/blank? tags)]
(swap! ref-tags set/union (set tags)) (swap! ref-tags set/union (set tags))
(map (fn [tag] {:block/name (util/page-name-sanity-lc tag) (map (fn [tag] {:block/name (util/page-name-sanity-lc tag)
:block/original-name tag}) :block/original-name tag})
tags))))) tags)))))
namespace-pages (let [page (:block/original-name page-entity)] namespace-pages (let [page (:block/original-name page-entity)]
(when (text/namespace-page? page) (when (text/namespace-page? page)
(->> (gp-util/split-namespace-pages page) (->> (gp-util/split-namespace-pages page)
(map (fn [page] (map (fn [page]
(-> (gp-block/page-name->map page true) (-> (block/page-name->map page true)
(assoc :block/format format))))))) (assoc :block/format format)))))))
pages (->> (concat pages (->> (concat
[page-entity] [page-entity]
@ref-pages @ref-pages
(map (map
(fn [page] (fn [page]
{:block/original-name page {:block/original-name page
:block/name (util/page-name-sanity-lc page)}) :block/name (util/page-name-sanity-lc page)})
@ref-tags) @ref-tags)
namespace-pages) namespace-pages)
;; remove block references ;; remove block references
(remove vector?) (remove vector?)

7
src/main/frontend/handler/page.cljs
View File

@ -35,6 +35,7 @@
[logseq.graph-parser.util :as gp-util] [logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.config :as gp-config] [logseq.graph-parser.config :as gp-config]
[logseq.graph-parser.block :as gp-block] [logseq.graph-parser.block :as gp-block]
[frontend.format.block :as block]
[goog.functions :refer [debounce]])) [goog.functions :refer [debounce]]))
(defn- get-directory (defn- get-directory
@ -72,7 +73,9 @@
(let [p (common-handler/get-page-default-properties title) (let [p (common-handler/get-page-default-properties title)
ps (merge p properties) ps (merge p properties)
content (page-property/insert-properties format "" ps) content (page-property/insert-properties format "" ps)
refs (gp-block/get-page-refs-from-properties properties)] refs (gp-block/get-page-refs-from-properties properties
(db/get-db (state/get-current-repo))
(state/get-date-formatter))]
{:block/uuid (db/new-block-id) {:block/uuid (db/new-block-id)
:block/properties ps :block/properties ps
:block/properties-order (keys ps) :block/properties-order (keys ps)
@ -127,7 +130,7 @@
[title]) [title])
format (or format (state/get-preferred-format)) format (or format (state/get-preferred-format))
pages (map (fn [page] pages (map (fn [page]
(-> (gp-block/page-name->map page true) (-> (block/page-name->map page true)
(assoc :block/format format))) (assoc :block/format format)))
pages) pages)
txs (->> pages txs (->> pages

5
src/main/frontend/util.cljc
View File

@ -713,11 +713,6 @@
(defn drop-nth [n coll] (defn drop-nth [n coll]
(keep-indexed #(when (not= %1 n) %2) coll)) (keep-indexed #(when (not= %1 n) %2) coll))
(defn capitalize-all [s]
(some->> (string/split s #" ")
(map string/capitalize)
(string/join " ")))
#?(:cljs #?(:cljs
(defn react (defn react
[ref] [ref]

145
src/main/logseq/graph_parser/block.cljs → src/main/logseq/graph_parser/block.cljc
View File

@ -1,12 +1,11 @@
(ns logseq.graph-parser.block (ns ^:nbb-compatible logseq.graph-parser.block
;; Disable clj linters since we don't support clj
#?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
:unresolved-symbol {:level :off}}}})
"Block related code needed for graph-parser" "Block related code needed for graph-parser"
(:require [clojure.string :as string] (:require [clojure.string :as string]
[clojure.walk :as walk] [clojure.walk :as walk]
[frontend.config :as config]
[frontend.date :as date]
[frontend.db :as db]
[datascript.core :as d] [datascript.core :as d]
[frontend.state :as state]
[logseq.graph-parser.text :as text] [logseq.graph-parser.text :as text]
[logseq.graph-parser.utf8 :as utf8] [logseq.graph-parser.utf8 :as utf8]
[logseq.graph-parser.property :as gp-property] [logseq.graph-parser.property :as gp-property]
@ -14,7 +13,8 @@
[logseq.graph-parser.config :as gp-config] [logseq.graph-parser.config :as gp-config]
[logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.date-time-util :as date-time-util] [logseq.graph-parser.date-time-util :as date-time-util]
[lambdaisland.glogi :as log])) #?(:org.babashka/nbb [logseq.graph-parser.log :as log]
:default [lambdaisland.glogi :as log])))
(defn heading-block? (defn heading-block?
[block] [block]
@ -36,8 +36,8 @@
""))) "")))
(string/join)))) (string/join))))
(defn get-page-reference (defn- get-page-reference
[block] [block supported-formats]
(let [page (cond (let [page (cond
(and (vector? block) (= "Link" (first block))) (and (vector? block) (= "Link" (first block)))
(let [typ (first (:url (second block))) (let [typ (first (:url (second block)))
@ -65,7 +65,7 @@
(not (string/starts-with? value "file:")) (not (string/starts-with? value "file:"))
(not (gp-config/local-asset? value)) (not (gp-config/local-asset? value))
(or (= ext :excalidraw) (or (= ext :excalidraw)
(not (contains? (config/supported-formats) ext)))) (not (contains? supported-formats ext))))
value))) value)))
(and (and
@ -147,7 +147,7 @@
;; TODO: we should move this to mldoc ;; TODO: we should move this to mldoc
(defn extract-properties (defn extract-properties
[format properties] [format properties user-config]
(when (seq properties) (when (seq properties)
(let [properties (seq properties) (let [properties (seq properties)
page-refs (->> page-refs (->>
@ -177,7 +177,7 @@
(remove string/blank? v) (remove string/blank? v)
(if (string/blank? v) (if (string/blank? v)
nil nil
(text/parse-property format k v (state/get-config)))) (text/parse-property format k v user-config)))
k (keyword k) k (keyword k)
v (if (and v (if (and
(string? v) (string? v)
@ -226,12 +226,12 @@
(defn convert-page-if-journal (defn convert-page-if-journal
"Convert journal file name to user' custom date format" "Convert journal file name to user' custom date format"
[original-page-name] [original-page-name date-formatter]
(when original-page-name (when original-page-name
(let [page-name (gp-util/page-name-sanity-lc original-page-name) (let [page-name (gp-util/page-name-sanity-lc original-page-name)
day (date/journal-title->int page-name)] day (date-time-util/journal-title->int page-name (date-time-util/safe-journal-title-formatters date-formatter))]
(if day (if day
(let [original-page-name (date/int->journal-title day)] (let [original-page-name (date-time-util/int->journal-title day date-formatter)]
[original-page-name (gp-util/page-name-sanity-lc original-page-name) day]) [original-page-name (gp-util/page-name-sanity-lc original-page-name) day])
[original-page-name page-name day])))) [original-page-name page-name day]))))
@ -241,48 +241,46 @@
with-timestamp?: assign timestampes to the map structure. with-timestamp?: assign timestampes to the map structure.
Useful when creating new pages from references or namespaces, Useful when creating new pages from references or namespaces,
as there's no chance to introduce timestamps via editing in page" as there's no chance to introduce timestamps via editing in page"
([original-page-name with-id?] [original-page-name with-id? db with-timestamp? date-formatter]
(page-name->map original-page-name with-id? true)) (cond
([original-page-name with-id? with-timestamp?] (and original-page-name (string? original-page-name))
(cond (let [original-page-name (gp-util/remove-boundary-slashes original-page-name)
(and original-page-name (string? original-page-name)) [original-page-name page-name journal-day] (convert-page-if-journal original-page-name date-formatter)
(let [original-page-name (gp-util/remove-boundary-slashes original-page-name) namespace? (and (not (boolean (text/get-nested-page-name original-page-name)))
[original-page-name page-name journal-day] (convert-page-if-journal original-page-name) (text/namespace-page? original-page-name))
namespace? (and (not (boolean (text/get-nested-page-name original-page-name))) page-entity (some-> db (d/entity [:block/name page-name]))
(text/namespace-page? original-page-name)) original-page-name (or (:block/original-name page-entity) original-page-name)]
page-entity (db/entity [:block/name page-name]) (merge
original-page-name (or (:block/original-name page-entity) original-page-name)] {:block/name page-name
(merge :block/original-name original-page-name}
{:block/name page-name (when with-id?
:block/original-name original-page-name} (if page-entity
(when with-id? {:block/uuid (:block/uuid page-entity)}
(if page-entity {:block/uuid (d/squuid)}))
{:block/uuid (:block/uuid page-entity)} (when namespace?
{:block/uuid (d/squuid)})) (let [namespace (first (gp-util/split-last "/" original-page-name))]
(when namespace? (when-not (string/blank? namespace)
(let [namespace (first (gp-util/split-last "/" original-page-name))] {:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}})))
(when-not (string/blank? namespace) (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity
{:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}}))) (let [current-ms (date-time-util/time-ms)]
(when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity {:block/created-at current-ms
(let [current-ms (date-time-util/time-ms)] :block/updated-at current-ms}))
{:block/created-at current-ms (if journal-day
:block/updated-at current-ms})) {:block/journal? true
(if journal-day :block/journal-day journal-day}
{:block/journal? true {:block/journal? false})))
:block/journal-day journal-day}
{:block/journal? false})))
(and (map? original-page-name) (:block/uuid original-page-name)) (and (map? original-page-name) (:block/uuid original-page-name))
original-page-name original-page-name
(and (map? original-page-name) with-id?) (and (map? original-page-name) with-id?)
(assoc original-page-name :block/uuid (d/squuid)) (assoc original-page-name :block/uuid (d/squuid))
:else :else
nil))) nil))
(defn- with-page-refs (defn- with-page-refs
[{:keys [title body tags refs marker priority] :as block} with-id?] [{:keys [title body tags refs marker priority] :as block} with-id? supported-formats db date-formatter]
(let [refs (->> (concat tags refs [marker priority]) (let [refs (->> (concat tags refs [marker priority])
(remove string/blank?) (remove string/blank?)
(distinct)) (distinct))
@ -293,7 +291,7 @@
(when-not (and (vector? form) (when-not (and (vector? form)
(= (first form) "Custom") (= (first form) "Custom")
(= (second form) "query")) (= (second form) "query"))
(when-let [page (get-page-reference form)] (when-let [page (get-page-reference form supported-formats)]
(swap! refs conj page)) (swap! refs conj page))
(when-let [tag (get-tag form)] (when-let [tag (get-tag form)]
(let [tag (text/page-ref-un-brackets! tag)] (let [tag (text/page-ref-un-brackets! tag)]
@ -315,7 +313,7 @@
(distinct)) (distinct))
refs (->> (distinct (concat refs children-pages)) refs (->> (distinct (concat refs children-pages))
(remove nil?)) (remove nil?))
refs (map (fn [ref] (page-name->map ref with-id?)) refs)] refs (map (fn [ref] (page-name->map ref with-id? db true date-formatter)) refs)]
(assoc block :refs refs)))) (assoc block :refs refs))))
(defn- with-block-refs (defn- with-block-refs
@ -409,7 +407,7 @@
block)) block))
(defn- get-block-content (defn- get-block-content
[utf8-content block format meta] [utf8-content block format meta block-pattern]
(let [content (if-let [end-pos (:end_pos meta)] (let [content (if-let [end-pos (:end_pos meta)]
(utf8/substring utf8-content (utf8/substring utf8-content
(:start_pos meta) (:start_pos meta)
@ -417,7 +415,7 @@
(utf8/substring utf8-content (utf8/substring utf8-content
(:start_pos meta))) (:start_pos meta)))
content (when content content (when content
(let [content (text/remove-level-spaces content format (config/get-block-pattern format))] (let [content (text/remove-level-spaces content format block-pattern)]
(if (or (:pre-block? block) (if (or (:pre-block? block)
(= (:format block) :org)) (= (:format block) :org))
content content
@ -437,7 +435,7 @@
(d/squuid))) (d/squuid)))
(defn get-page-refs-from-properties (defn get-page-refs-from-properties
[properties] [properties db date-formatter]
(let [page-refs (mapcat (fn [v] (cond (let [page-refs (mapcat (fn [v] (cond
(coll? v) (coll? v)
v v
@ -448,18 +446,18 @@
:else :else
nil)) (vals properties)) nil)) (vals properties))
page-refs (remove string/blank? page-refs)] page-refs (remove string/blank? page-refs)]
(map (fn [page] (page-name->map page true)) page-refs))) (map (fn [page] (page-name->map page true db true date-formatter)) page-refs)))
(defn- with-page-block-refs (defn- with-page-block-refs
[block with-id?] [block with-id? supported-formats db date-formatter]
(some-> block (some-> block
(with-page-refs with-id?) (with-page-refs with-id? supported-formats db date-formatter)
with-block-refs with-block-refs
block-tags->pages block-tags->pages
(update :refs (fn [col] (remove nil? col))))) (update :refs (fn [col] (remove nil? col)))))
(defn- with-pre-block-if-exists (defn- with-pre-block-if-exists
[blocks body pre-block-properties encoded-content] [blocks body pre-block-properties encoded-content {:keys [supported-formats db date-formatter]}]
(let [first-block (first blocks) (let [first-block (first blocks)
first-block-start-pos (get-in first-block [:block/meta :start_pos]) first-block-start-pos (get-in first-block [:block/meta :start_pos])
@ -471,7 +469,7 @@
(let [content (utf8/substring encoded-content 0 first-block-start-pos) (let [content (utf8/substring encoded-content 0 first-block-start-pos)
{:keys [properties properties-order]} pre-block-properties {:keys [properties properties-order]} pre-block-properties
id (get-custom-id-or-new-id {:properties properties}) id (get-custom-id-or-new-id {:properties properties})
property-refs (->> (get-page-refs-from-properties properties) property-refs (->> (get-page-refs-from-properties properties db date-formatter)
(map :block/original-name)) (map :block/original-name))
block {:uuid id block {:uuid id
:content content :content content
@ -482,7 +480,7 @@
:pre-block? true :pre-block? true
:unordered true :unordered true
:body body} :body body}
block (with-page-block-refs block false)] block (with-page-block-refs block false supported-formats db date-formatter)]
(block-keywordize block)) (block-keywordize block))
(select-keys first-block [:block/format :block/page])) (select-keys first-block [:block/format :block/page]))
blocks) blocks)
@ -490,7 +488,7 @@
(with-path-refs blocks))) (with-path-refs blocks)))
(defn- construct-block (defn- construct-block
[block properties timestamps body encoded-content format pos-meta with-id?] [block properties timestamps body encoded-content format pos-meta with-id? {:keys [block-pattern supported-formats db date-formatter]}]
(let [id (get-custom-id-or-new-id properties) (let [id (get-custom-id-or-new-id properties)
ref-pages-in-properties (->> (:page-refs properties) ref-pages-in-properties (->> (:page-refs properties)
(remove string/blank?)) (remove string/blank?))
@ -518,12 +516,12 @@
(assoc block :collapsed? true) (assoc block :collapsed? true)
block) block)
block (assoc block block (assoc block
:content (get-block-content encoded-content block format pos-meta)) :content (get-block-content encoded-content block format pos-meta block-pattern))
block (if (seq timestamps) block (if (seq timestamps)
(merge block (timestamps->scheduled-and-deadline timestamps)) (merge block (timestamps->scheduled-and-deadline timestamps))
block) block)
block (assoc block :body body) block (assoc block :body body)
block (with-page-block-refs block with-id?) block (with-page-block-refs block with-id? supported-formats db date-formatter)
{:keys [created-at updated-at]} (:properties properties) {:keys [created-at updated-at]} (:properties properties)
block (cond-> block block (cond-> block
(and created-at (integer? created-at)) (and created-at (integer? created-at))
@ -539,8 +537,10 @@
`blocks`: mldoc ast. `blocks`: mldoc ast.
`content`: markdown or org-mode text. `content`: markdown or org-mode text.
`with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids. `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
`format`: content's format, it could be either :markdown or :org-mode." `format`: content's format, it could be either :markdown or :org-mode.
[blocks content with-id? format] `options`: Options supported are :user-config, :block-pattern :supported-formats,
:date-formatter and :db"
[blocks content with-id? format {:keys [user-config] :as options}]
{:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]} {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
(try (try
(let [encoded-content (utf8/encode content) (let [encoded-content (utf8/encode content)
@ -564,11 +564,11 @@
(recur headings (rest blocks) timestamps' properties body)) (recur headings (rest blocks) timestamps' properties body))
(gp-property/properties-ast? block) (gp-property/properties-ast? block)
(let [properties (extract-properties format (second block))] (let [properties (extract-properties format (second block) user-config)]
(recur headings (rest blocks) timestamps properties body)) (recur headings (rest blocks) timestamps properties body))
(heading-block? block) (heading-block? block)
(let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)] (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id? options)]
(recur (conj headings block) (rest blocks) {} {} [])) (recur (conj headings block) (rest blocks) {} {} []))
:else :else
@ -577,8 +577,7 @@
sanity-blocks-data) sanity-blocks-data)
body body
properties])) properties]))
result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)] result (with-pre-block-if-exists blocks body pre-block-properties encoded-content options)]
(map #(dissoc % :block/meta) result)) (map #(dissoc % :block/meta) result))
(catch js/Error e (catch :default e
(js/console.error "extract-blocks-failed") (log/error :extract-blocks-failure e))))
(log/error :exception e))))

44
src/main/logseq/graph_parser/date_time_util.cljs
View File

@ -1,9 +1,51 @@
(ns ^:nbb-compatible logseq.graph-parser.date-time-util (ns ^:nbb-compatible logseq.graph-parser.date-time-util
"cljs-time util fns for graph-parser" "cljs-time util fns for graph-parser"
(:require [cljs-time.coerce :as tc] (:require [cljs-time.coerce :as tc]
[cljs-time.core :as t])) [cljs-time.core :as t]
[cljs-time.format :as tf]
[clojure.string :as string]
[logseq.graph-parser.util :as gp-util]))
(defn time-ms (defn time-ms
"Copy of util/time-ms. Too basic to couple this to main app" "Copy of util/time-ms. Too basic to couple this to main app"
[] []
(tc/to-long (t/now))) (tc/to-long (t/now)))
;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000
(defn safe-journal-title-formatters
[date-formatter]
(->> [date-formatter "MMM do, yyyy" "yyyy-MM-dd" "yyyy_MM_dd"]
(remove string/blank?)
distinct))
(defn journal-title->
[journal-title then-fn formatters]
(when-not (string/blank? journal-title)
(when-let [time (->> (map
(fn [formatter]
(try
(tf/parse (tf/formatter formatter) (gp-util/capitalize-all journal-title))
(catch js/Error _e
nil)))
formatters)
(filter some?)
first)]
(then-fn time))))
(defn journal-title->int
[journal-title formatters]
(when journal-title
(let [journal-title (gp-util/capitalize-all journal-title)]
(journal-title-> journal-title
#(gp-util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %))
formatters))))
(defn format
[date date-formatter]
(when date-formatter
(tf/unparse (tf/formatter date-formatter) date)))
(defn int->journal-title
[day date-formatter]
(when day
(format (tf/parse (tf/formatter "yyyyMMdd") (str day)) date-formatter)))

6
src/main/logseq/graph_parser/util.cljs
View File

@ -155,3 +155,9 @@
"Sanitize the query string for a page name (mandate for :block/name)" "Sanitize the query string for a page name (mandate for :block/name)"
[s] [s]
(page-name-sanity (string/lower-case s))) (page-name-sanity (string/lower-case s)))
(defn capitalize-all
[s]
(some->> (string/split s #" ")
(map string/capitalize)
(string/join " ")))

4
src/test/logseq/graph_parser/block_test.cljs
View File

@ -3,7 +3,7 @@
[cljs.test :refer [deftest are]])) [cljs.test :refer [deftest are]]))
(deftest test-extract-properties (deftest test-extract-properties
(are [x y] (= (:properties (gp-block/extract-properties :markdown x)) y) (are [x y] (= (:properties (gp-block/extract-properties :markdown x {})) y)
[["year" "1000"]] {:year 1000} [["year" "1000"]] {:year 1000}
[["year" "\"1000\""]] {:year "\"1000\""} [["year" "\"1000\""]] {:year "\"1000\""}
[["background-color" "#000000"]] {:background-color "#000000"} [["background-color" "#000000"]] {:background-color "#000000"}
@ -23,7 +23,7 @@
[["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}} [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}}
[["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}}) [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}})
(are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x))) y) (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x {}))) y)
[["year" "1000"]] [] [["year" "1000"]] []
[["year" "\"1000\""]] [] [["year" "\"1000\""]] []
[["foo" "[[bar]] test"]] ["bar" "test"] [["foo" "[[bar]] test"]] ["bar" "test"]

4
src/test/logseq/graph_parser/nbb_test_runner.cljs
View File

@ -5,6 +5,7 @@
[logseq.graph-parser.text :as text] [logseq.graph-parser.text :as text]
[logseq.graph-parser.text-test] [logseq.graph-parser.text-test]
[logseq.graph-parser.mldoc-test] [logseq.graph-parser.mldoc-test]
[logseq.graph-parser.block-test]
[logseq.graph-parser.property-test])) [logseq.graph-parser.property-test]))
(defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m]
@ -19,4 +20,5 @@
(alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property)) (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property))
(t/run-tests 'logseq.graph-parser.mldoc-test (t/run-tests 'logseq.graph-parser.mldoc-test
'logseq.graph-parser.text-test 'logseq.graph-parser.text-test
'logseq.graph-parser.property-test)) 'logseq.graph-parser.property-test
'logseq.graph-parser.block-test))