Setup cljs tests for graph-parser

- Also reuse docs graph test assertions to ensure main app
and graph-parser are in sync.
- Removed promesa from extract tests
since they are no longer async
- Added cljs and npm deps that were missing and caught by cljs tests
pull/5420/head
Gabriel Horner 2022-05-25 17:38:06 -04:00
parent 15b8e5e9df
commit 1e299052ce
14 changed files with 638 additions and 317 deletions

View File

@ -79,8 +79,6 @@ jobs:
yarn cljs:test
node static/tests.js
- name: Run nbb tests for graph-parser
run: yarn nbb-logseq -cp deps/graph-parser/src:deps/graph-parser/test -m logseq.graph-parser.nbb-test-runner/run-tests
# In this job because it depends on an npm package
- name: Load nbb compatible namespaces
run: bb test:load-nbb-compatible-namespaces

View File

@ -19,9 +19,65 @@ env:
JAVA_VERSION: '8'
# This is the latest node version we can run.
NODE_VERSION: '16'
BABASHKA_VERSION: '0.8.1'
BABASHKA_VERSION: '0.8.2'
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up Node
uses: actions/setup-node@v2
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'yarn'
cache-dependency-path: deps/graph-parser/yarn.lock
- name: Set up Java
uses: actions/setup-java@v2
with:
distribution: 'zulu'
java-version: ${{ env.JAVA_VERSION }}
- name: Set up Clojure
uses: DeLaGuardo/setup-clojure@master
with:
cli: ${{ env.CLOJURE_VERSION }}
# - name: Setup Babashka
# uses: turtlequeue/setup-babashka@v1.3.0
# with:
# babashka-version: ${{ env.BABASHKA_VERSION }}
- name: Clojure cache
uses: actions/cache@v2
id: clojure-deps
with:
path: |
~/.m2/repository
~/.gitlibs
key: ${{ runner.os }}-clojure-deps-${{ hashFiles('deps.edn') }}
restore-keys: ${{ runner.os }}-clojure-deps-
- name: Fetch Clojure deps
if: steps.clojure-deps.outputs.cache-hit != 'true'
run: clojure -A:test -P
- name: Fetch yarn deps
run: cd deps/graph-parser && yarn install --frozen-lockfile
- name: Run ClojureScript tests
run: clojure -M:test
- name: Run nbb-logseq tests
run: cd deps/graph-parser && yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests
# # In this job because it depends on an npm package
# - name: Load nbb compatible namespaces
# run: bb test:load-nbb-compatible-namespaces
lint:
runs-on: ubuntu-latest

View File

@ -3,5 +3,7 @@
;; carve doesn't detect nbb only usage
logseq.graph-parser.log
;; Used by logseq but not worth splitting up
logseq.graph-parser.db.schema]
logseq.graph-parser.db.schema
;; Used in tests
logseq.graph-parser.test.docs-graph-helper]
:report {:format :ignore}}

View File

@ -1 +1,3 @@
.clj-kondo/.cache
/.clj-kondo/.cache
cljs-test-runner-out
/test/docs

View File

@ -1,14 +1,23 @@
{:paths ["src"
;; Needed for logseq tests to pull in logseq.graph-parser.test*
"test"]
{:paths ["src"]
:deps
;; Deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
{datascript/datascript {:mvn/version "1.3.8"}
frankiesardo/linked {:mvn/version "1.3.0"}
com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
:sha "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
com.lambdaisland/glogi {:mvn/version "1.1.144"}}
;; stubbed in nbb
com.lambdaisland/glogi {:mvn/version "1.1.144"}
;; built in to nbb
cljs-bean/cljs-bean {:mvn/version "1.5.0"}}
:aliases
{:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.04.25"}}
;; This runs tests with nodejs. Would be nice to run this with a headless env since
;; this is how its normally run in the app but this requires more setup with
;; karma and shadow-cljs.edn
{:test {:extra-paths ["test"]
:extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}
org.clojure/clojurescript {:mvn/version "1.11.54"}}
:main-opts ["-m" "cljs-test-runner.main"]}
:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.04.25"}}
:main-opts ["-m" "clj-kondo.main"]}}}

View File

@ -4,5 +4,8 @@
"private": true,
"devDependencies": {
"@logseq/nbb-logseq": "^0.5.103"
},
"dependencies": {
"mldoc": "^1.3.3"
}
}

View File

@ -2,7 +2,8 @@
"Property fns needed by graph-parser"
(:require [logseq.graph-parser.util :as gp-util]
[clojure.string :as string]
[goog.string :as gstring]))
[goog.string :as gstring]
[goog.string.format]))
(defn properties-ast?
[block]

View File

@ -0,0 +1,163 @@
(ns ^:nbb-compatible logseq.graph-parser.test.docs-graph-helper
"Helper fns for setting up and running tests against docs graph"
(:require ["fs" :as fs]
["child_process" :as child-process]
[cljs.test :refer [is testing]]
[clojure.string :as string]
[datascript.core :as d]))
;; Helper fns for test setup
;; =========================
(defn slurp
"Like clojure.core/slurp"
[file]
(str (fs/readFileSync file)))
(defn- sh
"Run shell cmd synchronously and print to inherited streams by default. Aims
to be similar to babashka.tasks/shell"
[cmd opts]
(child-process/spawnSync (first cmd)
(clj->js (rest cmd))
(clj->js (merge {:stdio "inherit"} opts))))
(defn build-graph-files
[dir]
(let [files (->> (str (.-stdout (sh ["git" "ls-files"]
{:cwd dir :stdio nil})))
string/split-lines
(filter #(re-find #"^(pages|journals)" %))
(map #(str dir "/" %)))]
(mapv #(hash-map :file/path % :file/content (slurp %)) files)))
(defn clone-docs-repo-if-not-exists
[dir]
(when-not (.existsSync fs dir)
(sh ["git" "clone" "--depth" "1" "-b" "v0.6.7" "-c" "advice.detachedHead=false"
"https://github.com/logseq/docs" dir] {})))
;; Fns for common test assertions
;; ==============================
(defn- get-top-block-properties
[db]
(->> (d/q '[:find (pull ?b [*])
:where
[?b :block/properties]
[(missing? $ ?b :block/name)]]
db)
(map first)
(map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
(apply merge-with +)
(filter #(>= (val %) 5))
(into {})))
(defn- get-all-page-properties
[db]
(->> (d/q '[:find (pull ?b [*])
:where
[?b :block/properties]
[?b :block/name]]
db)
(map first)
(map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
(apply merge-with +)
(into {})))
(defn docs-graph-assertions
"These are common assertions that should pass in both graph-parser and main
logseq app. It is important to run these in both contexts to ensure that the
functionality in frontend.handler.repo and logseq.graph-parser remain the
same"
[db files]
;; Counts assertions help check for no major regressions. These counts should
;; only increase over time as the docs graph rarely has deletions
(testing "Counts"
(is (= 206 (count files)) "Correct file count")
(is (= 40888 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 3597
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
"Correct referenced blocks count")
(is (= 21
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/content ?content]
[(clojure.string/includes? ?content "+BEGIN_QUERY")]]
db)))
"Advanced query count"))
(testing "Query based stats"
(is (= (set (map :file/path files))
(->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}])
:where [?b :block/name] [?b :block/file]]
db)
(map (comp #(get-in % [:block/file :file/path]) first))
set))
"Journal and pages files on disk should equal ones in db")
(is (= (count (filter #(re-find #"journals/" (:file/path %))
files))
(->> (d/q '[:find (count ?b)
:where
[?b :block/journal? true]
[?b :block/name]
[?b :block/file]]
db)
ffirst))
"Journal page count on disk equals count in db")
(is (= {"CANCELED" 2 "DONE" 6 "LATER" 4 "NOW" 5}
(->> (d/q '[:find (pull ?b [*]) :where [?b :block/marker] ]
db)
(map first)
(group-by :block/marker)
(map (fn [[k v]] [k (count v)]))
(into {})))
"Task marker counts")
(is (= {:markdown 3140 :org 460}
(->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db)
(map first)
(group-by :block/format)
(map (fn [[k v]] [k (count v)]))
(into {})))
"Block format counts")
(is (= {:title 98 :id 98
:updated-at 47 :created-at 47
:card-last-score 6 :card-repeats 6 :card-next-schedule 6
:card-last-interval 6 :card-ease-factor 6 :card-last-reviewed 6
:alias 6}
(get-top-block-properties db))
"Counts for top block properties")
(is (= {:title 98
:alias 6
:tags 2 :permalink 2
:name 1 :type 1 :related 1 :sample 1 :click 1 :id 1 :example 1}
(get-all-page-properties db))
"Counts for all page properties")
(is (= {:block/scheduled 2
:block/priority 4
:block/deadline 1
:block/collapsed? 22
:block/heading-level 57
:block/repeated? 1}
(->> [:block/scheduled :block/priority :block/deadline :block/collapsed?
:block/heading-level :block/repeated?]
(map (fn [attr]
[attr
(ffirst (d/q [:find (list 'count '?b) :where ['?b attr]]
db))]))
(into {})))
"Counts for blocks with common block attributes")
(is (= #{"term" "setting" "book" "Templates" "Query" "Query/table" "page"}
(->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db)
(map (comp :block/original-name first))
set))
"Has correct namespaces")))

View File

@ -1,12 +1,11 @@
(ns logseq.graph-parser.extract-test
(:require [cljs.test :refer [async deftest is]]
(:require [cljs.test :refer [deftest is]]
[logseq.graph-parser.extract :as extract]
[clojure.pprint :as pprint]
[promesa.core :as p]))
[clojure.pprint :as pprint]))
(defn- extract
[text]
(p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
(let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
result (last result)
lefts (map (juxt :block/parent :block/left) result)]
(if (not= (count lefts) (count (distinct lefts)))
@ -15,34 +14,24 @@
(throw (js/Error. ":block/parent && :block/left conflicts")))
(mapv :block/content result))))
(defn- async-test
[x y]
(async done
(p/then
(extract x)
(fn [v]
(is (= y v))
(done)))))
(deftest test-extract-blocks-pages
[]
(async-test
(is (= ["a" "b" "c"]
(extract
"- a
- b
- c"
["a" "b" "c"])
- c")))
(async-test
"## hello
(is (= ["## hello" "world" "nice" "nice" "bingo" "world"]
(extract "## hello
- world
- nice
- nice
- bingo
- world"
["## hello" "world" "nice" "nice" "bingo" "world"])
- world")))
(async-test
"# a
(is (= ["# a" "## b" "### c" "#### d" "### e" "f" "g" "h" "i" "j"]
(extract "# a
## b
### c
#### d
@ -51,17 +40,13 @@
- g
- h
- i
- j"
["# a" "## b" "### c" "#### d" "### e" "f" "g" "h" "i" "j"]))
- j"))))
(deftest test-regression-1902
[]
(async-test
(is (= ["line1" "line2" "line3" "line4"]
(extract
"- line1
- line2
- line3
- line4"
["line1" "line2" "line3" "line4"]))
#_(cljs.test/run-tests)
- line4"))))

View File

@ -98,7 +98,7 @@
: definition" md-config {})))))
(deftest ^:integration test->edn
(let [graph-dir "src/test/docs"
(let [graph-dir "test/docs"
_ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
files (docs-graph-helper/build-graph-files graph-dir)
asts-by-file (->> files

View File

@ -1,34 +0,0 @@
(ns ^:nbb-compatible logseq.graph-parser.test.docs-graph-helper
"Helper fns for running tests against docs graph"
(:require ["fs" :as fs]
["child_process" :as child-process]
[clojure.string :as string]))
(defn slurp
"Like clojure.core/slurp"
[file]
(str (fs/readFileSync file)))
(defn- sh
"Run shell cmd synchronously and print to inherited streams by default. Aims
to be similar to babashka.tasks/shell"
[cmd opts]
(child-process/spawnSync (first cmd)
(clj->js (rest cmd))
(clj->js (merge {:stdio "inherit"} opts))))
(defn build-graph-files
[dir]
(let [files (->> (str (.-stdout (sh ["git" "ls-files"]
{:cwd dir :stdio nil})))
string/split-lines
(filter #(re-find #"^(pages|journals)" %))
(map #(str dir "/" %)))]
(mapv #(hash-map :file/path % :file/content (slurp %)) files)))
(defn clone-docs-repo-if-not-exists
[dir]
(when-not (.existsSync fs dir)
(sh ["git" "clone" "--depth" "1" "-b" "v0.6.7" "-c" "advice.detachedHead=false"
"https://github.com/logseq/docs" dir] {})))

View File

@ -1,134 +1,17 @@
(ns logseq.graph-parser-test
"TODO: Should I reuse repo-test or split it?"
(:require [cljs.test :refer [deftest is testing]]
(:require [cljs.test :refer [deftest]]
[logseq.graph-parser :as graph-parser]
[logseq.graph-parser.db :as gp-db]
[logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
[datascript.core :as d]))
[logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]))
(defn- get-top-block-properties
[db]
(->> (d/q '[:find (pull ?b [*])
:where
[?b :block/properties]
[(missing? $ ?b :block/name)]]
db)
(map first)
(map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
(apply merge-with +)
(filter #(>= (val %) 5))
(into {})))
(defn- get-all-page-properties
[db]
(->> (d/q '[:find (pull ?b [*])
:where
[?b :block/properties]
[?b :block/name]]
db)
(map first)
(map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
(apply merge-with +)
(into {})))
;; Integration test that test parsing a large graph like docs
(deftest ^:integration parse-and-load-files-to-db
(let [graph-dir "src/test/docs"
(let [graph-dir "test/docs"
_ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
files (docs-graph-helper/build-graph-files graph-dir)
conn (gp-db/start-conn)
; _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false})
_ (graph-parser/parse conn files)
db @conn]
;; Counts assertions help check for no major regressions. These counts should
;; only increase over time as the docs graph rarely has deletions
(testing "Counts"
(is (= 206 (count files)) "Correct file count")
(is (= 40888 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 3597
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
"Correct referenced blocks count")
(is (= 21
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/content ?content]
[(clojure.string/includes? ?content "+BEGIN_QUERY")]]
db)))
"Advanced query count"))
(testing "Query based stats"
(is (= (set (map :file/path files))
(->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}])
:where [?b :block/name] [?b :block/file]]
db)
(map (comp #(get-in % [:block/file :file/path]) first))
set))
"Journal and pages files on disk should equal ones in db")
(is (= (count (filter #(re-find #"journals/" (:file/path %))
files))
(->> (d/q '[:find (count ?b)
:where
[?b :block/journal? true]
[?b :block/name]
[?b :block/file]]
db)
ffirst))
"Journal page count on disk equals count in db")
(is (= {"CANCELED" 2 "DONE" 6 "LATER" 4 "NOW" 5}
(->> (d/q '[:find (pull ?b [*]) :where [?b :block/marker] ]
db)
(map first)
(group-by :block/marker)
(map (fn [[k v]] [k (count v)]))
(into {})))
"Task marker counts")
(is (= {:markdown 3140 :org 460}
(->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db)
(map first)
(group-by :block/format)
(map (fn [[k v]] [k (count v)]))
(into {})))
"Block format counts")
(is (= {:title 98 :id 98
:updated-at 47 :created-at 47
:card-last-score 6 :card-repeats 6 :card-next-schedule 6
:card-last-interval 6 :card-ease-factor 6 :card-last-reviewed 6
:alias 6}
(get-top-block-properties db))
"Counts for top block properties")
(is (= {:title 98
:alias 6
:tags 2 :permalink 2
:name 1 :type 1 :related 1 :sample 1 :click 1 :id 1 :example 1}
(get-all-page-properties db))
"Counts for all page properties")
(is (= {:block/scheduled 2
:block/priority 4
:block/deadline 1
:block/collapsed? 22
:block/heading-level 57
:block/repeated? 1}
(->> [:block/scheduled :block/priority :block/deadline :block/collapsed?
:block/heading-level :block/repeated?]
(map (fn [attr]
[attr
(ffirst (d/q [:find (list 'count '?b) :where ['?b attr]]
db))]))
(into {})))
"Counts for blocks with common block attributes")
(is (= #{"term" "setting" "book" "Templates" "Query" "Query/table" "page"}
(->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db)
(map (comp :block/original-name first))
set))
"Has correct namespaces"))))
(docs-graph-helper/docs-graph-assertions db files)))

View File

@ -9,6 +9,16 @@
dependencies:
import-meta-resolve "^1.1.1"
ansi-regex@^2.0.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"
integrity sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==
ansi-regex@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.1.tgz#123d6479e92ad45ad897d4054e3c7ca7db4944e1"
integrity sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==
builtins@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/builtins/-/builtins-4.1.0.tgz#1edd016dd91ce771a1ed6fc3b2b71fb918953250"
@ -16,6 +26,80 @@ builtins@^4.0.0:
dependencies:
semver "^7.0.0"
camelcase@^5.0.0:
version "5.3.1"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
cliui@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49"
integrity sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==
dependencies:
string-width "^2.1.1"
strip-ansi "^4.0.0"
wrap-ansi "^2.0.0"
code-point-at@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=
cross-spawn@^6.0.0:
version "6.0.5"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==
dependencies:
nice-try "^1.0.4"
path-key "^2.0.1"
semver "^5.5.0"
shebang-command "^1.2.0"
which "^1.2.9"
decamelize@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=
end-of-stream@^1.1.0:
version "1.4.4"
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
dependencies:
once "^1.4.0"
execa@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==
dependencies:
cross-spawn "^6.0.0"
get-stream "^4.0.0"
is-stream "^1.1.0"
npm-run-path "^2.0.0"
p-finally "^1.0.0"
signal-exit "^3.0.0"
strip-eof "^1.0.0"
find-up@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
dependencies:
locate-path "^3.0.0"
get-caller-file@^1.0.1:
version "1.0.3"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a"
integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==
get-stream@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==
dependencies:
pump "^3.0.0"
import-meta-resolve@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-1.1.1.tgz#244fd542fd1fae73550d4f8b3cde3bba1d7b2b18"
@ -23,6 +107,48 @@ import-meta-resolve@^1.1.1:
dependencies:
builtins "^4.0.0"
invert-kv@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
is-fullwidth-code-point@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb"
integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs=
dependencies:
number-is-nan "^1.0.0"
is-fullwidth-code-point@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f"
integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=
is-stream@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ=
isexe@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
lcid@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/lcid/-/lcid-2.0.0.tgz#6ef5d2df60e52f82eb228a4c373e8d1f397253cf"
integrity sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==
dependencies:
invert-kv "^2.0.0"
locate-path@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e"
integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==
dependencies:
p-locate "^3.0.0"
path-exists "^3.0.0"
lru-cache@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
@ -30,6 +156,134 @@ lru-cache@^6.0.0:
dependencies:
yallist "^4.0.0"
map-age-cleaner@^0.1.1:
version "0.1.3"
resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a"
integrity sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==
dependencies:
p-defer "^1.0.0"
mem@^4.0.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/mem/-/mem-4.3.0.tgz#461af497bc4ae09608cdb2e60eefb69bff744178"
integrity sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==
dependencies:
map-age-cleaner "^0.1.1"
mimic-fn "^2.0.0"
p-is-promise "^2.0.0"
mimic-fn@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
mldoc@^1.3.3:
version "1.3.3"
resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.3.3.tgz#b7f39b48eb0ef3558619d3e3522265977bd78fe3"
integrity sha512-TzW06GBltdKxwWAxOvflPmIVedu6bzl9T4YoYqnDUyZ3kELFMllEgiYCh65PPW3xsRMA/5OcRQqqGZGiKEJEug==
dependencies:
yargs "^12.0.2"
nice-try@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
npm-run-path@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f"
integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=
dependencies:
path-key "^2.0.0"
number-is-nan@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=
once@^1.3.1, once@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
dependencies:
wrappy "1"
os-locale@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-3.1.0.tgz#a802a6ee17f24c10483ab9935719cef4ed16bf1a"
integrity sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==
dependencies:
execa "^1.0.0"
lcid "^2.0.0"
mem "^4.0.0"
p-defer@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-1.0.0.tgz#9f6eb182f6c9aa8cd743004a7d4f96b196b0fb0c"
integrity sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=
p-finally@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=
p-is-promise@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e"
integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==
p-limit@^2.0.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==
dependencies:
p-try "^2.0.0"
p-locate@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4"
integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==
dependencies:
p-limit "^2.0.0"
p-try@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
path-exists@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=
path-key@^2.0.0, path-key@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=
pump@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
dependencies:
end-of-stream "^1.1.0"
once "^1.3.1"
require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
require-main-filename@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1"
integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=
semver@^5.5.0:
version "5.7.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
semver@^7.0.0:
version "7.3.7"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
@ -37,7 +291,121 @@ semver@^7.0.0:
dependencies:
lru-cache "^6.0.0"
set-blocking@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
shebang-command@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea"
integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=
dependencies:
shebang-regex "^1.0.0"
shebang-regex@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=
signal-exit@^3.0.0:
version "3.0.7"
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==
string-width@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=
dependencies:
code-point-at "^1.0.0"
is-fullwidth-code-point "^1.0.0"
strip-ansi "^3.0.0"
string-width@^2.0.0, string-width@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==
dependencies:
is-fullwidth-code-point "^2.0.0"
strip-ansi "^4.0.0"
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=
dependencies:
ansi-regex "^2.0.0"
strip-ansi@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f"
integrity sha1-qEeQIusaw2iocTibY1JixQXuNo8=
dependencies:
ansi-regex "^3.0.0"
strip-eof@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=
which-module@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=
which@^1.2.9:
version "1.3.1"
resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
dependencies:
isexe "^2.0.0"
wrap-ansi@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"
integrity sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=
dependencies:
string-width "^1.0.1"
strip-ansi "^3.0.1"
wrappy@1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
"y18n@^3.2.1 || ^4.0.0":
version "4.0.3"
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf"
integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==
yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yargs-parser@^11.1.1:
version "11.1.1"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-11.1.1.tgz#879a0865973bca9f6bab5cbdf3b1c67ec7d3bcf4"
integrity sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==
dependencies:
camelcase "^5.0.0"
decamelize "^1.2.0"
yargs@^12.0.2:
version "12.0.5"
resolved "https://registry.yarnpkg.com/yargs/-/yargs-12.0.5.tgz#05f5997b609647b64f66b81e3b4b10a368e7ad13"
integrity sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==
dependencies:
cliui "^4.0.0"
decamelize "^1.2.0"
find-up "^3.0.0"
get-caller-file "^1.0.1"
os-locale "^3.0.0"
require-directory "^2.1.1"
require-main-filename "^1.0.1"
set-blocking "^2.0.0"
string-width "^2.0.0"
which-module "^2.0.0"
y18n "^3.2.1 || ^4.0.0"
yargs-parser "^11.1.1"

View File

@ -9,131 +9,16 @@
(use-fixtures :each {:before test-helper/start-test-db!
:after test-helper/destroy-test-db!})
(defn- get-top-block-properties
[db]
(->> (d/q '[:find (pull ?b [*])
:where
[?b :block/properties]
[(missing? $ ?b :block/name)]]
db)
(map first)
(map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
(apply merge-with +)
(filter #(>= (val %) 5))
(into {})))
(defn- get-all-page-properties
[db]
(->> (d/q '[:find (pull ?b [*])
:where
[?b :block/properties]
[?b :block/name]]
db)
(map first)
(map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1))))
(apply merge-with +)
(into {})))
;; Integration test that test parsing a large graph like docs
;; TODO: remove large-var and finish delete testing
(deftest ^:large-vars/cleanup-todo ^:integration parse-and-load-files-to-db
(deftest ^:integration parse-and-load-files-to-db
(let [graph-dir "src/test/docs"
_ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir)
files (docs-graph-helper/build-graph-files graph-dir)
_ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false})
db (conn/get-db test-helper/test-db)]
;; Counts assertions help check for no major regressions. These counts should
;; only increase over time as the docs graph rarely has deletions
(testing "Counts"
(is (= 206 (count files)) "Correct file count")
(is (= 40888 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 3597
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
"Correct referenced blocks count")
(is (= 21
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/content ?content]
[(clojure.string/includes? ?content "+BEGIN_QUERY")]]
db)))
"Advanced query count"))
(testing "Query based stats"
(is (= (set (map :file/path files))
(->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}])
:where [?b :block/name] [?b :block/file]]
db)
(map (comp #(get-in % [:block/file :file/path]) first))
set))
"Journal and pages files on disk should equal ones in db")
(is (= (count (filter #(re-find #"journals/" (:file/path %))
files))
(->> (d/q '[:find (count ?b)
:where
[?b :block/journal? true]
[?b :block/name]
[?b :block/file]]
db)
ffirst))
"Journal page count on disk equals count in db")
(is (= {"CANCELED" 2 "DONE" 6 "LATER" 4 "NOW" 5}
(->> (d/q '[:find (pull ?b [*]) :where [?b :block/marker] ]
db)
(map first)
(group-by :block/marker)
(map (fn [[k v]] [k (count v)]))
(into {})))
"Task marker counts")
(is (= {:markdown 3140 :org 460}
(->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db)
(map first)
(group-by :block/format)
(map (fn [[k v]] [k (count v)]))
(into {})))
"Block format counts")
(is (= {:title 98 :id 98
:updated-at 47 :created-at 47
:card-last-score 6 :card-repeats 6 :card-next-schedule 6
:card-last-interval 6 :card-ease-factor 6 :card-last-reviewed 6
:alias 6}
(get-top-block-properties db))
"Counts for top block properties")
(is (= {:title 98
:alias 6
:tags 2 :permalink 2
:name 1 :type 1 :related 1 :sample 1 :click 1 :id 1 :example 1}
(get-all-page-properties db))
"Counts for all page properties")
(is (= {:block/scheduled 2
:block/priority 4
:block/deadline 1
:block/collapsed? 22
:block/heading-level 57
:block/repeated? 1}
(->> [:block/scheduled :block/priority :block/deadline :block/collapsed?
:block/heading-level :block/repeated?]
(map (fn [attr]
[attr
(ffirst (d/q [:find (list 'count '?b) :where ['?b attr]]
db))]))
(into {})))
"Counts for blocks with common block attributes")
(is (= #{"term" "setting" "book" "Templates" "Query" "Query/table" "page"}
(->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db)
(map (comp :block/original-name first))
set))
"Has correct namespaces"))
#_:clj-kondo/ignore ;; buggy unresolved var
(docs-graph-helper/docs-graph-assertions db files)
(testing "Delete previous file data when re-parsing a file"
(repo-handler/parse-files-and-load-to-db! test-helper/test-db