mirror of https://github.com/logseq/logseq
feat(sync): support concurrent full-sync local->remote
parent
3517ae9fea
commit
85bb55a784
|
@ -35,7 +35,8 @@
|
|||
"diff-match-patch": "1.0.5",
|
||||
"https-proxy-agent": "5.0.0",
|
||||
"@sentry/electron": "2.5.1",
|
||||
"posthog-js": "1.10.2"
|
||||
"posthog-js": "1.10.2",
|
||||
"@andelf/rsapi": "0.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electron-forge/cli": "^6.0.0-beta.57",
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
(ns electron.file-sync-rsapi
|
||||
(:require ["rsapi" :as rsapi]))
|
||||
(:require ["@andelf/rsapi" :as rsapi]))
|
||||
|
||||
(defn get-local-files-meta [graph-uuid base-path file-paths]
|
||||
(rsapi/getLocalFilesMeta graph-uuid base-path (clj->js file-paths)))
|
||||
|
|
|
@ -619,6 +619,7 @@
|
|||
|
||||
(def local-changes-chan (chan 100))
|
||||
|
||||
;;; type = "change" | "add" | "unlink"
|
||||
(deftype FileChangeEvent [type dir path stat]
|
||||
IRelativePath
|
||||
(-relative-path [_] (remove-dir-prefix dir path))
|
||||
|
@ -633,6 +634,20 @@
|
|||
(-pr-writer [_ w _opts]
|
||||
(write-all w (str {:type type :base-path dir :path path}))))
|
||||
|
||||
(defn- partition-file-change-events
|
||||
"return transducer.
|
||||
partition `FileChangeEvent`s, at most N file-change-events in each partition.
|
||||
only one type in a partition."
|
||||
[n]
|
||||
(comp
|
||||
(partition-by (fn [^FileChangeEvent e]
|
||||
(case (.-type e)
|
||||
("add" "change") :add-or-change
|
||||
"unlink" :unlink)))
|
||||
(map #(partition-all n %))
|
||||
cat))
|
||||
|
||||
|
||||
(defn file-watch-handler
|
||||
[type {:keys [dir path _content stat] :as _payload}]
|
||||
(go
|
||||
|
@ -652,7 +667,7 @@
|
|||
(stop-local->remote! [this])
|
||||
(ratelimit [this from-chan] "get watched local file-change events from FROM-CHAN,
|
||||
return chan returning events with rate limited")
|
||||
(sync-local->remote! [this ^FileChangeEvent e])
|
||||
(sync-local->remote! [this es] "es is a sequence of `FileChangeEvent`, all items have same type.")
|
||||
(sync-local->remote-all-files! [this stop-chan] "compare all local files to remote ones, sync if not equal.
|
||||
ensure local-txid = remote-txid before calling this func"))
|
||||
|
||||
|
@ -756,28 +771,26 @@
|
|||
(async/close! c)))))
|
||||
c))
|
||||
|
||||
(sync-local->remote! [this ^FileChangeEvent e]
|
||||
(let [type (.-type e)]
|
||||
(if (contains-path? (get-ignore-files this) (relative-path e))
|
||||
(go {:succ true}) ; ignore
|
||||
(do
|
||||
(prn "sync-local->remote!" e)
|
||||
(let [path* (relative-path e)
|
||||
r
|
||||
(cond
|
||||
(or (= "add" type) (= "change" type))
|
||||
(update-remote-file rsapi graph-uuid base-path path* @*txid)
|
||||
(sync-local->remote! [this es]
|
||||
(if (empty? es)
|
||||
{:succ true}
|
||||
(let [type (.-type ^FileChangeEvent (first es))
|
||||
ignore-files (get-ignore-files this)
|
||||
es->paths-xf (comp
|
||||
(map #(relative-path %))
|
||||
(filter #(not (contains-path? ignore-files %))))
|
||||
paths (sequence es->paths-xf es)]
|
||||
(println "sync-local->remote" paths)
|
||||
(let [r (case type
|
||||
("add" "change")
|
||||
(update-remote-files rsapi graph-uuid base-path paths @*txid)
|
||||
|
||||
(= "unlink" type)
|
||||
(delete-remote-files rsapi graph-uuid base-path [path*] @*txid)
|
||||
|
||||
;; (= "rename" type)
|
||||
;; (rename-local-file)
|
||||
)]
|
||||
"unlink"
|
||||
(delete-remote-files rsapi graph-uuid base-path paths @*txid))]
|
||||
(go
|
||||
(let [_ (.add-current-local->remote-files! sync-state [path*])
|
||||
(let [_ (.add-current-local->remote-files! sync-state paths)
|
||||
r* (<! r)
|
||||
_ (.remove-current-local->remote-files! sync-state [path*])]
|
||||
_ (.remove-current-local->remote-files! sync-state paths)]
|
||||
(cond
|
||||
(need-sync-remote? r*)
|
||||
{:need-sync-remote true}
|
||||
|
@ -793,7 +806,7 @@
|
|||
:else
|
||||
(do
|
||||
(println "sync-local->remote unknown:" r*)
|
||||
{:unknown r*})))))))))
|
||||
{:unknown r*}))))))))
|
||||
|
||||
(sync-local->remote-all-files! [this stop-chan]
|
||||
(go
|
||||
|
@ -804,23 +817,29 @@
|
|||
diff-local-files (set/difference local-all-files-meta remote-all-files-meta)
|
||||
ignore-files (get-ignore-files this)
|
||||
monitored-dirs (get-monitored-dirs this)
|
||||
change-events (->> diff-local-files
|
||||
(mapv
|
||||
#(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %) nil))
|
||||
(filterv #(let [path (relative-path %)]
|
||||
change-events-partitions
|
||||
(sequence
|
||||
(comp
|
||||
;; convert to FileChangeEvent
|
||||
(map #(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %) nil))
|
||||
;; filter ignore-files & monitored-dirs
|
||||
(filter #(let [path (relative-path %)]
|
||||
(and (not (contains-path? ignore-files path))
|
||||
(contains-path? monitored-dirs path)))))]
|
||||
(println "[full-sync]" (count change-events) "files need to sync to remote")
|
||||
(loop [es change-events]
|
||||
(if (empty? es)
|
||||
(contains-path? monitored-dirs path))))
|
||||
;; partition FileChangeEvents
|
||||
(partition-file-change-events 5))
|
||||
diff-local-files)]
|
||||
(println "[full-sync]" (count (flatten change-events-partitions)) "files need to sync to remote")
|
||||
(loop [es-partitions change-events-partitions]
|
||||
(if (empty? es-partitions)
|
||||
{:succ true}
|
||||
(if (async/poll! stop-chan)
|
||||
{:stop true}
|
||||
(let [e (first es)
|
||||
{:keys [succ need-sync-remote unknown] :as r} (<! (sync-local->remote! this e))]
|
||||
(let [{:keys [succ need-sync-remote unknown] :as r}
|
||||
(<! (sync-local->remote! this (first es-partitions)))]
|
||||
(cond
|
||||
succ
|
||||
(recur (next es))
|
||||
(recur (next es-partitions))
|
||||
|
||||
(or need-sync-remote unknown) r)))))))))
|
||||
|
||||
|
@ -960,7 +979,7 @@
|
|||
(assert (some? local-change))
|
||||
(go
|
||||
(let [{:keys [succ need-sync-remote unknown]}
|
||||
(<! (sync-local->remote! local->remote-syncer local-change))]
|
||||
(<! (sync-local->remote! local->remote-syncer [local-change]))]
|
||||
(cond
|
||||
succ
|
||||
(.schedule this ::idle)
|
||||
|
|
|
@ -12,6 +12,36 @@
|
|||
resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-5.1.1.tgz#9274ec7460652f9c632c59addf24efb1684ef876"
|
||||
integrity sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ==
|
||||
|
||||
"@andelf/rsapi-darwin-arm64@0.0.2":
|
||||
version "0.0.2"
|
||||
resolved "https://registry.npmmirror.com/@andelf/rsapi-darwin-arm64/download/@andelf/rsapi-darwin-arm64-0.0.2.tgz#c89a025fbc2e632ef1ed2a96fa64787e29e4bab8"
|
||||
integrity sha512-WfsVD+5EZkGD7NMyJ0Akwrowgf3IF2BcckERwx0PHQzHi09NZLxZtMr2NN2s6q8QOvGUfbqHYEATZ8zaZMU7XQ==
|
||||
|
||||
"@andelf/rsapi-darwin-x64@0.0.2":
|
||||
version "0.0.2"
|
||||
resolved "https://registry.npmmirror.com/@andelf/rsapi-darwin-x64/download/@andelf/rsapi-darwin-x64-0.0.2.tgz#d9f71946122dc4bb056e512fa9fc820d30c7159b"
|
||||
integrity sha512-r5F9odlNt0dD2W4eKYlJnqqoim/5kn/lzch7kHmvR9SKbxIe8jn/XwGqk5Kj5zIlhwpksT24dF47ppnZ8gJZRw==
|
||||
|
||||
"@andelf/rsapi-linux-x64-gnu@0.0.2":
|
||||
version "0.0.2"
|
||||
resolved "https://registry.npmmirror.com/@andelf/rsapi-linux-x64-gnu/download/@andelf/rsapi-linux-x64-gnu-0.0.2.tgz#f90d8402cbb7e07fcc5defc5c3743d2a591cc22b"
|
||||
integrity sha512-olB2TYRzqB5e8brtyvj0+YnbdLyWCvsQjbDqV8nKyvUThSEvBPUgREBigdpb26wSETPCX7q/IEQNDDOLdXjzmQ==
|
||||
|
||||
"@andelf/rsapi-win32-x64-msvc@0.0.2":
|
||||
version "0.0.2"
|
||||
resolved "https://registry.npmmirror.com/@andelf/rsapi-win32-x64-msvc/download/@andelf/rsapi-win32-x64-msvc-0.0.2.tgz#cee77358a4a3601d8e5a6aa5199ef537eadbd6f3"
|
||||
integrity sha512-qgVZY4lnnlVFu5cNV/pEkvlI/hGokpfpIAGAipYLKVcAmuZ/H9+3aD8PcSi4uRIuexFlSBIcNBi7FtnNahMUcA==
|
||||
|
||||
"@andelf/rsapi@0.0.2":
|
||||
version "0.0.2"
|
||||
resolved "https://registry.npmmirror.com/@andelf/rsapi/download/@andelf/rsapi-0.0.2.tgz#8b24e9792123131f5f4529f00fb8f380333acc26"
|
||||
integrity sha512-/CVyYM3+fRFvO6Dqyd8TJv6IOZsaFpKAkX07Vs2FbnSzR50XtGujYpcjyKjmq3zLPmtJYuvD80qwKqixvzCPPw==
|
||||
optionalDependencies:
|
||||
"@andelf/rsapi-darwin-arm64" "0.0.2"
|
||||
"@andelf/rsapi-darwin-x64" "0.0.2"
|
||||
"@andelf/rsapi-linux-x64-gnu" "0.0.2"
|
||||
"@andelf/rsapi-win32-x64-msvc" "0.0.2"
|
||||
|
||||
"@develar/schema-utils@~2.1.0":
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/@develar/schema-utils/-/schema-utils-2.1.0.tgz#eceb1695bfbed6f6bb84666d5d3abe5e1fd54e17"
|
||||
|
|
Loading…
Reference in New Issue