From 26be1ff9693ea4bb24964d1e214763e91fa16136 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sat, 23 Jan 2021 01:14:45 +0800 Subject: [PATCH] electron: file watcher wip --- package.json | 1 + resources/js/preload.js | 5 ++ src/electron/electron/handler.cljs | 52 +++++++++++- src/main/frontend/config.cljs | 4 + src/main/frontend/fs.cljs | 4 + src/main/frontend/fs/bfs.cljs | 2 + src/main/frontend/fs/nfs.cljs | 6 +- src/main/frontend/fs/node.cljs | 4 +- src/main/frontend/fs/protocol.cljs | 3 +- src/main/frontend/fs/watcher_handler.cljs | 38 +++++++++ src/main/frontend/handler.cljs | 8 +- src/main/frontend/handler/file.cljs | 11 +++ src/main/frontend/handler/web/nfs.cljs | 97 ++++++++++++----------- yarn.lock | 20 +++++ 14 files changed, 202 insertions(+), 53 deletions(-) create mode 100644 src/main/frontend/fs/watcher_handler.cljs diff --git a/package.json b/package.json index 26852af50..ddadc6980 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "cljs:build-electron": "clojure -A:cljs compile app electron" }, "dependencies": { + "chokidar": "^3.5.1", "codemirror": "^5.58.1", "diff": "5.0.0", "diff-match-patch": "^1.0.5", diff --git a/resources/js/preload.js b/resources/js/preload.js index f941b1b67..b987a63fa 100644 --- a/resources/js/preload.js +++ b/resources/js/preload.js @@ -5,6 +5,11 @@ contextBridge.exposeInMainWorld('apis', { return await ipcRenderer.invoke('main', arg) }, + on: (channel, callback) => { + const newCallback = (_, data) => callback(data); + ipcRenderer.on(channel, newCallback); + }, + checkForUpdates: async (...args) => { await ipcRenderer.invoke('check-for-updates', ...args) }, diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index e5a50863a..29a2748f5 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -3,8 +3,10 @@ [cljs-bean.core :as bean] ["fs" :as fs] ["path" :as path] + ["chokidar" :as watcher] [promesa.core :as p] - [goog.object :as gobj])) + [goog.object :as gobj] + [clojure.string :as string])) (defmulti handle (fn [_window args] (keyword (first args)))) @@ -64,6 +66,54 @@ (defmethod handle :getFiles [window [_ path]] (get-files path)) +(defn- get-file-ext + [file] + (last (string/split file #"\."))) + +(defonce file-watcher-chan "file-watcher") +(defn send-file-watcher! [win type payload] + (prn "file watch: " {:type type + :payload payload}) + (.. win -webContents + (send file-watcher-chan + (bean/->js {:type type :payload payload})))) + +(defn watch-dir! + [win dir] + (let [watcher (.watch watcher dir + (clj->js + {:ignored #"^\." + :persistent true + :awaitWriteFinish true}))] + (.on watcher "add" + (fn [path] + (send-file-watcher! win "add" + {:dir dir + :path path + :content (read-file path) + :stat (fs/statSync path)}))) + (.on watcher "change" + (fn [path] + (send-file-watcher! win "change" + {:dir dir + :path path + :content (read-file path) + :stat (fs/statSync path)}))) + (.on watcher "unlink" + (fn [path] + (send-file-watcher! win "unlink" + {:dir dir + :path path}))) + (.on watcher "error" + (fn [path] + (println "Watch error happend: " + {:path path}))) + true)) + +(defmethod handle :addDirWatcher [window [_ dir]] + (when dir + (watch-dir! window dir))) + (defmethod handle :default [args] (println "Error: no ipc handler for: " (bean/->js args))) diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index b25c70b80..f05954697 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -299,6 +299,10 @@ [s] (string/replace s local-db-prefix "")) +(defn get-local-repo + [dir] + (str local-db-prefix dir)) + (defn get-repo-dir [repo-url] (if (util/electron?) diff --git a/src/main/frontend/fs.cljs b/src/main/frontend/fs.cljs index 801d0d29d..1e7ff186c 100644 --- a/src/main/frontend/fs.cljs +++ b/src/main/frontend/fs.cljs @@ -105,6 +105,10 @@ [(:path dir) paths]) result)))) +(defn watch-dir! + [dir] + (protocol/watch-dir! node-record dir)) + (defn mkdir-if-not-exists [dir] (when dir diff --git a/src/main/frontend/fs/bfs.cljs b/src/main/frontend/fs/bfs.cljs index c6913ad05..1b43241d8 100644 --- a/src/main/frontend/fs/bfs.cljs +++ b/src/main/frontend/fs/bfs.cljs @@ -35,4 +35,6 @@ (open-dir [this ok-handler] nil) (get-files [this path-or-handle ok-handler] + nil) + (watch-dir! [this dir] nil)) diff --git a/src/main/frontend/fs/nfs.cljs b/src/main/frontend/fs/nfs.cljs index 14eb4f185..bd7a80236 100644 --- a/src/main/frontend/fs/nfs.cljs +++ b/src/main/frontend/fs/nfs.cljs @@ -198,4 +198,8 @@ (utils/openDirectory #js {:recursive true} ok-handler)) (get-files [this path-or-handle ok-handler] - (utils/getFiles path-or-handle true ok-handler))) + (utils/getFiles path-or-handle true ok-handler)) + + ;; TODO: + (watch-dir! [this dir] + nil)) diff --git a/src/main/frontend/fs/node.cljs b/src/main/frontend/fs/node.cljs index f0e7f1918..6d450c1fd 100644 --- a/src/main/frontend/fs/node.cljs +++ b/src/main/frontend/fs/node.cljs @@ -44,4 +44,6 @@ (open-dir [this ok-handler] (ipc/ipc "openDir" {})) (get-files [this path-or-handle ok-handler] - (ipc/ipc "getFiles" path-or-handle))) + (ipc/ipc "getFiles" path-or-handle)) + (watch-dir! [this dir] + (ipc/ipc "addDirWatcher" dir))) diff --git a/src/main/frontend/fs/protocol.cljs b/src/main/frontend/fs/protocol.cljs index 022db2ba7..916dfaf5d 100644 --- a/src/main/frontend/fs/protocol.cljs +++ b/src/main/frontend/fs/protocol.cljs @@ -10,4 +10,5 @@ (rename! [this repo old-path new-path]) (stat [this dir path]) (open-dir [this ok-handler]) - (get-files [this path-or-handle ok-handler])) + (get-files [this path-or-handle ok-handler]) + (watch-dir! [this dir])) diff --git a/src/main/frontend/fs/watcher_handler.cljs b/src/main/frontend/fs/watcher_handler.cljs new file mode 100644 index 000000000..57626be00 --- /dev/null +++ b/src/main/frontend/fs/watcher_handler.cljs @@ -0,0 +1,38 @@ +(ns frontend.fs.watcher-handler + (:require [clojure.core.async :as async] + [lambdaisland.glogi :as log] + [frontend.handler.file :as file-handler] + [frontend.handler.page :as page-handler] + [frontend.config :as config] + [cljs-bean.core :as bean])) + +(defn handle-changed! + [type {:keys [dir path content stat] :as payload}] + (when dir + (let [repo (config/get-local-repo dir)] + (prn "handle file notifier: " + {:repo repo + :type type + :payload payload}) + ;; (cond + ;; (contains? #{"add" "change"} type) + ;; ;; TODO: check content and mtime + ;; (file-handler/alter-file repo path content {}) + + ;; (= "unlink" type) + ;; ;; TODO: Remove page and blocks too + ;; ;; what if it's a mistaken, should we put it to .trash to have a way to restore it back? + ;; (file-handler/remove-file! repo path) + + ;; :else + ;; (log/error :fs/watcher-no-handler {:type type + ;; :payload payload})) + ))) + +(defn run-dirs-watcher! + [] + ;; TODO: move "file-watcher" to electron.ipc.channels + (js/window.apis.on "file-watcher" + (fn [data] + (let [{:keys [type payload]} (bean/->clj data)] + (handle-changed! type payload))))) diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index 3cf6d9b74..b4c4c95a1 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -19,6 +19,7 @@ [frontend.handler.ui :as ui-handler] [frontend.handler.export :as export-handler] [frontend.handler.web.nfs :as nfs] + [frontend.fs.watcher-handler :as fs-watcher-handler] [frontend.ui :as ui] [goog.object :as gobj] [frontend.idb :as idb] @@ -105,7 +106,8 @@ (fn [] (js/console.error "Failed to request GitHub app tokens.")))) - (watch-for-date!))) + (watch-for-date!) + (file-handler/watch-for-local-dirs!))) (p/catch (fn [error] (log/error :db/restore-failed error))))))] ;; clear this interval @@ -156,4 +158,6 @@ (reset! db/*sync-search-indice-f search/sync-search-indice!) (db/run-batch-txs!) (file-handler/run-writes-chan!) - (editor-handler/periodically-save!))) + (editor-handler/periodically-save!) + (when (util/electron?) + (fs-watcher-handler/run-dirs-watcher!)))) diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs index 94136babd..12b97f9eb 100644 --- a/src/main/frontend/handler/file.cljs +++ b/src/main/frontend/handler/file.cljs @@ -306,3 +306,14 @@ (> (state/get-repos) + (filter (fn [repo] + (config/local-db? (:url repo))))) + directories (map (fn [repo] (config/get-repo-dir (:url repo))) repos)] + (doseq [dir directories] + (prn "watch for dir changes: " dir) + (fs/watch-dir! dir))))) diff --git a/src/main/frontend/handler/web/nfs.cljs b/src/main/frontend/handler/web/nfs.cljs index 82a26b52c..8e4688898 100644 --- a/src/main/frontend/handler/web/nfs.cljs +++ b/src/main/frontend/handler/web/nfs.cljs @@ -205,6 +205,54 @@ :modified modified :deleted deleted})) +(defn- handle-diffs! + [repo nfs? old-files new-files handle-path path-handles re-index?] + (let [get-file-f (fn [path files] (some #(when (= (:file/path %) path) %) files)) + {:keys [added modified deleted] :as diffs} (compute-diffs old-files new-files) + ;; Use the same labels as isomorphic-git + rename-f (fn [typ col] (mapv (fn [file] {:type typ :path file}) col)) + _ (when (and nfs? (seq deleted)) + (let [deleted (doall + (-> (map (fn [path] (if (= "/" (first path)) + path + (str "/" path))) deleted) + (distinct)))] + (p/all (map (fn [path] + (let [handle-path (str handle-path path)] + (idb/remove-item! handle-path) + (nfs/remove-nfs-file-handle! handle-path))) deleted)))) + added-or-modified (set (concat added modified)) + _ (when (and nfs? (seq added-or-modified)) + (p/all (map (fn [path] + (when-let [handle (get @path-handles path)] + (idb/set-item! (str handle-path path) handle))) added-or-modified)))] + (-> (p/all (map (fn [path] + (when-let [file (get-file-f path new-files)] + (p/let [content (if nfs? + (.text (:file/file file)) + (:file/content file))] + (assoc file :file/content content)))) added-or-modified)) + (p/then (fn [result] + (let [files (map #(dissoc % :file/file :file/handle) result) + non-modified? (fn [file] + (let [content (:file/content file) + old-content (:file/content (get-file-f (:file/path file) old-files))] + (= content old-content))) + non-modified-files (->> (filter non-modified? files) + (map :file/path)) + [modified-files modified] (if re-index? + [files (set modified)] + [(remove non-modified? files) (set/difference (set modified) (set non-modified-files))]) + diffs (concat + (rename-f "remove" deleted) + (rename-f "add" added) + (rename-f "modify" modified))] + (when (or (and (seq diffs) (seq modified-files)) + (seq diffs)) + (repo-handler/load-repo-to-db! repo + {:diffs diffs + :nfs-files modified-files})))))))) + (defn- reload-dir! ([repo] (reload-dir! repo false)) @@ -236,53 +284,8 @@ (contains? file-paths (string/replace-first path (str dir-name "/") "")))) (into {}))))) - (set-files! @path-handles)) - get-file-f (fn [path files] (some #(when (= (:file/path %) path) %) files)) - {:keys [added modified deleted] :as diffs} (compute-diffs old-files new-files) - ;; Use the same labels as isomorphic-git - rename-f (fn [typ col] (mapv (fn [file] {:type typ :path file}) col)) - _ (when (and nfs? (seq deleted)) - (let [deleted (doall - (-> (map (fn [path] (if (= "/" (first path)) - path - (str "/" path))) deleted) - (distinct)))] - (p/all (map (fn [path] - (let [handle-path (str handle-path path)] - (idb/remove-item! handle-path) - (nfs/remove-nfs-file-handle! handle-path))) deleted)))) - added-or-modified (set (concat added modified)) - _ (when (and nfs? (seq added-or-modified)) - (p/all (map (fn [path] - (when-let [handle (get @path-handles path)] - (idb/set-item! (str handle-path path) handle))) added-or-modified)))] - (-> (p/all (map (fn [path] - (when-let [file (get-file-f path new-files)] - (p/let [content (if nfs? - (.text (:file/file file)) - (:file/content file))] - (assoc file :file/content content)))) added-or-modified)) - (p/then (fn [result] - (let [files (map #(dissoc % :file/file :file/handle) result) - non-modified? (fn [file] - (let [content (:file/content file) - old-content (:file/content (get-file-f (:file/path file) old-files))] - (= content old-content))) - non-modified-files (->> (filter non-modified? files) - (map :file/path)) - [modified-files modified] (if re-index? - [files (set modified)] - [(remove non-modified? files) (set/difference (set modified) (set non-modified-files))]) - diffs (concat - (rename-f "remove" deleted) - (rename-f "add" added) - (rename-f "modify" modified))] - (when (or (and (seq diffs) (seq modified-files)) - (seq diffs) ; delete -) - (repo-handler/load-repo-to-db! repo - {:diffs diffs - :nfs-files modified-files}))))))))) + (set-files! @path-handles))] + (handle-diffs! repo nfs? old-files new-files handle-path path-handles re-index?)))) (p/catch (fn [error] (log/error :nfs/load-files-error error))) (p/finally (fn [_] diff --git a/yarn.lock b/yarn.lock index 777553013..4f74fd6ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1097,6 +1097,21 @@ chokidar@^3.3.0, chokidar@^3.3.1: optionalDependencies: fsevents "~2.1.2" +chokidar@^3.5.1: + version "3.5.1" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a" + integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw== + dependencies: + anymatch "~3.1.1" + braces "~3.0.2" + glob-parent "~5.1.0" + is-binary-path "~2.1.0" + is-glob "~4.0.1" + normalize-path "~3.0.0" + readdirp "~3.5.0" + optionalDependencies: + fsevents "~2.3.1" + cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" @@ -2374,6 +2389,11 @@ fsevents@~2.1.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fsevents@~2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.1.tgz#b209ab14c61012636c8863507edf7fb68cc54e9f" + integrity sha512-YR47Eg4hChJGAB1O3yEAOkGO+rlzutoICGqGo9EZ4lKWokzZRSyIW1QmTzqjtw8MJdj9srP869CuWw/hyzSiBw== + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"