greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: chrome native compatible fs api

feat/pwa
Tienson Qin 2020-11-23 17:10:29 +08:00
parent 3a81b5e856
commit 50a7a7c585
7 changed files with 152 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/frontend/config.cljs
View File

@ -272,7 +272,8 @@
#{"now" "later" "todo" "doing" "done" "wait" "waiting"
"canceled" "cancelled" "started" "in-progress"})
(defonce local-db-prefix "logseq-local-")
(defonce local-db-prefix "logseq_local_")
(defonce local-handle-prefix (str "handle/" local-db-prefix))
(defn local-db?
[s]

2
src/main/frontend/db.cljs
View File

@ -27,8 +27,6 @@
;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
(defonce query-state (atom {}))
(defonce async-chan (atom nil))
(defn get-repo-path
[url]
(if (util/starts-with? url "http")

108
src/main/frontend/fs.cljs
View File

@ -4,8 +4,21 @@
[clojure.string :as string]
[frontend.idb :as idb]
[promesa.core :as p]
[goog.object :as gobj]
["/frontend/utils" :as utils]))
;; We need to cache the file handles in the memory so that
;; the browser will not keep asking permissions.
(defonce nfs-file-handles-cache (atom {}))
(defn get-nfs-file-handle
[handle-path]
(get @nfs-file-handles-cache handle-path))
(defn add-nfs-file-handle!
[handle-path handle]
(swap! nfs-file-handles-cache assoc handle-path handle))
;; TODO:
;; We need to support several platforms:
;; 1. Chrome native file system API (lighting-fs wip)
@ -31,13 +44,15 @@
(cond
(local-db? dir)
(let [[root new-dir] (rest (string/split dir "/"))
root-handle (str "handle-" root)]
root-handle (str "handle/" root)]
(p/let [handle (idb/get-item root-handle)]
(when (and handle new-dir
(not (string/blank? new-dir)))
(-> (p/let [handle (.getDirectoryHandle ^js handle new-dir
#js {:create true})
_ (idb/set-item! (str root-handle "/" new-dir) handle)]
handle-path (str root-handle "/" new-dir)
_ (idb/set-item! handle-path handle)]
(add-nfs-file-handle! handle-path handle)
(println "Stored handle: " (str root-handle "/" new-dir)))
(p/catch (fn [error]
(println "mkdir error: " error)
@ -51,35 +66,52 @@
(defn readdir
[dir]
(when (and dir js/window.pfs)
(js/window.pfs.readdir dir)))
(cond
(local-db? dir)
(let [prefix (str "handle/" dir)
cached-files (keys @nfs-file-handles-cache)]
(p/resolved
(->> (filter #(string/starts-with? % (str prefix "/")) cached-files)
(map (fn [path]
(string/replace path prefix ""))))))
(and dir js/window.pfs)
(js/window.pfs.readdir dir)
:else
nil))
(defn unlink
[path opts]
(js/window.pfs.unlink path opts))
(cond
(local-db? path)
(let [[dir basename] (util/get-dir-and-basename path)]
(p/let [handle (idb/get-item (str "handle" dir))]
(.removeEntry ^js handle basename)))
(defn rename
[old-path new-path]
(js/window.pfs.rename old-path new-path))
:else
(js/window.pfs.unlink path opts)))
(defn rmdir
"Remove the directory recursively."
[dir]
(js/window.workerThread.rimraf dir))
(cond
(local-db? dir)
nil
:else
(js/window.workerThread.rimraf dir)))
(defn read-file
([dir path]
(read-file dir path (clj->js {:encoding "utf8"})))
([dir path option]
(js/window.pfs.readFile (str dir "/" path) option)))
(cond
(local-db? dir)
nil
(defonce nfs-file-handles-cache (atom {}))
(defn get-nfs-file-handle
[handle-path]
(get @nfs-file-handles-cache handle-path))
(defn add-nfs-file-handle!
[handle-path handle]
(swap! nfs-file-handles-cache assoc handle-path handle))
:else
(js/window.pfs.readFile (str dir "/" path) option))))
(defn write-file
[dir path content]
@ -90,10 +122,13 @@
sub-dir (->> (butlast parts)
(remove string/blank?)
(string/join "/"))
handle-path (str "handle-"
handle-path (str "handle/"
(subs dir 1)
(if sub-dir
(str "/" sub-dir)))
handle-path (if (= "/" (last handle-path))
(subs handle-path 0 (dec (count handle-path)))
handle-path)
basename-handle-path (str handle-path "/" basename)
file-handle-cache (get-nfs-file-handle basename-handle-path)]
(p/let [file-handle (or file-handle-cache (idb/get-item basename-handle-path))]
@ -119,9 +154,41 @@
:else
nil))
(defn rename
[old-path new-path]
(cond
(local-db? old-path)
;; create new file
;; delete old file
(p/let [[dir basename] (util/get-dir-and-basename old-path)
[_ new-basename] (util/get-dir-and-basename new-path)
handle (idb/get-item (str "handle" old-path))
file (.getFile handle)
content (.text file)
_ (write-file dir new-basename content)]
(unlink old-path nil))
:else
(js/window.pfs.rename old-path new-path)))
(defn stat
[dir path]
(js/window.pfs.stat (str dir "/" path)))
(cond
(local-db? dir)
(if-let [file (get-nfs-file-handle (str "handle/"
(string/replace-first dir "/" "")
"/"
(string/replace-first path "/" "")))]
(p/let [file (.getFile file)]
(let [get-attr #(gobj/get file %)]
{:file/last-modified-at (get-attr "lastModified")
:file/size (get-attr "size")
:file/type (get-attr "type")}))
(p/rejected "File not exists"))
:else
;; FIXME: same format
(js/window.pfs.stat (str dir "/" (string/replace-first path "/" "")))))
(defn create-if-not-exists
([dir path]
@ -143,6 +210,3 @@
(stat dir path)
(fn [_stat] true)
(fn [_e] false)))
(comment
(def dir "/notes"))

95
src/main/frontend/handler/web/nfs.cljs
View File

@ -16,51 +16,56 @@
(defn ls-dir-files
[]
(->
(p/let [result (utils/openDirectory #js {:recursive true})
root-handle (nth result 0)
dir-name (gobj/get root-handle "name")
repo (str config/local-db-prefix dir-name)
root-handle-path (str "handle-" repo)
_ (idb/set-item! root-handle-path root-handle)
_ (fs/add-nfs-file-handle! root-handle-path root-handle)
result (nth result 1)
result (flatten (bean/->clj result))
files (doall
(map (fn [file]
(let [handle (gobj/get file "handle")
get-attr #(gobj/get file %)]
{:file/path (get-attr "webkitRelativePath")
:file/last-modified-at (get-attr "lastModified")
:file/size (get-attr "size")
:file/type (get-attr "type")
:file/file file
:file/handle handle})) result))
text-files (filter (fn [file] (contains? #{"org" "md" "markdown"} (util/get-file-ext (:file/path file)))) files)]
(doseq [file text-files]
(let [handle-path (str "handle-" repo "/" (:file/path file))
handle (:file/handle file)]
(idb/set-item! handle-path handle)
(fs/add-nfs-file-handle! handle-path handle)))
(-> (p/all (map (fn [file]
(p/let [content (.text (:file/file file))]
(assoc file :file/content content))) text-files))
(p/then (fn [result]
(let [files (map #(dissoc % :file/file) result)]
(repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
(repo-handler/load-repo-to-db! repo
{:first-clone? true
:nfs-files (map :file/path files)
:nfs-contents (mapv (fn [f] [(:file/path f) (:file/content f)]) files)
:additional-files-info files})
;; create default directories and files
)))
(p/catch (fn [error]
(println "Load files content error: ")
(js/console.dir error)))))
(p/catch (fn [error]
(println "Open directory error: ")
(js/console.dir error)))))
(let [path-handles (atom {})]
(->
(p/let [result (utils/openDirectory #js {:recursive true}
(fn [path handle]
(swap! path-handles assoc path handle)))
root-handle (nth result 0)
dir-name (gobj/get root-handle "name")
repo (str config/local-db-prefix dir-name)
root-handle-path (str config/local-handle-prefix dir-name)
_ (idb/set-item! root-handle-path root-handle)
_ (fs/add-nfs-file-handle! root-handle-path root-handle)
_ (doseq [[path handle] @path-handles]
(let [handle-path (str config/local-handle-prefix path)]
(idb/set-item! handle-path handle)
(fs/add-nfs-file-handle! handle-path handle)))
result (nth result 1)
result (flatten (bean/->clj result))
files (doall
(map (fn [file]
(let [handle (gobj/get file "handle")
get-attr #(gobj/get file %)]
{:file/path (get-attr "webkitRelativePath")
:file/last-modified-at (get-attr "lastModified")
:file/size (get-attr "size")
:file/type (get-attr "type")
:file/file file
:file/handle handle})) result))
markup-files (filter (fn [file]
(contains? config/markup-formats
(keyword (util/get-file-ext (:file/path file)))))
files)]
(-> (p/all (map (fn [file]
(p/let [content (.text (:file/file file))]
(assoc file :file/content content))) markup-files))
(p/then (fn [result]
(let [files (map #(dissoc % :file/file) result)]
(repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
(repo-handler/load-repo-to-db! repo
{:first-clone? true
:nfs-files (map :file/path files)
:nfs-contents (mapv (fn [f] [(:file/path f) (:file/content f)]) files)
:additional-files-info files})
;; create default directories and files
)))
(p/catch (fn [error]
(println "Load files content error: ")
(js/console.dir error)))))
(p/catch (fn [error]
(println "Open directory error: ")
(js/console.dir error))))))
(defn open-file-picker
"Shows a file picker that lets a user select a single existing file, returning a handle for the selected file. "

7
src/main/frontend/protocol/fs.cljs
View File

@ -1,7 +0,0 @@
(ns frontend.protocol.fs)
(defprotocol Fs
(load-directory! [this])
(write-file! [this path content])
(delete-file! [this path])
(get-file-stats! [this path]))

8
src/main/frontend/util.cljs
View File

@ -941,6 +941,14 @@
[file]
(last (string/split file #"\.")))
(defn get-dir-and-basename
[path]
(let [parts (string/split path "/")
basename (last parts)
dir (->> (butlast parts)
(string/join "/"))]
[dir basename]))
(defn get-relative-path
[current-file-path another-file-path]
(let [directories-f #(butlast (string/split % "/"))

11
src/main/frontend/utils.js
View File

@ -74,13 +74,13 @@ export var getSelectionText = function() {
// Modified from https://github.com/GoogleChromeLabs/browser-nativefs
// because shadow-cljs doesn't handle this babel transform
const getFiles = async function (dirHandle, recursive) {
const getFiles = async function (dirHandle, recursive, cb, path = dirHandle.name) {
const dirs = [];
const files = [];
const path = dirHandle.name;
for await (const entry of dirHandle.values()) {
const nestedPath = `${path}/${entry.name}`;
if (entry.kind === 'file') {
cb(nestedPath, entry);
files.push(
entry.getFile().then((file) => {
Object.defineProperty(file, 'webkitRelativePath', {
@ -98,17 +98,18 @@ const getFiles = async function (dirHandle, recursive) {
)
);
} else if (entry.kind === 'directory' && recursive) {
dirs.push(getFiles(entry, recursive, nestedPath));
cb(nestedPath, entry);
dirs.push(getFiles(entry, recursive, cb, nestedPath));
}
}
return [(await Promise.all(dirs)), (await Promise.all(files))];
};
export var openDirectory = async function (options = {}) {
export var openDirectory = async function (options = {}, cb) {
options.recursive = options.recursive || false;
const handle = await window.showDirectoryPicker({ mode: 'readwrite' });
return [handle, getFiles(handle, options.recursive)];
return [handle, getFiles(handle, options.recursive, cb)];
};
export var writeFile = async function (fileHandle, contents) {