feat(ios): add FileSync plugin

pull/5215/head^2
Andelf 2022-04-27 02:06:28 +08:00
parent 8162394698
commit ece4f0ba8c
14 changed files with 1079 additions and 23 deletions

View File

@ -25,8 +25,13 @@
D32752BE275496C60039291C /* CloudKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D32752BD275496C60039291C /* CloudKit.framework */; };
D3D62A0A275C92880003FBDC /* FileContainer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3D62A09275C92880003FBDC /* FileContainer.swift */; };
D3D62A0C275C928F0003FBDC /* FileContainer.m in Sources */ = {isa = PBXBuildFile; fileRef = D3D62A0B275C928F0003FBDC /* FileContainer.m */; };
FE443F1C27FF5420007ECE65 /* Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE443F1B27FF5420007ECE65 /* Extensions.swift */; };
FE443F1E27FF54AA007ECE65 /* Payload.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE443F1D27FF54AA007ECE65 /* Payload.swift */; };
FE443F2027FF54C9007ECE65 /* SyncClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE443F1F27FF54C9007ECE65 /* SyncClient.swift */; };
FE647FF427BDFEDE00F3206B /* FsWatcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE647FF327BDFEDE00F3206B /* FsWatcher.swift */; };
FE647FF627BDFEF500F3206B /* FsWatcher.m in Sources */ = {isa = PBXBuildFile; fileRef = FE647FF527BDFEF500F3206B /* FsWatcher.m */; };
FE8C946B27FD762700C8017B /* FileSync.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE8C946927FD762700C8017B /* FileSync.swift */; };
FE8C946C27FD762700C8017B /* FileSync.m in Sources */ = {isa = PBXBuildFile; fileRef = FE8C946A27FD762700C8017B /* FileSync.m */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@ -81,8 +86,13 @@
D3D62A09275C92880003FBDC /* FileContainer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FileContainer.swift; sourceTree = "<group>"; };
D3D62A0B275C928F0003FBDC /* FileContainer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FileContainer.m; sourceTree = "<group>"; };
DE5650F4AD4E2242AB9C012D /* Pods-Logseq.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Logseq.debug.xcconfig"; path = "Target Support Files/Pods-Logseq/Pods-Logseq.debug.xcconfig"; sourceTree = "<group>"; };
FE443F1B27FF5420007ECE65 /* Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Extensions.swift; sourceTree = "<group>"; };
FE443F1D27FF54AA007ECE65 /* Payload.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Payload.swift; sourceTree = "<group>"; };
FE443F1F27FF54C9007ECE65 /* SyncClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncClient.swift; sourceTree = "<group>"; };
FE647FF327BDFEDE00F3206B /* FsWatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FsWatcher.swift; sourceTree = "<group>"; };
FE647FF527BDFEF500F3206B /* FsWatcher.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FsWatcher.m; sourceTree = "<group>"; };
FE8C946927FD762700C8017B /* FileSync.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FileSync.swift; sourceTree = "<group>"; };
FE8C946A27FD762700C8017B /* FileSync.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FileSync.m; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@ -142,6 +152,7 @@
50B271D01FEDC1A000F3C39B /* public */,
7435D10B2704659F00AB88E0 /* FolderPicker.swift */,
FE647FF327BDFEDE00F3206B /* FsWatcher.swift */,
FE443F1A27FF53A2007ECE65 /* FileSync */,
FE647FF527BDFEF500F3206B /* FsWatcher.m */,
7435D10E2704660B00AB88E0 /* FolderPicker.m */,
D3D62A09275C92880003FBDC /* FileContainer.swift */,
@ -180,6 +191,18 @@
path = Pods;
sourceTree = "<group>";
};
FE443F1A27FF53A2007ECE65 /* FileSync */ = {
isa = PBXGroup;
children = (
FE8C946927FD762700C8017B /* FileSync.swift */,
FE443F1F27FF54C9007ECE65 /* SyncClient.swift */,
FE443F1D27FF54AA007ECE65 /* Payload.swift */,
FE443F1B27FF5420007ECE65 /* Extensions.swift */,
FE8C946A27FD762700C8017B /* FileSync.m */,
);
path = FileSync;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
@ -331,13 +354,18 @@
files = (
504EC3081FED79650016851F /* AppDelegate.swift in Sources */,
5FD5BB71278579F5008E6875 /* DownloadiCloudFiles.swift in Sources */,
FE443F1E27FF54AA007ECE65 /* Payload.swift in Sources */,
FE8C946B27FD762700C8017B /* FileSync.swift in Sources */,
FE647FF427BDFEDE00F3206B /* FsWatcher.swift in Sources */,
5FD5BB73278579FF008E6875 /* DownloadiCloudFiles.m in Sources */,
D3D62A0A275C92880003FBDC /* FileContainer.swift in Sources */,
D3D62A0C275C928F0003FBDC /* FileContainer.m in Sources */,
FE443F1C27FF5420007ECE65 /* Extensions.swift in Sources */,
FE8C946C27FD762700C8017B /* FileSync.m in Sources */,
7435D10F2704660B00AB88E0 /* FolderPicker.m in Sources */,
7435D10C2704659F00AB88E0 /* FolderPicker.swift in Sources */,
FE647FF627BDFEF500F3206B /* FsWatcher.m in Sources */,
FE443F2027FF54C9007ECE65 /* SyncClient.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};

View File

@ -0,0 +1,235 @@
//
// Extensions.swift
// Logseq
//
// Created by Mono Wang on 4/8/R4.
//
import Foundation
import CryptoKit
import var CommonCrypto.CC_MD5_DIGEST_LENGTH
import func CommonCrypto.CC_MD5
import typealias CommonCrypto.CC_LONG
// via https://github.com/krzyzanowskim/CryptoSwift
extension Array where Element == UInt8 {
public init(hex: String) {
self = Array.init()
self.reserveCapacity(hex.unicodeScalars.lazy.underestimatedCount)
var buffer: UInt8?
var skip = hex.hasPrefix("0x") ? 2 : 0
for char in hex.unicodeScalars.lazy {
guard skip == 0 else {
skip -= 1
continue
}
guard char.value >= 48 && char.value <= 102 else {
removeAll()
return
}
let v: UInt8
let c: UInt8 = UInt8(char.value)
switch c {
case let c where c <= 57:
v = c - 48
case let c where c >= 65 && c <= 70:
v = c - 55
case let c where c >= 97:
v = c - 87
default:
removeAll()
return
}
if let b = buffer {
append(b << 4 | v)
buffer = nil
} else {
buffer = v
}
}
if let b = buffer {
append(b)
}
}
}
@available(iOS 13.0, *)
extension SymmetricKey {
public init(passwordString keyString: String) throws {
let size = SymmetricKeySize.bits256
guard var keyData = keyString.data(using: .utf8) else {
print("Could not create raw Data from String.")
throw CryptoKitError.incorrectParameterSize
}
let keySizeBytes = size.bitCount / 8
keyData = keyData.subdata(in: 0..<keySizeBytes)
guard keyData.count >= keySizeBytes else { throw CryptoKitError.incorrectKeySize }
print("debug key \(keyData) \(keyData.hexDescription)")
self.init(data: keyData)
}
}
extension Data {
public init?(hexEncoded: String) {
self.init(Array<UInt8>(hex: hexEncoded))
}
var hexDescription: String {
return map { String(format: "%02hhx", $0) }.joined()
}
@available(iOS 13.0, *)
func aesEncrypt(keyString: String) throws -> Data {
let key = try? SymmetricKey(passwordString: keyString)
let nonce = Data(hexEncoded: "131348c0987c7eece60fc0bc") // = initialization vector
let tag = Data(hexEncoded: "5baa85ff3e7eda3204744ec74b71d523")
print("debug tag \(tag?.hexDescription) nonce \(nonce?.hexDescription)")
let sealedData = try! AES.GCM.seal(self, using: key!, nonce: AES.GCM.Nonce(data: nonce!), authenticating: tag!)
print("debug encrypted \(sealedData)")
guard let encryptedContent = sealedData.combined else {
throw CryptoKitError.underlyingCoreCryptoError(error: 2)
}
print("debug encrypted \(encryptedContent)")
print("debug encrypted \(encryptedContent.hexDescription)")
print("debug tag \(sealedData.tag.hexDescription)")
return encryptedContent
}
@available(iOS 13.0, *)
func aesDecrypt(keyString: String) throws -> Data {
let key = try! SymmetricKey(passwordString: keyString)
let tag = Data(hexEncoded: "5baa85ff3e7eda3204744ec74b71d523")
guard let sealedBox = try? AES.GCM.SealedBox(combined: self) else {
throw CryptoKitError.authenticationFailure
}
guard let decryptedData = try? AES.GCM.open(sealedBox, using: key, authenticating: tag!) else {
throw CryptoKitError.authenticationFailure
}
return decryptedData
}
}
extension String {
var MD5: String {
// TODO: incremental hash
if #available(iOS 13.0, *) {
let computed = Insecure.MD5.hash(data: self.data(using: .utf8)!)
return computed.map { String(format: "%02hhx", $0) }.joined()
} else {
// Fallback on earlier versions, no CryptoKit
let length = Int(CC_MD5_DIGEST_LENGTH)
let messageData = self.data(using:.utf8)!
var digestData = Data(count: length)
_ = digestData.withUnsafeMutableBytes { digestBytes -> UInt8 in
messageData.withUnsafeBytes { messageBytes -> UInt8 in
if let messageBytesBaseAddress = messageBytes.baseAddress, let digestBytesBlindMemory = digestBytes.bindMemory(to: UInt8.self).baseAddress {
let messageLength = CC_LONG(messageData.count)
CC_MD5(messageBytesBaseAddress, messageLength, digestBytesBlindMemory)
}
return 0
}
}
return digestData.map { String(format: "%02hhx", $0) }.joined()
}
}
func encodeAsFname() -> String {
var allowed = NSMutableCharacterSet.urlPathAllowed
allowed.remove(charactersIn: "&$@=;:+ ,?%#")
return self.addingPercentEncoding(withAllowedCharacters: allowed) ?? self
}
func decodeFromFname() -> String {
return self.removingPercentEncoding ?? self
}
static func random(length: Int) -> String {
let letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
return String((0..<length).map{ _ in letters.randomElement()! })
}
}
extension URL {
func relativePath(from base: URL) -> String? {
// Ensure that both URLs represent files:
guard self.isFileURL && base.isFileURL else {
return nil
}
// Remove/replace "." and "..", make paths absolute:
let destComponents = self.standardized.pathComponents
let baseComponents = base.standardized.pathComponents
// Find number of common path components:
var i = 0
while i < destComponents.count && i < baseComponents.count
&& destComponents[i] == baseComponents[i] {
i += 1
}
// Build relative path:
var relComponents = Array(repeating: "..", count: baseComponents.count - i)
relComponents.append(contentsOf: destComponents[i...])
return relComponents.joined(separator: "/")
}
func download(toFile file: URL, completion: @escaping (Error?) -> Void) {
// Download the remote URL to a file
let task = URLSession.shared.downloadTask(with: self) {
(tempURL, response, error) in
// Early exit on error
guard let tempURL = tempURL else {
completion(error)
return
}
if let response = response! as? HTTPURLResponse {
if response.statusCode == 404 {
completion(NSError(domain: "",
code: response.statusCode,
userInfo: [NSLocalizedDescriptionKey: "remote file not found"]))
return
}
if response.statusCode != 200 {
completion(NSError(domain: "",
code: response.statusCode,
userInfo: [NSLocalizedDescriptionKey: "invalid http status code"]))
return
}
}
do {
// Remove any existing document at file
if FileManager.default.fileExists(atPath: file.path) {
try FileManager.default.removeItem(at: file)
}
// Copy the tempURL to file
try FileManager.default.copyItem(
at: tempURL,
to: file
)
completion(nil)
}
// Handle potential file system errors
catch {
completion(error)
}
}
// Start the download
task.resume()
}
}

View File

@ -0,0 +1,19 @@
//
// FileSync.m
// Logseq
//
// Created by Mono Wang on 2/24/R4.
//
#import <Capacitor/Capacitor.h>
CAP_PLUGIN(FileSync, "FileSync",
CAP_PLUGIN_METHOD(setEnv, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(getLocalFilesMeta, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(getLocalAllFilesMeta, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(renameLocalFile, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(deleteLocalFiles, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(updateLocalFiles, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(deleteRemoteFiles, CAPPluginReturnPromise);
CAP_PLUGIN_METHOD(updateRemoteFiles, CAPPluginReturnPromise);
)

View File

@ -0,0 +1,290 @@
//
// FileSync.swift
// Logseq
//
// Created by Mono Wang on 2/24/R4.
//
import Capacitor
import Foundation
import AWSMobileClient
import CryptoKit
// MARK: Global variables
// Defualts to dev
var URL_BASE = URL(string: "https://api.logseq.com/file-sync/")!
var BUCKET: String = "logseq-file-sync-bucket"
var REGION: String = "us-east-2"
// MARK: FileSync Plugin
@objc(FileSync)
public class FileSync: CAPPlugin, SyncDebugDelegate {
override public func load() {
print("debug File sync iOS plugin loaded!")
AWSMobileClient.default().initialize { (userState, error) in
guard error == nil else {
print("error initializing AWSMobileClient. Error: \(error!.localizedDescription)")
return
}
}
}
// NOTE: for debug, or an activity indicator
public func debugNotification(_ message: [String: Any]) {
self.notifyListeners("debug", data: message)
}
@objc func setEnv(_ call: CAPPluginCall) {
guard let env = call.getString("env") else {
call.reject("required parameter: env")
return
}
switch env {
case "production", "product", "prod":
URL_BASE = URL(string: "https://api-prod.logseq.com/file-sync/")!
BUCKET = "logseq-file-sync-bucket-prod"
REGION = "us-east-1"
case "development", "develop", "dev":
URL_BASE = URL(string: "https://api.logseq.com/file-sync/")!
BUCKET = "logseq-file-sync-bucket"
REGION = "us-east-2"
default:
call.reject("invalid env: \(env)")
return
}
self.debugNotification(["event": "setenv:\(env)"])
call.resolve(["ok": true])
}
@objc func getLocalFilesMeta(_ call: CAPPluginCall) {
guard let basePath = call.getString("basePath"),
let filePaths = call.getArray("filePaths") as? [String] else {
call.reject("required paremeters: basePath, filePaths")
return
}
guard let baseURL = URL(string: basePath) else {
call.reject("invalid basePath")
return
}
var fileMd5Digests: [String: [String: Any]] = [:]
for filePath in filePaths {
let url = baseURL.appendingPathComponent(filePath)
if let content = try? String(contentsOf: url, encoding: .utf8) {
fileMd5Digests[filePath] = ["md5": content.MD5,
"size": content.lengthOfBytes(using: .utf8)]
}
}
call.resolve(["result": fileMd5Digests])
}
@objc func getLocalAllFilesMeta(_ call: CAPPluginCall) {
guard let basePath = call.getString("basePath"),
let baseURL = URL(string: basePath) else {
call.reject("invalid basePath")
return
}
var fileMd5Digests: [String: [String: Any]] = [:]
if let enumerator = FileManager.default.enumerator(at: baseURL, includingPropertiesForKeys: [.isRegularFileKey], options: [.skipsPackageDescendants, .skipsHiddenFiles]) {
for case let fileURL as URL in enumerator {
if !fileURL.isSkipped() {
if let content = try? String(contentsOf: fileURL, encoding: .utf8) {
fileMd5Digests[fileURL.relativePath(from: baseURL)!] = ["md5": content.MD5,
"size": content.lengthOfBytes(using: .utf8)]
}
} else if fileURL.isICloudPlaceholder() {
try? FileManager.default.startDownloadingUbiquitousItem(at: fileURL)
}
}
}
call.resolve(["result": fileMd5Digests])
}
@objc func renameLocalFile(_ call: CAPPluginCall) {
guard let basePath = call.getString("basePath"),
let baseURL = URL(string: basePath) else {
call.reject("invalid basePath")
return
}
guard let from = call.getString("from") else {
call.reject("invalid from file")
return
}
guard let to = call.getString("to") else {
call.reject("invalid to file")
return
}
let fromUrl = baseURL.appendingPathComponent(from)
let toUrl = baseURL.appendingPathComponent(to)
do {
try FileManager.default.moveItem(at: fromUrl, to: toUrl)
} catch {
call.reject("can not rename file: \(error.localizedDescription)")
return
}
call.resolve(["ok": true])
}
@objc func deleteLocalFiles(_ call: CAPPluginCall) {
guard let baseURL = call.getString("basePath").flatMap({path in URL(string: path)}),
let filePaths = call.getArray("filePaths") as? [String] else {
call.reject("required paremeters: basePath, filePaths")
return
}
for filePath in filePaths {
let fileUrl = baseURL.appendingPathComponent(filePath)
try? FileManager.default.removeItem(at: fileUrl) // ignore any delete errors
}
call.resolve(["ok": true])
}
/// remote -> local
@objc func updateLocalFiles(_ call: CAPPluginCall) {
guard let baseURL = call.getString("basePath").flatMap({path in URL(string: path)}),
let filePaths = call.getArray("filePaths") as? [String],
let graphUUID = call.getString("graphUUID") ,
let token = call.getString("token") else {
call.reject("required paremeters: basePath, filePaths, graphUUID, token")
return
}
let client = SyncClient(token: token, graphUUID: graphUUID)
client.delegate = self // receives notification
client.getFiles(at: filePaths) { (fileURLs, error) in
if let error = error {
print("debug getFiles error \(error)")
self.debugNotification(["event": "download:error", "data": ["message": "error while getting files \(filePaths)"]])
call.reject(error.localizedDescription)
} else {
// handle multiple completionHandlers
let group = DispatchGroup()
var downloaded: [String] = []
for (filePath, remoteFileURL) in fileURLs {
group.enter()
// NOTE: fileURLs from getFiles API is percent-encoded
let localFileURL = baseURL.appendingPathComponent(filePath.decodeFromFname())
remoteFileURL.download(toFile: localFileURL) {error in
if let error = error {
self.debugNotification(["event": "download:error", "data": ["message": "error while downloading \(filePath): \(error)"]])
print("debug download \(error) in \(filePath)")
} else {
self.debugNotification(["event": "download:file", "data": ["file": filePath]])
downloaded.append(filePath)
}
group.leave()
}
}
group.notify(queue: .main) {
self.debugNotification(["event": "download:done"])
call.resolve(["ok": true, "data": downloaded])
}
}
}
}
@objc func deleteRemoteFiles(_ call: CAPPluginCall) {
guard let filePaths = call.getArray("filePaths") as? [String],
let graphUUID = call.getString("graphUUID"),
let token = call.getString("token"),
let txid = call.getInt("txid") else {
call.reject("required paremeters: filePaths, graphUUID, token, txid")
return
}
guard !filePaths.isEmpty else {
call.reject("empty filePaths")
return
}
let client = SyncClient(token: token, graphUUID: graphUUID, txid: txid)
client.deleteFiles(filePaths) { txid, error in
guard error == nil else {
call.reject("delete \(error!)")
return
}
guard let txid = txid else {
call.reject("missing txid")
return
}
call.resolve(["ok": true, "txid": txid])
}
}
/// local -> remote
@objc func updateRemoteFiles(_ call: CAPPluginCall) {
guard let baseURL = call.getString("basePath").flatMap({path in URL(string: path)}),
let filePaths = call.getArray("filePaths") as? [String],
let graphUUID = call.getString("graphUUID"),
let token = call.getString("token"),
let txid = call.getInt("txid") else {
call.reject("required paremeters: basePath, filePaths, graphUUID, token, txid")
return
}
guard !filePaths.isEmpty else {
return call.reject("empty filePaths")
}
print("debug begin updateRemoteFiles \(filePaths)")
let client = SyncClient(token: token, graphUUID: graphUUID, txid: txid)
client.delegate = self
// 1. refresh_temp_credential
client.getTempCredential() { (credentials, error) in
guard error == nil else {
self.debugNotification(["event": "upload:error", "data": ["message": "error while refreshing credential: \(error!)"]])
call.reject("error(getTempCredential): \(error!)")
return
}
var files: [String: URL] = [:]
for filePath in filePaths {
// NOTE: filePath from js may contain spaces
let fileURL = baseURL.appendingPathComponent(filePath)
files[filePath.encodeAsFname()] = fileURL
}
// 2. upload_temp_file
client.uploadTempFiles(files, credentials: credentials!) { (uploadedFileKeyDict, error) in
guard error == nil else {
self.debugNotification(["event": "upload:error", "data": ["message": "error while uploading temp files: \(error!)"]])
call.reject("error(uploadTempFiles): \(error!)")
return
}
// 3. update_files
guard !uploadedFileKeyDict.isEmpty else {
self.debugNotification(["event": "upload:error", "data": ["message": "no file to update"]])
call.reject("no file to update")
return
}
client.updateFiles(uploadedFileKeyDict) { (txid, error) in
guard error == nil else {
self.debugNotification(["event": "upload:error", "data": ["message": "error while updating files: \(error!)"]])
call.reject("error updateFiles: \(error!)")
return
}
guard let txid = txid else {
call.reject("error: missing txid")
return
}
self.debugNotification(["event": "upload:done", "data": ["files": filePaths, "txid": txid]])
call.resolve(["ok": true, "files": uploadedFileKeyDict, "txid": txid])
}
}
}
}
}

View File

@ -0,0 +1,36 @@
//
// Payload.swift
// Logseq
//
// Created by Mono Wang on 4/8/R4.
//
import Foundation
struct GetFilesResponse: Decodable {
let PresignedFileUrls: [String: String]
}
struct DeleteFilesResponse: Decodable {
let TXId: Int
let DeleteSuccFiles: [String]
let DeleteFailedFiles: [String: String]
}
public struct S3Credential: Decodable {
let AccessKeyId: String
let Expiration: String
let SecretKey: String
let SessionToken: String
}
struct GetTempCredentialResponse: Decodable {
let Credentials: S3Credential
let S3Prefix: String
}
struct UpdateFilesResponse: Decodable {
let TXId: Int
let UpdateSuccFiles: [String]
let UpdateFailedFiles: [String: String]
}

View File

@ -0,0 +1,326 @@
//
// SyncClient.swift
// Logseq
//
// Created by Mono Wang on 4/8/R4.
//
import Foundation
import AWSMobileClient
import AWSS3
public protocol SyncDebugDelegate {
func debugNotification(_ message: [String: Any])
}
public class SyncClient {
private var token: String
private var graphUUID: String?
private var txid: Int = 0
private var s3prefix: String?
public var delegate: SyncDebugDelegate? = nil
public init(token: String) {
self.token = token
}
public init(token: String, graphUUID: String) {
self.token = token
self.graphUUID = graphUUID
}
public init(token: String, graphUUID: String, txid: Int) {
self.token = token
self.graphUUID = graphUUID
self.txid = txid
}
// get_files
// => file_path, file_url
public func getFiles(at filePaths: [String], completionHandler: @escaping ([String: URL], Error?) -> Void) {
let url = URL_BASE.appendingPathComponent("get_files")
var request = URLRequest(url: url)
request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type")
request.setValue("Logseq-sync/0.1", forHTTPHeaderField: "User-Agent")
request.setValue("Bearer \(self.token)", forHTTPHeaderField: "Authorization")
let payload = [
"GraphUUID": self.graphUUID ?? "",
"Files": filePaths.map { filePath in filePath.encodeAsFname()}
] as [String : Any]
let bodyData = try? JSONSerialization.data(
withJSONObject: payload,
options: []
)
request.httpMethod = "POST"
request.httpBody = bodyData
let task = URLSession.shared.dataTask(with: request) { (data, response, error) in
guard error == nil else {
completionHandler([:], error)
return
}
if (response as? HTTPURLResponse)?.statusCode != 200 {
let body = String(data: data!, encoding: .utf8) ?? "";
completionHandler([:], NSError(domain: "", code: 400, userInfo: [NSLocalizedDescriptionKey: "http error \(body)"]))
return
}
if let data = data {
let resp = try? JSONDecoder().decode([String:[String:String]].self, from: data)
let files = resp?["PresignedFileUrls"] ?? [:]
self.delegate?.debugNotification(["event": "download:prepare"])
completionHandler(files.mapValues({ url in URL(string: url)!}), nil)
} else {
// Handle unexpected error
completionHandler([:], NSError(domain: "", code: 400, userInfo: [NSLocalizedDescriptionKey: "unexpected error"]))
}
}
task.resume()
}
public func deleteFiles(_ filePaths: [String], completionHandler: @escaping (Int?, Error?) -> Void) {
let url = URL_BASE.appendingPathComponent("delete_files")
var request = URLRequest(url: url)
request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type")
request.setValue("Logseq-sync/0.1", forHTTPHeaderField: "User-Agent")
request.setValue("Bearer \(self.token)", forHTTPHeaderField: "Authorization")
let payload = [
"GraphUUID": self.graphUUID ?? "",
"Files": filePaths.map { filePath in filePath.encodeAsFname()},
"TXId": self.txid,
] as [String : Any]
let bodyData = try? JSONSerialization.data(
withJSONObject: payload,
options: []
)
request.httpMethod = "POST"
request.httpBody = bodyData
let task = URLSession.shared.dataTask(with: request) { (data, response, error) in
guard error == nil else {
completionHandler(nil, error)
return
}
if let response = response as? HTTPURLResponse {
let body = String(data: data!, encoding: .utf8) ?? ""
if response.statusCode == 409 {
if body.contains("txid_to_validate") {
completionHandler(nil, NSError(domain: "",
code: 409,
userInfo: [NSLocalizedDescriptionKey: "invalid txid: \(body)"]))
return
}
// fallthrough
}
if response.statusCode != 200 {
completionHandler(nil, NSError(domain: "",
code: response.statusCode,
userInfo: [NSLocalizedDescriptionKey: "invalid http status \(response.statusCode): \(body)"]))
return
}
}
if let data = data {
do {
let resp = try JSONDecoder().decode(DeleteFilesResponse.self, from: data)
// TODO: handle api resp?
self.delegate?.debugNotification(["event": "delete"])
completionHandler(resp.TXId, nil)
} catch {
completionHandler(nil, error)
}
} else {
// Handle unexpected error
completionHandler(nil, NSError(domain: "", code: 400, userInfo: [NSLocalizedDescriptionKey: "unexpected error"]))
}
}
task.resume()
}
// (txid, error)
public func updateFiles(_ fileKeyDict: [String: String], completionHandler: @escaping (Int?, Error?) -> Void) {
let url = URL_BASE.appendingPathComponent("update_files")
var request = URLRequest(url: url)
request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type")
request.setValue("Logseq-sync/0.1", forHTTPHeaderField: "User-Agent")
request.setValue("Bearer \(self.token)", forHTTPHeaderField: "Authorization")
let payload = [
"GraphUUID": self.graphUUID ?? "",
"Files": Dictionary(uniqueKeysWithValues: fileKeyDict.map { ($0, $1) }) as [String: String] as Any,
"TXId": self.txid,
] as [String : Any]
let bodyData = try? JSONSerialization.data(
withJSONObject: payload,
options: []
)
request.httpMethod = "POST"
request.httpBody = bodyData
let task = URLSession.shared.dataTask(with: request) { (data, response, error) in
guard error == nil else {
completionHandler(nil, error)
return
}
if let response = response as? HTTPURLResponse {
let body = String(data: data!, encoding: .utf8) ?? ""
if response.statusCode == 409 {
if body.contains("txid_to_validate") {
completionHandler(nil, NSError(domain: "",
code: 409,
userInfo: [NSLocalizedDescriptionKey: "invalid txid: \(body)"]))
return
}
// fallthrough
}
if response.statusCode != 200 {
completionHandler(nil, NSError(domain: "",
code: response.statusCode,
userInfo: [NSLocalizedDescriptionKey: "invalid http status \(response.statusCode): \(body)"]))
return
}
}
if let data = data {
let resp = try? JSONDecoder().decode(UpdateFilesResponse.self, from: data)
if resp?.UpdateFailedFiles.isEmpty ?? true {
completionHandler(resp?.TXId, nil)
} else {
completionHandler(nil, NSError(domain: "", code: 400, userInfo: [NSLocalizedDescriptionKey: "update fail for some files: \(resp?.UpdateFailedFiles.debugDescription)"]))
}
} else {
// Handle unexpected error
completionHandler(nil, NSError(domain: "", code: 400, userInfo: [NSLocalizedDescriptionKey: "unexpected error"]))
}
}
task.resume()
}
public func getTempCredential(completionHandler: @escaping (S3Credential?, Error?) -> Void) {
let url = URL_BASE.appendingPathComponent("get_temp_credential")
var request = URLRequest(url: url)
request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type")
request.setValue("Logseq-sync/0.1", forHTTPHeaderField: "User-Agent")
request.setValue("Bearer \(self.token)", forHTTPHeaderField: "Authorization")
request.httpMethod = "POST"
request.httpBody = Data()
let task = URLSession.shared.dataTask(with: request) { (data, response, error) in
guard error == nil else {
completionHandler(nil, error)
return
}
if let response = response as? HTTPURLResponse {
let body = String(data: data!, encoding: .utf8) ?? ""
if response.statusCode == 401 {
completionHandler(nil, NSError(domain: "", code: 401, userInfo: [NSLocalizedDescriptionKey: "unauthorized"]))
return
}
if response.statusCode != 200 {
completionHandler(nil, NSError(domain: "",
code: response.statusCode,
userInfo: [NSLocalizedDescriptionKey: "invalid http status \(response.statusCode): \(body)"]))
return
}
}
if let data = data {
let resp = try? JSONDecoder().decode(GetTempCredentialResponse.self, from: data)
// NOTE: remove BUCKET prefix here.
self.s3prefix = resp?.S3Prefix.replacingOccurrences(of: "\(BUCKET)/", with: "")
self.delegate?.debugNotification(["event": "upload:prepare"])
completionHandler(resp?.Credentials, nil)
} else {
// Handle unexpected error
completionHandler(nil, NSError(domain: "", code: 400, userInfo: [NSLocalizedDescriptionKey: "unexpected error"]))
}
}
task.resume()
}
// [filePath, Key]
public func uploadTempFiles(_ files: [String: URL], credentials: S3Credential, completionHandler: @escaping ([String: String], Error?) -> Void) {
let credentialsProvider = AWSBasicSessionCredentialsProvider(
accessKey: credentials.AccessKeyId, secretKey: credentials.SecretKey, sessionToken: credentials.SessionToken)
let configuration = AWSServiceConfiguration(region: .USEast2, credentialsProvider: credentialsProvider)
configuration?.timeoutIntervalForRequest = 5.0
configuration?.timeoutIntervalForResource = 5.0
let tuConf = AWSS3TransferUtilityConfiguration()
tuConf.bucket = BUCKET
//x tuConf.isAccelerateModeEnabled = true
let transferKey = String.random(length: 10)
AWSS3TransferUtility.register(
with: configuration!,
transferUtilityConfiguration: tuConf,
forKey: transferKey
) { (error) in
if let error = error {
print("error while register tu \(error)")
}
}
let transferUtility = AWSS3TransferUtility.s3TransferUtility(forKey: transferKey)
let uploadExpression = AWSS3TransferUtilityUploadExpression()
let group = DispatchGroup()
var keyFileDict: [String: String] = [:]
var fileKeyDict: [String: String] = [:]
let uploadCompletionHandler = { (task: AWSS3TransferUtilityUploadTask, error: Error?) -> Void in
// ignore any errors in first level of handler
if let error = error {
self.delegate?.debugNotification(["event": "upload:error", "data": ["key": task.key, "error": error.localizedDescription]])
}
if let HTTPResponse = task.response {
if HTTPResponse.statusCode != 200 || task.status != .completed {
print("debug uploading error \(HTTPResponse)")
}
}
// only save successful keys
let filePath = keyFileDict[task.key]!
fileKeyDict[filePath] = task.key
keyFileDict.removeValue(forKey: task.key)
self.delegate?.debugNotification(["event": "upload:file", "data": ["file": filePath, "key": task.key]])
group.leave() // notify finish upload
}
for (filePath, fileLocalURL) in files {
print("debug, upload temp \(fileLocalURL) \(filePath)")
guard let rawData = try? Data(contentsOf: fileLocalURL) else { continue }
group.enter()
let randFileName = String.random(length: 15).appending(".").appending(fileLocalURL.pathExtension)
let key = "\(self.s3prefix!)/ios\(randFileName)"
keyFileDict[key] = filePath
transferUtility?.uploadData(rawData, key: key, contentType: "application/octet-stream", expression: uploadExpression, completionHandler: uploadCompletionHandler)
.continueWith(block: { (task) in
if let error = task.error {
completionHandler([:], error)
}
return nil
})
}
group.notify(queue: .main) {
AWSS3TransferUtility.remove(forKey: transferKey)
completionHandler(fileKeyDict, nil)
}
}
}

View File

@ -84,6 +84,10 @@ extension URL {
if self.lastPathComponent.starts(with: ".") {
return true
}
// NOTE: used by file-sync
if self.lastPathComponent == "graphs-txid.edn" {
return true
}
let allowedPathExtensions: Set = ["md", "markdown", "org", "css", "edn", "excalidraw"]
if allowedPathExtensions.contains(self.pathExtension.lowercased()) {
return false
@ -167,7 +171,7 @@ public class PollingWatcher {
}
private func tick() {
let startTime = DispatchTime.now()
// let startTime = DispatchTime.now()
if let enumerator = FileManager.default.enumerator(
at: url,
@ -205,9 +209,9 @@ public class PollingWatcher {
self.updateMetaDb(with: newMetaDb)
}
let elapsedNanoseconds = DispatchTime.now().uptimeNanoseconds - startTime.uptimeNanoseconds
let elapsedInMs = Double(elapsedNanoseconds) / 1_000_000
print("debug ticker elapsed=\(elapsedInMs)ms")
// let elapsedNanoseconds = DispatchTime.now().uptimeNanoseconds - startTime.uptimeNanoseconds
// let elapsedInMs = Double(elapsedNanoseconds) / 1_000_000
// print("debug ticker elapsed=\(elapsedInMs)ms")
if #available(iOS 13.0, *) {
timer?.schedule(deadline: .now().advanced(by: .seconds(2)), leeway: .milliseconds(100))

View File

@ -23,4 +23,6 @@ end
target 'Logseq' do
capacitor_pods
# Add your Pods here
pod 'AWSMobileClient'
pod 'AWSS3'
end

View File

@ -41,7 +41,7 @@
(ui/dropdown-with-links
(fn [{:keys [toggle-fn]}]
[:a.button
{:on-click toggle-fn}
{:on-click toggle-fn}
[:span.text-sm.font-medium (user-handler/email)]])
[{:title (t :logout)
:options {:on-click user-handler/logout}}]

View File

@ -8,10 +8,12 @@
[clojure.set :as set]
[clojure.string :as string]
[electron.ipc :as ipc]
[goog.string :as gstring]
[frontend.config :as config]
[frontend.debug :as debug]
[frontend.handler.user :as user]
[frontend.state :as state]
[frontend.mobile.util :as mobile-util]
[frontend.util :as util]
[frontend.util.persist-var :as persist-var]
[frontend.handler.notification :as notification]
@ -158,7 +160,8 @@
(go
(let [resp (http/post (str "https://" config/API-DOMAIN "/file-sync/" api-name)
{:oauth-token token
:body (js/JSON.stringify (clj->js body))})]
:body (js/JSON.stringify (clj->js body))
:with-credentials? false})]
{:resp (<! resp)
:api-name api-name
:body body})))
@ -179,7 +182,9 @@
(:resp resp))))))
(defn- remove-dir-prefix [dir path]
(let [r (string/replace path (js/RegExp. (str "^" dir)) "")]
(let [is-mobile-url? (string/starts-with? dir "file://")
dir (if is-mobile-url? (gstring/urlDecode dir) dir)
r (string/replace path (js/RegExp. (str "^" (gstring/regExpEscape dir))) "")]
(if (string/starts-with? r "/")
(string/replace-first r "/" "")
r)))
@ -419,7 +424,7 @@
(string/index-of (str (ex-cause r)) "operation timed out")
(> n 0))
(do
(prn (str "retry(" n ") ..."))
(print (str "retry(" n ") ..."))
(recur (dec n)))
r))))
@ -490,7 +495,108 @@
(retry-rsapi
#(p->c (ipc/ipc "delete-remote-files" graph-uuid base-path filepaths local-txid token))))))))
(def rsapi (->RSAPI))
(deftype CapacitorAPI []
IToken
(get-token [this]
(go
(or (state/get-auth-id-token)
(<! (.refresh-token this)))))
(refresh-token [_]
(go
(<! (user/refresh-id-token&access-token))
(state/get-auth-id-token)))
IRSAPI
(set-env [_ prod?]
(go (<! (p->c (.setEnv mobile-util/file-sync (clj->js {:env (if prod? "prod" "dev")}))))))
(get-local-all-files-meta [_ _graph-uuid base-path]
(go
(let [r (<! (p->c (.getLocalAllFilesMeta mobile-util/file-sync (clj->js {:basePath base-path}))))]
(if (instance? ExceptionInfo r)
r
(->> (.-result r)
js->clj
(map (fn [[path metadata]]
(->FileMetadata (get metadata "size") (get metadata "md5") path nil false nil)))
set)))))
(get-local-files-meta [_ _graph-uuid base-path filepaths]
(go
(let [r (<! (p->c (.getLocalFilesMeta mobile-util/file-sync
(clj->js {:basePath base-path
:filePaths filepaths}))))]
(if (instance? ExceptionInfo r)
r
(->> (.-result r)
js->clj
(map (fn [[path metadata]]
(->FileMetadata (get metadata "size") (get metadata "md5") path nil false nil)))
set)))))
(rename-local-file [_ _graph-uuid base-path from to]
(go
(<! (p->c (.renameLocalFile mobile-util/file-sync
(clj->js {:basePath base-path
:from from
:to to}))))))
(update-local-files [this graph-uuid base-path filepaths]
(go
(let [token (<! (get-token this))
r (<! (retry-rsapi
#(p->c (.updateLocalFiles mobile-util/file-sync (clj->js {:graphUUID graph-uuid
:basePath base-path
:filePaths filepaths
:token token})))))]
(when (state/developer-mode?) (check-files-exists base-path filepaths))
r)))
(delete-local-files [_ _graph-uuid base-path filepaths]
(go
(let [r (<! (retry-rsapi #(p->c (.deleteLocalFiles mobile-util/file-sync
(clj->js {:basePath base-path
:filePaths filepaths})))))]
(when (state/developer-mode?) (check-files-not-exists base-path filepaths))
r)))
(update-remote-file [this graph-uuid base-path filepath local-txid]
(update-remote-files this graph-uuid base-path [filepath] local-txid))
(update-remote-files [this graph-uuid base-path filepaths local-txid]
(go
(let [token (<! (get-token this))
r (<! (p->c (.updateRemoteFiles mobile-util/file-sync
(clj->js {:graphUUID graph-uuid
:basePath base-path
:filePaths filepaths
:txid local-txid
:token token}))))]
(prn ::debug-update-remote-files r)
(if (instance? ExceptionInfo r)
r
(get (js->clj r) "txid")))))
(delete-remote-files [this graph-uuid _base-path filepaths local-txid]
(let [token (<! (get-token this))
r (<! (p->c (.deleteRemoteFiles mobile-util/file-sync
(clj->js {:graphUUID graph-uuid
:filePaths filepaths
:txid local-txid
:token token}))))]
(if (instance? ExceptionInfo r)
r
(get (js->clj r) "txid")))))
(def rsapi (cond
(util/electron?)
(->RSAPI)
(mobile-util/native-ios?)
(->CapacitorAPI)
:else
nil))
(deftype RemoteAPI []
Object
@ -713,11 +819,11 @@
(defn file-watch-handler
"file-watcher callback"
[type {:keys [dir path _content stat] :as _payload}]
(go
(when (some-> (state/get-file-sync-state)
sync-state--stopped?
not)
(>! local-changes-chan (->FileChangeEvent type dir path stat)))))
(when (some-> (state/get-file-sync-state)
sync-state--stopped?
not)
(go (>! local-changes-chan (->FileChangeEvent type dir path stat)))))
;;; ### remote->local syncer & local->remote syncer

View File

@ -341,11 +341,13 @@
false)))))
(defmethod handle :file-watcher/changed [[_ ^js event]]
(fs-watcher/handle-changed!
(.-event event)
(update (js->clj event :keywordize-keys true)
:path
js/decodeURI)))
(let [type (.-event event)
payload (js->clj event :keywordize-keys true)
payload' (-> payload
(update :path js/decodeURI))]
(prn ::fs-watcher payload)
(fs-watcher/handle-changed! type payload')
(sync/file-watch-handler type payload')))
(defmethod handle :rebuild-slash-commands-list [[_]]
(page-handler/rebuild-slash-commands-list!))

View File

@ -81,7 +81,8 @@
(defn login-callback [code]
(go
(let [resp (<! (http/get (str "https://" config/API-DOMAIN "/auth_callback?code=" code)))]
(let [resp (<! (http/get (str "https://" config/API-DOMAIN "/auth_callback?code=" code)
{:with-credentials? false}))]
(if (= 200 (:status resp))
(-> resp
(:body)
@ -99,7 +100,8 @@
[]
(when-let [refresh-token (state/get-auth-refresh-token)]
(go
(let [resp (<! (http/get (str "https://" config/API-DOMAIN "/auth_refresh_token?refresh_token=" refresh-token)))]
(let [resp (<! (http/get (str "https://" config/API-DOMAIN "/auth_refresh_token?refresh_token=" refresh-token)
{:with-credentials? false}))]
(if (= 400 (:status resp))
;; invalid refresh_token
(do

View File

@ -48,7 +48,11 @@
(js/window.history.back))))))
(when (mobile-util/native-ios?)
(ios-init))
(ios-init)
(.removeAllListeners mobile-util/file-sync)
(.addListener mobile-util/file-sync "debug"
(fn [event]
(js/console.log "🔄" event))))
(when (mobile-util/is-native-platform?)
(.addListener mobile-util/fs-watcher "watcher"

View File

@ -23,7 +23,9 @@
(defonce folder-picker (registerPlugin "FolderPicker"))
(when (native-ios?)
(defonce download-icloud-files (registerPlugin "DownloadiCloudFiles"))
(defonce ios-file-container (registerPlugin "FileContainer")))
(defonce ios-file-container (registerPlugin "FileContainer"))
(defonce file-sync (registerPlugin "FileSync")))
;; NOTE: both iOS and android share the same FsWatcher API
(when (is-native-platform?)
(defonce fs-watcher (registerPlugin "FsWatcher")))