From: Claudio Cambra Date: Sat, 18 Mar 2023 17:04:19 +0000 (+0100) Subject: Move remote sync functionality out of enumerator and into separate NextcloudSyncEngin... X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~10^2~55^2~37 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=35570a3160bacb11b52e952cd00b90cdaea51a0e;p=nextcloud-desktop.git Move remote sync functionality out of enumerator and into separate NextcloudSyncEngine class Signed-off-by: Claudio Cambra --- diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift deleted file mode 100644 index 8d2fba570..000000000 --- a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright (C) 2023 by Claudio Cambra - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -import FileProvider -import NextcloudKit -import OSLog - -extension FileProviderEnumerator { - func fullRecursiveScan(ncAccount: NextcloudAccount, - ncKit: NextcloudKit, - scanChangesOnly: Bool, - singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?, - _ error: NKError?) -> Void, - completionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable], - _ newMetadatas: [NextcloudItemMetadataTable], - _ updatedMetadatas: [NextcloudItemMetadataTable], - _ deletedMetadatas: [NextcloudItemMetadataTable], - _ error: NKError?) -> Void) { - - let rootContainerDirectoryMetadata = NextcloudItemMetadataTable() - rootContainerDirectoryMetadata.directory = true - rootContainerDirectoryMetadata.ocId = NSFileProviderItemIdentifier.rootContainer.rawValue - - // Create a serial dispatch queue - let dispatchQueue = DispatchQueue(label: "recursiveChangeEnumerationQueue", qos: .background) - - dispatchQueue.async { - let results = self.scanRecursively(rootContainerDirectoryMetadata, - ncAccount: ncAccount, - ncKit: ncKit, - scanChangesOnly: scanChangesOnly, - singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler) - - // Run a check to ensure files deleted in one location are not updated in another (e.g. when moved) - // The recursive scan provides us with updated/deleted metadatas only on a folder by folder basis; - // so we need to check we are not simultaneously marking a moved file as deleted and updated - var checkedDeletedMetadatas = results.deletedMetadatas - - for updatedMetadata in results.updatedMetadatas { - guard let matchingDeletedMetadataIdx = checkedDeletedMetadatas.firstIndex(where: { $0.ocId == updatedMetadata.ocId } ) else { - continue; - } - - checkedDeletedMetadatas.remove(at: matchingDeletedMetadataIdx) - } - - DispatchQueue.main.async { - completionHandler(results.metadatas, results.newMetadatas, results.updatedMetadatas, checkedDeletedMetadatas, results.error) - } - } - } - - private func scanRecursively(_ directoryMetadata: NextcloudItemMetadataTable, - ncAccount: NextcloudAccount, - ncKit: NextcloudKit, - scanChangesOnly: Bool, - singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?, - _ error: NKError?) -> Void) -> (metadatas: [NextcloudItemMetadataTable], - newMetadatas: [NextcloudItemMetadataTable], - updatedMetadatas: [NextcloudItemMetadataTable], - deletedMetadatas: [NextcloudItemMetadataTable], - error: NKError?) { - - if self.isInvalidated { - DispatchQueue.main.async { - singleFolderScanCompleteCompletionHandler(nil, nil) - } - return ([], [], [], [], nil) - } - - assert(directoryMetadata.directory, "Can only recursively scan a directory.") - - // Scanned in this directory - var currentMetadatas: [NextcloudItemMetadataTable] = [] - - // Will include results of recursive calls - var allMetadatas: [NextcloudItemMetadataTable] = [] - var allNewMetadatas: [NextcloudItemMetadataTable] = [] - var allUpdatedMetadatas: [NextcloudItemMetadataTable] = [] - var allDeletedMetadatas: [NextcloudItemMetadataTable] = [] - - let dbManager = NextcloudFilesDatabaseManager.shared - let dispatchGroup = DispatchGroup() // TODO: Maybe own thread? - - dispatchGroup.enter() - - var criticalError: NKError? - let itemServerUrl = directoryMetadata.ocId == NSFileProviderItemIdentifier.rootContainer.rawValue ? - ncAccount.davFilesUrl : directoryMetadata.serverUrl + "/" + directoryMetadata.fileName - - Logger.enumeration.debug("About to read: \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash))") - - FileProviderEnumerator.readServerUrl(itemServerUrl, ncAccount: ncAccount, ncKit: ncKit, stopAtMatchingEtags: scanChangesOnly) { metadatas, newMetadatas, updatedMetadatas, deletedMetadatas, readError in - - if readError != nil { - let nkReadError = NKError(error: readError!) - - // Is the error is that we have found matching etags on this item, then ignore it - // if we are doing a full rescan - guard nkReadError.isNoChangesError && scanChangesOnly else { - Logger.enumeration.error("Finishing enumeration of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) with \(readError!.localizedDescription, privacy: .public)") - - if nkReadError.isNotFoundError { - Logger.enumeration.info("404 error means item no longer exists. Deleting metadata and reporting as deletion without error") - - if let deletedMetadatas = dbManager.deleteDirectoryAndSubdirectoriesMetadata(ocId: directoryMetadata.ocId) { - allDeletedMetadatas += deletedMetadatas - } else { - Logger.enumeration.error("An error occurred while trying to delete directory and children not found in recursive scan") - } - - } else if nkReadError.isNoChangesError { // All is well, just no changed etags - Logger.enumeration.info("Error was to say no changed files -- not bad error. No need to check children.") - - } else if nkReadError.isUnauthenticatedError || nkReadError.isCouldntConnectError { - // If it is a critical error then stop, if not then continue - Logger.enumeration.error("Error will affect next enumerated items, so stopping enumeration.") - criticalError = nkReadError - - } - - dispatchGroup.leave() - return - } - } - - Logger.enumeration.info("Finished reading serverUrl: \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - - if let metadatas = metadatas { - currentMetadatas = metadatas - allMetadatas += metadatas - } else { - Logger.enumeration.warning("WARNING: Nil metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - } - - if let newMetadatas = newMetadatas { - allNewMetadatas += newMetadatas - } else { - Logger.enumeration.warning("WARNING: Nil new metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - } - - if let updatedMetadatas = updatedMetadatas { - allUpdatedMetadatas += updatedMetadatas - } else { - Logger.enumeration.warning("WARNING: Nil updated metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - } - - if let deletedMetadatas = deletedMetadatas { - allDeletedMetadatas += deletedMetadatas - } else { - Logger.enumeration.warning("WARNING: Nil deleted metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - } - - dispatchGroup.leave() - } - - dispatchGroup.wait() - - guard criticalError == nil else { - DispatchQueue.main.async { - singleFolderScanCompleteCompletionHandler(nil, criticalError) - } - return ([], [], [], [], error: criticalError) - } - - DispatchQueue.main.async { - singleFolderScanCompleteCompletionHandler(currentMetadatas, nil) - } - - var childDirectoriesToScan: [NextcloudItemMetadataTable] = [] - var candidateMetadatas: [NextcloudItemMetadataTable] - - if scanChangesOnly { - candidateMetadatas = allUpdatedMetadatas - } else { - candidateMetadatas = allMetadatas - } - - for candidateMetadata in candidateMetadatas { - if candidateMetadata.directory { - childDirectoriesToScan.append(candidateMetadata) - } - } - - if childDirectoriesToScan.isEmpty { - return (metadatas: allMetadatas, newMetadatas: allNewMetadatas, updatedMetadatas: allUpdatedMetadatas, deletedMetadatas: allDeletedMetadatas, nil) - } - - for childDirectory in childDirectoriesToScan { - let childScanResult = scanRecursively(childDirectory, - ncAccount: ncAccount, - ncKit: ncKit, - scanChangesOnly: scanChangesOnly, - singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler) - - allMetadatas += childScanResult.metadatas - allNewMetadatas += childScanResult.newMetadatas - allUpdatedMetadatas += childScanResult.updatedMetadatas - allDeletedMetadatas += childScanResult.deletedMetadatas - } - - return (metadatas: allMetadatas, newMetadatas: allNewMetadatas, updatedMetadatas: allUpdatedMetadatas, deletedMetadatas: allDeletedMetadatas, nil) - } - - static func readServerUrl(_ serverUrl: String, ncAccount: NextcloudAccount, ncKit: NextcloudKit, stopAtMatchingEtags: Bool = false, completionHandler: @escaping (_ metadatas: [NextcloudItemMetadataTable]?, _ newMetadatas: [NextcloudItemMetadataTable]?, _ updatedMetadatas: [NextcloudItemMetadataTable]?, _ deletedMetadatas: [NextcloudItemMetadataTable]?, _ readError: Error?) -> Void) { - let dbManager = NextcloudFilesDatabaseManager.shared - let ncKitAccount = ncAccount.ncKitAccount - - Logger.enumeration.debug("Starting to read serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) at depth 0. NCKit info: userId: \(ncKit.nkCommonInstance.user), password: \(ncKit.nkCommonInstance.password == "" ? "EMPTY PASSWORD" : "NOT EMPTY PASSWORD"), urlBase: \(ncKit.nkCommonInstance.urlBase), ncVersion: \(ncKit.nkCommonInstance.nextcloudVersion)") - - ncKit.readFileOrFolder(serverUrlFileName: serverUrl, depth: "0", showHiddenFiles: true) { account, files, _, error in - guard error == .success else { - Logger.enumeration.error("0 depth readFileOrFolder of url: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) did not complete successfully, received error: \(error.errorDescription, privacy: .public)") - completionHandler(nil, nil, nil, nil, error.error) - return - } - - guard let receivedItem = files.first else { - Logger.enumeration.error("Received no items from readFileOrFolder of \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)), not much we can do...") - completionHandler(nil, nil, nil, nil, error.error) - return - } - - guard receivedItem.directory else { - Logger.enumeration.debug("Read item is a file. Converting NKfile for serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - let itemMetadata = dbManager.convertNKFileToItemMetadata(receivedItem, account: ncKitAccount) - dbManager.addItemMetadata(itemMetadata) // TODO: Return some value when it is an update - completionHandler([itemMetadata], nil, nil, nil, error.error) - return - } - - if stopAtMatchingEtags, - let directoryMetadata = dbManager.directoryMetadata(account: ncKitAccount, serverUrl: serverUrl) { - - let directoryEtag = directoryMetadata.etag - - guard directoryEtag == "" || directoryEtag != receivedItem.etag else { - Logger.enumeration.debug("Read server url called with flag to stop enumerating at matching etags. Returning and providing soft error.") - - let description = "Fetched directory etag is same as that stored locally. Not fetching child items." - let nkError = NKError(errorCode: NKError.noChangesErrorCode, errorDescription: description) - - let metadatas = dbManager.itemMetadatas(account: account, serverUrl: serverUrl) - - completionHandler(metadatas, nil, nil, nil, nkError.error) - return - } - } - - Logger.enumeration.debug("Starting to read serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) at depth 1") - - ncKit.readFileOrFolder(serverUrlFileName: serverUrl, depth: "1", showHiddenFiles: true) { account, files, _, error in - guard error == .success else { - Logger.enumeration.error("1 depth readFileOrFolder of url: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) did not complete successfully, received error: \(error.errorDescription, privacy: .public)") - completionHandler(nil, nil, nil, nil, error.error) - return - } - - Logger.enumeration.debug("Starting async conversion of NKFiles for serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - DispatchQueue.main.async { - dbManager.convertNKFilesFromDirectoryReadToItemMetadatas(files, account: ncKitAccount) { directoryMetadata, childDirectoriesMetadata, metadatas in - - // STORE DATA FOR CURRENTLY SCANNED DIRECTORY - // We have now scanned this directory's contents, so update with etag in order to not check again if not needed - // unless it's the root container - if serverUrl != ncAccount.davFilesUrl { - dbManager.addItemMetadata(directoryMetadata) - } - - // Don't update the etags for folders as we haven't checked their contents. - // When we do a recursive check, if we update the etags now, we will think - // that our local copies are up to date -- instead, leave them as the old. - // They will get updated when they are the subject of a readServerUrl call. - // (See above) - let changedMetadatas = dbManager.updateItemMetadatas(account: ncKitAccount, serverUrl: serverUrl, updatedMetadatas: metadatas, updateDirectoryEtags: false) - - completionHandler(metadatas, changedMetadatas.newMetadatas, changedMetadatas.updatedMetadatas, changedMetadatas.deletedMetadatas, nil) - } - } - } - } - } -} diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift index b94eb43ad..103179530 100644 --- a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift +++ b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift @@ -30,6 +30,8 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { var serverUrl: String = "" var isInvalidated = false + let syncEngine = NextcloudSyncEngine() + private static func isSystemIdentifier(_ identifier: NSFileProviderItemIdentifier) -> Bool { return identifier == .rootContainer || identifier == .trashContainer || @@ -61,6 +63,8 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { } func invalidate() { + Logger.enumeration.debug("Enumerator is being invalidated for item with identifier: \(self.enumeratedItemIdentifier.rawValue, privacy: .public)") + syncEngine.invalidate() self.isInvalidated = true } @@ -92,7 +96,7 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { // We enumerate items as we get the server data for two reasons: // A) we avoid having a gigantic chunk of files to enumerate to the observer at the end // B) we don't need to worry about resolving which files are truly deleted vs moved at the end - fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false, singleFolderScanCompleteCompletionHandler: { metadatas, error in + syncEngine.fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false, singleFolderScanCompleteCompletionHandler: { metadatas, error in guard error == nil else { Logger.enumeration.error("There was an error during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)") @@ -156,7 +160,7 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { Logger.enumeration.debug("Enumerating initial page for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))") - FileProviderEnumerator.readServerUrl(serverUrl, ncAccount: ncAccount, ncKit: ncKit) { _, _, _, _, readError in + NextcloudSyncEngine.readServerUrl(serverUrl, ncAccount: ncAccount, ncKit: ncKit) { _, _, _, _, readError in guard readError == nil else { Logger.enumeration.error("Finishing enumeration for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) with error \(readError!.localizedDescription, privacy: .public)") @@ -219,7 +223,7 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { // Unlike when enumerating items we can't progressively enumerate items as we need to wait to resolve which items are truly deleted and which // have just been moved elsewhere. - fullRecursiveScan(ncAccount: self.ncAccount, + syncEngine.fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: true, singleFolderScanCompleteCompletionHandler: { _, _ in }) { _, newMetadatas, updatedMetadatas, deletedMetadatas, error in @@ -258,7 +262,7 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { // No matter what happens here we finish enumeration in some way, either from the error // handling below or from the completeChangesObserver - FileProviderEnumerator.readServerUrl(serverUrl, ncAccount: ncAccount, ncKit: ncKit, stopAtMatchingEtags: true) { _, newMetadatas, updatedMetadatas, deletedMetadatas, readError in + NextcloudSyncEngine.readServerUrl(serverUrl, ncAccount: ncAccount, ncKit: ncKit, stopAtMatchingEtags: true) { _, newMetadatas, updatedMetadatas, deletedMetadatas, readError in // If we get a 404 we might add more deleted metadatas var currentDeletedMetadatas: [NextcloudItemMetadataTable] = [] diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/NextcloudSyncEngine.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/NextcloudSyncEngine.swift new file mode 100644 index 000000000..87f64ebf8 --- /dev/null +++ b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/NextcloudSyncEngine.swift @@ -0,0 +1,301 @@ +/* + * Copyright (C) 2023 by Claudio Cambra + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +import FileProvider +import NextcloudKit +import OSLog + +class NextcloudSyncEngine : NSObject { + + var isInvalidated = false + + func invalidate() { + self.isInvalidated = true + } + + func fullRecursiveScan(ncAccount: NextcloudAccount, + ncKit: NextcloudKit, + scanChangesOnly: Bool, + singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?, + _ error: NKError?) -> Void, + completionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable], + _ newMetadatas: [NextcloudItemMetadataTable], + _ updatedMetadatas: [NextcloudItemMetadataTable], + _ deletedMetadatas: [NextcloudItemMetadataTable], + _ error: NKError?) -> Void) { + + let rootContainerDirectoryMetadata = NextcloudItemMetadataTable() + rootContainerDirectoryMetadata.directory = true + rootContainerDirectoryMetadata.ocId = NSFileProviderItemIdentifier.rootContainer.rawValue + + // Create a serial dispatch queue + let dispatchQueue = DispatchQueue(label: "recursiveChangeEnumerationQueue", qos: .userInitiated) + + dispatchQueue.async { + let results = self.scanRecursively(rootContainerDirectoryMetadata, + ncAccount: ncAccount, + ncKit: ncKit, + scanChangesOnly: scanChangesOnly, + singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler) + + // Run a check to ensure files deleted in one location are not updated in another (e.g. when moved) + // The recursive scan provides us with updated/deleted metadatas only on a folder by folder basis; + // so we need to check we are not simultaneously marking a moved file as deleted and updated + var checkedDeletedMetadatas = results.deletedMetadatas + + for updatedMetadata in results.updatedMetadatas { + guard let matchingDeletedMetadataIdx = checkedDeletedMetadatas.firstIndex(where: { $0.ocId == updatedMetadata.ocId } ) else { + continue; + } + + checkedDeletedMetadatas.remove(at: matchingDeletedMetadataIdx) + } + + DispatchQueue.main.async { + completionHandler(results.metadatas, results.newMetadatas, results.updatedMetadatas, checkedDeletedMetadatas, results.error) + } + } + } + + private func scanRecursively(_ directoryMetadata: NextcloudItemMetadataTable, + ncAccount: NextcloudAccount, + ncKit: NextcloudKit, + scanChangesOnly: Bool, + singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?, + _ error: NKError?) -> Void) -> (metadatas: [NextcloudItemMetadataTable], + newMetadatas: [NextcloudItemMetadataTable], + updatedMetadatas: [NextcloudItemMetadataTable], + deletedMetadatas: [NextcloudItemMetadataTable], + error: NKError?) { + + if self.isInvalidated { + DispatchQueue.main.async { + singleFolderScanCompleteCompletionHandler(nil, nil) + } + return ([], [], [], [], nil) + } + + assert(directoryMetadata.directory, "Can only recursively scan a directory.") + + // Scanned in this directory + var currentMetadatas: [NextcloudItemMetadataTable] = [] + + // Will include results of recursive calls + var allMetadatas: [NextcloudItemMetadataTable] = [] + var allNewMetadatas: [NextcloudItemMetadataTable] = [] + var allUpdatedMetadatas: [NextcloudItemMetadataTable] = [] + var allDeletedMetadatas: [NextcloudItemMetadataTable] = [] + + let dbManager = NextcloudFilesDatabaseManager.shared + let dispatchGroup = DispatchGroup() // TODO: Maybe own thread? + + dispatchGroup.enter() + + var criticalError: NKError? + let itemServerUrl = directoryMetadata.ocId == NSFileProviderItemIdentifier.rootContainer.rawValue ? + ncAccount.davFilesUrl : directoryMetadata.serverUrl + "/" + directoryMetadata.fileName + + Logger.enumeration.debug("About to read: \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash))") + + NextcloudSyncEngine.readServerUrl(itemServerUrl, ncAccount: ncAccount, ncKit: ncKit, stopAtMatchingEtags: scanChangesOnly) { metadatas, newMetadatas, updatedMetadatas, deletedMetadatas, readError in + + if readError != nil { + let nkReadError = NKError(error: readError!) + + // Is the error is that we have found matching etags on this item, then ignore it + // if we are doing a full rescan + guard nkReadError.isNoChangesError && scanChangesOnly else { + Logger.enumeration.error("Finishing enumeration of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) with \(readError!.localizedDescription, privacy: .public)") + + if nkReadError.isNotFoundError { + Logger.enumeration.info("404 error means item no longer exists. Deleting metadata and reporting as deletion without error") + + if let deletedMetadatas = dbManager.deleteDirectoryAndSubdirectoriesMetadata(ocId: directoryMetadata.ocId) { + allDeletedMetadatas += deletedMetadatas + } else { + Logger.enumeration.error("An error occurred while trying to delete directory and children not found in recursive scan") + } + + } else if nkReadError.isNoChangesError { // All is well, just no changed etags + Logger.enumeration.info("Error was to say no changed files -- not bad error. No need to check children.") + + } else if nkReadError.isUnauthenticatedError || nkReadError.isCouldntConnectError { + // If it is a critical error then stop, if not then continue + Logger.enumeration.error("Error will affect next enumerated items, so stopping enumeration.") + criticalError = nkReadError + + } + + dispatchGroup.leave() + return + } + } + + Logger.enumeration.info("Finished reading serverUrl: \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + + if let metadatas = metadatas { + currentMetadatas = metadatas + allMetadatas += metadatas + } else { + Logger.enumeration.warning("WARNING: Nil metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + } + + if let newMetadatas = newMetadatas { + allNewMetadatas += newMetadatas + } else { + Logger.enumeration.warning("WARNING: Nil new metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + } + + if let updatedMetadatas = updatedMetadatas { + allUpdatedMetadatas += updatedMetadatas + } else { + Logger.enumeration.warning("WARNING: Nil updated metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + } + + if let deletedMetadatas = deletedMetadatas { + allDeletedMetadatas += deletedMetadatas + } else { + Logger.enumeration.warning("WARNING: Nil deleted metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + } + + dispatchGroup.leave() + } + + dispatchGroup.wait() + + guard criticalError == nil else { + DispatchQueue.main.async { + singleFolderScanCompleteCompletionHandler(nil, criticalError) + } + return ([], [], [], [], error: criticalError) + } + + DispatchQueue.main.async { + singleFolderScanCompleteCompletionHandler(currentMetadatas, nil) + } + + var childDirectoriesToScan: [NextcloudItemMetadataTable] = [] + var candidateMetadatas: [NextcloudItemMetadataTable] + + if scanChangesOnly { + candidateMetadatas = allUpdatedMetadatas + } else { + candidateMetadatas = allMetadatas + } + + for candidateMetadata in candidateMetadatas { + if candidateMetadata.directory { + childDirectoriesToScan.append(candidateMetadata) + } + } + + if childDirectoriesToScan.isEmpty { + return (metadatas: allMetadatas, newMetadatas: allNewMetadatas, updatedMetadatas: allUpdatedMetadatas, deletedMetadatas: allDeletedMetadatas, nil) + } + + for childDirectory in childDirectoriesToScan { + let childScanResult = scanRecursively(childDirectory, + ncAccount: ncAccount, + ncKit: ncKit, + scanChangesOnly: scanChangesOnly, + singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler) + + allMetadatas += childScanResult.metadatas + allNewMetadatas += childScanResult.newMetadatas + allUpdatedMetadatas += childScanResult.updatedMetadatas + allDeletedMetadatas += childScanResult.deletedMetadatas + } + + return (metadatas: allMetadatas, newMetadatas: allNewMetadatas, updatedMetadatas: allUpdatedMetadatas, deletedMetadatas: allDeletedMetadatas, nil) + } + + static func readServerUrl(_ serverUrl: String, ncAccount: NextcloudAccount, ncKit: NextcloudKit, stopAtMatchingEtags: Bool = false, completionHandler: @escaping (_ metadatas: [NextcloudItemMetadataTable]?, _ newMetadatas: [NextcloudItemMetadataTable]?, _ updatedMetadatas: [NextcloudItemMetadataTable]?, _ deletedMetadatas: [NextcloudItemMetadataTable]?, _ readError: Error?) -> Void) { + let dbManager = NextcloudFilesDatabaseManager.shared + let ncKitAccount = ncAccount.ncKitAccount + + Logger.enumeration.debug("Starting to read serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) at depth 0. NCKit info: userId: \(ncKit.nkCommonInstance.user), password: \(ncKit.nkCommonInstance.password == "" ? "EMPTY PASSWORD" : "NOT EMPTY PASSWORD"), urlBase: \(ncKit.nkCommonInstance.urlBase), ncVersion: \(ncKit.nkCommonInstance.nextcloudVersion)") + + ncKit.readFileOrFolder(serverUrlFileName: serverUrl, depth: "0", showHiddenFiles: true) { account, files, _, error in + guard error == .success else { + Logger.enumeration.error("0 depth readFileOrFolder of url: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) did not complete successfully, received error: \(error.errorDescription, privacy: .public)") + completionHandler(nil, nil, nil, nil, error.error) + return + } + + guard let receivedItem = files.first else { + Logger.enumeration.error("Received no items from readFileOrFolder of \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)), not much we can do...") + completionHandler(nil, nil, nil, nil, error.error) + return + } + + guard receivedItem.directory else { + Logger.enumeration.debug("Read item is a file. Converting NKfile for serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + let itemMetadata = dbManager.convertNKFileToItemMetadata(receivedItem, account: ncKitAccount) + dbManager.addItemMetadata(itemMetadata) // TODO: Return some value when it is an update + completionHandler([itemMetadata], nil, nil, nil, error.error) + return + } + + if stopAtMatchingEtags, + let directoryMetadata = dbManager.directoryMetadata(account: ncKitAccount, serverUrl: serverUrl) { + + let directoryEtag = directoryMetadata.etag + + guard directoryEtag == "" || directoryEtag != receivedItem.etag else { + Logger.enumeration.debug("Read server url called with flag to stop enumerating at matching etags. Returning and providing soft error.") + + let description = "Fetched directory etag is same as that stored locally. Not fetching child items." + let nkError = NKError(errorCode: NKError.noChangesErrorCode, errorDescription: description) + + let metadatas = dbManager.itemMetadatas(account: account, serverUrl: serverUrl) + + completionHandler(metadatas, nil, nil, nil, nkError.error) + return + } + } + + Logger.enumeration.debug("Starting to read serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) at depth 1") + + ncKit.readFileOrFolder(serverUrlFileName: serverUrl, depth: "1", showHiddenFiles: true) { account, files, _, error in + guard error == .success else { + Logger.enumeration.error("1 depth readFileOrFolder of url: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) did not complete successfully, received error: \(error.errorDescription, privacy: .public)") + completionHandler(nil, nil, nil, nil, error.error) + return + } + + Logger.enumeration.debug("Starting async conversion of NKFiles for serverUrl: \(serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") + DispatchQueue.main.async { + dbManager.convertNKFilesFromDirectoryReadToItemMetadatas(files, account: ncKitAccount) { directoryMetadata, childDirectoriesMetadata, metadatas in + + // STORE DATA FOR CURRENTLY SCANNED DIRECTORY + // We have now scanned this directory's contents, so update with etag in order to not check again if not needed + // unless it's the root container + if serverUrl != ncAccount.davFilesUrl { + dbManager.addItemMetadata(directoryMetadata) + } + + // Don't update the etags for folders as we haven't checked their contents. + // When we do a recursive check, if we update the etags now, we will think + // that our local copies are up to date -- instead, leave them as the old. + // They will get updated when they are the subject of a readServerUrl call. + // (See above) + let changedMetadatas = dbManager.updateItemMetadatas(account: ncKitAccount, serverUrl: serverUrl, updatedMetadatas: metadatas, updateDirectoryEtags: false) + + completionHandler(metadatas, changedMetadatas.newMetadatas, changedMetadatas.updatedMetadatas, changedMetadatas.deletedMetadatas, nil) + } + } + } + } + } +} diff --git a/shell_integration/MacOSX/NextcloudIntegration/NextcloudIntegration.xcodeproj/project.pbxproj b/shell_integration/MacOSX/NextcloudIntegration/NextcloudIntegration.xcodeproj/project.pbxproj index 17c5d351a..0ba89693b 100644 --- a/shell_integration/MacOSX/NextcloudIntegration/NextcloudIntegration.xcodeproj/project.pbxproj +++ b/shell_integration/MacOSX/NextcloudIntegration/NextcloudIntegration.xcodeproj/project.pbxproj @@ -36,7 +36,7 @@ 53903D37295618A400D0B308 /* LineProcessor.h in Headers */ = {isa = PBXBuildFile; fileRef = 53903D36295618A400D0B308 /* LineProcessor.h */; settings = {ATTRIBUTES = (Public, ); }; }; 539158AC27BE71A900816F56 /* FinderSyncSocketLineProcessor.m in Sources */ = {isa = PBXBuildFile; fileRef = 539158AB27BE71A900816F56 /* FinderSyncSocketLineProcessor.m */; }; 53D056312970594F00988392 /* LocalFilesUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53D056302970594F00988392 /* LocalFilesUtils.swift */; }; - 53ED472029C5E64200795DB1 /* FileProviderEnumerator+RemoteSync.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53ED471F29C5E64200795DB1 /* FileProviderEnumerator+RemoteSync.swift */; }; + 53ED472029C5E64200795DB1 /* NextcloudSyncEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53ED471F29C5E64200795DB1 /* NextcloudSyncEngine.swift */; }; C2B573BA1B1CD91E00303B36 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = C2B573B91B1CD91E00303B36 /* main.m */; }; C2B573D21B1CD94B00303B36 /* main.m in Resources */ = {isa = PBXBuildFile; fileRef = C2B573B91B1CD91E00303B36 /* main.m */; }; C2B573DE1B1CD9CE00303B36 /* FinderSync.m in Sources */ = {isa = PBXBuildFile; fileRef = C2B573DD1B1CD9CE00303B36 /* FinderSync.m */; }; @@ -160,7 +160,7 @@ 539158B127BE891500816F56 /* LocalSocketClient.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = LocalSocketClient.h; sourceTree = ""; }; 539158B227BEC98A00816F56 /* LocalSocketClient.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LocalSocketClient.m; sourceTree = ""; }; 53D056302970594F00988392 /* LocalFilesUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalFilesUtils.swift; sourceTree = ""; }; - 53ED471F29C5E64200795DB1 /* FileProviderEnumerator+RemoteSync.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "FileProviderEnumerator+RemoteSync.swift"; sourceTree = ""; }; + 53ED471F29C5E64200795DB1 /* NextcloudSyncEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NextcloudSyncEngine.swift; sourceTree = ""; }; C2B573B11B1CD91E00303B36 /* desktopclient.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = desktopclient.app; sourceTree = BUILT_PRODUCTS_DIR; }; C2B573B51B1CD91E00303B36 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; C2B573B91B1CD91E00303B36 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; @@ -249,13 +249,13 @@ 5318AD8F29BF406500CBB71C /* Database */, 5352E85929B7BFB4002CE85C /* Extensions */, 538E397027F4765000FA63D5 /* FileProviderEnumerator.swift */, - 53ED471F29C5E64200795DB1 /* FileProviderEnumerator+RemoteSync.swift */, 538E396C27F4765000FA63D5 /* FileProviderExtension.swift */, 538E396E27F4765000FA63D5 /* FileProviderItem.swift */, 5318AD9629BF493600CBB71C /* FileProviderMaterialisedEnumerationObserver.swift */, 536EFBF6295CF58100F4CB13 /* FileProviderSocketLineProcessor.swift */, 53D056302970594F00988392 /* LocalFilesUtils.swift */, 536EFC35295E3C1100F4CB13 /* NextcloudAccount.swift */, + 53ED471F29C5E64200795DB1 /* NextcloudSyncEngine.swift */, 538E397327F4765000FA63D5 /* FileProviderExt.entitlements */, 538E397227F4765000FA63D5 /* Info.plist */, ); @@ -564,7 +564,7 @@ 536EFC36295E3C1100F4CB13 /* NextcloudAccount.swift in Sources */, 538E396D27F4765000FA63D5 /* FileProviderExtension.swift in Sources */, 536EFBF7295CF58100F4CB13 /* FileProviderSocketLineProcessor.swift in Sources */, - 53ED472029C5E64200795DB1 /* FileProviderEnumerator+RemoteSync.swift in Sources */, + 53ED472029C5E64200795DB1 /* NextcloudSyncEngine.swift in Sources */, 5318AD9929BF58D000CBB71C /* NKError+Extensions.swift in Sources */, 5318AD9529BF438F00CBB71C /* NextcloudLocalFileMetadataTable.swift in Sources */, 535AE30E29C0A2CC0042A9BA /* Logger+Extensions.swift in Sources */,