From 1524fc01f0fb113e693cb7c0c86cef533f504c0c Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Sat, 18 Mar 2023 16:44:30 +0100 Subject: [PATCH] Progressively enumerate items as we get them for working set Signed-off-by: Claudio Cambra --- .../FileProviderEnumerator+RemoteSync.swift | 51 +++++++++++++++++-- .../FileProviderEnumerator.swift | 30 +++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift index efc5ba69e..8d2fba570 100644 --- a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift +++ b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift @@ -17,7 +17,16 @@ import NextcloudKit import OSLog extension FileProviderEnumerator { - func fullRecursiveScan(ncAccount: NextcloudAccount, ncKit: NextcloudKit, scanChangesOnly: Bool, completionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable], _ newMetadatas: [NextcloudItemMetadataTable], _ updatedMetadatas: [NextcloudItemMetadataTable], _ deletedMetadatas: [NextcloudItemMetadataTable], _ error: NKError?) -> Void) { + func fullRecursiveScan(ncAccount: NextcloudAccount, + ncKit: NextcloudKit, + scanChangesOnly: Bool, + singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?, + _ error: NKError?) -> Void, + completionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable], + _ newMetadatas: [NextcloudItemMetadataTable], + _ updatedMetadatas: [NextcloudItemMetadataTable], + _ deletedMetadatas: [NextcloudItemMetadataTable], + _ error: NKError?) -> Void) { let rootContainerDirectoryMetadata = NextcloudItemMetadataTable() rootContainerDirectoryMetadata.directory = true @@ -27,7 +36,11 @@ extension FileProviderEnumerator { let dispatchQueue = DispatchQueue(label: "recursiveChangeEnumerationQueue", qos: .background) dispatchQueue.async { - let results = self.scanRecursively(rootContainerDirectoryMetadata, ncAccount: ncAccount, ncKit: ncKit, scanChangesOnly: scanChangesOnly) + let results = self.scanRecursively(rootContainerDirectoryMetadata, + ncAccount: ncAccount, + ncKit: ncKit, + scanChangesOnly: scanChangesOnly, + singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler) // Run a check to ensure files deleted in one location are not updated in another (e.g. when moved) // The recursive scan provides us with updated/deleted metadatas only on a folder by folder basis; @@ -48,14 +61,30 @@ extension FileProviderEnumerator { } } - private func scanRecursively(_ directoryMetadata: NextcloudItemMetadataTable, ncAccount: NextcloudAccount, ncKit: NextcloudKit, scanChangesOnly: Bool) -> (metadatas: [NextcloudItemMetadataTable], newMetadatas: [NextcloudItemMetadataTable], updatedMetadatas: [NextcloudItemMetadataTable], deletedMetadatas: [NextcloudItemMetadataTable], error: NKError?) { + private func scanRecursively(_ directoryMetadata: NextcloudItemMetadataTable, + ncAccount: NextcloudAccount, + ncKit: NextcloudKit, + scanChangesOnly: Bool, + singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?, + _ error: NKError?) -> Void) -> (metadatas: [NextcloudItemMetadataTable], + newMetadatas: [NextcloudItemMetadataTable], + updatedMetadatas: [NextcloudItemMetadataTable], + deletedMetadatas: [NextcloudItemMetadataTable], + error: NKError?) { if self.isInvalidated { + DispatchQueue.main.async { + singleFolderScanCompleteCompletionHandler(nil, nil) + } return ([], [], [], [], nil) } assert(directoryMetadata.directory, "Can only recursively scan a directory.") + // Scanned in this directory + var currentMetadatas: [NextcloudItemMetadataTable] = [] + + // Will include results of recursive calls var allMetadatas: [NextcloudItemMetadataTable] = [] var allNewMetadatas: [NextcloudItemMetadataTable] = [] var allUpdatedMetadatas: [NextcloudItemMetadataTable] = [] @@ -109,6 +138,7 @@ extension FileProviderEnumerator { Logger.enumeration.info("Finished reading serverUrl: \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") if let metadatas = metadatas { + currentMetadatas = metadatas allMetadatas += metadatas } else { Logger.enumeration.warning("WARNING: Nil metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") @@ -137,10 +167,17 @@ extension FileProviderEnumerator { dispatchGroup.wait() - if criticalError != nil { + guard criticalError == nil else { + DispatchQueue.main.async { + singleFolderScanCompleteCompletionHandler(nil, criticalError) + } return ([], [], [], [], error: criticalError) } + DispatchQueue.main.async { + singleFolderScanCompleteCompletionHandler(currentMetadatas, nil) + } + var childDirectoriesToScan: [NextcloudItemMetadataTable] = [] var candidateMetadatas: [NextcloudItemMetadataTable] @@ -161,7 +198,11 @@ extension FileProviderEnumerator { } for childDirectory in childDirectoriesToScan { - let childScanResult = scanRecursively(childDirectory, ncAccount: ncAccount, ncKit: ncKit, scanChangesOnly: scanChangesOnly) + let childScanResult = scanRecursively(childDirectory, + ncAccount: ncAccount, + ncKit: ncKit, + scanChangesOnly: scanChangesOnly, + singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler) allMetadatas += childScanResult.metadatas allNewMetadatas += childScanResult.newMetadatas diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift index b7a15d117..b94eb43ad 100644 --- a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift +++ b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift @@ -89,7 +89,26 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { if page == NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage || page == NSFileProviderPage.initialPageSortedByName as NSFileProviderPage { - fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false) { metadatas, _, _, _, error in + // We enumerate items as we get the server data for two reasons: + // A) we avoid having a gigantic chunk of files to enumerate to the observer at the end + // B) we don't need to worry about resolving which files are truly deleted vs moved at the end + fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false, singleFolderScanCompleteCompletionHandler: { metadatas, error in + + guard error == nil else { + Logger.enumeration.error("There was an error during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)") + observer.finishEnumeratingWithError(error!.toFileProviderError()) + return; + } + + guard let metadatas = metadatas else { + Logger.enumeration.warning("Received nil metadatas during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)") + return + } + + let items = FileProviderEnumerator.metadatasToFileProviderItems(metadatas, ncKit: self.ncKit) + observer.didEnumerate(items) + + }) { metadatas, _, _, _, error in if self.isInvalidated { Logger.enumeration.info("Enumerator invalidated during working set item enumeration. For user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") @@ -105,7 +124,7 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { Logger.enumeration.info("Finished recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)). Enumerating items.") - FileProviderEnumerator.completeEnumerationObserver(observer, ncKit: self.ncKit, numPage: 1, itemMetadatas: metadatas) + observer.finishEnumerating(upTo: FileProviderEnumerator.fileProviderPageforNumPage(1)) } return @@ -198,7 +217,12 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator { if enumeratedItemIdentifier == .workingSet { Logger.enumeration.debug("Enumerating changes in working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") - fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: true) { _, newMetadatas, updatedMetadatas, deletedMetadatas, error in + // Unlike when enumerating items we can't progressively enumerate items as we need to wait to resolve which items are truly deleted and which + // have just been moved elsewhere. + fullRecursiveScan(ncAccount: self.ncAccount, + ncKit: self.ncKit, + scanChangesOnly: true, + singleFolderScanCompleteCompletionHandler: { _, _ in }) { _, newMetadatas, updatedMetadatas, deletedMetadatas, error in if self.isInvalidated { Logger.enumeration.info("Enumerator invalidated during working set change scan. For user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))") -- 2.30.2