Progressively enumerate items as we get them for working set

Signed-off-by: Claudio Cambra <claudio.cambra@nextcloud.com>

diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift
index efc5ba69e95dac6db66146a1cdd7ff8668248c57..8d2fba570dcd590c4d9a57b62601abb171dafb52 100644 (file)
--- a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift
+++ b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+RemoteSync.swift
@@ -17,7 +17,16 @@ import NextcloudKit
 import OSLog
 
 extension FileProviderEnumerator {
-    func fullRecursiveScan(ncAccount: NextcloudAccount, ncKit: NextcloudKit, scanChangesOnly: Bool, completionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable], _ newMetadatas: [NextcloudItemMetadataTable], _ updatedMetadatas: [NextcloudItemMetadataTable], _ deletedMetadatas: [NextcloudItemMetadataTable], _ error: NKError?) -> Void) {
+    func fullRecursiveScan(ncAccount: NextcloudAccount,
+                           ncKit: NextcloudKit,
+                           scanChangesOnly: Bool,
+                           singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?,
+                                                                                _ error: NKError?) -> Void,
+                           completionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable],
+                                                        _ newMetadatas: [NextcloudItemMetadataTable],
+                                                        _ updatedMetadatas: [NextcloudItemMetadataTable],
+                                                        _ deletedMetadatas: [NextcloudItemMetadataTable],
+                                                        _ error: NKError?) -> Void) {
 
         let rootContainerDirectoryMetadata = NextcloudItemMetadataTable()
         rootContainerDirectoryMetadata.directory = true
@@ -27,7 +36,11 @@ extension FileProviderEnumerator {
         let dispatchQueue = DispatchQueue(label: "recursiveChangeEnumerationQueue", qos: .background)
 
         dispatchQueue.async {
-            let results = self.scanRecursively(rootContainerDirectoryMetadata, ncAccount: ncAccount, ncKit: ncKit, scanChangesOnly: scanChangesOnly)
+            let results = self.scanRecursively(rootContainerDirectoryMetadata,
+                                               ncAccount: ncAccount,
+                                               ncKit: ncKit,
+                                               scanChangesOnly: scanChangesOnly,
+                                               singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler)
 
             // Run a check to ensure files deleted in one location are not updated in another (e.g. when moved)
             // The recursive scan provides us with updated/deleted metadatas only on a folder by folder basis;
@@ -48,14 +61,30 @@ extension FileProviderEnumerator {
         }
     }
 
-    private func scanRecursively(_ directoryMetadata: NextcloudItemMetadataTable, ncAccount: NextcloudAccount, ncKit: NextcloudKit, scanChangesOnly: Bool) -> (metadatas: [NextcloudItemMetadataTable], newMetadatas: [NextcloudItemMetadataTable], updatedMetadatas: [NextcloudItemMetadataTable], deletedMetadatas: [NextcloudItemMetadataTable], error: NKError?) {
+    private func scanRecursively(_ directoryMetadata: NextcloudItemMetadataTable,
+                                 ncAccount: NextcloudAccount,
+                                 ncKit: NextcloudKit,
+                                 scanChangesOnly: Bool,
+                                 singleFolderScanCompleteCompletionHandler: @escaping(_ metadatas: [NextcloudItemMetadataTable]?,
+                                                                                      _ error: NKError?) -> Void) -> (metadatas: [NextcloudItemMetadataTable],
+                                                                                                                                               newMetadatas: [NextcloudItemMetadataTable],
+                                                                                                                                               updatedMetadatas: [NextcloudItemMetadataTable],
+                                                                                                                                               deletedMetadatas: [NextcloudItemMetadataTable],
+                                                                                                                                               error: NKError?) {
 
         if self.isInvalidated {
+            DispatchQueue.main.async {
+                singleFolderScanCompleteCompletionHandler(nil, nil)
+            }
             return ([], [], [], [], nil)
         }
 
         assert(directoryMetadata.directory, "Can only recursively scan a directory.")
 
+        // Scanned in this directory
+        var currentMetadatas: [NextcloudItemMetadataTable] = []
+
+        // Will include results of recursive calls
         var allMetadatas: [NextcloudItemMetadataTable] = []
         var allNewMetadatas: [NextcloudItemMetadataTable] = []
         var allUpdatedMetadatas: [NextcloudItemMetadataTable] = []
@@ -109,6 +138,7 @@ extension FileProviderEnumerator {
             Logger.enumeration.info("Finished reading serverUrl: \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
 
             if let metadatas = metadatas {
+                currentMetadatas = metadatas
                 allMetadatas += metadatas
             } else {
                 Logger.enumeration.warning("WARNING: Nil metadatas received for reading of changes at \(itemServerUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
@@ -137,10 +167,17 @@ extension FileProviderEnumerator {
 
         dispatchGroup.wait()
 
-        if criticalError != nil {
+        guard criticalError == nil else {
+            DispatchQueue.main.async {
+                singleFolderScanCompleteCompletionHandler(nil, criticalError)
+            }
             return ([], [], [], [], error: criticalError)
         }
 
+        DispatchQueue.main.async {
+            singleFolderScanCompleteCompletionHandler(currentMetadatas, nil)
+        }
+
         var childDirectoriesToScan: [NextcloudItemMetadataTable] = []
         var candidateMetadatas: [NextcloudItemMetadataTable]
 
@@ -161,7 +198,11 @@ extension FileProviderEnumerator {
         }
 
         for childDirectory in childDirectoriesToScan {
-            let childScanResult = scanRecursively(childDirectory, ncAccount: ncAccount, ncKit: ncKit, scanChangesOnly: scanChangesOnly)
+            let childScanResult = scanRecursively(childDirectory,
+                                                  ncAccount: ncAccount,
+                                                  ncKit: ncKit,
+                                                  scanChangesOnly: scanChangesOnly,
+                                                  singleFolderScanCompleteCompletionHandler: singleFolderScanCompleteCompletionHandler)
 
             allMetadatas += childScanResult.metadatas
             allNewMetadatas += childScanResult.newMetadatas

diff --git a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift
index b7a15d1174f10062ad3219935bd2416f55d9422a..b94eb43ad8ef8875d8026b01fb0ea5499358cb56 100644 (file)
--- a/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift
+++ b/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift
@@ -89,7 +89,26 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator {
             if page == NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage ||
                 page == NSFileProviderPage.initialPageSortedByName as NSFileProviderPage {
 
-                fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false) { metadatas, _, _, _, error in
+                // We enumerate items as we get the server data for two reasons:
+                // A) we avoid having a gigantic chunk of files to enumerate to the observer at the end
+                // B) we don't need to worry about resolving which files are truly deleted vs moved at the end
+                fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false, singleFolderScanCompleteCompletionHandler: { metadatas, error in
+
+                    guard error == nil else {
+                        Logger.enumeration.error("There was an error during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)")
+                        observer.finishEnumeratingWithError(error!.toFileProviderError())
+                        return;
+                    }
+
+                    guard let metadatas = metadatas else {
+                        Logger.enumeration.warning("Received nil metadatas during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)")
+                        return
+                    }
+
+                    let items = FileProviderEnumerator.metadatasToFileProviderItems(metadatas, ncKit: self.ncKit)
+                    observer.didEnumerate(items)
+
+                }) { metadatas, _, _, _, error in
 
                     if self.isInvalidated {
                         Logger.enumeration.info("Enumerator invalidated during working set item enumeration. For user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
@@ -105,7 +124,7 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator {
 
                     Logger.enumeration.info("Finished recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)). Enumerating items.")
 
-                    FileProviderEnumerator.completeEnumerationObserver(observer, ncKit: self.ncKit, numPage: 1, itemMetadatas: metadatas)
+                    observer.finishEnumerating(upTo: FileProviderEnumerator.fileProviderPageforNumPage(1))
                 }
 
                 return
@@ -198,7 +217,12 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator {
         if enumeratedItemIdentifier == .workingSet {
             Logger.enumeration.debug("Enumerating changes in working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
 
-            fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: true) { _, newMetadatas, updatedMetadatas, deletedMetadatas, error in
+            // Unlike when enumerating items we can't progressively enumerate items as we need to wait to resolve which items are truly deleted and which
+            // have just been moved elsewhere.
+            fullRecursiveScan(ncAccount: self.ncAccount,
+                              ncKit: self.ncKit,
+                              scanChangesOnly: true,
+                              singleFolderScanCompleteCompletionHandler: { _, _ in }) { _, newMetadatas, updatedMetadatas, deletedMetadatas, error in
 
                 if self.isInvalidated {
                     Logger.enumeration.info("Enumerator invalidated during working set change scan. For user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")