repo-pull: Add a queue for scanning

On systems with slow disks, the recursive scanning of directories can
be expensive -- it takes upwards of 2 minutes on our systems. This can
block the main loop for such a long time that it allows the download to
time out...

As such, move all the scanning of objects to a queue, processed from
an idle, to make sure that we don't block the main loop when scanning.

https://bugzilla.gnome.org/show_bug.cgi?id=753336

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index b8658c7a53c88e8a22e9660e6d8c254377d2b241..74acad5c5f60494daf395e496d951c094f386f6e 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -98,6 +98,9 @@ typedef struct {
 
   GError      **async_error;
   gboolean      caught_error;
+
+  GQueue scan_object_queue;
+  GSource *idle_src;
 } OtPullData;
 
 typedef struct {
@@ -117,17 +120,26 @@ typedef struct {
   char *expected_checksum;
 } FetchStaticDeltaData;
 
+typedef struct {
+  guchar csum[32];
+  OstreeObjectType objtype;
+  guint recursion_depth;
+} ScanObjectQueueData;
+
 static SoupURI *
 suburi_new (SoupURI   *base,
             const char *first,
             ...) G_GNUC_NULL_TERMINATED;
 
-static gboolean scan_one_metadata_object (OtPullData         *pull_data,
-                                          const char         *csum,
-                                          OstreeObjectType    objtype,
-                                          guint               recursion_depth,
-                                          GCancellable       *cancellable,
-                                          GError            **error);
+static void queue_scan_one_metadata_object (OtPullData         *pull_data,
+                                            const char         *csum,
+                                            OstreeObjectType    objtype,
+                                            guint               recursion_depth);
+
+static void queue_scan_one_metadata_object_c (OtPullData         *pull_data,
+                                              const guchar       *csum,
+                                              OstreeObjectType    objtype,
+                                              guint               recursion_depth);
 
 static gboolean scan_one_metadata_object_c (OtPullData         *pull_data,
                                             const guchar       *csum,
@@ -242,7 +254,8 @@ pull_termination_condition (OtPullData          *pull_data)
   gboolean current_write_idle = (pull_data->n_outstanding_metadata_write_requests == 0 &&
                                  pull_data->n_outstanding_content_write_requests == 0 &&
                                  pull_data->n_outstanding_deltapart_write_requests == 0 );
-  gboolean current_idle = current_fetch_idle && current_write_idle;
+  gboolean current_scan_idle = g_queue_is_empty (&pull_data->scan_object_queue);
+  gboolean current_idle = current_fetch_idle && current_write_idle && current_scan_idle;
 
   if (pull_data->caught_error)
     return TRUE;
@@ -282,6 +295,47 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
     }
 }
 
+static gboolean
+idle_worker (gpointer user_data)
+{
+  OtPullData *pull_data = user_data;
+  ScanObjectQueueData *scan_data;
+  GError *error = NULL;
+
+  scan_data = g_queue_pop_head (&pull_data->scan_object_queue);
+  if (!scan_data)
+    {
+      g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
+      return G_SOURCE_REMOVE;
+    }
+
+  scan_one_metadata_object_c (pull_data,
+                              scan_data->csum,
+                              scan_data->objtype,
+                              scan_data->recursion_depth,
+                              pull_data->cancellable,
+                              &error);
+  check_outstanding_requests_handle_error (pull_data, error);
+
+  g_free (scan_data);
+  return G_SOURCE_CONTINUE;
+}
+
+static void
+ensure_idle_queued (OtPullData *pull_data)
+{
+  GSource *idle_src;
+
+  if (pull_data->idle_src)
+    return;
+
+  idle_src = g_idle_source_new ();
+  g_source_set_callback (idle_src, idle_worker, pull_data, NULL);
+  g_source_attach (idle_src, pull_data->main_context);
+  g_source_unref (idle_src);
+  pull_data->idle_src = idle_src;
+}
+
 typedef struct {
   OtPullData     *pull_data;
   GInputStream   *result_stream;
@@ -454,16 +508,11 @@ scan_dirtree_object (OtPullData   *pull_data,
 
       if (subdir_target && strcmp (subdir_target, dirname) != 0)
         continue;
-      
-      if (!scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_csum),
-                                       OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
-                                       cancellable, error))
-        goto out;
-      
-      if (!scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (meta_csum),
-                                       OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
-                                       cancellable, error))
-        goto out;
+
+      queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_csum),
+                                        OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1);
+      queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (meta_csum),
+                                        OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1);
     }
 
   ret = TRUE;
@@ -696,9 +745,7 @@ on_metadata_written (GObject           *object,
       goto out;
     }
 
-  if (!scan_one_metadata_object_c (pull_data, csum, objtype, 0,
-                                   pull_data->cancellable, error))
-    goto out;
+  queue_scan_one_metadata_object_c (pull_data, csum, objtype, 0);
 
  out:
   pull_data->n_outstanding_metadata_write_requests--;
@@ -994,11 +1041,8 @@ scan_commit_object (OtPullData         *pull_data,
   have_parent = g_variant_n_children (parent_csum) > 0;
   if (have_parent && pull_data->maxdepth == -1)
     {
-      if (!scan_one_metadata_object_c (pull_data,
-                                       ostree_checksum_bytes_peek (parent_csum),
-                                       OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1,
-                                       cancellable, error))
-        goto out;
+      queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (parent_csum),
+                                        OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1);
     }
   else if (have_parent && depth > 0)
     {
@@ -1022,48 +1066,49 @@ scan_commit_object (OtPullData         *pull_data,
         {
           g_hash_table_insert (pull_data->commit_to_depth, g_strdup (parent_checksum),
                                GINT_TO_POINTER (parent_depth));
-          if (!scan_one_metadata_object_c (pull_data,
-                                           ostree_checksum_bytes_peek (parent_csum),
-                                           OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1,
-                                           cancellable, error))
-            goto out;
+          queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (parent_csum),
+                                            OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1);
         }
     }
 
   g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
   g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
 
-  if (!scan_one_metadata_object_c (pull_data,
-                                   ostree_checksum_bytes_peek (tree_contents_csum),
-                                   OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
-                                   cancellable, error))
-    goto out;
-
-  if (!scan_one_metadata_object_c (pull_data,
-                                   ostree_checksum_bytes_peek (tree_meta_csum),
-                                   OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
-                                   cancellable, error))
-    goto out;
+  queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
+                                    OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1);
+  queue_scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
+                                    OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1);
   
   ret = TRUE;
  out:
   return ret;
 }
 
-static gboolean
-scan_one_metadata_object (OtPullData         *pull_data,
-                          const char         *csum,
-                          OstreeObjectType    objtype,
-                          guint               recursion_depth,
-                          GCancellable       *cancellable,
-                          GError            **error)
+static void
+queue_scan_one_metadata_object (OtPullData         *pull_data,
+                                const char         *csum,
+                                OstreeObjectType    objtype,
+                                guint               recursion_depth)
 {
   guchar buf[32];
   ostree_checksum_inplace_to_bytes (csum, buf);
-  
-  return scan_one_metadata_object_c (pull_data, buf, objtype,
-                                     recursion_depth,
-                                     cancellable, error);
+  queue_scan_one_metadata_object_c (pull_data, buf, objtype, recursion_depth);
+}
+
+static void
+queue_scan_one_metadata_object_c (OtPullData         *pull_data,
+                                  const guchar         *csum,
+                                  OstreeObjectType    objtype,
+                                  guint               recursion_depth)
+{
+  ScanObjectQueueData *scan_data = g_new0 (ScanObjectQueueData, 1);
+
+  memcpy (scan_data->csum, csum, sizeof (scan_data->csum));
+  scan_data->objtype = objtype;
+  scan_data->recursion_depth = recursion_depth;
+
+  g_queue_push_tail (&pull_data->scan_object_queue, scan_data);
+  ensure_idle_queued (pull_data);
 }
 
 static gboolean
@@ -1675,6 +1720,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
                                                          (GDestroyNotify)g_free, NULL);
   pull_data->dir = g_strdup (dir_to_pull);
+  g_queue_init (&pull_data->scan_object_queue);
 
   pull_data->start_time = g_get_monotonic_time ();
 
@@ -2021,9 +2067,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   while (g_hash_table_iter_next (&hash_iter, &key, &value))
     {
       const char *commit = value;
-      if (!scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT,
-                                     0, pull_data->cancellable, error))
-        goto out;
+      queue_scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT, 0);
     }
 
   g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
@@ -2050,9 +2094,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       if (!delta_superblock)
         {
           g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
-          if (!scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT,
-                                         0, pull_data->cancellable, error))
-            goto out;
+          queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0);
         }
       else
         {
@@ -2077,6 +2119,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   /* Now await work completion */
   while (!pull_termination_condition (pull_data))
     g_main_context_iteration (pull_data->main_context, TRUE);
+
   if (pull_data->caught_error)
     goto out;
   
@@ -2203,6 +2246,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
+  g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
   g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
   return ret;
 }