fetcher: Avoid too large queues for metadata processing

We added backoff/queueing for fetching via HTTP, but we have
another queue in the metadata scanning which can also grow
up to the number of outstanding objects, which can be large.

Capping the scanning operation when we have hit our operation
limit will avoid potentially large amounts of allocations in the
case of e.g. a slow network.

Closes: https://github.com/ostreedev/ostree/issues/2732

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 86b4358a12579cb0c7117c939addc3cac83b97d6..f9dd6eea9738cb7d241e4a1245df0f7859ae87e5 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -150,6 +150,7 @@ static void enqueue_one_static_delta_superblock_request_s (OtPullData          *
                                                            FetchDeltaSuperData *fetch_data);
 static void enqueue_one_static_delta_part_request_s (OtPullData           *pull_data,
                                                      FetchStaticDeltaData *fetch_data);
+static void ensure_idle_queued (OtPullData *pull_data);
 
 static gboolean scan_one_metadata_object (OtPullData                 *pull_data,
                                           const char                 *checksum,
@@ -385,6 +386,9 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
           g_free (checksum);
         }
 
+      /* Finally, if we still have capacity, scan more metadata objects */
+      if (!g_queue_is_empty (&pull_data->scan_object_queue))
+        ensure_idle_queued (pull_data);
     }
 }
 
@@ -461,6 +465,10 @@ ensure_idle_queued (OtPullData *pull_data)
   if (pull_data->idle_src)
     return;
 
+  /* If the operation queue is full, there's no point in blocking further. */
+  if (fetcher_queue_is_full (pull_data))
+    return;
+
   idle_src = g_idle_source_new ();
   g_source_set_callback (idle_src, idle_worker, pull_data, NULL);
   g_source_attach (idle_src, pull_data->main_context);