From: Colin Walters <walters@verbum.org>
Date: Mon, 14 Nov 2022 19:06:05 +0000 (-0500)
Subject: fetcher: Avoid too large queues for metadata processing
X-Git-Tag: archive/raspbian/2023.7-3+rpi1~1^2~9^2~3^2~12^2
X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=97e9bc7fd4a73c5c63e5390b3935cf986432ab6e;p=ostree.git

fetcher: Avoid too large queues for metadata processing

We added backoff/queueing for fetching via HTTP, but we have
another queue in the metadata scanning which can also grow
up to the number of outstanding objects, which can be large.

Capping the scanning operation when we have hit our operation
limit will avoid potentially large amounts of allocations in the
case of e.g. a slow network.

Closes: https://github.com/ostreedev/ostree/issues/2732
---

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 86b4358a..f9dd6eea 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -150,6 +150,7 @@ static void enqueue_one_static_delta_superblock_request_s (OtPullData          *
                                                            FetchDeltaSuperData *fetch_data);
 static void enqueue_one_static_delta_part_request_s (OtPullData           *pull_data,
                                                      FetchStaticDeltaData *fetch_data);
+static void ensure_idle_queued (OtPullData *pull_data);
 
 static gboolean scan_one_metadata_object (OtPullData                 *pull_data,
                                           const char                 *checksum,
@@ -385,6 +386,9 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
           g_free (checksum);
         }
 
+      /* Finally, if we still have capacity, scan more metadata objects */
+      if (!g_queue_is_empty (&pull_data->scan_object_queue))
+        ensure_idle_queued (pull_data);
     }
 }
 
@@ -461,6 +465,10 @@ ensure_idle_queued (OtPullData *pull_data)
   if (pull_data->idle_src)
     return;
 
+  /* If the operation queue is full, there's no point in blocking further. */
+  if (fetcher_queue_is_full (pull_data))
+    return;
+
   idle_src = g_idle_source_new ();
   g_source_set_callback (idle_src, idle_worker, pull_data, NULL);
   g_source_attach (idle_src, pull_data->main_context);