pull: Add summary-{,sig-}bytes options to ostree_repo_pull()

These allow the `summary` and `summary.sig` files to be cached at a
higher layer (for example, flatpak) between related pull operations (for
example, within a single flatpak transaction). This avoids
re-downloading `summary.sig` multiple times throughout a transaction,
which increases the transaction’s latency and introduces the possibility
for inconsistency between parts of the transaction if the server changes
its `summary` file part-way through.

In particular, this should speed up flatpak transactions on machines
with high latency network connections, where network round trips have a
high impact on the latency of an overall operation.

Signed-off-by: Philip Withnall <withnall@endlessm.com>

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 18fe9d276b16ba7eb8c79e99732f6cd376309fdc..894e4b1fe3259b6849af5f0a4c7997a2cadb9325 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -3311,6 +3311,14 @@ initiate_request (OtPullData                 *pull_data,
  *     not being pulled will be ignored and any ref without a keyring remote
  *     will be verified with the keyring of the remote being pulled from.
  *     Since: 2019.2
+ *   * `summary-bytes` (`ay'): Contents of the `summary` file to use. If this is
+ *     specified, `summary-sig-bytes` must also be specified. This is
+ *     useful if doing multiple pull operations in a transaction, using
+ *     ostree_repo_remote_fetch_summary_with_options() beforehand to download
+ *     the `summary` and `summary.sig` once for the entire transaction. If not
+ *     specified, the `summary` will be downloaded from the remote. Since: 2020.5
+ *   * `summary-sig-bytes` (`ay`): Contents of the `summary.sig` file. If this
+ *     is specified, `summary-bytes` must also be specified. Since: 2020.5
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -3356,6 +3364,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   int i;
   g_autofree char **opt_localcache_repos = NULL;
   g_autoptr(GVariantIter) ref_keyring_map_iter = NULL;
+  g_autoptr(GVariant) summary_bytes_v = NULL;
+  g_autoptr(GVariant) summary_sig_bytes_v = NULL;
   /* If refs or collection-refs has exactly one value, this will point to that
    * value, otherwise NULL. Used for logging.
    */
@@ -3402,6 +3412,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries);
       opt_ref_keyring_map_set =
        g_variant_lookup (options, "ref-keyring-map", "a(sss)", &ref_keyring_map_iter);
+      (void) g_variant_lookup (options, "summary-bytes", "@ay", &summary_bytes_v);
+      (void) g_variant_lookup (options, "summary-sig-bytes", "@ay", &summary_sig_bytes_v);
 
       if (pull_data->remote_refspec_name != NULL)
         pull_data->remote_name = g_strdup (pull_data->remote_refspec_name);
@@ -3439,6 +3451,10 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
    */
   g_return_val_if_fail (!pull_data->dry_run || pull_data->require_static_deltas, FALSE);
 
+  /* summary-bytes and summary-sig-bytes must both be specified, or neither be
+   * specified, so we know they’re consistent */
+  g_return_val_if_fail ((summary_bytes_v == NULL) == (summary_sig_bytes_v == NULL), FALSE);
+
   pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
   pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
   /* See our processing of OSTREE_REPO_PULL_FLAGS_UNTRUSTED below */
@@ -3651,6 +3667,10 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       if (!metalink_uri)
         goto out;
 
+      /* FIXME: Use summary_bytes_v/summary_sig_bytes_v to avoid unnecessary
+       * re-downloads here. Would require additional support for caching the
+       * metalink file or mirror list. */
+
       metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
                                        OSTREE_MAX_METADATA_SIZE, metalink_uri,
                                        pull_data->n_network_retries);
@@ -3841,7 +3861,25 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_autoptr(GVariant) additional_metadata = NULL;
     gboolean summary_from_cache = FALSE;
 
-    if (!pull_data->summary_data_sig)
+    if (summary_sig_bytes_v)
+      {
+        /* Must both be specified */
+        g_assert (summary_bytes_v);
+
+        bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v);
+        bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v);
+
+        if (!bytes_sig || !bytes_summary)
+          {
+            g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                         "summary-bytes or summary-sig-bytes set to invalid value");
+            goto out;
+          }
+
+        g_debug ("Loaded %s summary from options", remote_name_or_baseurl);
+      }
+
+    if (!bytes_sig)
       {
         if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
                                                          pull_data->meta_mirrorlist,
@@ -3854,6 +3892,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       }
 
     if (bytes_sig &&
+        !bytes_summary &&
         !pull_data->remote_repo_local &&
         !_ostree_repo_load_cache_summary_if_same_sig (self,
                                                       remote_name_or_baseurl,
@@ -3863,7 +3902,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                                       error))
       goto out;
 
-    if (bytes_summary)
+    if (bytes_summary && !summary_bytes_v)
       {
         g_debug ("Loaded %s summary from cache", remote_name_or_baseurl);
         summary_from_cache = TRUE;