pull: Add a --dry-run option for static deltas

One of the design goals with deltas was not just wire efficiency,
but also having all the data up front about how much data would
be transferred before starting.

Let's expose that better by adding a `dry-run` option to the pull API.
This requires static deltas to be useful.  Basically we simply call
the progress callback once with the data from the superblock.

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index a7db45f9b26843f96743e86ec9757c7535792561..efa424f6eb8698c8d3e942e19d96e27bcbb505d4 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -48,6 +48,8 @@ typedef struct {
   GCancellable *cancellable;
   OstreeAsyncProgress *progress;
 
+  gboolean      dry_run;
+  gboolean      dry_run_emitted_progress;
   gboolean      legacy_transaction_resuming;
   enum {
     OSTREE_PULL_PHASE_FETCHING_REFS,
@@ -78,6 +80,7 @@ typedef struct {
   guint             n_outstanding_deltapart_write_requests;
   guint             n_total_deltaparts;
   guint64           total_deltapart_size;
+  guint64           total_deltapart_usize;
   gint              n_requested_metadata;
   gint              n_requested_content;
   guint             n_fetched_deltaparts;
@@ -227,6 +230,8 @@ update_progress (gpointer user_data)
                                   pull_data->n_total_deltaparts);
   ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size",
                                     pull_data->total_deltapart_size);
+  ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize",
+                                    pull_data->total_deltapart_usize);
   ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks",
                                   pull_data->static_delta_superblocks->len);
 
@@ -243,6 +248,9 @@ update_progress (gpointer user_data)
   else
     ostree_async_progress_set_status (pull_data->progress, NULL);
 
+  if (pull_data->dry_run)
+    pull_data->dry_run_emitted_progress = TRUE;
+
   return TRUE;
 }
 
@@ -262,6 +270,9 @@ pull_termination_condition (OtPullData          *pull_data)
   if (pull_data->caught_error)
     return TRUE;
 
+  if (pull_data->dry_run)
+    return pull_data->dry_run_emitted_progress;
+
   switch (pull_data->phase)
     {
     case OSTREE_PULL_PHASE_FETCHING_REFS:
@@ -1451,11 +1462,18 @@ process_one_static_delta_fallback (OtPullData   *pull_data,
   if (!ostree_validate_structureof_csum_v (csum_v, error))
     goto out;
 
+  pull_data->total_deltapart_size += compressed_size;
+  pull_data->total_deltapart_usize += uncompressed_size;
+
+  if (pull_data->dry_run)
+    {
+      ret = TRUE;
+      goto out;
+    }
+
   objtype = (OstreeObjectType)objtype_y;
   checksum = ostree_checksum_from_bytes_v (csum_v);
 
-  pull_data->total_deltapart_size += compressed_size;
-
   if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
                                &is_stored,
                                cancellable, error))
@@ -1524,6 +1542,7 @@ process_one_static_delta (OtPullData   *pull_data,
     }
 
   /* Write the to-commit object */
+  if (!pull_data->dry_run)
   {
     g_autoptr(GVariant) to_csum_v = NULL;
     g_autofree char *to_checksum = NULL;
@@ -1626,7 +1645,11 @@ process_one_static_delta (OtPullData   *pull_data,
       }
 
       pull_data->total_deltapart_size += size;
+      pull_data->total_deltapart_usize += usize;
 
+      if (pull_data->dry_run)
+        continue;
+      
       fetch_data = g_new0 (FetchStaticDeltaData, 1);
       fetch_data->pull_data = pull_data;
       fetch_data->objects = g_variant_ref (objects);
@@ -1780,6 +1803,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas);
       (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas);
       (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
+      (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
     }
 
   g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -1790,6 +1814,10 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
 
   g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE);
+  /* We only do dry runs with static deltas, because we don't really have any
+   * in-advance information for bare fetches.
+   */
+  g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE);
 
   pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
   pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
@@ -2243,7 +2271,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
   if (pull_data->progress)
     {
-      update_timeout = g_timeout_source_new_seconds (1);
+      update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
       g_source_set_priority (update_timeout, G_PRIORITY_HIGH);
       g_source_set_callback (update_timeout, update_progress, pull_data, NULL);
       g_source_attach (update_timeout, pull_data->main_context);
@@ -2256,6 +2284,12 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
   if (pull_data->caught_error)
     goto out;
+
+  if (pull_data->dry_run)
+    {
+      ret = TRUE;
+      goto out;
+    }
   
   g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0);
   g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0);

diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c
index 7fa673cdbeba72cdb652fcadd229b07ff40390df..7c91890f0d44b0b08657ed4835b2abb319150c07 100644 (file)
--- a/src/ostree/ot-builtin-pull.c
+++ b/src/ostree/ot-builtin-pull.c
@@ -30,6 +30,7 @@
 static gboolean opt_disable_fsync;
 static gboolean opt_mirror;
 static gboolean opt_commit_only;
+static gboolean opt_dry_run;
 static gboolean opt_disable_static_deltas;
 static gboolean opt_require_static_deltas;
 static char* opt_subpath;
@@ -42,6 +43,7 @@ static GOptionEntry options[] = {
    { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL },
    { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL },
    { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL },
+   { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL },
    { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" },
    { NULL }
  };
@@ -62,6 +64,39 @@ gpg_verify_result_cb (OstreeRepo *repo,
   gs_console_begin_status_line (console, "", NULL, NULL);
 }
 
+static gboolean printed_console_progress;
+
+static void
+dry_run_console_progress_changed (OstreeAsyncProgress *progress,
+                                  gpointer             user_data)
+{
+  guint fetched_delta_parts, total_delta_parts;
+  guint64 total_delta_part_size, total_delta_part_usize;
+  GString *buf;
+
+  g_assert (!printed_console_progress);
+  printed_console_progress = TRUE;
+
+  fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts");
+  total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts");
+  total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size");
+  total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize");
+
+  buf = g_string_new ("");
+
+  { g_autofree char *formatted_size =
+      g_format_size (total_delta_part_size);
+    g_autofree char *formatted_usize =
+      g_format_size (total_delta_part_usize);
+
+    g_string_append_printf (buf, "Delta update: %u/%u parts, %s to transfer, %s uncompressed",
+                            fetched_delta_parts, total_delta_parts,
+                            formatted_size, formatted_usize);
+  }
+  g_print ("%s\n", buf->str);
+  g_string_free (buf, TRUE);
+}
+
 gboolean
 ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error)
 {
@@ -99,6 +134,13 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
   if (opt_commit_only)
     pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY;
 
+  if (opt_dry_run && !opt_require_static_deltas)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "--dry-run requires --require-static-deltas");
+      goto out;
+    }
+
   if (strchr (argv[1], ':') == NULL)
     {
       remote = g_strdup (argv[1]);
@@ -149,11 +191,21 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
       g_ptr_array_add (refs_to_fetch, NULL);
     }
 
-  console = gs_console_get ();
-  if (console)
+  if (!opt_dry_run)
     {
-      gs_console_begin_status_line (console, "", NULL, NULL);
-      progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console);
+      console = gs_console_get ();
+      if (console)
+        {
+          gs_console_begin_status_line (console, "", NULL, NULL);
+          progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console);
+          signal_handler_id = g_signal_connect (repo, "gpg-verify-result",
+                                                G_CALLBACK (gpg_verify_result_cb),
+                                                console);
+        }
+    }
+  else
+    {
+      progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, console);
       signal_handler_id = g_signal_connect (repo, "gpg-verify-result",
                                             G_CALLBACK (gpg_verify_result_cb),
                                             console);
@@ -180,6 +232,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
     g_variant_builder_add (&builder, "{s@v}", "require-static-deltas",
                            g_variant_new_variant (g_variant_new_boolean (opt_require_static_deltas)));
 
+    g_variant_builder_add (&builder, "{s@v}", "dry-run",
+                           g_variant_new_variant (g_variant_new_boolean (opt_dry_run)));
+
     if (override_commit_ids)
       g_variant_builder_add (&builder, "{s@v}", "override-commit-ids",
                              g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len)));
@@ -192,6 +247,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
   if (progress)
     ostree_async_progress_finish (progress);
 
+  if (opt_dry_run)
+    g_assert (printed_console_progress);
+
   ret = TRUE;
  out:
   if (signal_handler_id > 0)

diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index 04272d5fc86be0f19692e07cb0b18bf854f79dc8..f73b681834ab433d566df2786139b338ff04cbaa 100755 (executable)
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -107,12 +107,24 @@ rm main-files -rf
 # Generate delta that we'll use
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
 prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^)
+new_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main)
 ostree --repo=ostree-srv/gnomerepo summary -u
 
+cd ${test_tmpdir}
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt
+assert_file_has_content out.txt 'Delta update: 0/1 parts'
+rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
+assert_streq "${prev_rev}" "${rev}"
+${CMD_PREFIX} ostree --repo=repo fsck
+
 cd ${test_tmpdir}
 repo_init
 ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
 ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main
+rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
+assert_streq "${new_rev}" "${rev}"
 ${CMD_PREFIX} ostree --repo=repo fsck
 
 cd ${test_tmpdir}
@@ -140,6 +152,8 @@ fi
 assert_file_has_content err.txt "deltas required, but none found"
 ${CMD_PREFIX} ostree --repo=repo fsck
 
+echo "ok delta required but don't exist"
+
 cd ${test_tmpdir}
 rm main-files -rf
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files