repo: Add archive/zlib-level option, drop default compression to 6

The gzip default is 6.  When I was writing this code, I chose 9 under
the assumption that for long-term archival, the extra compression was
worth it.

Turns out level 9 is really, really not worth it.  Here's run at level 9
compressing the current Fedora Atomic Host into archive:

```
ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host
real    2m38.115s
user    2m31.210s
sys     0m3.114s
617M    repo
```

And here's the new default level of 6:

```
ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host
real    0m53.712s
user    0m43.727s
sys     0m3.601s
619M    repo
619M    total
```

As you can see, we run almost *three times* faster, and we take up *less
than one percent* more space.

Conclusion: Using level 9 is dumb.  And here's a run at compression level 1:

```
ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host
real    0m24.073s
user    0m17.574s
sys     0m2.636s
643M    repo
643M    total
```

I would argue actually many people would prefer even this for "devel" repos.
For production repos, you want static deltas anyways.  (However, perhaps
we should support a model where generating a delta involves re-compressing
fallback objects with a bit stronger compression level).

Anyways, let's make everyone's life better and switch the default to 6.

Closes: #671
Approved by: jlebon

diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h
index 0c5fb0eb463d5c2ab77fb5e8df6c1676f1d153cf..cfd8a998cbd30acdde73997070d9ba60365cf76f 100644 (file)
--- a/src/libostree/ostree-core-private.h
+++ b/src/libostree/ostree-core-private.h
@@ -24,6 +24,9 @@
 
 G_BEGIN_DECLS
 
+/* It's what gzip does, 9 is too slow */
+#define OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL (6)
+
 /* This file contains private implementation data format definitions
  * read by multiple implementation .c files.
  */
@@ -143,4 +146,16 @@ _ostree_detached_metadata_append_gpg_sig (GVariant   *existing_metadata,
 GFile *
 _ostree_get_default_sysroot_path (void);
 
+_OSTREE_PUBLIC
+gboolean
+_ostree_raw_file_to_archive_stream (GInputStream       *input,
+                                    GFileInfo          *file_info,
+                                    GVariant           *xattrs,
+                                    guint               compression_level,
+                                    GInputStream      **out_input,
+                                    GCancellable       *cancellable,
+                                    GError            **error);
+
+
+
 G_END_DECLS

diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index e3f0a7716d20fb095cc1b1e543709ada6aec5d42..af36d98b2c9ce0d5131f87aa369c4ecebe91317f 100644 (file)
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -453,6 +453,34 @@ header_and_input_to_stream (GVariant           *file_header,
   return TRUE;
 }
 
+gboolean
+_ostree_raw_file_to_archive_stream (GInputStream       *input,
+                                    GFileInfo          *file_info,
+                                    GVariant           *xattrs,
+                                    guint               compression_level,
+                                    GInputStream      **out_input,
+                                    GCancellable       *cancellable,
+                                    GError            **error)
+{
+  g_autoptr(GVariant) file_header = NULL;
+  g_autoptr(GInputStream) zlib_input = NULL;
+
+  file_header = _ostree_zlib_file_header_new (file_info, xattrs);
+  if (input != NULL)
+    {
+      g_autoptr(GConverter) zlib_compressor = NULL;
+
+      zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, compression_level));
+      zlib_input = g_converter_input_stream_new (input, zlib_compressor);
+    }
+  return header_and_input_to_stream (file_header,
+                                     zlib_input,
+                                     out_input,
+                                     NULL,
+                                     cancellable,
+                                     error);
+}
+
 /**
  * ostree_raw_file_to_archive_z2_stream:
  * @input: File raw content stream
@@ -473,23 +501,9 @@ ostree_raw_file_to_archive_z2_stream (GInputStream       *input,
                                       GCancellable       *cancellable,
                                       GError            **error)
 {
-  g_autoptr(GVariant) file_header = NULL;
-  g_autoptr(GInputStream) zlib_input = NULL;
-
-  file_header = _ostree_zlib_file_header_new (file_info, xattrs);
-  if (input != NULL)
-    {
-      g_autoptr(GConverter) zlib_compressor = NULL;
-
-      zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9));
-      zlib_input = g_converter_input_stream_new (input, zlib_compressor);
-    }
-  return header_and_input_to_stream (file_header,
-                                     zlib_input,
-                                     out_input,
-                                     NULL,
-                                     cancellable,
-                                     error);
+  return _ostree_raw_file_to_archive_stream (input, file_info, xattrs,
+                                             OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL,
+                                             out_input, cancellable, error);
 }
 
 /**

diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index eb9733e8f8ed8e5bd2bedb46f329061b9ce229a3..07940f4868ee6e3ad687695436812191dcf145ff 100644 (file)
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -719,7 +719,7 @@ write_object (OstreeRepo         *self,
 
           if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
             {
-              zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9);
+              zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, self->zlib_compression_level);
               compressed_out_stream = g_converter_output_stream_new (temp_out, zlib_compressor);
               /* Don't close the base; we'll do that later */
               g_filter_output_stream_set_close_base_stream ((GFilterOutputStream*)compressed_out_stream, FALSE);

diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index a4e59e44bce8836599f35dac65fe6ce99624b4a6..cfc178f30de837dfd86600708a7305867fd65da6 100644 (file)
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -89,6 +89,7 @@ struct OstreeRepo {
   GError *writable_error;
   gboolean in_transaction;
   gboolean disable_fsync;
+  guint zlib_compression_level;
   GHashTable *loose_object_devino_hash;
   GHashTable *updated_uncompressed_dirs;
   GHashTable *object_sizes;

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index fd49f0fc2c9cb623bb8fb98c61e6d20a5b587180..4ac39d11043cd9bceb1f50d551ba2b7164438278 100644 (file)
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -2083,6 +2083,19 @@ reload_core_config (OstreeRepo          *self,
     self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10);
   }
 
+  { g_autofree char *compression_level_str = NULL;
+
+    /* gzip defaults to 6 */
+    (void)ot_keyfile_get_value_with_default (self->config, "archive", "zlib-level", NULL,
+                                             &compression_level_str, NULL);
+
+    if (compression_level_str)
+      /* Ensure level is in [1,9] */
+      self->zlib_compression_level = MAX (1, MIN (9, g_ascii_strtoull (compression_level_str, NULL, 10)));
+    else
+      self->zlib_compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL;
+  }
+
   if (!ot_keyfile_get_value_with_default (self->config, "core", "parent",
                                           NULL, &parent_repo_path, error))
     return FALSE;