Add public API for fsck, use it before loading metadata
authorColin Walters <walters@verbum.org>
Tue, 5 Dec 2017 19:27:15 +0000 (14:27 -0500)
committerAtomic Bot <atomic-devel@projectatomic.io>
Tue, 12 Dec 2017 14:03:09 +0000 (14:03 +0000)
A while ago I did `truncate -s 0 /path/to/repo/00/123.commit`, and expected a
checksum error, but I actually got a validation error due to us loading the
commit into a variant and trying to parse out the parent checksum, etc.

I first started by changing the `load_and_fsck_one_object()` function to
checksum before loading, but the problem is that we do a traverse of all objects
first. Fixing this is going to require an `OSTREE_REPO_COMMIT_TRAVER_FLAG_FSCK`
or something.

In the meantime at least though, let's add a public API to fsck a single object
which *does* checksum cleanly before parsing the object, and change the `fsck`
command to use it.

We then change the fsck binary to do this while iterating over the refs
and finding the commit object.  This way we'll at least get a checksum
first for commit objects, even if not dirtree/dirmeta.

Closes: #1364
Approved by: jlebon

apidoc/ostree-sections.txt
src/libostree/libostree-devel.sym
src/libostree/ostree-repo.c
src/libostree/ostree-repo.h
src/ostree/ot-builtin-fsck.c
tests/pull-test.sh
tests/test-corruption.sh
tests/test-pull-corruption.sh

index b37c8914af1b9402d2fbba70fab3c48679aebdb0..4e474d8db3c7156517b2bdbc4ec066cce3989391 100644 (file)
@@ -348,6 +348,7 @@ ostree_repo_import_object_from_with_trust
 ostree_repo_import_archive_to_mtree
 ostree_repo_export_tree_to_archive
 ostree_repo_delete_object
+ostree_repo_fsck_object
 OstreeRepoCommitFilterResult
 OstreeRepoCommitFilter
 OstreeRepoCommitModifier
index df18b9ab49aef5495860a013ccb2e5ed764244fe..36926ce16d63e9a46b9a80863534bcf105488f69 100644 (file)
@@ -20,6 +20,7 @@
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
 
 LIBOSTREE_2017.15 {
+  ostree_repo_fsck_object;
 } LIBOSTREE_2017.14;
 
 /* Stub section for the stable release *after* this development one; don't
index 9591d01cc3142d5493576b1171c999eacefc4f08..08a6276b0e5e54793053f299b5ffb9feb4a50c4f 100644 (file)
@@ -3914,6 +3914,115 @@ ostree_repo_delete_object (OstreeRepo           *self,
   return TRUE;
 }
 
+static gboolean
+fsck_metadata_object (OstreeRepo           *self,
+                      OstreeObjectType      objtype,
+                      const char           *sha256,
+                      GCancellable         *cancellable,
+                      GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck ", sha256, ".", ostree_object_type_to_string (objtype));
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GVariant) metadata = NULL;
+  if (!load_metadata_internal (self, objtype, sha256, TRUE,
+                               &metadata, NULL, NULL, NULL,
+                               cancellable, error))
+    return FALSE;
+
+  g_auto(OtChecksum) hasher = { 0, };
+  ot_checksum_init (&hasher);
+  ot_checksum_update (&hasher, g_variant_get_data (metadata), g_variant_get_size (metadata));
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&hasher, actual_checksum, sizeof (actual_checksum));
+  if (!_ostree_compare_object_checksum (objtype, sha256, actual_checksum, error))
+    return FALSE;
+
+  switch (objtype)
+    {
+    case OSTREE_OBJECT_TYPE_COMMIT:
+      if (!ostree_validate_structureof_commit (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_TREE:
+      if (!ostree_validate_structureof_dirtree (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_META:
+      if (!ostree_validate_structureof_dirmeta (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT:
+    case OSTREE_OBJECT_TYPE_COMMIT_META:
+      /* TODO */
+      break;
+    case OSTREE_OBJECT_TYPE_FILE:
+      g_assert_not_reached ();
+      break;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+fsck_content_object (OstreeRepo           *self,
+                     const char           *sha256,
+                     GCancellable         *cancellable,
+                     GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck content object ", sha256);
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GInputStream) input = NULL;
+  g_autoptr(GFileInfo) file_info = NULL;
+  g_autoptr(GVariant) xattrs = NULL;
+
+  if (!ostree_repo_load_file (self, sha256, &input, &file_info, &xattrs,
+                              cancellable, error))
+    return FALSE;
+
+  /* TODO more consistency checks here */
+  const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+  if (!ostree_validate_structureof_file_mode (mode, error))
+    return FALSE;
+
+  g_autofree guchar *computed_csum = NULL;
+  if (!ostree_checksum_file_from_input (file_info, xattrs, input,
+                                        OSTREE_OBJECT_TYPE_FILE, &computed_csum,
+                                        cancellable, error))
+    return FALSE;
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ostree_checksum_inplace_from_bytes (computed_csum, actual_checksum);
+  return _ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE, sha256, actual_checksum, error);
+}
+
+/**
+ * ostree_repo_fsck_object:
+ * @self: Repo
+ * @objtype: Object type
+ * @sha256: Checksum
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Verify consistency of the object; this performs checks only relevant to the
+ * immediate object itself, such as checksumming. This API call will not itself
+ * traverse metadata objects for example.
+ *
+ * Since: 2017.15
+ */
+gboolean
+ostree_repo_fsck_object (OstreeRepo           *self,
+                         OstreeObjectType      objtype,
+                         const char           *sha256,
+                         GCancellable         *cancellable,
+                         GError              **error)
+{
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+    return fsck_metadata_object (self, objtype, sha256, cancellable, error);
+  else
+    return fsck_content_object (self, sha256, cancellable, error);
+}
+
 /**
  * ostree_repo_import_object_from:
  * @self: Destination repo
index 7ce47a56534eecfb0767b58f6bc866336f95d941..5ef12bb9edcf5427b0e603b221b48f189c08952c 100644 (file)
@@ -650,10 +650,17 @@ gboolean      ostree_repo_import_object_from_with_trust (OstreeRepo           *s
 _OSTREE_PUBLIC
 gboolean      ostree_repo_delete_object (OstreeRepo           *self,
                                          OstreeObjectType      objtype,
-                                         const char           *sha256, 
+                                         const char           *sha256,
                                          GCancellable         *cancellable,
                                          GError              **error);
 
+_OSTREE_PUBLIC
+gboolean      ostree_repo_fsck_object (OstreeRepo           *self,
+                                       OstreeObjectType      objtype,
+                                       const char           *sha256,
+                                       GCancellable         *cancellable,
+                                       GError              **error);
+
 /** 
  * OstreeRepoCommitFilterResult:
  * @OSTREE_REPO_COMMIT_FILTER_ALLOW: Do commit this object
index 116fdc6b38cbd54927119dc1d950d058f3b52372..eeaa23a8121b653078961e202076d695a1b9ae47 100644 (file)
@@ -44,118 +44,36 @@ static GOptionEntry options[] = {
 };
 
 static gboolean
-load_and_fsck_one_object (OstreeRepo            *repo,
-                          const char            *checksum,
-                          OstreeObjectType       objtype,
-                          gboolean              *out_found_corruption,
-                          GCancellable          *cancellable,
-                          GError               **error)
+fsck_one_object (OstreeRepo            *repo,
+                 const char            *checksum,
+                 OstreeObjectType       objtype,
+                 gboolean              *out_found_corruption,
+                 GCancellable          *cancellable,
+                 GError               **error)
 {
-  gboolean missing = FALSE;
-  g_autoptr(GVariant) metadata = NULL;
-  g_autoptr(GInputStream) input = NULL;
-  g_autoptr(GFileInfo) file_info = NULL;
-  g_autoptr(GVariant) xattrs = NULL;
   g_autoptr(GError) temp_error = NULL;
-
-  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+  if (!ostree_repo_fsck_object (repo, objtype, checksum, cancellable, &temp_error))
     {
-      if (!ostree_repo_load_variant (repo, objtype,
-                                     checksum, &metadata, &temp_error))
+      if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
-          if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-            {
-              g_clear_error (&temp_error);
-              g_printerr ("Object missing: %s.%s\n", checksum,
-                          ostree_object_type_to_string (objtype));
-              missing = TRUE;
-            }
-          else
-            {
-              g_propagate_error (error, g_steal_pointer (&temp_error));
-              return glnx_prefix_error (error, "Loading metadata object %s", checksum);
-            }
+          g_clear_error (&temp_error);
+          g_printerr ("Object missing: %s.%s\n", checksum,
+                      ostree_object_type_to_string (objtype));
+          *out_found_corruption = TRUE;
         }
       else
         {
-          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
-            {
-              if (!ostree_validate_structureof_commit (metadata, error))
-                return glnx_prefix_error (error, "While validating commit metadata '%s'", checksum);
-            }
-          else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
-            {
-              if (!ostree_validate_structureof_dirtree (metadata, error))
-                return glnx_prefix_error (error, "While validating directory tree '%s'", checksum);
-            }
-          else if (objtype == OSTREE_OBJECT_TYPE_DIR_META)
-            {
-              if (!ostree_validate_structureof_dirmeta (metadata, error))
-                return glnx_prefix_error (error, "While validating directory metadata '%s'", checksum);
-            }
-
-          input = g_memory_input_stream_new_from_data (g_variant_get_data (metadata),
-                                                       g_variant_get_size (metadata),
-                                                       NULL);
-
-        }
-    }
-  else
-    {
-      guint32 mode;
-      g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
-      if (!ostree_repo_load_file (repo, checksum, &input, &file_info,
-                                  &xattrs, cancellable, &temp_error))
-        {
-          if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-            {
-              g_clear_error (&temp_error);
-              g_printerr ("Object missing: %s.%s\n", checksum,
-                          ostree_object_type_to_string (objtype));
-              missing = TRUE;
-            }
-          else
-            {
-              g_propagate_error (error, g_steal_pointer (&temp_error));
-              return glnx_prefix_error (error, "Loading file object %s", checksum);
-            }
-        }
-      else
-        {
-          mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
-          if (!ostree_validate_structureof_file_mode (mode, error))
-            return glnx_prefix_error (error, "While validating file '%s'", checksum);
-        }
-    }
-
-  if (missing)
-    {
-      *out_found_corruption = TRUE;
-    }
-  else
-    {
-      g_autofree guchar *computed_csum = NULL;
-      g_autofree char *tmp_checksum = NULL;
-
-      if (!ostree_checksum_file_from_input (file_info, xattrs, input,
-                                            objtype, &computed_csum,
-                                            cancellable, error))
-        return FALSE;
-
-      tmp_checksum = ostree_checksum_from_bytes (computed_csum);
-      if (strcmp (checksum, tmp_checksum) != 0)
-        {
-          g_autofree char *msg = g_strdup_printf ("corrupted object %s.%s; actual checksum: %s",
-                                               checksum, ostree_object_type_to_string (objtype),
-                                               tmp_checksum);
           if (opt_delete)
             {
-              g_printerr ("%s\n", msg);
+              g_printerr ("%s\n", temp_error->message);
               (void) ostree_repo_delete_object (repo, objtype, checksum, cancellable, NULL);
               *out_found_corruption = TRUE;
             }
           else
-            return glnx_throw (error, "%s", msg);
+            {
+              g_propagate_error (error, g_steal_pointer (&temp_error));
+              return FALSE;
+            }
         }
     }
 
@@ -201,8 +119,8 @@ fsck_reachable_objects_from_commits (OstreeRepo            *repo,
 
       ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
 
-      if (!load_and_fsck_one_object (repo, checksum, objtype, out_found_corruption,
-                                     cancellable, error))
+      if (!fsck_one_object (repo, checksum, objtype, out_found_corruption,
+                            cancellable, error))
         return FALSE;
 
       if (mod == 0 || (i % mod == 0))
@@ -239,6 +157,12 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation,
     {
       const char *refname = key;
       const char *checksum = value;
+
+      if (!fsck_one_object (repo, checksum, OSTREE_OBJECT_TYPE_COMMIT,
+                            &found_corruption,
+                            cancellable, error))
+        return FALSE;
+
       g_autoptr(GVariant) commit = NULL;
       if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
                                      checksum, &commit, error))
index c09feb30fd7515a3605b51b5ac1b1884bfdd57f2..e6317fbf547244ded0db87194c63408e28019bae 100644 (file)
@@ -183,7 +183,7 @@ if ! skip_one_without_user_xattrs; then
         if ${CMD_PREFIX} ostree --repo=cacherepo fsck 2>err.txt; then
             fatal "corrupt repo fsck?"
         fi
-        assert_file_has_content err.txt "corrupted.*${checksum}"
+        assert_file_has_content err.txt "Corrupted.*${checksum}"
         rm ostree-srv/corruptrepo -rf
         ostree_repo_init ostree-srv/corruptrepo --mode=archive
         ${CMD_PREFIX} ostree --repo=ostree-srv/corruptrepo pull-local cacherepo main
index 8e2aba56278ba133655c6343e65ebe2a8f0d94b0..52a8189a6614efc71948c71bfdaaa8aaa893196b 100755 (executable)
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+# Copyright (C) 2011,2017 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..3"
+echo "1..4"
 
 . $(dirname $0)/libtest.sh
 
@@ -35,6 +35,18 @@ $OSTREE fsck -q
 
 echo "ok chmod"
 
+cd ${test_tmpdir}
+rm repo files -rf
+setup_test_repository "bare"
+rev=$($OSTREE rev-parse test2)
+echo -n > repo/objects/${rev:0:2}/${rev:2}.commit
+if $OSTREE fsck -q 2>err.txt; then
+    fatal "fsck unexpectedly succeeded"
+fi
+assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected"
+
+echo "ok metadata checksum"
+
 cd ${test_tmpdir}
 rm repo files -rf
 setup_test_repository "bare"
index ea29a87c5e5b4455d2c5aba32b535d68bb639efc..3c8d4c9a7092d3103121a2f1a43a4a27475e69c0 100755 (executable)
@@ -76,7 +76,7 @@ if ! skip_one_without_user_xattrs; then
     if ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo fsck 2>err.txt; then
         assert_not_reached "fsck with corrupted commit worked?"
     fi
-    assert_file_has_content err.txt "corrupted object ${corruptrev}\.commit"
+    assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected='${corruptrev}' actual='${rev}'"
 
     # Do a pull-local; this should succeed since we don't verify checksums
     # for local repos by default.