upgrade: Handle upgrade to v6 when the repository already contains v6 unlocked files...

Closes https://github.com/datalad/datalad/issues/1020

The use of runWriter in scanUnlockedFiles broke due to this change;
it failed with blocked indefinitely in mvar, because the database write
handle was taken while linkFromAnnex needed to also write to it (to update
the inode cache). So, switched to using a separate runWriter for each call
to addAssociatedFileFast. A little less efficient, but not greatly; the
writes should all still be cached.

diff --git a/Annex/WorkTree.hs b/Annex/WorkTree.hs
index fe42ab726bda530faacd5c7ba2593622312c91ca..d62e15aee63d5dd64fe23255ef1c40f134a60d50 100644 (file)
--- a/Annex/WorkTree.hs
+++ b/Annex/WorkTree.hs
@@ -11,7 +11,10 @@ import Annex.Common
 import Annex.Link
 import Annex.CatFile
 import Annex.Version
+import Annex.Content
+import Annex.ReplaceFile
 import Config
+import Git.FilePath
 import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.LsTree
@@ -54,24 +57,37 @@ ifAnnexed file yes no = maybe no yes =<< lookupFile file
  - This is expensive, and so normally the associated files are updated
  - incrementally when changes are noticed. So, this only needs to be done
  - when initializing/upgrading a v6 mode repository.
+ -
+ - Also, the content for the unlocked file may already be present as
+ - an annex object. If so, make the unlocked file use that content.
  -}
 scanUnlockedFiles :: Annex ()
-scanUnlockedFiles = whenM (isJust <$> inRepo Git.Branch.current) $ 
-       Database.Keys.runWriter $ \h -> do
-               showSideAction "scanning for unlocked files"
-               liftIO $ Database.Keys.SQL.dropAllAssociatedFiles h
-               (l, cleanup) <- inRepo $ Git.LsTree.lsTree Git.Ref.headRef
-               forM_ l $ \i -> 
-                       when (isregfile i) $
-                               maybe noop (add h i)
-                                       =<< catKey (Git.LsTree.sha i)
-               liftIO $ void cleanup
+scanUnlockedFiles = whenM (isJust <$> inRepo Git.Branch.current) $ do
+       showSideAction "scanning for unlocked files"
+       Database.Keys.runWriter $
+               liftIO . Database.Keys.SQL.dropAllAssociatedFiles
+       (l, cleanup) <- inRepo $ Git.LsTree.lsTree Git.Ref.headRef
+       forM_ l $ \i -> 
+               when (isregfile i) $
+                       maybe noop (add i)
+                               =<< catKey (Git.LsTree.sha i)
+       liftIO $ void cleanup
   where
        isregfile i = case Git.Types.toBlobType (Git.LsTree.mode i) of
                Just Git.Types.FileBlob -> True
                Just Git.Types.ExecutableBlob -> True
                _ -> False
-       add h i k = liftIO $ Database.Keys.SQL.addAssociatedFileFast 
-               (toIKey k)
-               (Git.LsTree.file i)
-               h
+       add i k = do
+               let tf = Git.LsTree.file i
+               Database.Keys.runWriter $
+                       liftIO . Database.Keys.SQL.addAssociatedFileFast (toIKey k) tf
+               whenM (inAnnex k) $ do
+                       f <- fromRepo $ fromTopFilePath tf
+                       destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f
+                       replaceFile f $ \tmp -> do
+                               r <- linkFromAnnex k tmp destmode
+                               case r of
+                                       LinkAnnexOk -> return ()
+                                       LinkAnnexNoop -> return ()
+                                       LinkAnnexFailed -> liftIO $
+                                               writePointerFile tmp k destmode

diff --git a/CHANGELOG b/CHANGELOG
index 7dd7b8d2bfdbf98cb8b24bdb40102a5310c67c63..db291e15a3e983cb7fead74b51372d7e2bca998b 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,6 +2,8 @@ git-annex (6.20161013) UNRELEASED; urgency=medium
 
   * lock, smudge: Fix edge cases where data loss could occur in v6 mode
     when the keys database was not populated.
+  * upgrade: Handle upgrade to v6 when the repository already contains
+    v6 unlocked files whose content is already present.
 
  -- Joey Hess <id@joeyh.name>  Mon, 17 Oct 2016 12:46:54 -0400
 

diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index dfd8350f10c2ec18d71487ca69a2df89305e5892..0ac221c1f5ce2639d3418aac8c1b2f2f89da2f17 100644 (file)
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -56,7 +56,7 @@ performNew dest key = do
                                case r of
                                        LinkAnnexOk -> return ()
                                        LinkAnnexNoop -> return ()
-                                       _ -> error "unlock failed"
+                                       LinkAnnexFailed -> error "unlock failed"
                        , liftIO $ writePointerFile tmp key destmode
                        )
        next $ cleanupNew dest key destmode

diff --git a/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn b/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
index 08df3b13cf44d9954484f4e9d43700a55be134bb..57a17445563a931b9d4e58e28b681e9d1becb758 100644 (file)
--- a/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
+++ b/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
@@ -35,4 +35,13 @@ $> ./v6-push.sh 6 2>&1 | grep '>>>'
 > 
 > I am not sure yet why the keys database lacked an entry for the file;
 > perhaps something to do with it being a v6 unlocked file in a v5
-> repository. --[[Joey]]
+> repository.
+> 
+> Ok.. Seems that cloning to a v5 repository, and then copying/getting
+> objects into it, and then upgrading to v6 reproduces the problem with the
+> keys database. The inode cache does not get populated for unlocked files
+> on upgrade. Also, unlocked files stay as pointer files even when their
+> content is present in annex/objects. Fixed the upgrade process to handle
+> this case.
+> 
+> [[fixed|done]]] --[[Joey]]