migrate: support adding size back to URL keys

migrate: Support adding size to URL keys that were added with --relaxed, by
running eg: git-annex migrate --backend=URL foo

Since url keys cannot be generated, that used to fail. Make it notice that
the backend is not changed, and just get the size of the content.

Sponsored-by: Brock Spratlen on Patreon

diff --git a/Annex/Content.hs b/Annex/Content.hs
index c3766c0abcdd5ee95a1f98fc4e42032af3ccccea..04fc474d168e1857d4657dcdb1197a08bc22ce9d 100644 (file)
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -1,6 +1,6 @@
 {- git-annex file content managing
  -
- - Copyright 2010-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -66,6 +66,7 @@ module Annex.Content (
        getKeyStatus,
        getKeyFileStatus,
        cleanObjectDirs,
+       contentSize,
 ) where
 
 import System.IO.Unsafe (unsafeInterleaveIO)
@@ -916,3 +917,25 @@ getKeyFileStatus key file = do
                                )
                _ -> return s
 
+{- Gets the size of the content of a key when it is present.
+ - Useful when the key does not have keySize set. 
+ -
+ - When the object file appears possibly modified with annex.thin set, does
+ - not do an expensive verification that the content is good, just returns
+ - Nothing.
+ -}
+contentSize :: Key -> Annex (Maybe FileSize)
+contentSize key = catchDefaultIO Nothing $
+       withObjectLoc key $ \loc ->
+               withTSDelta (liftIO . genInodeCache loc) >>= \case
+                       Just ic -> ifM (unmodified ic)
+                               ( return (Just (inodeCacheFileSize ic))
+                               , return Nothing
+                               )
+                       Nothing -> return Nothing
+  where
+       unmodified ic =
+               ifM (annexThin <$> Annex.getGitConfig)
+                       ( isUnmodifiedCheap' key ic
+                       , return True
+                       )

diff --git a/Annex/Content/Presence.hs b/Annex/Content/Presence.hs
index d3aea8715116410b6e335f0e5c83fbb412325a83..2eb0016ddd105d8656748efec891166b3e8b529f 100644 (file)
--- a/Annex/Content/Presence.hs
+++ b/Annex/Content/Presence.hs
@@ -18,6 +18,7 @@ module Annex.Content.Presence (
        isUnmodified,
        isUnmodified',
        isUnmodifiedCheap,
+       isUnmodifiedCheap',
        withContentLockFile,
        contentLockFile,
 ) where
@@ -206,7 +207,7 @@ isUnmodified' = isUnmodifiedLowLevel Database.Keys.addInodeCaches
  - within a small time window (eg 1 second).
  -}
 isUnmodifiedCheap :: Key -> RawFilePath -> Annex Bool
-isUnmodifiedCheap key f = maybe (return False) (isUnmodifiedCheap' key) 
+isUnmodifiedCheap key f = maybe (pure False) (isUnmodifiedCheap' key) 
        =<< withTSDelta (liftIO . genInodeCache f)
 
 isUnmodifiedCheap' :: Key -> InodeCache -> Annex Bool

diff --git a/CHANGELOG b/CHANGELOG
index ba666913ad5d8349e8d0a9f09e4e5863de7513c5..862e5944926f60503413b753935bbdad4eb61c1f 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -9,6 +9,8 @@ git-annex (10.20231130) UNRELEASED; urgency=medium
     that have been migrated.
   * Added annex.syncmigrations config that can be set to false to prevent
     pull and sync from migrating object content.
+  * migrate: Support adding size to URL keys that were added with
+    --relaxed, by running eg: git-annex migrate --backend=URL foo
   * Make git-annex get/copy/move --from foo override configuration of 
     remote.foo.annex-ignore, as documented.
   * Support git-annex copy/move --from-anywhere --to remote.

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 344287f506c49a6c8ce8522fe03cdbade2d3f665..c07377aae288ffe2ffce17a8d8c5a5d5e318a13f 100644 (file)
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -88,29 +88,26 @@ start o ksha si file key = do
                Just oldbackend -> do
                        exists <- inAnnex key
                        newbackend <- chooseBackend file
-                       if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists
+                       if (newbackend /= oldbackend || upgradableKey oldbackend || forced) && exists
                                then go False oldbackend newbackend
-                               else if removeSize o && exists
-                                       then go True oldbackend oldbackend
+                               else if cantweaksize newbackend oldbackend && exists
+                                       then go True oldbackend newbackend
                                        else stop
   where
-       go onlyremovesize oldbackend newbackend = do
+       go onlytweaksize oldbackend newbackend = do
                keyrec <- case ksha of
                        Just (KeySha s) -> pure (MigrationRecord s)
                        Nothing -> error "internal"
                starting "migrate" (mkActionItem (key, file)) si $
-                       perform onlyremovesize o file key keyrec oldbackend newbackend
+                       perform onlytweaksize o file key keyrec oldbackend newbackend
 
-{- Checks if a key is upgradable to a newer representation.
- - 
- - Reasons for migration:
- -  - Ideally, all keys have file size metadata. Old keys may not.
- -  - Something has changed in the backend, such as a bug fix.
- -}
-upgradableKey :: Backend -> Key -> Bool
-upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
-  where
-       backendupgradable = maybe False (\a -> a key) (canUpgradeKey backend)
+       cantweaksize newbackend oldbackend
+               | removeSize o = isJust (fromKey keySize key)
+               | newbackend /= oldbackend = False
+               | isNothing (fromKey keySize key) = True
+               | otherwise = False
+
+       upgradableKey oldbackend = maybe False (\a -> a key) (canUpgradeKey oldbackend)
 
 {- Store the old backend's key in the new backend
  - The old backend's key is not dropped from it, because there may
@@ -122,13 +119,13 @@ upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
  - generated.
  -}
 perform :: Bool -> MigrateOptions -> RawFilePath -> Key -> MigrationRecord -> Backend -> Backend -> CommandPerform
-perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
+perform onlytweaksize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
   where
        go Nothing = stop
        go (Just (newkey, knowngoodcontent))
-               | knowngoodcontent = finish (removesize newkey)
+               | knowngoodcontent = finish =<< tweaksize newkey
                | otherwise = stopUnless checkcontent $
-                       finish (removesize newkey)
+                       finish =<< tweaksize newkey
        checkcontent = Command.Fsck.checkBackend oldbackend oldkey KeyPresent afile
        finish newkey = ifM (Command.ReKey.linkKey file oldkey newkey)
                ( do
@@ -142,7 +139,7 @@ perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< ge
                                logMigration oldkeyrec
                , giveup "failed creating link from old to new key"
                )
-       genkey _ | onlyremovesize = return $ Just (oldkey, False)
+       genkey _ | onlytweaksize = return $ Just (oldkey, False)
        genkey Nothing = do
                content <- calcRepo $ gitAnnexLocation oldkey
                let source = KeySource
@@ -155,9 +152,17 @@ perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< ge
        genkey (Just fm) = fm oldkey newbackend afile >>= \case
                Just newkey -> return (Just (newkey, True))
                Nothing -> genkey Nothing
-       removesize k
-               | removeSize o = alterKey k $ \kd -> kd { keySize = Nothing } 
-               | otherwise = k
+       tweaksize k
+               | removeSize o = pure (removesize k)
+               | onlytweaksize = addsize k
+               | otherwise = pure k
+       removesize k = alterKey k $ \kd -> kd { keySize = Nothing }
+       addsize k
+               | fromKey keySize k == Nothing = 
+                       contentSize k >>= return . \case
+                               Just sz -> alterKey k $ \kd -> kd { keySize = Just sz }
+                               Nothing -> k
+               | otherwise = return k
        afile = AssociatedFile (Just file)
 
 update :: Key -> Key -> CommandStart

diff --git a/doc/git-annex-migrate.mdwn b/doc/git-annex-migrate.mdwn
index e5e4fe0beceebe8f576a3840da3d9f5e6ba48c83..235a32c5e3f6a456bc7d3619d5592f2789a3e496 100644 (file)
--- a/doc/git-annex-migrate.mdwn
+++ b/doc/git-annex-migrate.mdwn
@@ -82,6 +82,10 @@ format.
 
        git-annex migrate --remove-size --backend=URL somefile
 
+  To add back the size to an URL key, use this:
+
+    git-annex migrate --backend=URL somefile
+
 * `--json`
 
   Enable JSON output. This is intended to be parsed by programs that use