From 1e31bf81221d774fc26dac4fb15d722660dd0312 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Nov 2023 16:32:32 -0400 Subject: [PATCH] copy/move --from-anywhere --to remote Implementation was simple because it's equivilant to --from=foo --to remote for each other remote, followed by --to remote when there's a local copy. (Or, in the edge case of --from-anywhere --to=here, it's the same as --to=here.) Note that, when the local repo does not have a copy, fromToPerform gets it from a remote, sends it to the destination, and drops the local copy. Another call to that for a second remote will notice that the dest now has a copy, and simply drop from the second remote, avoiding a second transfer. Also note that, when numcopies doesn't allow dropping it from everywhere, it will drop it from the cheapest remotes first (maybe not ideal) up to more expensive remotes, and finally from the local repo. So the local repo will generally end up holding a copy. Maybe not ideal in all cases either, but it seems no worse to do that than to end up with a copy undropped from a remote. And I'm not entirely happy with the output, eg: copy bigfile (from r3...) ok copy bigfile ok That makes sense if you think of the second line as being the same as what is output by `git-annex copy bigfile --to bar`, but it's less clear in this context. Maybe add "(from here...)"? Also the --json output doesn't have a machine-readable field for the "from" uuid, and maybe it should? Sponsored-by: Dartmouth College's DANDI project --- CHANGELOG | 1 + CmdLine/GitAnnex/Options.hs | 33 ++++++++++++++----- Command/Copy.hs | 14 ++++---- Command/Move.hs | 29 ++++++++++++++++ ..._c374eb44ea08f220dbcce5ecb88403fb._comment | 16 +++++++++ doc/git-annex-copy.mdwn | 5 +++ doc/git-annex-move.mdwn | 5 +++ 7 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment diff --git a/CHANGELOG b/CHANGELOG index 3fdedb9023..de7bca3074 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ git-annex (10.20231130) UNRELEASED; urgency=medium * Make git-annex get/copy/move --from foo override configuration of remote.foo.annex-ignore, as documented. + * Support git-annex copy/move --from-anywhere --to remote. -- Joey Hess Thu, 30 Nov 2023 14:48:12 -0400 diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs index b5170f3131..b3c21aeece 100644 --- a/CmdLine/GitAnnex/Options.hs +++ b/CmdLine/GitAnnex/Options.hs @@ -162,40 +162,55 @@ parseToOption = strOption <> completeRemotes ) +parseFromAnywhereOption :: Parser Bool +parseFromAnywhereOption = switch + ( long "from-anywhere" + <> help "from any remote" + ) + parseRemoteOption :: Parser RemoteName parseRemoteOption = strOption ( long "remote" <> metavar paramRemote <> completeRemotes ) --- | From or to a remote, or both, or a special --to=here +-- | --from or --to a remote, or both, or a special --to=here, +-- or --from-anywhere --to remote. data FromToHereOptions = FromOrToRemote FromToOptions | ToHere | FromRemoteToRemote (DeferredParse Remote) (DeferredParse Remote) + | FromAnywhereToRemote (DeferredParse Remote) parseFromToHereOptions :: Parser (Maybe FromToHereOptions) parseFromToHereOptions = go <$> optional parseFromOption <*> optional parseToOption + <*> parseFromAnywhereOption where - go (Just from) (Just to) = Just $ FromRemoteToRemote + go _ (Just to) True = Just $ FromAnywhereToRemote + (mkParseRemoteOption to) + go (Just from) (Just to) _ = Just $ FromRemoteToRemote (mkParseRemoteOption from) (mkParseRemoteOption to) - go (Just from) Nothing = Just $ FromOrToRemote + go (Just from) Nothing _ = Just $ FromOrToRemote (FromRemote $ mkParseRemoteOption from) - go Nothing (Just to) = Just $ case to of + go Nothing (Just to) _ = Just $ case to of "here" -> ToHere "." -> ToHere _ -> FromOrToRemote $ ToRemote $ mkParseRemoteOption to - go Nothing Nothing = Nothing + go Nothing Nothing _ = Nothing instance DeferredParseClass FromToHereOptions where - finishParse (FromOrToRemote v) = FromOrToRemote <$> finishParse v + finishParse (FromOrToRemote v) = + FromOrToRemote <$> finishParse v finishParse ToHere = pure ToHere - finishParse (FromRemoteToRemote v1 v2) = FromRemoteToRemote - <$> finishParse v1 - <*> finishParse v2 + finishParse (FromRemoteToRemote v1 v2) = + FromRemoteToRemote + <$> finishParse v1 + <*> finishParse v2 + finishParse (FromAnywhereToRemote v) = + FromAnywhereToRemote <$> finishParse v -- Options for acting on keys, rather than work tree files. data KeyOptions diff --git a/Command/Copy.hs b/Command/Copy.hs index 88d645a693..67971af2f4 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -69,6 +69,7 @@ seek' o fto = startConcurrency (Command.Move.stages fto) $ do FromOrToRemote (ToRemote _) -> Just True ToHere -> Just False FromRemoteToRemote _ _ -> Nothing + FromAnywhereToRemote _ -> Nothing , usesLocationLog = True } keyaction = Command.Move.startKey fto Command.Move.RemoveNever @@ -84,12 +85,13 @@ start o fto si file key = stopUnless shouldCopy $ | autoMode o = want <||> numCopiesCheck file key (<) | otherwise = return True want = case fto of - FromOrToRemote (ToRemote dest) -> - (Remote.uuid <$> getParsed dest) >>= checkwantsend + FromOrToRemote (ToRemote dest) -> checkwantsend dest FromOrToRemote (FromRemote _) -> checkwantget ToHere -> checkwantget - FromRemoteToRemote _ dest -> - (Remote.uuid <$> getParsed dest) >>= checkwantsend - - checkwantsend = wantGetBy False (Just key) (AssociatedFile (Just file)) + FromRemoteToRemote _ dest -> checkwantsend dest + FromAnywhereToRemote dest -> checkwantsend dest + + checkwantsend dest = + (Remote.uuid <$> getParsed dest) >>= + wantGetBy False (Just key) (AssociatedFile (Just file)) checkwantget = wantGet False (Just key) (AssociatedFile (Just file)) diff --git a/Command/Move.hs b/Command/Move.hs index 59bdf9ecad..0bc707df85 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -81,6 +81,7 @@ seek' o fto = startConcurrency (stages fto) $ do FromOrToRemote (ToRemote _) -> Just True ToHere -> Nothing FromRemoteToRemote _ _ -> Nothing + FromAnywhereToRemote _ -> Nothing , usesLocationLog = True } keyaction = startKey fto (removeWhen o) @@ -91,6 +92,7 @@ stages (FromOrToRemote (FromRemote _)) = transferStages stages (FromOrToRemote (ToRemote _)) = commandStages stages ToHere = transferStages stages (FromRemoteToRemote _ _) = transferStages +stages (FromAnywhereToRemote _) = transferStages start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart start fromto removewhen si f k = start' fromto removewhen afile si k ai @@ -118,6 +120,9 @@ start' fromto removewhen afile si key ai = src' <- getParsed src dest' <- getParsed dest fromToStart removewhen afile key ai si src' dest' + FromAnywhereToRemote dest -> do + dest' <- getParsed dest + fromAnywhereToStart removewhen afile key ai si dest' describeMoveAction :: RemoveWhen -> String describeMoveAction RemoveNever = "copy" @@ -353,6 +358,30 @@ fromToStart removewhen afile key ai si src dest = then not <$> expectedPresent dest key else return True +fromAnywhereToStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> CommandStart +fromAnywhereToStart removewhen afile key ai si dest = + stopUnless somethingtodo $ do + u <- getUUID + if u == Remote.uuid dest + then toHereStart removewhen afile key ai si + else startingNoMessage (OnlyActionOn key ai) $ do + rs <- filter (/= dest) + <$> Remote.keyPossibilities (Remote.IncludeIgnored False) key + forM_ rs $ \r -> + includeCommandAction $ + starting (describeMoveAction removewhen) ai si $ + fromToPerform r dest removewhen key afile + whenM (inAnnex key) $ + void $ includeCommandAction $ + toStart removewhen afile key ai si dest + next $ return True + where + somethingtodo = do + fast <- Annex.getRead Annex.fast + if fast && removewhen == RemoveNever + then not <$> expectedPresent dest key + else return True + {- When there is a local copy, transfer it to the dest, and drop from the src. - - When the dest has a copy, drop it from the src. diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment new file mode 100644 index 0000000000..38e8ea92d2 --- /dev/null +++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2023-11-30T18:26:30Z" + content=""" +I like the idea of `copy --from-anywhere --to=remote` and just +use the lowest cost remote (when not in local repo). Like `git-annex get` +and `git-annex copy --to=here`. + +Hmm, if there's a remote that is too expensive to want to use in such a +copy, it would be possible to use `-c remote.foo.annex-ignore=true` +to make it avoid using that remote. As can also be done in the case of +`git-annex get`, although that was not documented well. + +I've implemented --from-anywhere.. +"""]] diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn index 57905b672c..add7a96f06 100644 --- a/doc/git-annex-copy.mdwn +++ b/doc/git-annex-copy.mdwn @@ -41,6 +41,11 @@ Paths of files or directories to operate on can be specified. then deleting the content from the local repository (if it was not present to start with). +* `--from-anywhere --to=remote` + + Copy to the remote files from the local repository as well as from any reachable + remotes. + * `--jobs=N` `-JN` Enables parallel transfers with up to the specified number of jobs diff --git a/doc/git-annex-move.mdwn b/doc/git-annex-move.mdwn index b30163c891..c93953f985 100644 --- a/doc/git-annex-move.mdwn +++ b/doc/git-annex-move.mdwn @@ -38,6 +38,11 @@ Paths of files or directories to operate on can be specified. then deleting the content from the local repository (if it was not present to start with). +* `--from-anywhere --to=remote` + + Move to the remote files from the local repository and from all + reachable remotes. + * `--force` Override numcopies and required content checking, and always remove -- 2.30.2