From ca31d7e54fefe6e88880caaa12a090041cef7ba9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 18 Dec 2020 17:04:46 -0400 Subject: [PATCH] refactor That code was not borg specific, and I can see making more remotes for other backup software. --- Remote/Borg.hs | 44 ++--------------------------- Remote/Helper/ThirdParty.hs | 56 +++++++++++++++++++++++++++++++++++++ git-annex.cabal | 1 + 3 files changed, 59 insertions(+), 42 deletions(-) create mode 100644 Remote/Helper/ThirdParty.hs diff --git a/Remote/Borg.hs b/Remote/Borg.hs index 823cceca21..3f0c2b68cc 100644 --- a/Remote/Borg.hs +++ b/Remote/Borg.hs @@ -21,10 +21,9 @@ import Remote.Helper.Special import Remote.Helper.ExportImport import Annex.UUID import Types.ProposedAccepted -import Crypto (isEncKey) import Utility.Metered +import qualified Remote.Helper.ThirdParty as ThirdParty -import qualified System.FilePath.ByteString as P import qualified Data.Map as M type BorgRepo = String @@ -70,7 +69,7 @@ gen r u rc gc rs = do , exportActions = exportUnsupported , importActions = ImportActions { listImportableContents = listImportableContentsM borgrepo - , importKey = Just importKeyM + , importKey = Just ThirdParty.importKey , retrieveExportWithContentIdentifier = retrieveExportWithContentIdentifierM borgrepo , checkPresentExportWithContentIdentifier = checkPresentExportWithContentIdentifierM borgrepo -- This remote is thirdPartyPopulated, so these @@ -123,45 +122,6 @@ borgLocal = notElem ':' listImportableContentsM :: BorgRepo -> Annex (Maybe (ImportableContents (ContentIdentifier, ByteSize))) listImportableContentsM borgrepo = error "TODO" --- Since this remote is thirdPartyPopulated, this needs to --- find only those ImportLocations that are annex object files. All other --- files in the borg backup are ignored. -importKeyM :: ImportLocation -> ContentIdentifier -> ByteSize -> MeterUpdate -> Annex (Maybe Key) -importKeyM loc cid sz _ = return $ case deserializeKey' f of - Just k - -- Annex objects always are in a subdirectory with the same - -- name as the filename. If this is not the case for the file - -- that was backed up, it is probably not a valid annex object. - -- Eg, it could be something in annex/bad/, or annex/tmp/. - -- Or it could be a file that only happens to have a name - -- like an annex object. - -- (This does unfortunately prevent recognizing files that are - -- part of special remotes that don't use that layout. The most - -- likely special remote to be in a backup, the directory - -- special remote, does use that layout at least.) - | lastMaybe (P.splitDirectories (P.dropFileName p)) /= Just f -> Nothing - -- Chunked or encrypted keys used in special remotes are not - -- supported. - | isChunkKey k || isEncKey k -> Nothing - -- Check that the size of the key is the same as the size of the - -- file stored in borg. This is a cheap way to make sure it's - -- probabably the actual content of the file. We don't fully - -- verify the content here because that could be a very - -- expensive operation for a large repository; if the user - -- wants to detect every possible data corruption problem - -- (eg, wrong data read off disk during backup, or the object - -- was corrupt in the git-annex repo and that bad object got - -- backed up), they can fsck the remote. - | otherwise -> case fromKey keySize k of - Just sz' - | sz' == sz -> Just k - | otherwise -> Nothing - Nothing -> Just k - Nothing -> Nothing - where - p = fromImportLocation loc - f = P.takeFileName p - retrieveExportWithContentIdentifierM :: BorgRepo -> ImportLocation -> ContentIdentifier -> FilePath -> Annex Key -> MeterUpdate -> Annex Key retrieveExportWithContentIdentifierM borgrepo loc cid dest k p = error "TODO" diff --git a/Remote/Helper/ThirdParty.hs b/Remote/Helper/ThirdParty.hs new file mode 100644 index 0000000000..6ce106fa12 --- /dev/null +++ b/Remote/Helper/ThirdParty.hs @@ -0,0 +1,56 @@ +{- Helpers for thirdPartyPopulated remotes + - + - Copyright 2020 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Remote.Helper.ThirdParty where + +import Annex.Common +import Types.Remote +import Types.Import +import Crypto (isEncKey) +import Utility.Metered + +import qualified System.FilePath.ByteString as P + +-- When a remote is thirdPartyPopulated, and contains a backup of a +-- git-annex repository or some special remotes, this can be used to +-- find only those ImportLocations that are annex object files. +-- All other ImportLocations are ignored. +importKey :: ImportLocation -> ContentIdentifier -> ByteSize -> MeterUpdate -> Annex (Maybe Key) +importKey loc _cid sz _ = return $ case deserializeKey' f of + Just k + -- Annex objects always are in a subdirectory with the same + -- name as the filename. If this is not the case for the file + -- that was backed up, it is probably not a valid annex object. + -- Eg, it could be something in annex/bad/, or annex/tmp/. + -- Or it could be a file that only happens to have a name + -- like an annex object. + -- (This does unfortunately prevent recognizing files that are + -- part of special remotes that don't use that layout. The most + -- likely special remote to be in a backup, the directory + -- special remote, does use that layout at least.) + | lastMaybe (P.splitDirectories (P.dropFileName p)) /= Just f -> Nothing + -- Chunked or encrypted keys used in special remotes are not + -- supported. + | isChunkKey k || isEncKey k -> Nothing + -- Check that the size of the key is the same as the size of the + -- file stored in the backup. This is a cheap way to make sure it's + -- probabably the actual content of the file. We don't fully + -- verify the content here because that could be a very + -- expensive operation for a large repository; if the user + -- wants to detect every possible data corruption problem + -- (eg, wrong data read off disk during backup, or the object + -- was corrupt in the git-annex repo and that bad object got + -- backed up), they can fsck the remote. + | otherwise -> case fromKey keySize k of + Just sz' + | sz' == sz -> Just k + | otherwise -> Nothing + Nothing -> Just k + Nothing -> Nothing + where + p = fromImportLocation loc + f = P.takeFileName p diff --git a/git-annex.cabal b/git-annex.cabal index e035eedd28..f39146f25b 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -963,6 +963,7 @@ Executable git-annex Remote.Helper.Messages Remote.Helper.P2P Remote.Helper.ReadOnly + Remote.Helper.ThirdParty Remote.Helper.Special Remote.Helper.Ssh Remote.HttpAlso -- 2.30.2