import: Retry downloads that fail

Also, using the transfer machinery for this makes eg, git-annex info show
in-progress imports, and makes --notify-start/finish work.

diff --git a/Annex/Import.hs b/Annex/Import.hs
index 44877bc0c8f92165394f77516d47c03a6be16ce4..d6b3f65bb5947cce2adbf633855b36352ae7e453 100644 (file)
--- a/Annex/Import.hs
+++ b/Annex/Import.hs
@@ -35,6 +35,7 @@ import Annex.Content
 import Annex.Export
 import Annex.RemoteTrackingBranch
 import Annex.HashObject
+import Annex.Transfer
 import Command
 import Backend
 import Types.Key
@@ -411,8 +412,13 @@ importKeys remote importtreeconfig importcontent importablecontents = do
                                        const runimport
        
        dodownload cidmap db (loc, (cid, sz)) largematcher = do
+               f <- locworktreefile loc
+               let af = AssociatedFile (Just f)
                let downloader tmpfile p = do
-                       k <- Remote.retrieveExportWithContentIdentifier ia loc cid tmpfile (mkkey loc tmpfile largematcher) p
+                       k <- Remote.retrieveExportWithContentIdentifier
+                               ia loc cid tmpfile 
+                               (mkkey f tmpfile largematcher)
+                               p
                        case keyGitSha k of
                                Nothing -> do
                                        ok <- moveAnnex k tmpfile
@@ -431,16 +437,17 @@ importKeys remote importtreeconfig importcontent importablecontents = do
                                warning (show e)
                                return Nothing
                checkDiskSpaceToGet tmpkey Nothing $
-                       withTmp tmpkey $ \tmpfile ->
-                               metered Nothing tmpkey $
-                                       const (rundownload tmpfile)
+                       notifyTransfer Download af $
+                               download (Remote.uuid remote) tmpkey af stdRetry $ \p ->
+                                       withTmp tmpkey $ \tmpfile ->
+                                               metered (Just p) tmpkey $
+                                                       const (rundownload tmpfile)
          where
                tmpkey = importKey cid sz
-               
+       
        ia = Remote.importActions remote
        
-       mkkey loc tmpfile largematcher = do
-               f <- fromRepo $ fromTopFilePath $ locworktreefilename loc
+       mkkey f tmpfile largematcher = do
                matcher <- largematcher (fromRawFilePath f)
                let mi = MatchingFile FileInfo
                        { matchFile = f
@@ -458,10 +465,11 @@ importKeys remote importtreeconfig importcontent importablecontents = do
                                fst <$> genKey ks nullMeterUpdate backend
                        else gitShaKey <$> hashFile tmpfile
 
-       locworktreefilename loc = asTopFilePath $ case importtreeconfig of
-               ImportTree -> fromImportLocation loc
-               ImportSubTree subdir _ ->
-                       getTopFilePath subdir P.</> fromImportLocation loc
+       locworktreefile loc = fromRepo $ fromTopFilePath $ asTopFilePath $
+               case importtreeconfig of
+                       ImportTree -> fromImportLocation loc
+                       ImportSubTree subdir _ ->
+                               getTopFilePath subdir P.</> fromImportLocation loc
 
        getcidkey cidmap db cid = liftIO $
                CIDDb.getContentIdentifierKeys db rs cid >>= \case

diff --git a/CHANGELOG b/CHANGELOG
index dfcc4313605c652e66cdd4879adbfaa2e25a6a6c..d9385aecc6c9b94d9a1c35dcfb6b7b81485964d4 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -28,6 +28,7 @@ git-annex (8.20200815) UNRELEASED; urgency=medium
     to 5, to avoid some unusual edge cases where too much retrying could
     result in far more data transfer than makes sense.
   * Retry transfers to exporttree=yes remotes same as for other remotes.
+  * import: Retry downloads that fail, same as is done for downloads generally.
 
  -- Joey Hess <id@joeyh.name>  Fri, 14 Aug 2020 14:57:45 -0400
 

diff --git a/doc/todo/tolerate_intermittent_errors_upon___34__export__34_____40__and_probably_copy__41___to_S3/comment_3_fb3e0585f6700050db7a06e27d8e1099._comment b/doc/todo/tolerate_intermittent_errors_upon___34__export__34_____40__and_probably_copy__41___to_S3/comment_3_fb3e0585f6700050db7a06e27d8e1099._comment
new file mode 100644 (file)
index 0000000..e4fea93
--- /dev/null
+++ b/doc/todo/tolerate_intermittent_errors_upon___34__export__34_____40__and_probably_copy__41___to_S3/comment_3_fb3e0585f6700050db7a06e27d8e1099._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2020-09-04T17:40:07Z"
+ content="""
+Both retry now.
+"""]]