Bugfix to getting content from an export remote with -J, when the export database...

diff --git a/CHANGELOG b/CHANGELOG
index 97009669981efea3bc448e9af3f327f951cf54d5..443bdbeaba4f9c22ec26f4a643678760d048b604 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -34,6 +34,8 @@ git-annex (8.20200221) UNRELEASED; urgency=medium
   * Auto upgrades from older repo versions, like v5, now jump right to v8.
   * Extended annex.security.allowed-ip-addresses to let specific ports
     of an IP address to be used, while denying use of other ports.
+  * Bugfix to getting content from an export remote with -J, when the
+    export database was not yet populated.
 
  -- Joey Hess <id@joeyh.name>  Wed, 19 Feb 2020 12:48:58 -0400
 

diff --git a/Remote/Helper/ExportImport.hs b/Remote/Helper/ExportImport.hs
index a412f143fea7a4a73e4536e9c86f8ba3b46fa772..d6e680c6db288b0d593b6de26688d17d2ea7c09f 100644 (file)
--- a/Remote/Helper/ExportImport.hs
+++ b/Remote/Helper/ExportImport.hs
@@ -160,7 +160,6 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
                                { storeExport = \f k loc p -> do
                                        db <- getciddb ciddbv
                                        exportdb <- getexportdb exportdbv
-                                       updateexportdb exportdb exportdbv
                                        oldks <- liftIO $ Export.getExportTreeKey exportdb loc
                                        oldcids <- liftIO $ concat
                                                <$> mapM (ContentIdentifier.getContentIdentifiers db rs) oldks
@@ -277,8 +276,7 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
                lcklckv <- liftIO newEmptyTMVarIO
                dbv <- liftIO newEmptyTMVarIO
                exportinconflict <- liftIO $ newTVarIO False
-               exportupdated <- liftIO $ newTMVarIO ()
-               return (dbv, lcklckv, exportinconflict, exportupdated)
+               return (dbv, lcklckv, exportinconflict)
 
        -- Only open the database once it's needed.
        getciddb (dbtv, lcklckv) =
@@ -300,13 +298,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
                                )
        
        -- Only open the database once it's needed.
-       getexportdb (dbv, lcklckv, _, _) = 
+       --
+       -- After opening the database, check if the export log is
+       -- different than the database, and update the database, to notice
+       -- when an export has been updated from another repository.
+       getexportdb (dbv, lcklckv, exportinconflict) = 
                liftIO (atomically (tryReadTMVar dbv)) >>= \case
                        Just db -> return db
                        -- let only one thread take the lock
                        Nothing -> ifM (liftIO $ atomically $ tryPutTMVar lcklckv ())
                                ( do
                                        db <- Export.openDb (uuid r)
+                                       updateexportdb db exportinconflict
                                        liftIO $ atomically $ putTMVar dbv db
                                        return db
                                -- loser waits for winner to open the db and
@@ -314,24 +317,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
                                , liftIO $ atomically (readTMVar dbv)
                                )
 
-       getexportinconflict (_, _, v, _) = v
+       getexportinconflict (_, _, v) = v
 
-       -- Check once if the export log is different than the database and
-       -- updates the database, to notice when an export has been
-       -- updated from another repository.                             
-       updateexportdb db (_, _, exportinconflict, exportupdated) =
-               liftIO (atomically (tryTakeTMVar exportupdated)) >>= \case
-                       Just () -> Export.updateExportTreeFromLog db >>= \case
-                               Export.ExportUpdateSuccess -> return ()
-                               Export.ExportUpdateConflict -> do
-                                       warnExportImportConflict r
-                                       liftIO $ atomically $
-                                               writeTVar exportinconflict True
-                       Nothing -> return ()
+       updateexportdb db exportinconflict =
+               Export.updateExportTreeFromLog db >>= \case
+                       Export.ExportUpdateSuccess -> return ()
+                       Export.ExportUpdateConflict -> do
+                               warnExportImportConflict r
+                               liftIO $ atomically $
+                                       writeTVar exportinconflict True
                
        getexportlocs dbv k = do
                db <- getexportdb dbv
-               updateexportdb db dbv
                liftIO $ Export.getExportTree db k
 
        retrieveKeyFileFromExport dbv k _af dest p = unVerified $

diff --git a/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn
index 4387afa3b29e5b1e871df43d359b3ca30bd9b668..5b3d7c31b88561d9ecc48094945510fd24360af3 100644 (file)
--- a/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn
+++ b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn
@@ -89,3 +89,5 @@ get sub-01/meg/sub-01_task-audiovisual_run-01_meg.fif (from s3-PUBLIC...)
 
 [[!meta author=yoh]]
 [[!tag projects/repronim]]
+
+> [[fixed|done]] --[[Joey]]

diff --git a/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment
new file mode 100644 (file)
index 0000000..a1e27d0
--- /dev/null
+++ b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2020-02-26T18:22:46Z"
+ content="""
+How do I produce such a repo, I thought that git-annex has fixed the 
+problem that made it not include the S3 versioning information?
+I don't want to see a lot of repos being created with that information
+missing.
+
+----
+
+Anyway, the S3 version is is a red herring, the failure is actually
+caused by the export db not getting populated from the git-annex branch
+before some threads try to use it. Remote.Helper.ExportImport has a
+updateexportdb that lets one thread update the db, but other threads
+don't block waiting for it. Easily fixed.
+"""]]