From e520341500dce9f708a4de674f19047c5d325744 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Feb 2020 14:57:29 -0400 Subject: [PATCH] Bugfix to getting content from an export remote with -J, when the export database was not yet populated. --- CHANGELOG | 2 ++ Remote/Helper/ExportImport.hs | 33 +++++++++---------- ...t_where_versioning_info_was_forgotten.mdwn | 2 ++ ..._4a4d5b62af4a42ab9072ccaa6d8bed7d._comment | 18 ++++++++++ 4 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment diff --git a/CHANGELOG b/CHANGELOG index 9700966998..443bdbeaba 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -34,6 +34,8 @@ git-annex (8.20200221) UNRELEASED; urgency=medium * Auto upgrades from older repo versions, like v5, now jump right to v8. * Extended annex.security.allowed-ip-addresses to let specific ports of an IP address to be used, while denying use of other ports. + * Bugfix to getting content from an export remote with -J, when the + export database was not yet populated. -- Joey Hess Wed, 19 Feb 2020 12:48:58 -0400 diff --git a/Remote/Helper/ExportImport.hs b/Remote/Helper/ExportImport.hs index a412f143fe..d6e680c6db 100644 --- a/Remote/Helper/ExportImport.hs +++ b/Remote/Helper/ExportImport.hs @@ -160,7 +160,6 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o { storeExport = \f k loc p -> do db <- getciddb ciddbv exportdb <- getexportdb exportdbv - updateexportdb exportdb exportdbv oldks <- liftIO $ Export.getExportTreeKey exportdb loc oldcids <- liftIO $ concat <$> mapM (ContentIdentifier.getContentIdentifiers db rs) oldks @@ -277,8 +276,7 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o lcklckv <- liftIO newEmptyTMVarIO dbv <- liftIO newEmptyTMVarIO exportinconflict <- liftIO $ newTVarIO False - exportupdated <- liftIO $ newTMVarIO () - return (dbv, lcklckv, exportinconflict, exportupdated) + return (dbv, lcklckv, exportinconflict) -- Only open the database once it's needed. getciddb (dbtv, lcklckv) = @@ -300,13 +298,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o ) -- Only open the database once it's needed. - getexportdb (dbv, lcklckv, _, _) = + -- + -- After opening the database, check if the export log is + -- different than the database, and update the database, to notice + -- when an export has been updated from another repository. + getexportdb (dbv, lcklckv, exportinconflict) = liftIO (atomically (tryReadTMVar dbv)) >>= \case Just db -> return db -- let only one thread take the lock Nothing -> ifM (liftIO $ atomically $ tryPutTMVar lcklckv ()) ( do db <- Export.openDb (uuid r) + updateexportdb db exportinconflict liftIO $ atomically $ putTMVar dbv db return db -- loser waits for winner to open the db and @@ -314,24 +317,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o , liftIO $ atomically (readTMVar dbv) ) - getexportinconflict (_, _, v, _) = v + getexportinconflict (_, _, v) = v - -- Check once if the export log is different than the database and - -- updates the database, to notice when an export has been - -- updated from another repository. - updateexportdb db (_, _, exportinconflict, exportupdated) = - liftIO (atomically (tryTakeTMVar exportupdated)) >>= \case - Just () -> Export.updateExportTreeFromLog db >>= \case - Export.ExportUpdateSuccess -> return () - Export.ExportUpdateConflict -> do - warnExportImportConflict r - liftIO $ atomically $ - writeTVar exportinconflict True - Nothing -> return () + updateexportdb db exportinconflict = + Export.updateExportTreeFromLog db >>= \case + Export.ExportUpdateSuccess -> return () + Export.ExportUpdateConflict -> do + warnExportImportConflict r + liftIO $ atomically $ + writeTVar exportinconflict True getexportlocs dbv k = do db <- getexportdb dbv - updateexportdb db dbv liftIO $ Export.getExportTree db k retrieveKeyFileFromExport dbv k _af dest p = unVerified $ diff --git a/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn index 4387afa3b2..5b3d7c31b8 100644 --- a/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn +++ b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn @@ -89,3 +89,5 @@ get sub-01/meg/sub-01_task-audiovisual_run-01_meg.fif (from s3-PUBLIC...) [[!meta author=yoh]] [[!tag projects/repronim]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment new file mode 100644 index 0000000000..a1e27d03d6 --- /dev/null +++ b/doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2020-02-26T18:22:46Z" + content=""" +How do I produce such a repo, I thought that git-annex has fixed the +problem that made it not include the S3 versioning information? +I don't want to see a lot of repos being created with that information +missing. + +---- + +Anyway, the S3 version is is a red herring, the failure is actually +caused by the export db not getting populated from the git-annex branch +before some threads try to use it. Remote.Helper.ExportImport has a +updateexportdb that lets one thread update the db, but other threads +don't block waiting for it. Easily fixed. +"""]] -- 2.30.2