closeDb,
getRepoSizes,
setRepoSizes,
- getLiveSizeChanges,
+ getLiveRepoSizes,
startingLiveSizeChange,
successfullyFinishedLiveSizeChange,
- staleLiveSizeChange,
- getSizeChanges,
+ removeStaleLiveSizeChange,
) where
import Annex.Common
import Database.Persist.Sql hiding (Key)
import Database.Persist.TH
import qualified System.FilePath.ByteString as P
-import qualified Data.Map as M
+import qualified Data.Map.Strict as M
+import qualified Data.Set as S
share [mkPersist sqlSettings, mkMigrate "migrateRepoSizes"] [persistLowerCase|
-- Corresponds to location log information from the git-annex branch.
repo UUID
rollingtotal FileSize
UniqueRepoRollingTotal repo
+-- The most recent size changes that were removed from LiveSizeChanges
+-- upon successful completion.
+RecentChanges
+ repo UUID
+ key Key
+ change SizeChange
+ UniqueRecentChange repo key
|]
{- Gets a handle to the database. It's cached in Annex state. -}
closeDb (RepoSizeHandle (Just h)) = liftIO $ H.closeDb h
closeDb (RepoSizeHandle Nothing) = noop
+{- Gets the sizes of repositories as of a commit to the git-annex
+ - branch. -}
getRepoSizes :: RepoSizeHandle -> IO (M.Map UUID RepoSize, Maybe Sha)
getRepoSizes (RepoSizeHandle (Just h)) = H.queryDb h $ do
- sizemap <- M.fromList . map conv <$> getRepoSizes'
+ sizemap <- M.fromList <$> getRepoSizes'
annexbranchsha <- getAnnexBranchCommit
return (sizemap, annexbranchsha)
+getRepoSizes (RepoSizeHandle Nothing) = return (mempty, Nothing)
+
+getRepoSizes' :: SqlPersistM [(UUID, RepoSize)]
+getRepoSizes' = map conv <$> selectList [] []
where
conv entity =
let RepoSizes u sz = entityVal entity
in (u, RepoSize sz)
-getRepoSizes (RepoSizeHandle Nothing) = return (mempty, Nothing)
-
-getRepoSizes' :: SqlPersistM [Entity RepoSizes]
-getRepoSizes' = selectList [] []
getAnnexBranchCommit :: SqlPersistM (Maybe Sha)
getAnnexBranchCommit = do
setRepoSizes :: RepoSizeHandle -> M.Map UUID RepoSize -> Sha -> IO ()
setRepoSizes (RepoSizeHandle (Just h)) sizemap branchcommitsha =
H.commitDb h $ do
- l <- getRepoSizes'
- forM_ (map entityVal l) $ \(RepoSizes u _) ->
+ l <- getRepoSizes'
+ forM_ (map fst l) $ \u ->
unless (M.member u sizemap) $
unsetRepoSize u
forM_ (M.toList sizemap) $
deleteWhere ([] :: [Filter AnnexBranch])
void $ insertUniqueFast $ AnnexBranch $ toSSha branchcommitsha
-{- If there is already a size change for the same UUID, Key,
- - and SizeChangeId, it is overwritten with the new size change. -}
startingLiveSizeChange :: RepoSizeHandle -> UUID -> Key -> SizeChange -> SizeChangeId -> IO ()
startingLiveSizeChange (RepoSizeHandle (Just h)) u k sc sid =
H.commitDb h $ void $ upsertBy
successfullyFinishedLiveSizeChange :: RepoSizeHandle -> UUID -> Key -> SizeChange -> SizeChangeId -> IO ()
successfullyFinishedLiveSizeChange (RepoSizeHandle (Just h)) u k sc sid =
H.commitDb h $ do
- -- Update the rolling total and remove the live change in the
- -- same transaction.
+ -- Update the rolling total, add as a recent change,
+ -- and remove the live change in the same transaction.
rollingtotal <- getSizeChangeFor u
- setSizeChangeFor u (updaterollingtotal rollingtotal)
+ setSizeChangeFor u (updateRollingTotal rollingtotal sc k)
+ addRecentChange u k sc
removeLiveSizeChange u k sc sid
where
updaterollingtotal t = case sc of
ksz = fromMaybe 0 $ fromKey keySize k
successfullyFinishedLiveSizeChange (RepoSizeHandle Nothing) _ _ _ _ = noop
-staleLiveSizeChange :: RepoSizeHandle -> UUID -> Key -> SizeChange -> SizeChangeId -> IO ()
-staleLiveSizeChange (RepoSizeHandle (Just h)) u k sc sid =
+updateRollingTotal :: FileSize -> SizeChange -> Key -> FileSize
+updateRollingTotal t sc k = case sc of
+ AddingKey -> t + ksz
+ RemovingKey -> t - ksz
+ where
+ ksz = fromMaybe 0 $ fromKey keySize k
+
+removeStaleLiveSizeChange :: RepoSizeHandle -> UUID -> Key -> SizeChange -> SizeChangeId -> IO ()
+removeStaleLiveSizeChange (RepoSizeHandle (Just h)) u k sc sid =
H.commitDb h $ removeLiveSizeChange u k sc sid
-staleLiveSizeChange (RepoSizeHandle Nothing) _ _ _ _ = noop
+removeStaleLiveSizeChange (RepoSizeHandle Nothing) _ _ _ _ = noop
removeLiveSizeChange :: UUID -> Key -> SizeChange -> SizeChangeId -> SqlPersistM ()
removeLiveSizeChange u k sc sid =
, LiveSizeChangesChange ==. sc
]
-getLiveSizeChanges :: RepoSizeHandle -> IO (M.Map UUID (Key, SizeChange, SizeChangeId))
-getLiveSizeChanges (RepoSizeHandle (Just h)) = H.queryDb h $ do
- m <- M.fromList . map conv <$> getLiveSizeChanges'
- return m
+getLiveSizeChanges :: SqlPersistM (M.Map UUID [(Key, (SizeChange, SizeChangeId))])
+getLiveSizeChanges = M.fromListWith (++) . map conv <$> selectList [] []
where
conv entity =
let LiveSizeChanges u k sid sc = entityVal entity
- in (u, (k, sc, sid))
-getLiveSizeChanges (RepoSizeHandle Nothing) = return mempty
-
-getLiveSizeChanges' :: SqlPersistM [Entity LiveSizeChanges]
-getLiveSizeChanges' = selectList [] []
+ in (u, [(k, (sc, sid))])
-getSizeChanges :: RepoSizeHandle -> IO (M.Map UUID FileSize)
-getSizeChanges (RepoSizeHandle (Just h)) = H.queryDb h getSizeChanges'
-getSizeChanges (RepoSizeHandle Nothing) = return mempty
-
-getSizeChanges' :: SqlPersistM (M.Map UUID FileSize)
-getSizeChanges' = M.fromList . map conv <$> selectList [] []
+getSizeChanges :: SqlPersistM (M.Map UUID FileSize)
+getSizeChanges = M.fromList . map conv <$> selectList [] []
where
conv entity =
let SizeChanges u n = entityVal entity
(UniqueRepoRollingTotal u)
(SizeChanges u sz)
[SizeChangesRollingtotal =. sz]
+
+addRecentChange :: UUID -> Key -> SizeChange -> SqlPersistM ()
+addRecentChange u k sc =
+ void $ upsertBy
+ (UniqueRecentChange u k)
+ (RecentChanges u k sc)
+ [RecentChangesChange =. sc]
+
+getRecentChange :: UUID -> Key -> SqlPersistM (Maybe SizeChange)
+getRecentChange u k = do
+ l <- selectList
+ [ RecentChangesRepo ==. u
+ , RecentChangesKey ==. k
+ ] []
+ return $ case l of
+ (s:_) -> Just $ recentChangesChange $ entityVal s
+ [] -> Nothing
+
+{- Gets the sizes of Repos as of a commit to the git-annex branch
+ - (which is not necessarily the current commit), adjusted with all
+ - live changes that have happened since then or are happening now.
+ -
+ - This does not necessarily include all changes that have been journalled,
+ - only ones that had startingLiveSizeChange called for them will be
+ - included. Also live changes or recent changes that were to a UUID not in
+ - the RepoSizes map are not included.
+ -
+ - In the unlikely case where two live changes are occurring, one
+ - adding a key and the other removing the same key, the one
+ - adding the key is used, in order to err on the side of a larger
+ - RepoSize.
+ -
+ - Omits live changes that are redundant due to a recent change already
+ - being recorded for the same change.
+ -
+ - This is only expensive when there are a lot of live changes happening at
+ - the same time.
+ -}
+getLiveRepoSizes :: RepoSizeHandle -> IO (M.Map UUID RepoSize, Maybe Sha)
+getLiveRepoSizes (RepoSizeHandle (Just h)) = H.queryDb h $ do
+ sizechanges <- getSizeChanges
+ livechanges <- getLiveSizeChanges
+ reposizes <- getRepoSizes'
+ annexbranchsha <- getAnnexBranchCommit
+ m <- M.fromList <$> forM reposizes (go sizechanges livechanges)
+ return (m, annexbranchsha)
+ where
+ go
+ :: M.Map UUID FileSize
+ -> M.Map UUID [(Key, (SizeChange, SizeChangeId))]
+ -> (UUID, RepoSize)
+ -> SqlPersistM (UUID, RepoSize)
+ go sizechanges livechanges (u, RepoSize startsize) = do
+ let livechangesbykey =
+ M.fromListWith (++) $ maybe [] (\v -> [v]) $
+ M.lookup u livechanges
+ livechanges' <- combinelikelivechanges <$>
+ filterM (nonredundantlivechange livechangesbykey u)
+ (fromMaybe [] $ M.lookup u livechanges)
+ let sizechange = foldl'
+ (\t (k, sc) -> updateRollingTotal t sc k)
+ (fromMaybe 0 (M.lookup u sizechanges))
+ livechanges'
+ return (u, RepoSize (startsize + sizechange))
+
+ combinelikelivechanges =
+ S.elems
+ . S.fromList
+ . map (\(k, (sc, _)) -> (k, sc))
+
+ nonredundantlivechange
+ :: M.Map Key [(SizeChange, SizeChangeId)]
+ -> UUID
+ -> (Key, (SizeChange, SizeChangeId))
+ -> SqlPersistM Bool
+ nonredundantlivechange livechangesbykey u (k, (sc, cid))
+ | null (competinglivechanges livechangesbykey k sc cid) =
+ getRecentChange u k >>= pure . \case
+ Nothing -> True
+ Just sc' -> sc /= sc'
+ | otherwise = pure False
+
+ competinglivechanges
+ :: M.Map Key [(SizeChange, SizeChangeId)]
+ -> Key
+ -> SizeChange
+ -> SizeChangeId
+ -> [(SizeChange, SizeChangeId)]
+ competinglivechanges livechangesbykey k RemovingKey cid =
+ filter (\(sc', cid') -> cid /= cid' && sc' == AddingKey)
+ (fromMaybe [] $ M.lookup k livechangesbykey)
+ competinglivechanges _ _ AddingKey _ = []
+getLiveRepoSizes (RepoSizeHandle Nothing) = return mempty
When updating location log for a key, when there is actually a change,
update the db, remove the live update (done) and update the sizechanges
- table in the same transaction.
+ table in the same transaction (done).
Two concurrent processes might both start the same action, eg dropping
a key, and both succeed, and so both update the location log. One needs
* Still implementing LiveUpdate. Check for TODO XXX markers
+* Concurrency issue noted in commit db89e39df606b6ec292e0f1c3a7a60e317ac60f1
+
+ But: There will be a window where the redundant LiveUpdate is still
+ visible in the db, and processes can see it, combine it with the
+ rollingtotal, and arrive at the wrong size. This is a small window, but
+ it still ought to be addressed. Unsure if it would always be safe to
+ remove the redundant LiveUpdate? Consider the case where two drops and a
+ get are all running concurrently somehow, and the order they finish is
+ [drop, get, drop]. The second drop seems redundant to the first, but
+ it would not be safe to remove it. While this seems unlikely, it's hard
+ to rule out that a get and drop at different stages can both be running
+ at the same time.
+
+ It also is possible for a redundant LiveUpdate to get added to the db
+ just after the rollingtotal was updated. In this case, combining the LiveUpdate
+ with the rollingtotal again yields the wrong reposize.
+
+ So is the rollingtotal doomed to not be accurate?
+
+ A separate table could be kept of recent updates. When combining a LiveUpdate
+ with the rollingtotal to get a reposize, first check if the LiveUpdate is
+ redundant given a recent update. When updating the RepoSizes table, clear the
+ recent updates table and the rolling totals table (in the same transaction).
+ This recent updates table could get fairly large, but only needs to be queried
+ for each current LiveUpdate, of which there are not ususally many running.
+
+ When does a recent update mean a LiveUpdate is redundant? In the case of two drops,
+ the second is clearly redundant. But what about two gets and a drop? In this
+ case, after the first get, we don't know what order operations will
+ happen in. So the fact that the first get is in the recent updates table
+ should not make the second get be treated as redundant.
+
+ So, look up each LiveUpdate in the recent updates table. When the same
+ operation is found there, look to see if there is any other LiveUpdate of
+ the same key and uuid, but with a different SizeChange. Only when there is
+ not is the LiveUpdate redundant.
+
+ What if the recent updates table contains a get and a drop of the same
+ key. Now a get is running. Is it redundant? Perhaps the recent updates
+ table needs timestamps. More simply, when adding a drop to the recent
+ updates table, any existing get of the same key should be removed.
+
* In the case where a copy to a remote fails (due eg to annex.diskreserve),
the LiveUpdate thread can not get a chance to catch its exception when
the LiveUpdate is gced, before git-annex exits. In this case, the
I'd think, but I tried manually doing a performGC at git-annex shutdown
and it didn't help.
+ getLiveRepoSizes is an unfinished try at implementing the above.
+
+* Something needs to empty SizeChanges and RecentChanges when
+ setRepoSizes is called. While avoiding races.
+
* The assistant is using NoLiveUpdate, but it should be posssible to plumb
a LiveUpdate through it from preferred content checking to location log
updating.
overLocationLogs. In the other path it does not, and this should be fixed
for consistency and correctness.
+* getLiveRepoSizes has a filterM getRecentChange over the live updates.
+ This could be optimised to a single sql join. There are usually not many
+ live updates, but sometimes there will be a great many recent changes,
+ so it might be worth doing this optimisation.
+
## completed items for August's work on balanced preferred content
* Balanced preferred content basic implementation, including --rebalance
projects / git-annex.git / commitdiff