git-annex log --received modifier option

Only counting received and not dropped makes this show the bandwidth of
data coming into the repository, although only in a sense. Since
git-annex branch updates only happen at the end of a command, and we
don't know when a command started, it's only an approximation of the
actual bandwidth. (A previous git-annex branch update made have
happened in a different repository.)

It would be possible to also add a --dropped option, but I don't know
how useful that would be?

Sponsored-by: Nicholas Golder-Manning on Patreon

diff --git a/CHANGELOG b/CHANGELOG
index df257afc458b37c07683543d7b2ab47ee0282f53..b322f8f6bee2eff46de19eb7f1ac967a91781294 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -15,6 +15,8 @@ git-annex (10.20230927) UNRELEASED; urgency=medium
   * info: Added calculation of combined annex size of all repositories.
   * log: Added options --sizesof, --sizes and --totalsizes that
     display how the size of repositories changed over time.
+  * log: Added options --interval, --bytes, --received 
+    to tune the output of the above added options.
 
  -- Joey Hess <id@joeyh.name>  Tue, 10 Oct 2023 13:17:31 -0400
 

diff --git a/Command/Log.hs b/Command/Log.hs
index 3eba85040dce88c2c787d8c557bc36111452c5ad..f02d213644b79e6c201afa5d1de7898904659f33 100644 (file)
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -50,7 +50,8 @@ data LogOptions = LogOptions
        , sizesOfOption :: Maybe (DeferredParse UUID)
        , sizesOption :: Bool
        , totalSizesOption :: Bool
-       , whenOption :: Maybe Duration
+       , intervalOption :: Maybe Duration
+       , receivedOption :: Bool
        , rawDateOption :: Bool
        , bytesOption :: Bool
        , gourceOption :: Bool
@@ -83,6 +84,10 @@ optParser desc = LogOptions
                ( long "interval" <> metavar paramTime
                <> help "minimum time between displays of changed size"
                ))
+       <*> switch
+               ( long "received"
+               <> help "display received data per interval rather than repository sizes"
+               )
        <*> switch
                ( long "raw-date"
                <> help "display seconds from unix epoch"
@@ -295,7 +300,7 @@ sizeHistoryInfo mu o = do
        uuidmap <- getuuidmap
        zone <- liftIO getCurrentTimeZone
        liftIO $ displayheader uuidmap
-       let dispst = (zone, False, epoch, Nothing)
+       let dispst = (zone, False, epoch, Nothing, mempty)
        (l, cleanup) <- getlog
        g <- Annex.gitRepo
        liftIO $ catObjectStream g $ \feeder closer reader -> do
@@ -379,7 +384,9 @@ sizeHistoryInfo mu o = do
                combinedlog = PLog.compactLog (oldlog ++ newlog)
                combinedlocs = S.fromList (presentlocs combinedlog)
                addedlocs = S.difference combinedlocs oldlocs
-               removedlocs = S.difference oldlocs combinedlocs
+               removedlocs
+                       | receivedOption o = S.empty
+                       | otherwise = S.difference oldlocs combinedlocs
        
        addnew k sizemap locmap newlog = 
                ( updatesize sizemap (ksz k) locs
@@ -402,23 +409,36 @@ sizeHistoryInfo mu o = do
                                (M.elems uuidmap)
                | otherwise = return ()
 
-       displaysizes (zone, displayedyet, prevt, prevoutput) trustlog uuidmap sizemap t
-               | t - prevt >= dt
-                 && (displayedyet || any (/= 0) sizes) 
-                 && (prevoutput /= Just output) = do
+       displaysizes (zone, displayedyet, prevt, prevoutput, prevsizemap) trustlog uuidmap sizemap t
+               | t - prevt >= dt && changedoutput = do
                        displayts zone t output
-                       return (zone, True, t, Just output)
-               | t < prevt = return (zone, displayedyet, t, Just output)
-               | otherwise = return (zone, displayedyet, prevt, Just output)
+                       return (zone, True, t, Just output, sizemap')
+               | t < prevt = return (zone, displayedyet, t, Just output, prevsizemap)
+               | otherwise = return (zone, displayedyet, prevt, prevoutput, prevsizemap)
          where
                output = intercalate "," (map showsize sizes)
                us = case mu of
                        Just u -> [u]
                        Nothing -> M.keys uuidmap
                sizes
-                       | totalSizesOption o = [sum (M.elems sizemap')]
-                       | otherwise = map (\u -> fromMaybe 0 (M.lookup u sizemap')) us
-               dt = maybe 1 durationToPOSIXTime (whenOption o)
+                       | totalSizesOption o = [sum (M.elems sizedisplaymap)]
+                       | otherwise = map (\u -> fromMaybe 0 (M.lookup u sizedisplaymap)) us
+               dt = maybe 1 durationToPOSIXTime (intervalOption o)
+
+               changedoutput
+                       | receivedOption o = 
+                               any (/= 0) sizes 
+                                       || prevoutput /= Just output
+                       | otherwise = 
+                               (displayedyet || any (/= 0) sizes)
+                                       && (prevoutput /= Just output)
+
+               sizedisplaymap
+                       | receivedOption o = 
+                               M.unionWith posminus sizemap' prevsizemap
+                       | otherwise = sizemap'
+
+               posminus a b = max 0 (a - b)
 
                -- A verison of sizemap where uuids that are currently dead
                -- have 0 size.
@@ -433,7 +453,7 @@ sizeHistoryInfo mu o = do
                        then rawTimeStamp t
                        else showTimeStamp zone "%Y-%m-%dT%H:%M:%S" t
 
-       displayendsizes (zone , _, _, Just output) = do
+       displayendsizes (zone , _, _, Just output, _) = do
                now <- getPOSIXTime
                displayts zone now output
        displayendsizes _ = return ()

diff --git a/doc/git-annex-log.mdwn b/doc/git-annex-log.mdwn
index 65636e5c9c26ea5483eaac7ae58c7c32a8227410..9e24fac86f18d102fb6efcbdd8c150878ebd23d2 100644 (file)
--- a/doc/git-annex-log.mdwn
+++ b/doc/git-annex-log.mdwn
@@ -41,7 +41,7 @@ false, information may not have been committed to the branch yet.
 * `--sizesof=repository`
 
   Displays a history of the total size of the annexed files in a repository
-  as it changed over time from the creation of the repository to the present.
+  over time from the creation of the repository to the present.
 
   The repository can be "here" for the current repository, or the name of a
   remote, or a repository description or uuid.
@@ -65,19 +65,15 @@ false, information may not have been committed to the branch yet.
 
   When using `--sizesof`, `--sizes`, and `--totalsizes`, this 
   controls the minimum interval between displays of the size.
-  The default is to display each recorded change to the size.
+  The default is to display each new recorded size.
 
   The time is of the form "30d" or "1y".
 
-* `--since=date`, `--after=date`, `--until=date`, `--before=date`, `--max-count=N`
+* `--received`
 
-  These options are passed through to `git log`, and can be used to limit
-  how far back to search for location log changes.
-  
-  For example: `--since "1 month ago"`
-
-  These options do not have an affect when using `--sizesof`, `--sizes`, 
-  and `--totalsizes`.
+  Combine this option with `--sizesof` or `--sizes` to display
+  the amount of data received into repositories since the last
+  line was output.
 
 * `--bytes`
 
@@ -88,6 +84,16 @@ false, information may not have been committed to the branch yet.
   Rather than the normal display of a date in the local time zone,
   displays seconds since the unix epoch.
 
+* `--since=date`, `--after=date`, `--until=date`, `--before=date`, `--max-count=N`
+
+  These options are passed through to `git log`, and can be used to limit
+  how far back to search for location log changes.
+  
+  For example: `--since "1 month ago"`
+
+  These options do not have an affect when using `--sizesof`, `--sizes`, 
+  and `--totalsizes`.
+
 * `--gource`
 
   Generates output suitable for the `gource` visualization program.