From 69dd3dcc899366239f7ba68ab80680ecc116900f Mon Sep 17 00:00:00 2001 From: edef Date: Fri, 2 Dec 2022 12:57:48 +0000 Subject: [PATCH] submit BLAKE3 patch --- doc/bugs/blake3_hash_support.mdwn | 164 ++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 doc/bugs/blake3_hash_support.mdwn diff --git a/doc/bugs/blake3_hash_support.mdwn b/doc/bugs/blake3_hash_support.mdwn new file mode 100644 index 0000000000..41c40f8c4d --- /dev/null +++ b/doc/bugs/blake3_hash_support.mdwn @@ -0,0 +1,164 @@ +This is a patch that seems to work for my personal use. +BLAKE3 does support variable lengths, but my code does not implement support for anything other than 256-bit (32-byte) digests. +I'm familiar enough with the codebase to be sure whether adding variable length support later is a backwards compatibility hazard or not. + +[[!format patch """ +From efa115d94d1a5a52574d5760c6e951ed3c518667 Mon Sep 17 00:00:00 2001 +From: edef +Date: Fri, 2 Dec 2022 12:16:44 +0000 +Subject: [PATCH] support BLAKE3 + +This uses the blake3 package from Hackage, since cryptonite does not +have BLAKE3 support yet. + +diff --git a/Backend/Hash.hs b/Backend/Hash.hs +index 550d8fc6c..809a82599 100644 +--- a/Backend/Hash.hs ++++ b/Backend/Hash.hs +@@ -27,8 +27,11 @@ import qualified Data.ByteString as S + import qualified Data.ByteString.Short as S (toShort, fromShort) + import qualified Data.ByteString.Char8 as S8 + import qualified Data.ByteString.Lazy as L ++import Data.IORef ++import Control.Arrow + import Control.DeepSeq + import Control.Exception (evaluate) ++import qualified BLAKE3 + + data Hash + = MD5Hash +@@ -40,6 +43,7 @@ data Hash + | Blake2bpHash HashSize + | Blake2sHash HashSize + | Blake2spHash HashSize ++ | Blake3Hash + + cryptographicallySecure :: Hash -> Bool + cryptographicallySecure (SHA2Hash _) = True +@@ -49,6 +53,7 @@ cryptographicallySecure (Blake2bHash _) = True + cryptographicallySecure (Blake2bpHash _) = True + cryptographicallySecure (Blake2sHash _) = True + cryptographicallySecure (Blake2spHash _) = True ++cryptographicallySecure Blake3Hash = True + cryptographicallySecure SHA1Hash = False + cryptographicallySecure MD5Hash = False + +@@ -63,6 +68,7 @@ hashes = concat + , map (Blake2bpHash . HashSize) [512] + , map (Blake2sHash . HashSize) [256, 160, 224] + , map (Blake2spHash . HashSize) [256, 224] ++ , [Blake3Hash] + , [SHA1Hash] + , [MD5Hash] + ] +@@ -99,6 +105,7 @@ hashKeyVariety (Blake2bHash size) he = Blake2bKey size he + hashKeyVariety (Blake2bpHash size) he = Blake2bpKey size he + hashKeyVariety (Blake2sHash size) he = Blake2sKey size he + hashKeyVariety (Blake2spHash size) he = Blake2spKey size he ++hashKeyVariety Blake3Hash he = Blake3Key he + + {- A key is a hash of its contents. -} + keyValue :: Hash -> KeySource -> MeterUpdate -> Annex Key +@@ -219,6 +226,7 @@ hasher (Blake2bHash hashsize) = blake2bHasher hashsize + hasher (Blake2bpHash hashsize) = blake2bpHasher hashsize + hasher (Blake2sHash hashsize) = blake2sHasher hashsize + hasher (Blake2spHash hashsize) = blake2spHasher hashsize ++hasher Blake3Hash = blake3Hasher + + mkHasher :: HashAlgorithm h => (L.ByteString -> Digest h) -> Context h -> Hasher + mkHasher h c = (show . h, mkIncrementalVerifier c descChecksum . sameCheckSum) +@@ -272,6 +280,27 @@ blake2spHasher (HashSize hashsize) + | hashsize == 224 = mkHasher blake2sp_224 blake2sp_224_context + | otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize + ++blake3Hasher :: Hasher ++blake3Hasher = (hash, incremental) where ++ finalize :: BLAKE3.Hasher -> BLAKE3.Digest BLAKE3.DEFAULT_DIGEST_LEN ++ finalize = BLAKE3.finalize ++ ++ hash :: L.ByteString -> String ++ hash = show . finalize . L.foldlChunks ((. pure) . BLAKE3.update) BLAKE3.hasher ++ ++ incremental :: Key -> IO IncrementalVerifier ++ incremental k = do ++ v <- newIORef (Just (BLAKE3.hasher, 0)) ++ return $ IncrementalVerifier ++ { updateIncrementalVerifier = \b -> ++ modifyIORef' v . fmap $ flip BLAKE3.update [b] *** (fromIntegral (S.length b) +) ++ , finalizeIncrementalVerifier = ++ fmap (sameCheckSum k . show . finalize . fst) <$> readIORef v ++ , unableIncrementalVerifier = writeIORef v Nothing ++ , positionIncrementalVerifier = fmap snd <$> readIORef v ++ , descIncrementalVerifier = descChecksum ++ } ++ + sha1Hasher :: Hasher + sha1Hasher = mkHasher sha1 sha1_context + +diff --git a/Types/Key.hs b/Types/Key.hs +index 271723982..ea71f85ed 100644 +--- a/Types/Key.hs ++++ b/Types/Key.hs +@@ -214,6 +214,7 @@ data KeyVariety + | Blake2bpKey HashSize HasExt + | Blake2sKey HashSize HasExt + | Blake2spKey HashSize HasExt ++ | Blake3Key HasExt + | SHA1Key HasExt + | MD5Key HasExt + | WORMKey +@@ -247,6 +248,7 @@ hasExt (Blake2bKey _ (HasExt b)) = b + hasExt (Blake2bpKey _ (HasExt b)) = b + hasExt (Blake2sKey _ (HasExt b)) = b + hasExt (Blake2spKey _ (HasExt b)) = b ++hasExt (Blake3Key (HasExt b)) = b + hasExt (SHA1Key (HasExt b)) = b + hasExt (MD5Key (HasExt b)) = b + hasExt WORMKey = False +@@ -262,6 +264,7 @@ sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2 + sameExceptExt (Blake2bpKey sz1 _) (Blake2bpKey sz2 _) = sz1 == sz2 + sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2 + sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2 ++sameExceptExt (Blake3Key _) (Blake3Key _) = True + sameExceptExt (SHA1Key _) (SHA1Key _) = True + sameExceptExt (MD5Key _) (MD5Key _) = True + sameExceptExt _ _ = False +@@ -275,6 +278,7 @@ formatKeyVariety v = case v of + Blake2bpKey sz e -> adde e (addsz sz "BLAKE2BP") + Blake2sKey sz e -> adde e (addsz sz "BLAKE2S") + Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP") ++ Blake3Key e -> adde e "BLAKE3" + SHA1Key e -> adde e "SHA1" + MD5Key e -> adde e "MD5" + WORMKey -> "WORM" +@@ -337,6 +341,8 @@ parseKeyVariety "BLAKE2SP224" = Blake2spKey (HashSize 224) (HasExt False) + parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True) + parseKeyVariety "BLAKE2SP256" = Blake2spKey (HashSize 256) (HasExt False) + parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True) ++parseKeyVariety "BLAKE3" = Blake3Key (HasExt False) ++parseKeyVariety "BLAKE3E" = Blake3Key (HasExt True) + parseKeyVariety "SHA1" = SHA1Key (HasExt False) + parseKeyVariety "SHA1E" = SHA1Key (HasExt True) + parseKeyVariety "MD5" = MD5Key (HasExt False) +diff --git a/git-annex.cabal b/git-annex.cabal +index cd58a4ca3..7c251e33b 100644 +--- a/git-annex.cabal ++++ b/git-annex.cabal +@@ -362,6 +362,7 @@ Executable git-annex + securemem, + crypto-api, + cryptonite (>= 0.23), ++ blake3, + memory, + deepseq, + split, +diff --git a/stack.yaml b/stack.yaml +index 7dbfb657a..936ee841b 100644 +--- a/stack.yaml ++++ b/stack.yaml +@@ -25,3 +25,4 @@ extra-deps: + - base64-bytestring-1.0.0.3 + - bencode-0.6.1.1 + - http-client-0.7.9 ++- blake3-0.2@sha256:d1146b9a51ccfbb0532780778b6d016a614e3d44c05d8c1923dde9a8be869045,2448 +"""]] -- 2.30.2