1 /** 2 Computes MurmurHash hashes of arbitrary data. MurmurHash is a non-cryptographic 3 hash function suitable for general hash-based lookup. 4 This module conforms to the APIs defined in $(D std.digest.digest). 5 This module publicly imports $(D std.digest.digest) and can be used as a stand-alone module. 6 Note: The current implementation is optimized for little endian architectures. 7 It will exhibit different results on big endian architectures and a slightly less uniform distribution. 8 License: $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 9 Authors: Guillaume Chatelet 10 References: $(LINK2 https://code.google.com/p/smhasher/wiki/MurmurHash3, Reference implementation) 11 $(BR) $(LINK2 https://en.wikipedia.org/wiki/MurmurHash, Wikipedia on MurmurHash) 12 */ 13 /* Copyright Guillaume Chatelet 2016. 14 * Distributed under the Boost Software License, Version 1.0. 15 * (See LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 16 */ 17 //module std.digest.murmurhash; 18 module lincount.murmurhash; 19 20 static if (__VERSION__ < 2072) { 21 22 public import std.digest.digest; 23 24 @safe: 25 26 /// 27 unittest 28 { 29 // MurmurHash3_x86_32, MurmurHash3_x86_128 and MurmurHash3_x64_128 implement 30 // std.digest.digest Template API. 31 static assert(isDigest!MurmurHash3_x86_32); 32 // The convenient digest template allows for quick hashing of data. 33 auto hashed = digest!MurmurHash3_x86_32([1, 2, 3, 4]); 34 } 35 36 /// 37 unittest 38 { 39 // One can also hash ubyte data piecewise. 40 const(ubyte)[] data1 = [1, 2, 3]; 41 const(ubyte)[] data2 = [4, 5, 6, 7]; 42 MurmurHash3_x86_32 hasher; 43 hasher.put(data1); 44 hasher.put(data2); 45 auto hashed = hasher.finish(); 46 } 47 48 /// 49 unittest 50 { 51 // Using SMurmurHash3_x86_32, SMurmurHash3_x86_128 and SMurmurHash3_x64_128 52 // you gain full control over which part of the algorithm to run. 53 // This allows for maximum throughput but needs extra care. 54 55 // Data type must be the same as the hasher's element type: 56 // - uint for SMurmurHash3_x86_32 57 // - ulong[2] for SMurmurHash3_x86_128 and SMurmurHash3_x64_128 58 const(uint)[] data = [1, 2, 3, 4]; 59 // Note the hasher starts with S. 60 SMurmurHash3_x86_32 hasher; 61 // Push as many array of elements as you need. The less call the better performance wise. 62 hasher.putBlocks(data); 63 // Put remainder bytes if needed. This method can be called only once. 64 hasher.putRemainder(ubyte(1), ubyte(1), ubyte(1)); 65 // Call finalize to incorporate data length in the hash. 66 hasher.finalize(); 67 // Finally get the hashed value. 68 auto hashed = hasher.getBytes(); 69 } 70 71 /// Implements MurmurHash3_x86_32 $(D std.digest.digest) Template API. 72 alias MurmurHash3_x86_32 = Piecewise!SMurmurHash3_x86_32; 73 /// Implements MurmurHash3_x86_128 $(D std.digest.digest) Template API. 74 alias MurmurHash3_x86_128 = Piecewise!SMurmurHash3_x86_128; 75 /// Implements MurmurHash3_x64_128 $(D std.digest.digest) Template API. 76 alias MurmurHash3_x64_128 = Piecewise!SMurmurHash3_x64_128; 77 78 /// Implements MurmurHash3_x86_32 $(D std.digest.digest.Digest) OOO API. 79 alias MurmurHash3_x86_32Digest = WrapperDigest!MurmurHash3_x86_32; 80 /// Implements MurmurHash3_x86_128 $(D std.digest.digest.Digest) OOO API. 81 alias MurmurHash3_x86_128Digest = WrapperDigest!MurmurHash3_x86_128; 82 /// Implements MurmurHash3_x64_128 $(D std.digest.digest.Digest) OOO API. 83 alias MurmurHash3_x64_128Digest = WrapperDigest!MurmurHash3_x64_128; 84 85 // This definition of NO_UNALIGNED_ACCESS is too restrictive. Only a few old 86 // SPARC/ARM chips cannot do unaligned reads. 87 version(ARM) { version = NO_UNALIGNED_ACCESS; } 88 version(SPARC) { version = NO_UNALIGNED_ACCESS; } 89 90 /** 91 Pushes an array of blocks at once. It is more efficient to push as much data as 92 possible in a single call. 93 On platform that does not support unaligned reads (some old ARM chips), it is 94 forbidden to pass non aligned data. 95 */ 96 void putBlocks(H, Block = H.Block)(ref H hasher, scope const(Block[]) blocks...) pure nothrow @nogc 97 in 98 { 99 version(NO_UNALIGNED_ACCESS) assert(blocks.ptr % Block.alignof == 0); 100 } 101 body 102 { 103 foreach (const block; blocks) 104 { 105 hasher.putBlock(block); 106 } 107 hasher.size += blocks.length * Block.sizeof; 108 } 109 110 /** 111 Returns the current hashed value as an ubyte array. 112 */ 113 auto getBytes(H)(ref H hash) pure nothrow @nogc 114 { 115 static if (is(H.Block == uint)) 116 { 117 return cast(ubyte[H.Block.sizeof]) cast(uint[1])[hash.get()]; 118 } 119 else 120 { 121 return cast(ubyte[H.Block.sizeof]) hash.get(); 122 } 123 } 124 125 /** 126 MurmurHash3 for x86 processors producing a 32 bits value. 127 This is a lower level implementation that makes finalization optional and have slightly better performance. 128 Note that $(D putRemainder) can be called only once and that no subsequent calls to $(D putBlocks) is allowed. 129 */ 130 struct SMurmurHash3_x86_32 131 { 132 private: 133 enum uint c1 = 0xcc9e2d51; 134 enum uint c2 = 0x1b873593; 135 uint h1; 136 137 public: 138 alias Block = uint; /// The element size for x86_32 implementation. 139 size_t size; 140 141 this(uint seed) 142 { 143 h1 = seed; 144 } 145 146 @disable this(this); 147 148 /// Adds a single Block of data without increasing size. 149 /// Make sure to increase size by Block.sizeof for each call to putBlock. 150 void putBlock(uint block) pure nothrow @nogc 151 { 152 update(h1, block, 0, c1, c2, 15, 13, 0xe6546b64); 153 } 154 155 /// Put remainder bytes. This must be called only once after putBlock and before finalize. 156 void putRemainder(scope const(ubyte[]) data...) pure nothrow @nogc 157 { 158 assert(data.length < Block.sizeof); 159 assert(data.length >= 0); 160 size += data.length; 161 uint k1 = 0; 162 final switch (data.length & 3) 163 { 164 case 3: 165 k1 ^= data[2] << 16; 166 goto case; 167 case 2: 168 k1 ^= data[1] << 8; 169 goto case; 170 case 1: 171 k1 ^= data[0]; 172 h1 ^= shuffle(k1, c1, c2, 15); 173 goto case; 174 case 0: 175 } 176 } 177 178 /// Incorporate size and finalizes the hash. 179 void finalize() pure nothrow @nogc 180 { 181 h1 ^= size; 182 h1 = fmix(h1); 183 } 184 185 /// Returns the hash as an uint value. 186 Block get() pure nothrow @nogc 187 { 188 return h1; 189 } 190 } 191 192 version (unittest) 193 { 194 import std.string : representation; 195 196 auto hash(H, Block = H.Block)(string data) 197 { 198 H hasher; 199 immutable blocks = data.length / Block.sizeof; 200 hasher.putBlocks(cast(const(Block)[]) data[0 .. blocks * Block.sizeof]); 201 hasher.putRemainder(cast(const(ubyte)[]) data[blocks * Block.sizeof .. $]); 202 hasher.finalize(); 203 return hasher.getBytes(); 204 } 205 206 void checkResult(H)(in string[string] groundtruth) 207 { 208 foreach (data, expectedHash; groundtruth) 209 { 210 alias PiecewiseH = Piecewise!H; 211 assert(data.digest!PiecewiseH.toHexString() == expectedHash); 212 assert(data.hash!H.toHexString() == expectedHash); 213 PiecewiseH hasher; 214 foreach (element; data) 215 { 216 hasher.put(element); 217 } 218 assert(hasher.finish.toHexString() == expectedHash); 219 } 220 } 221 } 222 223 unittest 224 { 225 checkResult!SMurmurHash3_x86_32([ 226 "" : "00000000", 227 "a" : "B269253C", 228 "ab" : "5FD7BF9B", 229 "abc" : "FA93DDB3", 230 "abcd" : "6A67ED43", 231 "abcde" : "F69A9BE8", 232 "abcdef" : "85C08161", 233 "abcdefg" : "069B3C88", 234 "abcdefgh" : "C4CCDD49", 235 "abcdefghi" : "F0061442", 236 "abcdefghij" : "91779288", 237 "abcdefghijk" : "DF253B5F", 238 "abcdefghijkl" : "273D6FA3", 239 "abcdefghijklm" : "1B1612F2", 240 "abcdefghijklmn" : "F06D52F8", 241 "abcdefghijklmno" : "D2F7099D", 242 "abcdefghijklmnop" : "ED9162E7", 243 "abcdefghijklmnopq" : "4A5E65B6", 244 "abcdefghijklmnopqr" : "94A819C2", 245 "abcdefghijklmnopqrs" : "C15BBF85", 246 "abcdefghijklmnopqrst" : "9A711CBE", 247 "abcdefghijklmnopqrstu" : "ABE7195A", 248 "abcdefghijklmnopqrstuv" : "C73CB670", 249 "abcdefghijklmnopqrstuvw" : "1C4D1EA5", 250 "abcdefghijklmnopqrstuvwx" : "3939F9B0", 251 "abcdefghijklmnopqrstuvwxy" : "1A568338", 252 "abcdefghijklmnopqrstuvwxyz" : "6D034EA3"]); 253 } 254 255 /** 256 MurmurHash3 for x86 processors producing a 128 bits value. 257 This is a lower level implementation that makes finalization optional and have slightly better performance. 258 Note that $(D putRemainder) can be called only once and that no subsequent calls to $(D putBlocks) is allowed. 259 */ 260 struct SMurmurHash3_x86_128 261 { 262 private: 263 enum uint c1 = 0x239b961b; 264 enum uint c2 = 0xab0e9789; 265 enum uint c3 = 0x38b34ae5; 266 enum uint c4 = 0xa1e38b93; 267 uint h4, h3, h2, h1; 268 269 public: 270 alias Block = uint[4]; /// The element size for x86_128 implementation. 271 size_t size; 272 273 this(uint seed4, uint seed3, uint seed2, uint seed1) 274 { 275 h4 = seed4; 276 h3 = seed3; 277 h2 = seed2; 278 h1 = seed1; 279 } 280 281 this(uint seed) 282 { 283 h4 = h3 = h2 = h1 = seed; 284 } 285 286 @disable this(this); 287 288 /// Adds a single Block of data without increasing size. 289 /// Make sure to increase size by Block.sizeof for each call to putBlock. 290 void putBlock(Block block) pure nothrow @nogc 291 { 292 update(h1, block[0], h2, c1, c2, 15, 19, 0x561ccd1b); 293 update(h2, block[1], h3, c2, c3, 16, 17, 0x0bcaa747); 294 update(h3, block[2], h4, c3, c4, 17, 15, 0x96cd1c35); 295 update(h4, block[3], h1, c4, c1, 18, 13, 0x32ac3b17); 296 } 297 298 /// Put remainder bytes. This must be called only once after putBlock and before finalize. 299 void putRemainder(scope const(ubyte[]) data...) pure nothrow @nogc 300 { 301 assert(data.length < Block.sizeof); 302 assert(data.length >= 0); 303 size += data.length; 304 uint k1 = 0; 305 uint k2 = 0; 306 uint k3 = 0; 307 uint k4 = 0; 308 309 final switch (data.length & 15) 310 { 311 case 15: 312 k4 ^= data[14] << 16; 313 goto case; 314 case 14: 315 k4 ^= data[13] << 8; 316 goto case; 317 case 13: 318 k4 ^= data[12] << 0; 319 h4 ^= shuffle(k4, c4, c1, 18); 320 goto case; 321 case 12: 322 k3 ^= data[11] << 24; 323 goto case; 324 case 11: 325 k3 ^= data[10] << 16; 326 goto case; 327 case 10: 328 k3 ^= data[9] << 8; 329 goto case; 330 case 9: 331 k3 ^= data[8] << 0; 332 h3 ^= shuffle(k3, c3, c4, 17); 333 goto case; 334 case 8: 335 k2 ^= data[7] << 24; 336 goto case; 337 case 7: 338 k2 ^= data[6] << 16; 339 goto case; 340 case 6: 341 k2 ^= data[5] << 8; 342 goto case; 343 case 5: 344 k2 ^= data[4] << 0; 345 h2 ^= shuffle(k2, c2, c3, 16); 346 goto case; 347 case 4: 348 k1 ^= data[3] << 24; 349 goto case; 350 case 3: 351 k1 ^= data[2] << 16; 352 goto case; 353 case 2: 354 k1 ^= data[1] << 8; 355 goto case; 356 case 1: 357 k1 ^= data[0] << 0; 358 h1 ^= shuffle(k1, c1, c2, 15); 359 goto case; 360 case 0: 361 } 362 } 363 364 /// Incorporate size and finalizes the hash. 365 void finalize() pure nothrow @nogc 366 { 367 h1 ^= size; 368 h2 ^= size; 369 h3 ^= size; 370 h4 ^= size; 371 372 h1 += h2; 373 h1 += h3; 374 h1 += h4; 375 h2 += h1; 376 h3 += h1; 377 h4 += h1; 378 379 h1 = fmix(h1); 380 h2 = fmix(h2); 381 h3 = fmix(h3); 382 h4 = fmix(h4); 383 384 h1 += h2; 385 h1 += h3; 386 h1 += h4; 387 h2 += h1; 388 h3 += h1; 389 h4 += h1; 390 } 391 392 /// Returns the hash as an uint[4] value. 393 Block get() pure nothrow @nogc 394 { 395 return [h1, h2, h3, h4]; 396 } 397 } 398 399 unittest 400 { 401 checkResult!SMurmurHash3_x86_128([ 402 "" : "00000000000000000000000000000000", 403 "a" : "3C9394A71BB056551BB056551BB05655", 404 "ab" : "DF5184151030BE251030BE251030BE25", 405 "abc" : "D1C6CD75A506B0A2A506B0A2A506B0A2", 406 "abcd" : "AACCB6962EC6AF452EC6AF452EC6AF45", 407 "abcde" : "FB2E40C5BCC5245D7701725A7701725A", 408 "abcdef" : "0AB97CE12127AFA1F9DFBEA9F9DFBEA9", 409 "abcdefg" : "D941B590DE3A86092869774A2869774A", 410 "abcdefgh" : "3611F4AE8714B1AD92806CFA92806CFA", 411 "abcdefghi" : "1C8C05AD6F590622107DD2147C4194DD", 412 "abcdefghij" : "A72ED9F50E90379A2AAA92C77FF12F69", 413 "abcdefghijk" : "DDC9C8A01E111FCA2DF1FE8257975EBD", 414 "abcdefghijkl" : "FE038573C02482F4ADDFD42753E58CD2", 415 "abcdefghijklm" : "15A23AC1ECA1AEDB66351CF470DE2CD9", 416 "abcdefghijklmn" : "8E11EC75D71F5D60F4456F944D89D4F1", 417 "abcdefghijklmno" : "691D6DEEAED51A4A5714CE84A861A7AD", 418 "abcdefghijklmnop" : "2776D29F5612B990218BCEE445BA93D1", 419 "abcdefghijklmnopq" : "D3A445046F5C51642ADC6DD99D07111D", 420 "abcdefghijklmnopqr" : "AA5493A0DA291D966A9E7128585841D9", 421 "abcdefghijklmnopqrs" : "281B6A4F9C45B9BFC3B77850930F2C20", 422 "abcdefghijklmnopqrst" : "19342546A8216DB62873B49E545DCB1F", 423 "abcdefghijklmnopqrstu" : "A6C0F30D6C738620E7B9590D2E088D99", 424 "abcdefghijklmnopqrstuv" : "A7D421D9095CDCEA393CBBA908342384", 425 "abcdefghijklmnopqrstuvw" : "C3A93D572B014949317BAD7EE809158F", 426 "abcdefghijklmnopqrstuvwx" : "802381D77956833791F87149326E4801", 427 "abcdefghijklmnopqrstuvwxy" : "0AC619A5302315755A80D74ADEFAA842", 428 "abcdefghijklmnopqrstuvwxyz" : "1306343E662F6F666E56F6172C3DE344"]); 429 } 430 431 /** 432 MurmurHash3 for x86_64 processors producing a 128 bits value. 433 This is a lower level implementation that makes finalization optional and have slightly better performance. 434 Note that $(D putRemainder) can be called only once and that no subsequent calls to $(D putBlocks) is allowed. 435 */ 436 struct SMurmurHash3_x64_128 437 { 438 private: 439 enum ulong c1 = 0x87c37b91114253d5; 440 enum ulong c2 = 0x4cf5ad432745937f; 441 ulong h2, h1; 442 443 public: 444 alias Block = ulong[2]; /// The element size for x64_128 implementation. 445 size_t size; 446 447 this(ulong seed) 448 { 449 h2 = h1 = seed; 450 } 451 452 this(ulong seed2, ulong seed1) 453 { 454 h2 = seed2; 455 h1 = seed1; 456 } 457 458 @disable this(this); 459 460 /// Adds a single Block of data without increasing size. 461 /// Make sure to increase size by Block.sizeof for each call to putBlock. 462 void putBlock(Block block) pure nothrow @nogc 463 { 464 update(h1, block[0], h2, c1, c2, 31, 27, 0x52dce729); 465 update(h2, block[1], h1, c2, c1, 33, 31, 0x38495ab5); 466 } 467 468 /// Put remainder bytes. This must be called only once after putBlock and before finalize. 469 void putRemainder(scope const(ubyte[]) data...) pure nothrow @nogc 470 { 471 assert(data.length < Block.sizeof); 472 assert(data.length >= 0); 473 size += data.length; 474 ulong k1 = 0; 475 ulong k2 = 0; 476 final switch (data.length & 15) 477 { 478 case 15: 479 k2 ^= ulong(data[14]) << 48; 480 goto case; 481 case 14: 482 k2 ^= ulong(data[13]) << 40; 483 goto case; 484 case 13: 485 k2 ^= ulong(data[12]) << 32; 486 goto case; 487 case 12: 488 k2 ^= ulong(data[11]) << 24; 489 goto case; 490 case 11: 491 k2 ^= ulong(data[10]) << 16; 492 goto case; 493 case 10: 494 k2 ^= ulong(data[9]) << 8; 495 goto case; 496 case 9: 497 k2 ^= ulong(data[8]) << 0; 498 h2 ^= shuffle(k2, c2, c1, 33); 499 goto case; 500 case 8: 501 k1 ^= ulong(data[7]) << 56; 502 goto case; 503 case 7: 504 k1 ^= ulong(data[6]) << 48; 505 goto case; 506 case 6: 507 k1 ^= ulong(data[5]) << 40; 508 goto case; 509 case 5: 510 k1 ^= ulong(data[4]) << 32; 511 goto case; 512 case 4: 513 k1 ^= ulong(data[3]) << 24; 514 goto case; 515 case 3: 516 k1 ^= ulong(data[2]) << 16; 517 goto case; 518 case 2: 519 k1 ^= ulong(data[1]) << 8; 520 goto case; 521 case 1: 522 k1 ^= ulong(data[0]) << 0; 523 h1 ^= shuffle(k1, c1, c2, 31); 524 goto case; 525 case 0: 526 } 527 } 528 529 /// Incorporate size and finalizes the hash. 530 void finalize() pure nothrow @nogc 531 { 532 h1 ^= size; 533 h2 ^= size; 534 535 h1 += h2; 536 h2 += h1; 537 h1 = fmix(h1); 538 h2 = fmix(h2); 539 h1 += h2; 540 h2 += h1; 541 } 542 543 /// Returns the hash as an ulong[2] value. 544 Block get() pure nothrow @nogc 545 { 546 return [h1, h2]; 547 } 548 } 549 550 unittest 551 { 552 checkResult!SMurmurHash3_x64_128([ 553 "" : "00000000000000000000000000000000", 554 "a" : "897859F6655555855A890E51483AB5E6", 555 "ab" : "2E1BED16EA118B93ADD4529B01A75EE6", 556 "abc" : "6778AD3F3F3F96B4522DCA264174A23B", 557 "abcd" : "4FCD5646D6B77BB875E87360883E00F2", 558 "abcde" : "B8BB96F491D036208CECCF4BA0EEC7C5", 559 "abcdef" : "55BFA3ACBF867DE45C842133990971B0", 560 "abcdefg" : "99E49EC09F2FCDA6B6BB55B13AA23A1C", 561 "abcdefgh" : "028CEF37B00A8ACCA14069EB600D8948", 562 "abcdefghi" : "64793CF1CFC0470533E041B7F53DB579", 563 "abcdefghij" : "998C2F770D5BC1B6C91A658CDC854DA2", 564 "abcdefghijk" : "029D78DFB8D095A871E75A45E2317CBB", 565 "abcdefghijkl" : "94E17AE6B19BF38E1C62FF7232309E1F", 566 "abcdefghijklm" : "73FAC0A78D2848167FCCE70DFF7B652E", 567 "abcdefghijklmn" : "E075C3F5A794D09124336AD2276009EE", 568 "abcdefghijklmno" : "FB2F0C895124BE8A612A969C2D8C546A", 569 "abcdefghijklmnop" : "23B74C22A33CCAC41AEB31B395D63343", 570 "abcdefghijklmnopq" : "57A6BD887F746475E40D11A19D49DAEC", 571 "abcdefghijklmnopqr" : "508A7F90EC8CF0776BC7005A29A8D471", 572 "abcdefghijklmnopqrs" : "886D9EDE23BC901574946FB62A4D8AA6", 573 "abcdefghijklmnopqrst" : "F1E237F926370B314BD016572AF40996", 574 "abcdefghijklmnopqrstu" : "3CC9FF79E268D5C9FB3C9BE9C148CCD7", 575 "abcdefghijklmnopqrstuv" : "56F8ABF430E388956DA9F4A8741FDB46", 576 "abcdefghijklmnopqrstuvw" : "8E234F9DBA0A4840FFE9541CEBB7BE83", 577 "abcdefghijklmnopqrstuvwx" : "F72CDED40F96946408F22153A3CF0F79", 578 "abcdefghijklmnopqrstuvwxy" : "0F96072FA4CBE771DBBD9E398115EEED", 579 "abcdefghijklmnopqrstuvwxyz" : "A94A6F517E9D9C7429D5A7B6899CADE9"]); 580 } 581 582 unittest 583 { 584 // Pushing unaligned data and making sure the result is still coherent. 585 void testUnalignedHash(H)() 586 { 587 immutable ubyte[1025] data = 0xAC; 588 immutable alignedHash = digest!H(data[0 .. $ - 1]); // 0..1023 589 immutable unalignedHash = digest!H(data[1 .. $]); // 1..1024 590 assert(alignedHash == unalignedHash); 591 } 592 593 testUnalignedHash!MurmurHash3_x86_32(); 594 testUnalignedHash!MurmurHash3_x86_128(); 595 testUnalignedHash!MurmurHash3_x64_128(); 596 } 597 598 //private: 599 /* 600 This is a helper struct and is not intended to be used directly. MurmurHash 601 cannot put chunks smaller than Block.sizeof at a time. This struct stores 602 remainder bytes in a buffer and pushes it when the block is complete or during 603 finalization. 604 */ 605 struct Piecewise(Hasher) 606 { 607 enum blockSize = bits!Block; 608 609 alias Block = Hasher.Block; 610 union BufferUnion 611 { 612 Block block; 613 ubyte[Block.sizeof] data; 614 } 615 616 BufferUnion buffer; 617 size_t bufferSize; 618 Hasher hasher; 619 620 // Initialize 621 void start() 622 { 623 this = Piecewise.init; 624 } 625 626 /** 627 Adds data to the digester. This function can be called many times in a row 628 after start but before finish. 629 */ 630 void put(scope const(ubyte)[] data...) pure nothrow 631 { 632 // Buffer should never be full while entering this function. 633 assert(bufferSize < Block.sizeof); 634 635 // Check if we have some leftover data in the buffer. Then fill the first block buffer. 636 if (bufferSize + data.length < Block.sizeof) 637 { 638 buffer.data[bufferSize .. bufferSize + data.length] = data[]; 639 bufferSize += data.length; 640 return; 641 } 642 const bufferLeeway = Block.sizeof - bufferSize; 643 assert(bufferLeeway <= Block.sizeof); 644 buffer.data[bufferSize .. $] = data[0 .. bufferLeeway]; 645 hasher.putBlock(buffer.block); 646 data = data[bufferLeeway .. $]; 647 648 // Do main work: process chunks of Block.sizeof bytes. 649 const numBlocks = data.length / Block.sizeof; 650 const remainderStart = numBlocks * Block.sizeof; 651 version(NO_UNALIGNED_ACCESS) assert(data.ptr % Block.alignof == 0); 652 foreach (const Block block; cast(const(Block[]))(data[0 .. remainderStart])) 653 { 654 hasher.putBlock(block); 655 } 656 // +1 for bufferLeeway Block. 657 hasher.size += (numBlocks + 1) * Block.sizeof; 658 data = data[remainderStart .. $]; 659 660 // Now add remaining data to buffer. 661 assert(data.length < Block.sizeof); 662 bufferSize = data.length; 663 buffer.data[0 .. data.length] = data[]; 664 } 665 666 /** 667 Finalizes the computation of the hash and returns the computed value. 668 Note that $(D finish) can be called only once and that no subsequent calls 669 to $(D put) is allowed. 670 */ 671 ubyte[Block.sizeof] finish() pure nothrow 672 { 673 auto tail = getRemainder(); 674 if (tail.length > 0) 675 { 676 hasher.putRemainder(tail); 677 } 678 hasher.finalize(); 679 return hasher.getBytes(); 680 } 681 682 private: 683 const(ubyte)[] getRemainder() 684 { 685 return buffer.data[0 .. bufferSize]; 686 } 687 } 688 689 unittest 690 { 691 struct DummyHasher 692 { 693 alias Block = ubyte[2]; 694 const(Block)[] results; 695 size_t size; 696 697 void putBlock(Block value) pure nothrow 698 { 699 results ~= value; 700 } 701 702 void putRemainder(scope const(ubyte)[] data...) pure nothrow 703 { 704 } 705 706 void finalize() pure nothrow 707 { 708 } 709 710 Block getBytes() pure nothrow 711 { 712 return Block.init; 713 } 714 } 715 716 auto digester = Piecewise!DummyHasher(); 717 assert(digester.hasher.results == []); 718 assert(digester.getRemainder() == []); 719 digester.put(0); 720 assert(digester.hasher.results == []); 721 assert(digester.getRemainder() == [0]); 722 digester.put(1, 2); 723 assert(digester.hasher.results == [[0, 1]]); 724 assert(digester.getRemainder() == [2]); 725 digester.put(3, 4, 5); 726 assert(digester.hasher.results == [[0, 1], [2, 3], [4, 5]]); 727 assert(digester.getRemainder() == []); 728 } 729 730 template bits(T) 731 { 732 enum bits = T.sizeof * 8; 733 } 734 735 T rotl(T)(T x, uint y) 736 in 737 { 738 import std.traits : isUnsigned; 739 740 static assert(isUnsigned!T); 741 assert(y >= 0 && y <= bits!T); 742 } 743 body 744 { 745 return ((x << y) | (x >> (bits!T - y))); 746 } 747 748 T shuffle(T)(T k, T c1, T c2, ubyte r1) 749 { 750 import std.traits : isUnsigned; 751 752 static assert(isUnsigned!T); 753 k *= c1; 754 k = rotl(k, r1); 755 k *= c2; 756 return k; 757 } 758 759 void update(T)(ref T h, T k, T mixWith, T c1, T c2, ubyte r1, ubyte r2, T n) 760 { 761 import std.traits : isUnsigned; 762 763 static assert(isUnsigned!T); 764 h ^= shuffle(k, c1, c2, r1); 765 h = rotl(h, r2); 766 h += mixWith; 767 h = h * 5 + n; 768 } 769 770 uint fmix(uint h) pure nothrow @nogc 771 { 772 h ^= h >> 16; 773 h *= 0x85ebca6b; 774 h ^= h >> 13; 775 h *= 0xc2b2ae35; 776 h ^= h >> 16; 777 return h; 778 } 779 780 ulong fmix(ulong k) pure nothrow @nogc 781 { 782 k ^= k >> 33; 783 k *= 0xff51afd7ed558ccd; 784 k ^= k >> 33; 785 k *= 0xc4ceb9fe1a85ec53; 786 k ^= k >> 33; 787 return k; 788 } 789 790 }