diff --git a/wizard-work/beam/jump_ref.c b/wizard-work/beam/jump_ref.c new file mode 100644 index 0000000..e10fb04 --- /dev/null +++ b/wizard-work/beam/jump_ref.c @@ -0,0 +1,40 @@ +/* ref.c -- Lamping & Veach (2014), the reference form: 64-bit wrapping + * multiply, and the jump computed with a double, exactly as published in + * "A Fast, Minimal Memory, Consistent Hash Algorithm", figure 5. + * + * This file exists so the Python module's docstring claim ("results match the + * C reference") can be graded against an implementation the Python module did + * not write, rather than against itself. Nothing here mirrors the Python + * control flow: the loop, the multiply and the double division are the + * paper's, and the key comes from stdin, not from a hash fold. + * + * stdin: one " " pair per line, n <= 2^31-1 + * stdout: " " + */ +#include +#include +#include + +static int32_t jump_ref(uint64_t key, int32_t num_buckets) { + int64_t b = -1, j = 0; + while (j < num_buckets) { + b = j; + key = key * 2862933555777941757ULL + 1; + j = (int64_t)((double)(b + 1) * (double)(1LL << 31) / + (double)((key >> 33) + 1)); + } + return (int32_t)b; +} + +int main(void) { + unsigned long long k; + long n; + while (scanf("%llu %ld", &k, &n) == 2) { + if (n < 1 || n > 2147483647L) { + fprintf(stderr, "ref: n out of range: %ld\n", n); + return 2; + } + printf("%llu %ld %d\n", k, n, jump_ref((uint64_t)k, (int32_t)n)); + } + return 0; +} diff --git a/wizard-work/beam/test_jump_consistent_hash.py b/wizard-work/beam/test_jump_consistent_hash.py index 53b8eec..1326408 100644 --- a/wizard-work/beam/test_jump_consistent_hash.py +++ b/wizard-work/beam/test_jump_consistent_hash.py @@ -1,10 +1,28 @@ import random import unittest -from jump_consistent_hash import jump_hash, JumpConsistentHash +from jump_consistent_hash import jump_hash, JumpConsistentHash, _to_u64 class TestJumpHash(unittest.TestCase): + def test_golden_vectors(self): + # Golden vectors, produced by jump_ref.c in this directory: the + # Lamping & Veach figure-5 reference, compiled with `gcc -O2`, fed the + # key and bucket count and asked for the bucket. Regenerate with + # gcc -O2 -o /tmp/jump_ref jump_ref.c && echo "0 5" | /tmp/jump_ref + # These pin the MAPPING, not its statistics: a hash can be uniform, + # minimal-movement and deterministic while sending every key to a + # different bucket than the reference does. The last two cells pin a + # string key and its md5->u64 fold, because the fold is this module's + # own convention (the paper defines the algorithm on a u64) and it is + # the line that can move ~99% of keys without any property cell + # noticing. + self.assertEqual(jump_hash(0, 5), 0) + self.assertEqual(jump_hash(18446744073709551615, 100), 92) + self.assertEqual(_to_u64("key-0"), 18183421273640453746) + self.assertEqual(jump_hash("key-0", 100), 40) + self.assertEqual(jump_hash("order-42", 1000), 667) + def test_range(self): for n in (1, 2, 5, 10, 100, 1000): for i in range(200): @@ -37,6 +55,13 @@ class TestJumpHash(unittest.TestCase): # expected ~ 10000/(k+1); allow generous slack expected = 10000 / (k + 1) self.assertLess(moved, expected * 2.5, f"k={k} moved={moved}") + # ... and a lower bound, so a sticky hash (one that never moves a + # key, and therefore satisfies every "nothing extra moved" test) + # is graded here rather than only by uniformity. Measured ratios + # on this key set are 0.974-1.047, so 0.5 is slack, not a knob. + self.assertGreater(moved, expected * 0.5, + f"k={k} moved={moved} -- below half the " + f"expected {expected:.0f}: keys are not moving") def test_uniformity(self): n = 10 @@ -49,6 +74,23 @@ class TestJumpHash(unittest.TestCase): self.assertGreater(c, mean * 0.9) self.assertLess(c, mean * 1.1) + def test_uniformity_other_bucket_count(self): + # A second bucket count, so a hash tuned to n=10 (or a bug that only + # bites when n is not a round number) is not silently exempt. n=97 is + # prime and non-power-of-two. Band: +/-10% of the mean, measured + # spread on this key set is -5.28%/+4.57% (deterministic, fixed keys), + # so the band has ~2x headroom and still fails any gross bias. + n = 97 + keys = 200000 + counts = [0] * n + for i in range(keys): + counts[jump_hash(i, n)] += 1 + mean = keys / n + self.assertEqual(sum(counts), keys) + for b, c in enumerate(counts): + self.assertGreater(c, mean * 0.90, f"bucket {b} under-filled: {c}") + self.assertLess(c, mean * 1.10, f"bucket {b} over-filled: {c}") + def test_wrapper(self): jch = JumpConsistentHash(5) self.assertEqual(jch.bucket("a"), jump_hash("a", 5))