Implement consistent hashing with bounded loads

  • Updated on 28th Mar 2023

The article Consistent Hashing with Load Bounds discusses the problems solved by consistent hashing with load bounds, as well as the theoretical aspects such as algorithm principles. This article will use GPT to discuss how to implement the C# code.

If implementing consistent hashing with bounded load through C# code, one can use the MurmurHash3 hash function to generate hash values for each node in the system, and then map each key to the node whose hash value is closest to the hash value of the key. Below is an example implementation provided by GPT.

Bounded Load Consistent Hashing function is implemented using the C# code:

using System;
using System.Collections.Generic;

public class BoundedLoadConsistentHash<T>
{
    private readonly Func<T, uint> _hashFunction;
    private readonly SortedDictionary<uint, T> _ring = new SortedDictionary<uint, T>();
    private readonly Dictionary<T, int> _loadMap = new Dictionary<T, int>();
    private readonly int _maxLoad;

    public BoundedLoadConsistentHash(Func<T, uint> hashFunction, int maxLoad)
    {
        _hashFunction = hashFunction;
        _maxLoad = maxLoad;
    }

    public void Add(T node)
    {
        for (int i = 0; i < _maxLoad; i++)
        {
            uint hash = MurmurHash3.Hash(node.GetHashCode() + i);
            _ring[hash] = node;
            _loadMap[node] = i + 1;
        }
    }

    public void Remove(T node)
    {
        for (int i = 0; i < _maxLoad; i++)
        {
            uint hash = MurmurHash3.Hash(node.GetHashCode() + i);
            _ring.Remove(hash);
            _loadMap.Remove(node);
        }
    }

    public T GetNode(string key)
    {
        if (_ring.Count == 0)
        {
            throw new InvalidOperationException("No nodes available");
        }

        uint hash = MurmurHash3.Hash(key);
        foreach (var nodeHash in _ring.Keys)
        {
            if (hash <= nodeHash)
            {
                return _ring[nodeHash];
            }
        }

        return _ring[_ring.Keys[0]];
    }

    public int GetLoad(T node)
    {
        return _loadMap[node];
    }
}

The MurmurHash3 hash function is implemented using the C# code:

public static class MurmurHash3
{
    private const uint Seed = 0xc58f1a7b;

    public static uint Hash(int input)
    {
        return Hash(BitConverter.GetBytes(input));
    }

    public static uint Hash(string input)
    {
        return Hash(System.Text.Encoding.UTF8.GetBytes(input));
    }

    private static uint Hash(byte[] input)
    {
        const uint c1 = 0xcc9e2d51;
        const uint c2 = 0x1b873593;
        const int r1 = 15;
        const int r2 = 13;
        const uint m = 5;
        const uint n = 0xe6546b64;

        uint hash = Seed;

        int len = input.Length / 4;
        for (int i = 0; i < len; i++)
        {
            uint k = BitConverter.ToUInt32(input, i * 4);

            k *= c1;
            k = (k << r1) | (k >> (32 - r1));
            k *= c2;

            hash ^= k;
            hash = (hash << r2) | (hash >> (32 - r2));
            hash = hash * m + n;
        }

        uint remainingBytes = (uint)(input.Length & 3);
        if (remainingBytes > 0)
        {
            uint k = 0;
            for (int i = 0; i < remainingBytes; i++)
            {
                k <<= 8;
                k |= input[input.Length - remainingBytes + i];
            }

            k *= c1;
            k = (k << r1) | (k >> (32 - r1));
            k *= c2;

            hash ^= k;
        }

        hash ^= (uint)input.Length;
        hash ^= hash >> 16;
        hash *= 0x85ebca6b;
        hash ^= hash >> 13;
        hash *= 0xc2b2ae35;
        hash ^= hash >> 16;
        return hash;
    }
}

Examples:

class Program
{
    static void Main(string[] args)
    {
        BoundedLoadConsistentHash<string> ch = new BoundedLoadConsistentHash<string>(MurmurHash3.Hash, 3);
        ch.Add("node1");
        ch.Add("node2");
        ch.Add("node3");

        string key = "example_key";
        string node = ch.GetNode(key);
        int load = ch.GetLoad(node);

        Console.WriteLine("Key: {0}, Node: {1}, Load: {2}", key, node, load);
    }
}

In the code example above, an instance of ConsistentHash is created and three nodes are added to it. Then, the GetNode method is used to obtain the node to which a key belongs, and the GetLoad method is used to get the load of that node. By default, the maximum load for each node is 3, so when the load reaches 3, no more keys will be assigned to that node.