Library · Databases · storage structures
Cuckoo Filter
#pragma once
#include <bits/stdc++.h>
using namespace std;
class CuckooFilter {
private:
static const int BUCKET_SIZE = 4;
static const int EMPTY = 0;
int N, MAX_NUM_KICKS;
vector<array<int, BUCKET_SIZE>> B;
static size_t hash(const string& key) {
size_t h = 0;
for (char c : key) h = h * 31 + c;
return h;
}
static int fingerprint(const string& x) {
return ((hash(x) * 11400714819323198485ULL) >> 56) % 255 + 1;
}
public:
CuckooFilter(int n, int maxNumKicks) : MAX_NUM_KICKS(maxNumKicks) {
N = 1;
while (N < n) N *= 2;
B.resize(N);
}
int find(int i, int v) {
for (int j = 0; j < BUCKET_SIZE; j++) {
if (B[i][j] == v) return j;
}
return -1;
}
bool insert(const string& x) {
int f = fingerprint(x);
size_t i1 = hash(x) % N;
size_t i2 = (i1 ^ hash(to_string(f))) % N;
// if bucket[i1] or bucket[i2] has an empty entry
// add f to that bucket
int p1 = find(i1, EMPTY);
if (p1 != -1) {
B[i1][p1] = f;
return true;
}
int p2 = find(i2, EMPTY);
if (p2 != -1) {
B[i2][p2] = f;
return true;
}
// must relocate existing items
size_t i = rand() % 2 ? i1 : i2;
// remember each swap (bucket, entry) to undo on failure
vector<pair<size_t, int>> kicks;
// cap the kicks so a cycle can't loop forever
for (int n = 0; n < MAX_NUM_KICKS; n++) {
// randomly select an entry e from bucket[i]
int e = rand() % BUCKET_SIZE;
// swap f and the fingerprint stored in entry e
swap(f, B[i][e]);
kicks.push_back({i, e});
i = (i ^ hash(to_string(f))) % N;
// if bucket[i] has an empty entry
int p = find(i, EMPTY);
if (p != -1) {
B[i][p] = f;
return true;
}
}
// undo the swaps so no existing fingerprint is lost
for (int k = kicks.size() - 1; k >= 0; k--) {
swap(f, B[kicks[k].first][kicks[k].second]);
}
return false;
}
bool lookup(const string& x) {
int f = fingerprint(x);
size_t i1 = hash(x) % N;
size_t i2 = (i1 ^ hash(to_string(f))) % N;
// if bucket[i1] or bucket[i2] has f
return find(i1, f) != -1 || find(i2, f) != -1;
}
bool remove(const string& x) {
int f = fingerprint(x);
size_t i1 = hash(x) % N;
size_t i2 = (i1 ^ hash(to_string(f))) % N;
// if bucket[i1] or bucket[i2] has f
int p1 = find(i1, f);
if (p1 != -1) {
B[i1][p1] = EMPTY;
return true;
}
int p2 = find(i2, f);
if (p2 != -1) {
B[i2][p2] = EMPTY;
return true;
}
return false;
}
// statistics
double fill_rate() {
int used = 0;
for (auto& bucket : B) {
for (int v : bucket) used += v != EMPTY;
}
return (double)used / (N * BUCKET_SIZE);
}
double false_positive_rate() {
// a lookup compares f with up to 2 buckets, each entry matches with prob 1/255
double entries = 2 * BUCKET_SIZE * fill_rate();
return 1 - pow(1 - 1.0 / 255, entries);
}
};
import random
from utils.hash import djb2, fnv1a
class CuckooFilter:
BUCKET_SIZE = 4
EMPTY = 0
def __init__(self, n, max_kicks) -> None:
self.n = n
self.hash = djb2
self.fingerprint = fnv1a
self.max_kicks = max_kicks
self.B = [[0] * self.BUCKET_SIZE for _ in range(self.n)]
def _find(self, i: int, v: int) -> int:
for j in range(self.BUCKET_SIZE):
if self.B[i][j] == v: return j
return -1
def insert(self, key: str) -> bool:
f = self.fingerprint(key)
i1 = self.hash(key) % self.n
i2 = (i1 ^ self.hash(str(f))) % self.n
p1, p2 = self._find(i1, self.EMPTY), self._find(i2, self.EMPTY)
if p1 != -1:
self.B[i1][p1] = f
return True
if p2 != -1:
self.B[i2][p2] = f
return True
# randomize bucket i1 and i2
i = i1 if random.random() < 0.5 else i2
# remember each swap to undo on failure
kicks = []
for kick in range(self.max_kicks):
# randomize a position to swap
e = random.getrandbits(64) % self.BUCKET_SIZE
f, self.B[i][e] = self.B[i][e], f
kicks.append((i, e))
i = (i ^ self.hash(str(f))) % self.n
p = self._find(i, self.EMPTY)
if p != -1:
self.B[i][p] = f
return True
for k in reversed(range(len(kicks))):
f, self.B[kicks[k][0]][kicks[k][1]] = self.B[kicks[k][0]][kicks[k][1]], f
return False
def lookup(self, key: str) -> bool:
f = self.fingerprint(key)
i1 = self.hash(key) % self.n
i2 = (i1 ^ self.hash(str(f))) % self.n
return self._find(i1, f) != -1 or self._find(i2, f) != -1
def remove(self, key: str) -> bool:
f = self.fingerprint(key)
i1 = self.hash(key) % self.n
i2 = (i1 ^ self.hash(str(f))) % self.n
p1, p2 = self._find(i1, f), self._find(i2, f)
if p1 != -1:
self.B[i1][p1] = self.EMPTY
return True
if p2 != -1:
self.B[i2][p2] = self.EMPTY
return True
return False
def fill_rate(self) -> float:
used = sum(v != self.EMPTY for bucket in self.B for v in bucket)
return used / (self.n * self.BUCKET_SIZE)
def false_positive_rate(self) -> float:
# a lookup compares f with up to 2 buckets, each entry matches with prob 1/255
entries = 2 * self.BUCKET_SIZE * self.fill_rate()
return 1 - (1 - 1 / 255) ** entries