evolution/libibex/hash.c

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
 *
 * Copyright (C) 2000 Helix Code, Inc.
 *
 * Authors: Michael Zucchi <notzed@helixcode.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public License
 * as published by the Free Software Foundation; either version 2 of
 * the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with the Gnome Library; see the file COPYING.LIB.  If not,
 * write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

/* hash based index mechanism */

#include <stdio.h>
#include <stdlib.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>

#include "block.h"
#include "index.h"

#define d(x)

#define HASH_SIZE (1024)
#define KEY_THRESHOLD (sizeof(struct _hashkey) + 4) /* minimum number of free bytes we worry about
						      maintaining free blocks for */
#define ARRAY_LEN(a) (sizeof(a)/sizeof(a[0]))

typedef guint32 hashid_t;

struct _HASHCursor {
	struct _IBEXCursor cursor;

	hashid_t key;
	hashid_t block;
	unsigned int index;
	unsigned int size;
};

static struct _IBEXIndex *hash_create(struct _memcache *bc, int size);
static struct _IBEXIndex *hash_open(struct _memcache *bc, blockid_t root);
static int hash_sync(struct _IBEXIndex *index);
static int hash_close(struct _IBEXIndex *index);

static hashid_t hash_find(struct _IBEXIndex *index, const char *key, int keylen);
static void hash_remove(struct _IBEXIndex *index, const char *key, int keylen);
static hashid_t hash_insert(struct _IBEXIndex *index, const char *key, int keylen);
static char *hash_get_key(struct _IBEXIndex *index, hashid_t hashbucket, int *len);
static void hash_set_data_block(struct _IBEXIndex *index, hashid_t keyid, blockid_t blockid, blockid_t tail);
static blockid_t hash_get_data_block(struct _IBEXIndex *index, hashid_t keyid, blockid_t *tail);
static struct _IBEXCursor *hash_get_cursor(struct _IBEXIndex *index);

static struct _IBEXCursor *hash_cursor_create(struct _IBEXIndex *);
static void hash_cursor_close(struct _IBEXCursor *);
static guint32 hash_cursor_next(struct _IBEXCursor *);
static char *hash_cursor_next_key(struct _IBEXCursor *, int *keylenptr);

struct _IBEXIndexClass ibex_hash_class = {
	hash_create, hash_open,
	hash_sync, hash_close,
	hash_find,
	hash_remove,
	hash_insert,
	hash_get_key,
	hash_set_data_block,
	hash_get_data_block,
	hash_get_cursor,
};

struct _IBEXCursorClass ibex_hash_cursor_class = {
	hash_cursor_close,
	hash_cursor_next,
	hash_cursor_next_key
};

/* the reason we have the tail here is that otherwise we need to
   have a 32 bit blockid for the root node; which would make this
   structure the same size anyway, with about 24 wasted bits */
struct _hashkey {
	blockid_t next;		/* next in hash chain */
	blockid_t tail;
	unsigned int root:32-BLOCK_BITS;
	unsigned int keyoffset:BLOCK_BITS;
};

struct _hashblock {
	unsigned int next:32-BLOCK_BITS;	/* next block, linked list of all key blocks: block number */
	unsigned int used:BLOCK_BITS;		/* number of elements used */
	union {
		struct _hashkey keys[(BLOCK_SIZE-4)/sizeof(struct _hashkey)];
		char keydata[BLOCK_SIZE-4];
	} hashblock_u;
};
#define hb_keys hashblock_u.keys
#define hb_keydata hashblock_u.keydata

/* size of block overhead + 2 key block overhead */
#define MAX_KEYLEN (BLOCK_SIZE - 4 - 12 - 12)

/* root block for a hash index */
struct _hashroot {
	hashid_t free;		/* free list */
	guint32 size;		/* how big the hash table is */
	hashid_t keys;		/* linked list of blocks */
	hashid_t table[(BLOCK_SIZE-8)/sizeof(hashid_t)]; /* pointers to blocks of pointers */
};

struct _hashtableblock {
	hashid_t buckets[BLOCK_SIZE/sizeof(hashid_t)];
};

/* map a hash index to a block index */
#define HASH_INDEX(b) ((b) & (BLOCK_SIZE-1))
/* map a hash index to a block number */
#define HASH_BLOCK(b) ((b) & ~(BLOCK_SIZE-1))
/* map a block + index to a hash key */
#define HASH_KEY(b, i) (((b) & ~(BLOCK_SIZE-1)) | ((i) & (BLOCK_SIZE-1)))

/* taken from tdb/gdbm */
static unsigned int hash_key(const unsigned char *key, int keylen)
{
	char *newkey;
	newkey = alloca(keylen+1);
	memcpy(newkey, key, keylen);
	newkey[keylen]=0;
	return g_str_hash(newkey);
#if 0
	unsigned int value;	/* Used to compute the hash value.  */
	unsigned int  i;	/* Used to cycle through random values. */

	/* Set the initial value from the key size. */
	value = 0x238F13AF * keylen;
	for (i=0; i < keylen; i++) {
		value = (value + (key[i] << (i*5 % 24)));
	}

	value = (1103515243 * value + 12345);  

	return value;
#endif
}

/* create a new hash table, return a pointer to its root block */
static struct _IBEXIndex *
hash_create(struct _memcache *bc, int size)
{
	blockid_t root, block;
	struct _hashroot *hashroot;
	int i;
	struct _hashtableblock *table;
	struct _IBEXIndex *index;

	g_assert(size<=10240);

	d(printf("initialising hash table, size = %d\n", size));

	index = g_malloc0(sizeof(*index));
	index->blocks = bc;
	index->klass = &ibex_hash_class;
	root = ibex_block_get(bc);
	index->root = root;
	d(printf(" root = %d\n", root));
	hashroot = (struct _hashroot *)ibex_block_read(bc, root);
	hashroot->free = 0;
	hashroot->size = size;
	ibex_block_dirty((struct _block *)hashroot);
	for (i=0;i<size/(BLOCK_SIZE/sizeof(blockid_t));i++) {
		d(printf("initialising hash table index block %d\n", i));
		block = hashroot->table[i] = ibex_block_get(bc);
		table = (struct _hashtableblock *)ibex_block_read(bc, block);
		memset(table, 0, sizeof(table));
		ibex_block_dirty((struct _block *)table);
	}

	return index;
}

static struct _IBEXIndex *
hash_open(struct _memcache *bc, blockid_t root)
{
	struct _IBEXIndex *index;

	/* FIXME: check a 'magic', and the root for validity */

	index = g_malloc0(sizeof(*index));
	index->blocks = bc;
	index->root = root;
	index->klass = &ibex_hash_class;

	return index;
}


static int hash_sync(struct _IBEXIndex *index)
{
	/* nop, index always synced on disk (at least, to blocks) */
	return 0;
}

static int hash_close(struct _IBEXIndex *index)
{
#ifdef INDEX_STAT
	printf("Performed %d lookups, average %f depth\n", index->lookups, (double)index->lookup_total/index->lookups);
#endif
	g_free(index);
	return 0;
}

/* get an iterator class */
static struct _IBEXCursor *hash_get_cursor(struct _IBEXIndex *index)
{
	return hash_cursor_create(index);
}

/* convert a hashbucket id into a name */
static char *
hash_get_key(struct _IBEXIndex *index, hashid_t hashbucket, int *len)
{
	struct _hashblock *bucket;
	int ind;
	char *ret, *start, *end;

	if (hashbucket == 0) {
		if (len)
			*len = 0;
		return g_strdup("");
	}

	bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashbucket));
	ind = HASH_INDEX(hashbucket);

	g_assert(ind < bucket->used);

	start = &bucket->hb_keydata[bucket->hb_keys[ind].keyoffset];
	if (ind == 0) {
		end = &bucket->hb_keydata[sizeof(bucket->hb_keydata)/sizeof(bucket->hb_keydata[0])];
	} else {
		end = &bucket->hb_keydata[bucket->hb_keys[ind-1].keyoffset];
	}
	
	ret = g_malloc(end-start+1);
	memcpy(ret, start, end-start);
	ret[end-start]=0;
	if (len)
		*len = end-start;
	return ret;
}

/* sigh, this is fnugly code ... */
static hashid_t
hash_find(struct _IBEXIndex *index, const char *key, int keylen)
{
	struct _hashroot *hashroot;
	guint32 hash;
	int hashentry;
	blockid_t hashtable;
	hashid_t hashbucket;
	struct _hashtableblock *table;

	g_assert(index != 0);
	g_assert(index->root != 0);

	d(printf("finding hash %.*s\n", keylen, key));

	/* truncate the key to the maximum size */
	if (keylen > MAX_KEYLEN)
		keylen = MAX_KEYLEN;

	hashroot = (struct _hashroot *)ibex_block_read(index->blocks, index->root);

	/* find the table containing this entry */
	hash = hash_key(key, keylen) % hashroot->size;
	hashtable = hashroot->table[hash / (BLOCK_SIZE/sizeof(blockid_t))];
	g_assert(hashtable != 0);
	table = (struct _hashtableblock *)ibex_block_read(index->blocks, hashtable);
	hashentry = hash % (BLOCK_SIZE/sizeof(blockid_t));
	/* and its bucket */
	hashbucket = table->buckets[hashentry];

#ifdef INDEX_STAT
	index->lookups++;
#endif
	/* go down the bucket chain, reading each entry till we are done ... */
	while (hashbucket != 0) {
		struct _hashblock *bucket;
		char *start, *end;
		int ind;

#ifdef INDEX_STAT
		index->lookup_total++;
#endif

		d(printf(" checking bucket %d\n", hashbucket));

		/* get the real bucket id from the hashbucket id */
		bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashbucket));
		/* and get the key number within the block */
		ind = HASH_INDEX(hashbucket);

		g_assert(ind < bucket->used);

		start = &bucket->hb_keydata[bucket->hb_keys[ind].keyoffset];
		if (ind == 0) {
			end = &bucket->hb_keydata[sizeof(bucket->hb_keydata)/sizeof(bucket->hb_keydata[0])];
		} else {
			end = &bucket->hb_keydata[bucket->hb_keys[ind-1].keyoffset];
		}
		if ( (end-start) == keylen
		     && memcmp(start, key, keylen) == 0) {
			return hashbucket;
		}
		hashbucket = bucket->hb_keys[ind].next;
	}
	return 0;
}

static int
hash_info(struct _hashblock *bucket, int index)
{
	char *start, *end;

	g_assert(index < bucket->used);

	start = &bucket->hb_keydata[bucket->hb_keys[index].keyoffset];
	if (index == 0) {
		end = &bucket->hb_keydata[sizeof(bucket->hb_keydata)/sizeof(bucket->hb_keydata[0])];
	} else {
		end = &bucket->hb_keydata[bucket->hb_keys[index-1].keyoffset];
	}

	return end-start;
}


/* TODO: get rid of hash_compress/remove and just have one a-la the disktail code */

/* compresses the bucket 'bucket', removing data
   at index 'index' */
static void
hash_compress(struct _hashblock *bucket, int index)
{
	int i;
	char *start, *end, *newstart;

	/* get start/end of area to zap */
	start = &bucket->hb_keydata[bucket->hb_keys[index].keyoffset];
	if (index == 0) {
		end = &bucket->hb_keydata[sizeof(bucket->hb_keydata)/sizeof(bucket->hb_keydata[0])];
	} else {
		end = &bucket->hb_keydata[bucket->hb_keys[index-1].keyoffset];
	}

	if (start == end)
		return;

	/* fixup data */
	newstart = &bucket->hb_keydata[bucket->hb_keys[bucket->used-1].keyoffset];
	memmove(newstart+(end-start), newstart, start-newstart);

	/* fixup key pointers */
	for (i=index;i<bucket->used;i++) {
		bucket->hb_keys[i].keyoffset += (end-start);
	}
	ibex_block_dirty((struct _block *)bucket);
}

/* make room 'len' for the key 'index' */
/* assumes key 'index' is already empty (0 length) */
static void
hash_expand(struct _hashblock *bucket, int index, int len)
{
	int i;
	char *end, *newstart;

	/* get start/end of area to zap */
	if (index == 0) {
		end = &bucket->hb_keydata[sizeof(bucket->hb_keydata)/sizeof(bucket->hb_keydata[0])];
	} else {
		end = &bucket->hb_keydata[bucket->hb_keys[index-1].keyoffset];
	}

	/* fixup data */
	newstart = &bucket->hb_keydata[bucket->hb_keys[bucket->used-1].keyoffset];
	memmove(newstart-len, newstart, end-newstart);

	/* fixup key pointers */
	for (i=index;i<bucket->used;i++) {
		bucket->hb_keys[i].keyoffset -= len;
	}
	ibex_block_dirty((struct _block *)bucket);
}

static void
hash_remove(struct _IBEXIndex *index, const char *key, int keylen)
{
	struct _hashroot *hashroot;
	guint32 hash;
	int hashentry;
	blockid_t hashtable;
	hashid_t hashbucket, hashprev;
	struct _hashtableblock *table;

	g_assert(index != 0);
	g_assert(index->root != 0);

	d(printf("removing hash %.*s\n", keylen, key));

	/* truncate the key to the maximum size */
	if (keylen > MAX_KEYLEN)
		keylen = MAX_KEYLEN;

	hashroot = (struct _hashroot *)ibex_block_read(index->blocks, index->root);

	/* find the table containing this entry */
	hash = hash_key(key, keylen) % hashroot->size;
	hashtable = hashroot->table[hash / (BLOCK_SIZE/sizeof(blockid_t))];
	table = (struct _hashtableblock *)ibex_block_read(index->blocks, hashtable);
	hashentry = hash % (BLOCK_SIZE/sizeof(blockid_t));
	/* and its bucket */
	hashbucket = table->buckets[hashentry];

	/* go down the bucket chain, reading each entry till we are done ... */
	hashprev = 0;
	while (hashbucket != 0) {
		struct _hashblock *bucket;
		char *start, *end;
		int ind;

		d(printf(" checking bucket %d\n", hashbucket));

		/* get the real bucket id from the hashbucket id */
		bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashbucket));
		/* and get the key number within the block */
		ind = HASH_INDEX(hashbucket);

		g_assert(ind < bucket->used);

		start = &bucket->hb_keydata[bucket->hb_keys[ind].keyoffset];
		if (ind == 0) {
			end = &bucket->hb_keydata[sizeof(bucket->hb_keydata)/sizeof(bucket->hb_keydata[0])];
		} else {
			end = &bucket->hb_keydata[bucket->hb_keys[ind-1].keyoffset];
		}
		if ( (end-start) == keylen
		     && memcmp(start, key, keylen) == 0) {
			struct _hashblock *prevbucket;

			if (hashprev == 0) {
				/* unlink from hash chain */
				table->buckets[hashentry] = bucket->hb_keys[HASH_INDEX(hashbucket)].next;
				/* link into free list */
				bucket->hb_keys[HASH_INDEX(hashbucket)].next = hashroot->free;
				hashroot->free = hashbucket;
				/* compress away data */
				hash_compress(bucket, HASH_INDEX(hashbucket));
				ibex_block_dirty((struct _block *)bucket);
				ibex_block_dirty((struct _block *)table);
				ibex_block_dirty((struct _block *)hashroot);
			} else {
				prevbucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashprev));
				prevbucket->hb_keys[HASH_INDEX(hashprev)].next =
					bucket->hb_keys[ind].next;
				/* link into free list */
				bucket->hb_keys[ind].next = hashroot->free;
				hashroot->free = hashbucket;
				/* compress entry */
				hash_compress(bucket, ind);
				ibex_block_dirty((struct _block *)bucket);
				ibex_block_dirty((struct _block *)prevbucket);
				ibex_block_dirty((struct _block *)hashroot);
			}
			return;
		}
		hashprev = hashbucket;
		hashbucket = bucket->hb_keys[ind].next;
	}
}

/* set where the datablock is located */
static void
hash_set_data_block(struct _IBEXIndex *index, hashid_t keyid, blockid_t blockid, blockid_t tail)
{
	struct _hashblock *bucket;

	d(printf("setting data block hash %d to %d tail %d\n", keyid, blockid, tail));

	/* map to a block number */
	g_assert((blockid & (BLOCK_SIZE-1)) == 0);
	blockid >>= BLOCK_BITS;

	bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(keyid));
	if (bucket->hb_keys[HASH_INDEX(keyid)].root != blockid
	    || bucket->hb_keys[HASH_INDEX(keyid)].tail != tail) {
		bucket->hb_keys[HASH_INDEX(keyid)].tail = tail;
		bucket->hb_keys[HASH_INDEX(keyid)].root = blockid;
		ibex_block_dirty((struct _block *)bucket);
	}
}

static blockid_t
hash_get_data_block(struct _IBEXIndex *index, hashid_t keyid, blockid_t *tail)
{
	struct _hashblock *bucket;

	d(printf("getting data block hash %d\n", keyid));

	if (keyid == 0) {
		if (tail)
			*tail = 0;
		return 0;
	}

	bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(keyid));
	if (tail)
		*tail = bucket->hb_keys[HASH_INDEX(keyid)].tail;
	return bucket->hb_keys[HASH_INDEX(keyid)].root << BLOCK_BITS;
}

static hashid_t
hash_insert(struct _IBEXIndex *index, const char *key, int keylen)
{
	struct _hashroot *hashroot;
	guint32 hash;
	int hashentry;
	blockid_t hashtable;
	hashid_t hashbucket, keybucket, keyprev, keyfree;
	struct _hashtableblock *table;
	struct _hashblock *bucket;
	int count;

	g_assert(index != 0);
	g_assert(index->root != 0);

	/* truncate the key to the maximum size */
	if (keylen > MAX_KEYLEN)
		keylen = MAX_KEYLEN;

	d(printf("inserting hash %.*s\n", keylen, key));

	hashroot = (struct _hashroot *)ibex_block_read(index->blocks, index->root);

	/* find the table containing this entry */
	hash = hash_key(key, keylen) % hashroot->size;
	hashtable = hashroot->table[hash / (BLOCK_SIZE/sizeof(blockid_t))];
	table = (struct _hashtableblock *)ibex_block_read(index->blocks, hashtable);
	hashentry = hash % (BLOCK_SIZE/sizeof(blockid_t));
	/* and its bucket */
	hashbucket = table->buckets[hashentry];

	/* now look for a free slot, first try the free list */
	/* but dont try too hard if our key is just too long ... so just scan upto
	   4 blocks, but if we dont find a space, tough ... */
	keybucket = hashroot->free;
	keyprev = 0;
	count = 0;
	while (keybucket && count<4) {
		int space;
		
		d(printf(" checking free %d\n", keybucket));

		/* read the bucket containing this free key */
		bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(keybucket));

		/* check if there is enough space for the key */
		space =  &bucket->hb_keydata[bucket->hb_keys[bucket->used-1].keyoffset]
			- (char *)&bucket->hb_keys[bucket->used];
		if (space >= keylen) {
			hash_expand(bucket, HASH_INDEX(keybucket), keylen);
			memcpy(&bucket->hb_keydata[bucket->hb_keys[HASH_INDEX(keybucket)].keyoffset],
			       key, keylen);

			/* check if there is free space still in this node, and there are no other empty blocks */
			keyfree = bucket->hb_keys[HASH_INDEX(keybucket)].next;
			if ((space-keylen) >= KEY_THRESHOLD) {
				int i;
				int head = ARRAY_LEN(bucket->hb_keydata);
				int found = FALSE;

				for (i=0;i<bucket->used;i++) {
					if (bucket->hb_keys[i].keyoffset == head) {
						/* already have a free slot in this block, leave it */
						found = TRUE;
						break;
					}
					head = bucket->hb_keys[i].keyoffset;
				}
				if (!found) {
					/* we should link in a new free slot for this node */
					bucket->hb_keys[bucket->used].next = bucket->hb_keys[HASH_INDEX(keybucket)].next;
					bucket->hb_keys[bucket->used].keyoffset = bucket->hb_keys[bucket->used-1].keyoffset;
					keyfree = HASH_KEY(HASH_BLOCK(keybucket), bucket->used);
					bucket->used++;
				}
			}
			/* link 'keyfree' back to the parent ... */
			if (keyprev == 0) {
				hashroot->free = keyfree;
				ibex_block_dirty((struct _block *)hashroot);
			} else {
				struct _hashblock *prevbucket;
				prevbucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(keyprev));
				prevbucket->hb_keys[HASH_INDEX(keyprev)].next = keyfree;
				ibex_block_dirty((struct _block *)prevbucket);
			}

			/* link into the hash chain */
			bucket->hb_keys[HASH_INDEX(keybucket)].next = hashbucket;
			bucket->hb_keys[HASH_INDEX(keybucket)].root = 0;
			bucket->hb_keys[HASH_INDEX(keybucket)].tail = 0;
			table->buckets[hashentry] = keybucket;
			ibex_block_dirty((struct _block *)table);
			ibex_block_dirty((struct _block *)bucket);

			d(printf(" new key id %d\n", keybucket));
			d(printf(" new free id %d\n", hashroot->free));

			return keybucket;
		}

		count++;
		keyprev = keybucket;
		keybucket = bucket->hb_keys[HASH_INDEX(keybucket)].next;
	}

	/* else create a new block ... */
	keybucket = ibex_block_get(index->blocks);
	bucket = (struct _hashblock *)ibex_block_read(index->blocks, keybucket);

	d(printf("creating new key bucket %d\n", keybucket));

	memset(bucket, 0, sizeof(*bucket));

	bucket->used = 2;
	/* first block, is the new key */
	bucket->hb_keys[0].keyoffset = ARRAY_LEN(bucket->hb_keydata) - keylen;
	memcpy(&bucket->hb_keydata[bucket->hb_keys[0].keyoffset], key, keylen);
	bucket->hb_keys[0].next = hashbucket;
	bucket->hb_keys[0].root = 0;
	bucket->hb_keys[0].tail = 0;

	table->buckets[hashentry] = HASH_KEY(keybucket, 0);

	/* next block is a free block, link into free list */
	bucket->hb_keys[1].keyoffset = bucket->hb_keys[0].keyoffset;
	bucket->hb_keys[1].next = hashroot->free;
	hashroot->free = HASH_KEY(keybucket, 1);

	/* link new block into keys list */
	bucket->next = block_number(hashroot->keys);
	hashroot->keys = keybucket;

	ibex_block_dirty((struct _block *)hashroot);
	ibex_block_dirty((struct _block *)table);
	ibex_block_dirty((struct _block *)bucket);

	d(printf(" new key id %d\n", HASH_KEY(keybucket, 0)));
	d(printf(" new free id %d\n", hashroot->free));

	return HASH_KEY(keybucket, 0);
}

/* hash cursor functions */
static struct _IBEXCursor *
hash_cursor_create(struct _IBEXIndex *idx)
{
	struct _HASHCursor *idc;
	struct _hashroot *hashroot;

	idc = g_malloc(sizeof(*idc));
	idc->cursor.klass = &ibex_hash_cursor_class;
	idc->cursor.index = idx;
	idc->key = 0;
	idc->index = 0;

	hashroot = (struct _hashroot *)ibex_block_read(idx->blocks, idx->root);
	idc->size = hashroot->size;
	idc->block = hashroot->keys;
	
	return &idc->cursor;
}

static void
hash_cursor_close(struct _IBEXCursor *idc)
{
	g_free(idc);
}

static guint32
hash_cursor_next(struct _IBEXCursor *idc)
{
	struct _HASHCursor *hc = (struct _HASHCursor *)idc;
	struct _hashblock *bucket;

	while (hc->block != 0) {
		bucket = (struct _hashblock *)ibex_block_read(idc->index->blocks, hc->block);
		while (hc->index < bucket->used) {
			if (hash_info(bucket, hc->index) > 0) {
				hc->key = HASH_KEY(hc->block, hc->index);
				hc->index++;
				if (hc->index == bucket->used) {
					hc->index = 0;
					hc->block = block_location(bucket->next);
				}
				return hc->key;
			}
			hc->index++;
		}
		hc->index = 0;
		hc->block = block_location(bucket->next);
	}

	return hc->block;
}

static char *
hash_cursor_next_key(struct _IBEXCursor *idc, int *keylenptr)
{
	/* TODO: this could be made slightly mroe efficient going to the structs direct.
	   but i'm lazy today */
	return idc->index->klass->get_key(idc->index, idc->klass->next(idc), keylenptr);
}

/* debug */
void ibex_hash_dump(struct _IBEXIndex *index);
static void ibex_hash_dump_rec(struct _IBEXIndex *index, int *words, int *wordslen);

void ibex_hash_dump(struct _IBEXIndex *index)
{
	int words = 0, wordslen=0;

	ibex_hash_dump_rec(index, &words, &wordslen);

	printf("Total words = %d, bytes = %d, ave length = %f\n", words, wordslen, (double)wordslen/(double)words);
}


static void
ibex_hash_dump_rec(struct _IBEXIndex *index, int *words, int *wordslen)
{
	int i;
	struct _hashtableblock *table;
	struct _hashblock *bucket;
	struct _hashroot *hashroot;
	blockid_t hashtable;
	hashid_t hashbucket;
	extern void ibex_diskarray_dump(struct _memcache *blocks, blockid_t head, blockid_t tail);


	printf("Walking hash tree:\n");
	hashroot = (struct _hashroot *)ibex_block_read(index->blocks, index->root);
	for (i=0;i<hashroot->size;i++) {
		printf("Hash table chain: %d\n", i);
		hashtable = hashroot->table[i / (BLOCK_SIZE/sizeof(blockid_t))];
		table = (struct _hashtableblock *)ibex_block_read(index->blocks, hashtable);
		hashbucket = table->buckets[i % (BLOCK_SIZE/sizeof(blockid_t))];
		while (hashbucket) {
			int len;

			*words = *words + 1;

			bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashbucket));
			printf(" bucket %d: [used %d]", hashbucket, bucket->used);
			if (HASH_INDEX(hashbucket) == 0) {
				len = ARRAY_LEN(bucket->hb_keydata) -
					bucket->hb_keys[HASH_INDEX(hashbucket)].keyoffset;
			} else {
				len = bucket->hb_keys[HASH_INDEX(hashbucket)-1].keyoffset -
					bucket->hb_keys[HASH_INDEX(hashbucket)].keyoffset;
			}
			printf("'%.*s' = %d next=%d\n", len, &bucket->hb_keydata[bucket->hb_keys[HASH_INDEX(hashbucket)].keyoffset],
			       bucket->hb_keys[HASH_INDEX(hashbucket)].root,
			       bucket->hb_keys[HASH_INDEX(hashbucket)].next);

			*wordslen = *wordslen + len;

			ibex_diskarray_dump(index->blocks,
					    bucket->hb_keys[HASH_INDEX(hashbucket)].root << BLOCK_BITS,
					    bucket->hb_keys[HASH_INDEX(hashbucket)].tail);

			hashbucket = bucket->hb_keys[HASH_INDEX(hashbucket)].next;
		}
		/* make sure its still in the cache */
		hashroot = (struct _hashroot *)ibex_block_read(index->blocks, index->root);
	}

	hashbucket = hashroot->free;
	printf("Dumping free lists ..\n");
	while (hashbucket) {
		printf(" %d", hashbucket);
		bucket = (struct _hashblock *)ibex_block_read(index->blocks, HASH_BLOCK(hashbucket));
		hashbucket = bucket->hb_keys[HASH_INDEX(hashbucket)].next;
	}
	printf("\n");
}

#if 0
int main(int argc, char **argv)
{
	struct _memcache *bc;
	struct _IBEXIndex *hash;
	int i;

	bc = ibex_block_cache_open("index.db", O_CREAT|O_RDWR, 0600);
	hash = ibex_hash_class.create(bc, 1024);
	for (i=0;i<10000;i++) {
		char key[16];
		sprintf(key, "key %d", i);
		ibex_hash_class.insert(hash, key, strlen(key));
	}

	for (i=500;i<1000;i++) {
		char key[16];
		sprintf(key, "key %d", i);
		ibex_hash_class.remove(hash, key, strlen(key));
	}

	for (i=500;i<1000;i++) {
		char key[16];
		sprintf(key, "key %d", i);
		ibex_hash_class.insert(hash, key, strlen(key));
	}


	ibex_hash_dump(hash);

	for (i=0;i<2000;i++) {
		char key[16], *lookup;
		hashid_t keyid;
		blockid_t root, tail;

		sprintf(key, "key %d", i);
		keyid = ibex_hash_class.find(hash, key, strlen(key));
		lookup = ibex_hash_class.get_key(hash, keyid, 0);
		root = ibex_hash_class.get_data(hash, keyid, &tail);
		printf("key %s = %d = '%s' root:%d tail:%d \n", key, keyid, lookup, root, tail);
		g_free(lookup);
	}

	ibex_hash_class.close(hash);
	ibex_block_cache_close(bc);
	return 0;
}

#endif