diff -Nup 1.2.4/Makefile.am memcached-tag_1.2.4.1/Makefile.am --- 1.2.4/Makefile.am 2008-02-05 11:09:17.000000000 +0800 +++ memcached-tag_1.2.4.1/Makefile.am 2008-02-03 15:11:38.000000000 +0800 @@ -1,6 +1,6 @@ bin_PROGRAMS = memcached memcached-debug -memcached_SOURCES = memcached.c slabs.c slabs.h items.c items.h assoc.c assoc.h memcached.h thread.c stats.c stats.h +memcached_SOURCES = memcached.c slabs.c slabs.h items.c items.h assoc.c assoc.h memcached.h thread.c stats.c stats.h tag.c tag.h splaytree.c splaytree.h memcached_debug_SOURCES = $(memcached_SOURCES) memcached_CPPFLAGS = -DNDEBUG memcached_LDADD = @LIBOBJS@ diff -Nup 1.2.4/items.c memcached-tag_1.2.4.1/items.c --- 1.2.4/items.c 2008-02-05 11:09:17.000000000 +0800 +++ memcached-tag_1.2.4.1/items.c 2008-02-03 15:11:38.000000000 +0800 @@ -83,6 +83,8 @@ item *do_item_alloc(char *key, const siz item *it; char suffix[40]; size_t ntotal = item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix); + snode *key_sn = NULL; + int size = 0; unsigned int id = slabs_clsid(ntotal); if (id == 0) @@ -116,6 +118,21 @@ item *do_item_alloc(char *key, const siz stats.evictions++; STATS_UNLOCK(); } + //tag related + //reverse delete key + size = sizeof(snode) + search->nkey + 1; + key_sn = calloc(1, size); + if (NULL == key_sn) { + fprintf(stderr, "SERVER_ERROR out of memory"); + return; + } + key_sn->nstr = search->nkey; + strncpy(GET_name(key_sn), ITEM_key(search), search->nkey); + GET_name(key_sn)[search->nkey] = '\0'; /* because strncpy() sucks */ + tag_reverse_del_key(search->tags, key_sn); + free(key_sn); + key_sn = NULL; + //tag related do_item_unlink(search); break; } @@ -130,7 +147,7 @@ item *do_item_alloc(char *key, const siz assert(it != heads[it->slabs_clsid]); - it->next = it->prev = it->h_next = 0; + it->next = it->prev = it->h_next = it->tags = 0; it->refcount = 1; /* the caller will have a reference */ DEBUG_REFCNT(it, '*'); it->it_flags = 0; diff -Nup 1.2.4/memcached.c memcached-tag_1.2.4.1/memcached.c --- 1.2.4/memcached.c 2008-02-05 11:09:17.000000000 +0800 +++ memcached-tag_1.2.4.1/memcached.c 2008-02-03 15:11:38.000000000 +0800 @@ -145,6 +145,7 @@ static void stats_init(void) { stats.curr_items = stats.total_items = stats.curr_conns = stats.total_conns = stats.conn_structs = 0; stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = stats.evictions = 0; stats.curr_bytes = stats.bytes_read = stats.bytes_written = 0; + stats.tag_add_cmds = stats.tag_delete_cmds = 0; /* make the time we started always be 2 seconds before we really did, so time(0) - time.started is never zero. if so, things @@ -159,6 +160,7 @@ static void stats_reset(void) { stats.total_items = stats.total_conns = 0; stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = stats.evictions = 0; stats.bytes_read = stats.bytes_written = 0; + stats.tag_add_cmds = stats.tag_delete_cmds = 0; stats_prefix_clear(); STATS_UNLOCK(); } @@ -184,6 +186,7 @@ static void settings_init(void) { #endif settings.prefix_delimiter = ':'; settings.detail_enabled = 0; + settings.is_ignore_empty = false; //tag related } /* returns true if a deleted item's delete-locked-time is over, and it @@ -763,6 +766,8 @@ static void complete_nread(conn *c) { item *it = c->item; int comm = c->item_comm; int ret; + int length = -1; + int flag = -1; STATS_LOCK(); stats.set_cmds++; @@ -772,8 +777,23 @@ static void complete_nread(conn *c) { out_string(c, "CLIENT_ERROR bad data chunk"); } else { ret = store_item(it, comm); - if (ret == 1) - out_string(c, "STORED"); + //tag related + if (ret == 1) { + sscanf(ITEM_suffix(it), " %d %d", &flag, &length); + if (0 == length && true == settings.is_ignore_empty) { + STATS_LOCK(); + stats.set_cmds--; + STATS_UNLOCK(); + + item_unlink(c->item); //not store empty key + + out_string(c, "NOT_STORED"); + } + else { + out_string(c, "STORED"); + } + } + //tag related else if(ret == 2) out_string(c, "EXISTS"); else if(ret == 3) @@ -895,6 +915,7 @@ typedef struct token_s { #define COMMAND_TOKEN 0 #define SUBCOMMAND_TOKEN 1 #define KEY_TOKEN 1 +#define TAG_BEGINKEY_TOKEN 2 #define KEY_MAX_LENGTH 250 #define MAX_TOKENS 7 @@ -983,8 +1004,13 @@ inline static void process_stats_detail( char *stats = stats_prefix_dump(&len); write_and_free(c, stats, len); } + else if (strcmp(command, "tag_dump") == 0) { + int len; + char *stats = tag_dump(&len); + write_and_free(c, stats, len); + } else { - out_string(c, "CLIENT_ERROR usage: stats detail on|off|dump"); + out_string(c, "CLIENT_ERROR usage: stats detail on|off|dump|tag_dump"); } } @@ -1032,6 +1058,8 @@ static void process_stat(conn *c, token_ pos += sprintf(pos, "STAT cmd_set %llu\r\n", stats.set_cmds); pos += sprintf(pos, "STAT get_hits %llu\r\n", stats.get_hits); pos += sprintf(pos, "STAT get_misses %llu\r\n", stats.get_misses); + pos += sprintf(pos, "STAT cmd_tag_add %llu\r\n", stats.tag_add_cmds); //tag related + pos += sprintf(pos, "STAT cmd_tag_delete %llu\r\n", stats.tag_delete_cmds); //tag related pos += sprintf(pos, "STAT evictions %llu\r\n", stats.evictions); pos += sprintf(pos, "STAT bytes_read %llu\r\n", stats.bytes_read); pos += sprintf(pos, "STAT bytes_written %llu\r\n", stats.bytes_written); @@ -1408,6 +1436,62 @@ static void process_update_command(conn conn_set_state(c, conn_nread); } +//tag related +static void process_tag_add_command(conn *c, token_t *tokens, const size_t ntokens) { + char *tag_name = NULL; + size_t ntag = 0; + char *key_name = NULL; + size_t nkey = 0; + token_t *key_token = NULL; + int ret = 0; + + assert(c != NULL && tokens != NULL); + + tag_name = tokens[KEY_TOKEN].value; + ntag = tokens[KEY_TOKEN].length; + if (ntag > KEY_MAX_LENGTH) { + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + + STATS_LOCK(); + stats.tag_add_cmds++; + STATS_UNLOCK(); + + key_token = &tokens[TAG_BEGINKEY_TOKEN]; + do { + for(; key_token->length != 0; key_token++) { + key_name = key_token->value; + nkey = key_token->length; + + if (nkey > KEY_MAX_LENGTH || ntag > KEY_MAX_LENGTH) { + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + + ret = tag_insert(tag_name, ntag, key_name, nkey); + if (0 == ret) { + out_string(c, "SERVER_ERROR out of memory or param error"); + return; + } + } + + /* + * If the command string hasn't been fully processed, get the next set + * of tokens. + */ + if(key_token->value != NULL) { + tokenize_command(key_token->value, tokens, MAX_TOKENS); + key_token = tokens; + } + + } while(key_token->value != NULL); + + out_string(c, "TAG_STORED"); + return; +} +//tag related + static void process_arithmetic_command(conn *c, token_t *tokens, const size_t ntokens, const bool incr) { char temp[sizeof("18446744073709551615")]; item *it; @@ -1510,6 +1594,8 @@ static void process_delete_command(conn size_t nkey; item *it; time_t exptime = 0; + snode *ptag = NULL, *pnext = NULL, *key_sn = NULL; + int size = 0; assert(c != NULL); @@ -1549,6 +1635,22 @@ static void process_delete_command(conn it = item_get(key, nkey); if (it) { + //tag related + //reverse delete key + size = sizeof(snode) + nkey + 1; + key_sn = calloc(1, size); + if (NULL == key_sn) { + fprintf(stderr, "SERVER_ERROR out of memory"); + return; + } + key_sn->nstr = nkey; + strncpy(GET_name(key_sn), key, nkey); + GET_name(key_sn)[nkey] = '\0'; /* because strncpy() sucks */ + tag_reverse_del_key(it->tags, key_sn); + free(key_sn); + key_sn = NULL; + //tag related + if (exptime == 0) { item_unlink(it); item_remove(it); /* release our reference */ @@ -1562,6 +1664,35 @@ static void process_delete_command(conn } } +//tag related +static void process_tag_delete_command(conn *c, token_t *tokens, const size_t ntokens) { + char *tag_name = NULL; + size_t ntag = 0; + + assert(c != NULL && tokens != NULL); + + tag_name = tokens[KEY_TOKEN].value; + ntag = tokens[KEY_TOKEN].length; + if(ntag > KEY_MAX_LENGTH) { + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + + STATS_LOCK(); + stats.tag_delete_cmds++; + STATS_UNLOCK(); + + if (true == tag_delete(tag_name, ntag)) { + out_string(c, "TAG_DELETED"); + } + else { + out_string(c, "TAG_NOT_FOUND"); + } + + return; +} +//tag related + /* * Adds an item to the deferred-delete list so it can be reaped later. * @@ -1790,6 +1921,16 @@ static void process_command(conn *c, cha #endif } else if (ntokens == 3 && (strcmp(tokens[COMMAND_TOKEN].value, "verbosity") == 0)) { process_verbosity_command(c, tokens, ntokens); +//tag related + } else if (ntokens >= 4 && (strcmp(tokens[COMMAND_TOKEN].value, "tag_add") == 0)) { + + process_tag_add_command(c, tokens, ntokens); + + } else if (ntokens == 3 && (strcmp(tokens[COMMAND_TOKEN].value, "tag_delete") == 0)) { + + process_tag_delete_command(c, tokens, ntokens); + +//tag related } else { out_string(c, "ERROR"); } @@ -2552,7 +2693,8 @@ static void usage(void) { "-b run a managed instanced (mnemonic: buckets)\n" "-P save PID in , only used with -d option\n" "-f chunk size growth factor, default 1.25\n" - "-n minimum space allocated for key+value+flags, default 48\n"); + "-n minimum space allocated for key+value+flags, default 48\n" + "-e ignore key with empty value when set/add/replace\n"); #ifdef USE_THREADS printf("-t number of threads to use, default 4\n"); #endif @@ -2685,7 +2827,7 @@ int main (int argc, char **argv) { setbuf(stderr, NULL); /* process arguments */ - while ((c = getopt(argc, argv, "a:bp:s:U:m:Mc:khirvdl:u:P:f:s:n:t:D:")) != -1) { + while ((c = getopt(argc, argv, "a:bp:s:U:m:Mc:khirvdl:u:P:f:s:n:t:D:e")) != -1) { switch (c) { case 'a': /* access for unix domain socket, as octal mask (like chmod)*/ @@ -2774,6 +2916,9 @@ int main (int argc, char **argv) { settings.prefix_delimiter = optarg[0]; settings.detail_enabled = 1; break; + case 'e': //tag related + settings.is_ignore_empty = true; + break; default: fprintf(stderr, "Illegal argument \"%c\"\n", c); return 1; @@ -2896,6 +3041,7 @@ int main (int argc, char **argv) { stats_init(); assoc_init(); conn_init(); + tag_init(); //tag related /* Hacky suffix buffers. */ suffix_init(); slabs_init(settings.maxbytes, settings.factor); diff -Nup 1.2.4/memcached.h memcached-tag_1.2.4.1/memcached.h --- 1.2.4/memcached.h 2008-02-05 11:09:17.000000000 +0800 +++ memcached-tag_1.2.4.1/memcached.h 2008-02-03 15:11:38.000000000 +0800 @@ -56,6 +56,8 @@ # include #endif +#include "splaytree.h" + /** Time relative to server start. Smaller than time_t on 64-bit systems. */ typedef unsigned int rel_time_t; @@ -70,6 +72,8 @@ struct stats { uint64_t set_cmds; uint64_t get_hits; uint64_t get_misses; + uint64_t tag_add_cmds; //tag related + uint64_t tag_delete_cmds; //tag related uint64_t evictions; time_t started; /* when the process was started */ uint64_t bytes_read; @@ -95,11 +99,36 @@ struct settings { int num_threads; /* number of libevent threads to run */ char prefix_delimiter; /* character that marks a key prefix (for stats) */ int detail_enabled; /* nonzero if we're collecting detailed stats */ + bool is_ignore_empty; /* is ignore key with empty value */ }; extern struct stats stats; extern struct settings settings; +//tag related +typedef struct _str_node { + struct _str_node *next; + uint8_t nstr; /* length of key or tag */ + char unused[3]; + void * end[]; + /* then null-terminated key */ +} snode; + +#define GET_name(snode) ((char*)&((snode)->end[0])) + +typedef struct _strtag { + splay_tree *keys; /* key */ + struct _strtag *h_next; /* hash chain next */ + bool is_deling;/* is deleting */ + uint8_t ntag; /* length of tag */ + char unused[2]; + void * end[]; + /* then null-terminated tag */ +} tag; + +#define TAG_name(tag) ((char*)&((tag)->end[0])) +//tag related + #define ITEM_LINKED 1 #define ITEM_DELETED 2 @@ -110,6 +139,7 @@ typedef struct _stritem { struct _stritem *next; struct _stritem *prev; struct _stritem *h_next; /* hash chain next */ + splay_tree *tags; /* the tags include the item -- tag related */ rel_time_t time; /* least recent access */ rel_time_t exptime; /* expire time */ int nbytes; /* size of data */ @@ -249,6 +279,7 @@ conn *conn_new(const int sfd, const int #include "slabs.h" #include "assoc.h" #include "items.h" +#include "tag.h" /* @@ -325,6 +356,13 @@ int mt_store_item(item *item, int comm # define slabs_reassign(x,y) mt_slabs_reassign(x,y) # define slabs_stats(x) mt_slabs_stats(x) # define store_item(x,y) mt_store_item(x,y) +//tag related +# define tag_insert(x,y,z,a) mt_tag_insert(x,y,z,a) +# define tag_delete(x,y) mt_tag_delete(x,y) +# define tag_find(x,y) mt_tag_find(x,y) +# define tag_dump(x) mt_tag_dump(x) +# define tag_reverse_del_key(x,y) mt_tag_reverse_del_key(x,y) +//tag related # define STATS_LOCK() mt_stats_lock() # define STATS_UNLOCK() mt_stats_unlock() @@ -359,6 +397,13 @@ int mt_store_item(item *item, int comm # define slabs_stats(x) do_slabs_stats(x) # define store_item(x,y) do_store_item(x,y) # define thread_init(x,y) 0 +//tag related +# define tag_insert(x,y,z,a) do_tag_insert(x,y,z,a) +# define tag_delete(x,y) do_tag_delete(x,y) +# define tag_find(x,y) do_tag_find(x,y) +# define tag_dump(x) do_tag_dump(x) +# define tag_reverse_del_key(x,y) do_tag_reverse_del_key(x,y) +//tag related # define STATS_LOCK() /**/ # define STATS_UNLOCK() /**/ Common subdirectories: 1.2.4/scripts and memcached-tag_1.2.4.1/scripts diff -Nup 1.2.4/splaytree.c memcached-tag_1.2.4.1/splaytree.c --- 1.2.4/splaytree.c 1970-01-01 08:00:00.000000000 +0800 +++ memcached-tag_1.2.4.1/splaytree.c 2008-02-03 15:11:38.000000000 +0800 @@ -0,0 +1,263 @@ +/* + An implementation of top-down splaying with sizes + D. Sleator , January 1994. + + This extends top-down-splay.c to maintain a size field in each node. + This is the number of nodes in the subtree rooted there. This makes + it possible to efficiently compute the rank of a key. (The rank is + the number of nodes to the left of the given key.) It it also + possible to quickly find the node of a given rank. Both of these + operations are illustrated in the code below. The remainder of this + introduction is taken from top-down-splay.c. + + "Splay trees", or "self-adjusting search trees" are a simple and + efficient data structure for storing an ordered set. The data + structure consists of a binary tree, with no additional fields. It + allows searching, insertion, deletion, deletemin, deletemax, + splitting, joining, and many other operations, all with amortized + logarithmic performance. Since the trees adapt to the sequence of + requests, their performance on real access patterns is typically even + better. Splay trees are described in a number of texts and papers + [1,2,3,4]. + + The code here is adapted from simple top-down splay, at the bottom of + page 669 of [2]. It can be obtained via anonymous ftp from + spade.pc.cs.cmu.edu in directory /usr/sleator/public. + + The chief modification here is that the splay operation works even if the + item being splayed is not in the tree, and even if the tree root of the + tree is NULL. So the line: + + t = splay(i, t); + + causes it to search for item with key i in the tree rooted at t. If it's + there, it is splayed to the root. If it isn't there, then the node put + at the root is the last one before NULL that would have been reached in a + normal binary search for i. (It's a neighbor of i in the tree.) This + allows many other operations to be easily implemented, as shown below. + + [1] "Data Structures and Their Algorithms", Lewis and Denenberg, + Harper Collins, 1991, pp 243-251. + [2] "Self-adjusting Binary Search Trees" Sleator and Tarjan, + JACM Volume 32, No 3, July 1985, pp 652-686. + [3] "Data Structure and Algorithm Analysis", Mark Weiss, + Benjamin Cummins, 1992, pp 119-130. + [4] "Data Structures, Algorithms, and Performance", Derick Wood, + Addison-Wesley, 1993, pp 367-375 +*/ + +#include +#include +#include "splaytree.h" +#include "memcached.h" + +#define compare(i,j) ((i)-(j)) +/* This is the comparison. */ +/* Returns <0 if i0 if i>j */ + +#define node_size splaytree_size + +/* Splay using the key i (which may or may not be in the tree.) + * The starting root is t, and the tree used is defined by rat + * size fields are maintained */ +splay_tree * splaytree_splay (splay_tree *t, long long i) { + splay_tree N, *l, *r, *y; + long long comp, root_size, l_size, r_size; + + if (t == NULL) return t; + N.left = N.right = NULL; + l = r = &N; + root_size = node_size(t); + l_size = r_size = 0; + + for (;;) { + comp = compare(i, t->key); + if (comp < 0) { + if (t->left == NULL) break; + if (compare(i, t->left->key) < 0) { + y = t->left; /* rotate right */ + t->left = y->right; + y->right = t; + t->size = node_size(t->left) + node_size(t->right) + 1; + t = y; + if (t->left == NULL) break; + } + r->left = t; /* link right */ + r = t; + t = t->left; + r_size += 1+node_size(r->right); + } else if (comp > 0) { + if (t->right == NULL) break; + if (compare(i, t->right->key) > 0) { + y = t->right; /* rotate left */ + t->right = y->left; + y->left = t; + t->size = node_size(t->left) + node_size(t->right) + 1; + t = y; + if (t->right == NULL) break; + } + l->right = t; /* link left */ + l = t; + t = t->right; + l_size += 1+node_size(l->left); + } else { + break; + } + } + l_size += node_size(t->left); /* Now l_size and r_size are the sizes of */ + r_size += node_size(t->right); /* the left and right trees we just built.*/ + t->size = l_size + r_size + 1; + + l->right = r->left = NULL; + + /* The following two loops correct the size fields of the right path */ + /* from the left child of the root and the right path from the left */ + /* child of the root. */ + for (y = N.right; y != NULL; y = y->right) { + y->size = l_size; + l_size -= 1+node_size(y->left); + } + for (y = N.left; y != NULL; y = y->left) { + y->size = r_size; + r_size -= 1+node_size(y->right); + } + + l->right = t->left; /* assemble */ + r->left = t->right; + t->left = N.right; + t->right = N.left; + + return t; +} + +splay_tree * splaytree_insert(splay_tree * t, long long i, void *data) { +/* Insert key i into the tree t, if it is not already there. */ +/* Return a pointer to the resulting tree. */ + splay_tree * new; + snode *sn1 = NULL, *sn2 = NULL; + bool is_repeat = false; + + if (t != NULL) { + t = splaytree_splay(t, i); + if (compare(i, t->key)==0) { + //tag related + sn1 = (snode*)data; + sn2 = (snode*)t->data; + for(; NULL!=sn2; sn2=sn2->next) { + if (sn1->nstr == sn2->nstr + && 0 == memcmp(GET_name(sn1), GET_name(sn2), sn1->nstr)) { + is_repeat = true; + break; + } + } + if(false == is_repeat) { + //same hash value, but different string + sn1->next = t->data; + t->data = (void*)sn1; + } + else { + free(data); + data = NULL; + } + //tag related + return t; /* it's already there */ + } + } + new = (splay_tree *) malloc (sizeof (splay_tree)); + assert(new); + if (t == NULL) { + new->left = new->right = NULL; + } else if (compare(i, t->key) < 0) { + new->left = t->left; + new->right = t; + t->left = NULL; + t->size = 1+node_size(t->right); + } else { + new->right = t->right; + new->left = t; + t->right = NULL; + t->size = 1+node_size(t->left); + } + new->key = i; + new->data = data; + new->size = 1 + node_size(new->left) + node_size(new->right); + return new; +} + +splay_tree * splaytree_delete(splay_tree *t, long long i, void *data) { +/* Deletes i from the tree if it's there. */ +/* Return a pointer to the resulting tree. */ + splay_tree * x; + long long tsize; + snode *sn1 = NULL, *sn2 = NULL, *pre = NULL; + bool is_empty = false; + + if (t==NULL) return NULL; + tsize = t->size; + t = splaytree_splay(t, i); + if (compare(i, t->key) == 0) { /* found it */ + //tag related + sn1 = (snode*)data; + sn2 = (snode*)t->data; + if (sn1->nstr == sn2->nstr + && 0 == memcmp(GET_name(sn1), GET_name(sn2), sn1->nstr)) { + //head node + t->data = ((snode*)t->data)->next; + free(sn2); + sn2 = NULL; + if(NULL == t->data) + is_empty = true; + } + else { + //normal node + for (pre=sn2, sn2=sn2->next; NULL!=sn2; pre=sn2, sn2=sn2->next) { + if (sn1->nstr == sn2->nstr + && 0 == memcmp(GET_name(sn1), GET_name(sn2), sn1->nstr)) { + pre->next = sn2->next; + free(sn2); + sn2 = NULL; + break; + } + } + } + if (false == is_empty) { + return t; + } + //tag related + if (t->left == NULL) { + x = t->right; + } else { + x = splaytree_splay(t->left, i); + x->right = t->right; + } + free(t); + if (x != NULL) { + x->size = tsize-1; + } + return x; + } else { + return t; /* It wasn't there */ + } +} + +splay_tree *find_rank(long long r, splay_tree *t) { +/* Returns a pointer to the node in the tree with the given rank. */ +/* Returns NULL if there is no such node. */ +/* Does not change the tree. To guarantee logarithmic behavior, */ +/* the node found here should be splayed to the root. */ + long long lsize; + if ((r < 0) || (r >= node_size(t))) return NULL; + for (;;) { + lsize = node_size(t->left); + if (r < lsize) { + t = t->left; + } else if (r > lsize) { + r = r - lsize -1; + t = t->right; + } else { + return t; + } + } +} + + diff -Nup 1.2.4/splaytree.h memcached-tag_1.2.4.1/splaytree.h --- 1.2.4/splaytree.h 1970-01-01 08:00:00.000000000 +0800 +++ memcached-tag_1.2.4.1/splaytree.h 2008-02-03 15:11:38.000000000 +0800 @@ -0,0 +1,24 @@ +#ifndef _SPLAY_TREE_H_ +#define _SPLAY_TREE_H_ + +typedef struct tree_node { + struct tree_node * left, * right; + long long key; + long size; /* maintained to be the number of nodes rooted here */ + + void *data; +} splay_tree; + + +splay_tree * splaytree_splay (splay_tree *t, long long key); +splay_tree * splaytree_insert(splay_tree *t, long long key, void *data); +splay_tree * splaytree_delete(splay_tree *t, long long key, void *data); +splay_tree * splaytree_size(splay_tree *t); + +#define splaytree_size(x) (((x)==NULL) ? 0 : ((x)->size)) +/* This macro returns the size of a node. Unlike "x->size", */ +/* it works even if x=NULL. The test could be avoided by using */ +/* a special version of NULL which was a real node with size 0. */ + + +#endif Common subdirectories: 1.2.4/t and memcached-tag_1.2.4.1/t diff -Nup 1.2.4/tag.c memcached-tag_1.2.4.1/tag.c --- 1.2.4/tag.c 1970-01-01 08:00:00.000000000 +0800 +++ memcached-tag_1.2.4.1/tag.c 2008-02-03 15:11:38.000000000 +0800 @@ -0,0 +1,367 @@ +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Tag function for Memcached. + * + * $Id$ + */ +#include +#include +#include +#include +#include +#ifdef HAVE_MALLOC_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif +#include "memcached.h" +#include "splaytree.h" + +/******** function declaration ********/ + +void tag_init(void); +int do_tag_insert(char *tag_name, const uint8_t ntag, char *key_name, const uint8_t nkey); +bool do_tag_delete(const char *tag_name, const uint8_t ntag); +tag *do_tag_find(const char *tag_name, const uint8_t ntag); +static tag** _hashtag_before (const char *tag_name, const uint8_t ntag); +char *do_tag_dump(int *len); +void do_tag_reverse_del_key(splay_tree *tags_tree, snode* key_sn); + +/******** variable definition ********/ + +typedef unsigned long int ub4; /* unsigned 4-byte quantities */ +typedef unsigned char ub1; /* unsigned 1-byte quantities */ + +/* how many powers of 2's worth of buckets we use */ +static unsigned int tag_hashpower = 16; + +#define hashsize(n) ((ub4)1<<(n)) +#define hashmask(n) (hashsize(n)-1) + +/* Hash table. This is where we look. */ +static tag** hashtable = 0; + +/* Number of tags in the hash table. */ +static unsigned int hash_tags = 0; + +/******** function definition ********/ + +void tag_init(void) { + unsigned int hash_size = hashsize(tag_hashpower) * sizeof(void*); + hashtable = calloc(1, hash_size); + if (!hashtable) { + fprintf(stderr, "Failed to init hashtable.\n"); + exit(EXIT_FAILURE); + } +} + +/* +return : + 0 - failed + 1 - succ +*/ +int do_tag_insert(char *tag_name, const uint8_t ntag, char *key_name, const uint8_t nkey) { + long long hv = 0; + unsigned int bucket = 0; + snode *sn = NULL; + tag *ta = NULL; + item *it = NULL; + int size = 0; + + if (NULL == tag_name || '\0' == tag_name[0] || NULL == key_name || '\0' == key_name[0]) { + fprintf(stderr, "error params\n"); + return 0; + } + + //reverse tree : item->tag + it = assoc_find(key_name, nkey); + if (it) { + size = sizeof(snode) + ntag + 1; + sn = calloc(1, size); + if (NULL == sn) { + fprintf(stderr, "Out of memory in do_tag_insert 3\n"); + return 0; + } + sn->nstr = ntag; + strncpy(GET_name(sn), tag_name, ntag); + GET_name(sn)[ntag] = '\0'; /* because strncpy() sucks */ + + //add tag name into item struct + hv = hash(tag_name, ntag, 0); + it->tags = splaytree_insert(it->tags, hv, (void*)sn); + } + else { + //item not exist, just ignore it! return succ. + return 1; + } + + //tag->key + ta = do_tag_find(tag_name, ntag); + if (NULL == ta) { + size = sizeof(tag) + ntag + 1; + ta = calloc(1, size); + if (NULL == ta) { + fprintf(stderr, "Out of memory in do_tag_insert 1 \n"); + return 0; + } + ta->ntag = ntag; + strncpy(TAG_name(ta), tag_name, ntag); + TAG_name(ta)[ntag] = '\0'; /* because strncpy() sucks */ + + // add tag into hash + hv = hash(TAG_name(ta), ta->ntag, 0); + bucket = hv & hashmask(tag_hashpower - 1); + ta->h_next = hashtable[bucket]; + hashtable[bucket] = ta; + hash_tags++; + } + + size = sizeof(snode) + nkey + 1; + sn = calloc(1, size); + if (NULL == sn) { + fprintf(stderr, "Out of memory in do_tag_insert 2\n"); + return 0; + } + sn->nstr = nkey; + strncpy(GET_name(sn), key_name, nkey); + GET_name(sn)[nkey] = '\0'; /* because strncpy() sucks */ + + //add key name for tag + hv = hash(key_name, nkey, 0); + ta->keys = splaytree_insert(ta->keys, hv, (void*)sn); + + return 1; +} + +/* +return: + false - failed + true - succ +*/ +bool do_tag_delete(const char *tag_name, const uint8_t ntag) { + tag **before = NULL, *prev = NULL, *nxt = NULL, *pt = NULL; + snode *key_sn = NULL, *temp = NULL, *ptsn = NULL, *pnext = NULL; + item *it = NULL; + char *key_name = NULL, *other_tname = NULL; + uint8_t nkey = 0, other_ntag = 0; + int size = 0; + long long hv = 0; + splay_tree *root = NULL; + + if (NULL == tag_name || '\0' == tag_name[0]) { + fprintf(stderr, "error params in do_tag_delete\n"); + return 0; + } + + before = _hashtag_before(tag_name, ntag); + prev = *before; + if (prev) { + nxt = prev->h_next; + prev->h_next = 0; + root = prev->keys; + prev->is_deling = true; + + while(NULL != root) { + key_sn = (snode*)root->data; + + key_name = GET_name(key_sn); + nkey = key_sn->nstr; + + if (settings.detail_enabled) { + stats_prefix_record_delete(key_name); + } + + it = assoc_find(key_name, nkey); + if (it) { + //reverse del + do_tag_reverse_del_key(it->tags, key_sn); + + //delete item + item_unlink(it); + item_remove(it); // release our reference + } + + //delete key_name + root = splaytree_delete(root, root->key, root->data); + } + + //delete tag from hash + free(prev); + *before = prev = NULL; + + *before = nxt; + hash_tags--; + } + else { + return false; //not found + } + + return true; //delete +} + +void do_tag_reverse_del_key(splay_tree *tags_tree, snode* key_sn) { + tag *ta = NULL, **before = NULL, *prev = NULL, *nxt = NULL; + long long hv = 0; + char *tag_name = NULL, *key_name = NULL; + uint8_t ntag = 0, nkey = 0; + snode *sn = NULL; + + if (NULL == tags_tree) { + return; + } + do_tag_reverse_del_key(tags_tree->left, key_sn); + do_tag_reverse_del_key(tags_tree->right, key_sn); + + if (NULL == key_sn) { + fprintf(stderr, "error params in do_tag_reverse_del_key\n"); + return; + } + key_name = GET_name(key_sn); + nkey = key_sn->nstr; + + for(sn=(snode*)tags_tree->data; NULL!=sn; sn=sn->next) { + tag_name = GET_name(sn); + ntag = sn->nstr; + + ta = do_tag_find(tag_name, ntag); + if (ta) { + if (true == ta->is_deling) { + return; + } + hv = hash(key_name, nkey, 0); + ta->keys = splaytree_delete(ta->keys, hv, (void*)key_sn); + + //delete empty tag + if (NULL == ta->keys) { + before = _hashtag_before(tag_name, ntag); + prev = *before; + + if (prev) { + //delete tag from hash + nxt = prev->h_next; + + free(prev); + prev = NULL; + + *before = nxt; + hash_tags--; + } + } + } + } + + return; +} + +/* +return : + NULL - not found + other - found +*/ +/* returns the address of the tag pointer before the tag. if *tag == 0, + the tag wasn't found */ +static tag** _hashtag_before (const char *tag_name, const uint8_t ntag) { + long long hv = 0; + tag **pos = NULL; + unsigned int bucket = 0; + + if (NULL == tag_name || '\0' == tag_name[0]) { + fprintf(stderr, "error params\n"); + return 0; + } + + hv = hash(tag_name, ntag, 0); + bucket = hv & hashmask(tag_hashpower - 1); + pos = &hashtable[bucket]; + + while (*pos && + ((ntag != (*pos)->ntag) || memcmp(tag_name, TAG_name(*pos), ntag))) { + pos = &(*pos)->h_next; + } + + return pos; +} + +/* +return: + NULL - not found + other - found +*/ +tag *do_tag_find(const char *tag_name, const uint8_t ntag) { + long long hv = 0; + tag *ta = NULL; + unsigned int bucket = 0; + + if (NULL == tag_name || '\0' == tag_name[0]) { + fprintf(stderr, "error params\n"); + return 0; + } + + hv = hash(tag_name, ntag, 0); + bucket = hv & hashmask(tag_hashpower - 1); + ta = hashtable[bucket]; + + while (ta) { + if ((ntag == ta->ntag) && + (0 == memcmp(tag_name, TAG_name(ta), ntag))) { + return ta; + } + ta = ta->h_next; + } + return 0; +} + +#define MAX_BUF_SIZE 8192 +void spt_recursion(splay_tree *sp, char *buf, int *pos) { + snode *sn = NULL; + + if(NULL == sp || NULL == buf || NULL == pos) + return; + + for(sn=(snode*)sp->data; NULL!=sn; sn=sn->next) { + (*pos) += snprintf(buf+(*pos), MAX_BUF_SIZE-(*pos), " %s", GET_name(sn)); + } + + spt_recursion(sp->left, buf, pos); + spt_recursion(sp->right, buf, pos); + + return; +} + +/* +return: + NULL - fail + NOT-NULL - succ +*/ +char *do_tag_dump(int *len) { + char *buf = NULL; + tag *ta = NULL; + snode *sn = NULL; + splay_tree *sp = NULL; + int pos = 0; + int i = 0; + + buf = calloc(1, MAX_BUF_SIZE); + if (0 == buf) { + fprintf(stderr, "OUT OF MEMORY in do_tag_dump\n"); + return NULL; + } + + pos += snprintf(buf+pos, MAX_BUF_SIZE-pos, "\r\n"); + + for (i = 0; i < hashsize(tag_hashpower); i++) { + for (ta = hashtable[i]; NULL != ta; ta = ta->h_next) { + pos += snprintf(buf+pos, MAX_BUF_SIZE-pos, "%s :", TAG_name(ta)); + spt_recursion(ta->keys, buf, &pos); + pos += snprintf(buf+pos, MAX_BUF_SIZE-pos, "\r\n"); + } + } + + pos += snprintf(buf+pos, MAX_BUF_SIZE-pos, "END\r\n"); + + *len = pos; + + return buf; +} + diff -Nup 1.2.4/tag.h memcached-tag_1.2.4.1/tag.h --- 1.2.4/tag.h 1970-01-01 08:00:00.000000000 +0800 +++ memcached-tag_1.2.4.1/tag.h 2008-02-03 15:11:38.000000000 +0800 @@ -0,0 +1,15 @@ +#ifndef TAG_H +#define TAG_H + +#include "assoc.h" + +/******** function declaration ********/ +void tag_init(void); +int do_tag_insert(char *tag_name, const uint8_t ntag, char *key_name, const uint8_t nkey); +bool do_tag_delete(const char *tag_name, const uint8_t ntag); +tag *do_tag_find(const char *tag_name, const uint8_t ntag); +char *do_tag_dump(int *len); +void do_tag_reverse_del_key(splay_tree *tags_tree, snode* key_sn); + +#endif //TAG_H + diff -Nup 1.2.4/thread.c memcached-tag_1.2.4.1/thread.c --- 1.2.4/thread.c 2008-02-05 11:09:17.000000000 +0800 +++ memcached-tag_1.2.4.1/thread.c 2008-02-03 15:11:38.000000000 +0800 @@ -63,6 +63,11 @@ static pthread_mutex_t stats_lock; static CQ_ITEM *cqi_freelist; static pthread_mutex_t cqi_freelist_lock; +//tag related +/* Lock for tag */ +static pthread_mutex_t tag_lock; +//tag related + /* * Each libevent instance has a wakeup pipe, which other threads * can use to signal that they've put a new connection on its queue. @@ -636,6 +641,8 @@ void thread_init(int nthreads, struct ev pthread_mutex_init(&cqi_freelist_lock, NULL); cqi_freelist = NULL; + pthread_mutex_init(&tag_lock, NULL); //tag related + threads = malloc(sizeof(LIBEVENT_THREAD) * nthreads); if (! threads) { perror("Can't allocate thread descriptors"); @@ -672,4 +679,50 @@ void thread_init(int nthreads, struct ev pthread_mutex_unlock(&init_lock); } +//tag related +/******************************* TAG ******************************/ +int mt_tag_insert(char *tag_name, const size_t ntag, char *key_name, const size_t nkey) { + int ret = 0; + + pthread_mutex_lock(&tag_lock); + ret = do_tag_insert(tag_name, ntag, key_name, nkey); + pthread_mutex_unlock(&tag_lock); + return ret; +} + +bool mt_tag_delete(const char *tag_name, const size_t ntag) { + bool ret = false; + + pthread_mutex_lock(&tag_lock); + ret = do_tag_delete(tag_name, ntag); + pthread_mutex_unlock(&tag_lock); + return ret; +} + +tag* mt_tag_find(const char *tag_name, const size_t ntag) { + tag* ret = NULL; + + pthread_mutex_lock(&tag_lock); + ret = do_tag_find(tag_name, ntag); + pthread_mutex_unlock(&tag_lock); + return ret; +} + +char* mt_tag_dump(int *len) { + char* ret = NULL; + + pthread_mutex_lock(&tag_lock); + ret = do_tag_dump(len); + pthread_mutex_unlock(&tag_lock); + return ret; +} + +void mt_tag_reverse_del_key(splay_tree *tags_tree, snode* key_sn) { + pthread_mutex_lock(&tag_lock); + do_tag_reverse_del_key(tags_tree, key_sn); + pthread_mutex_unlock(&tag_lock); +} +//tag related + #endif +