from trunk: support for 32-bit tag numbers in rpc structures

svn:r593

ChangeLog

@@ -15,6 +15,8 @@ Changes in current version:
  o pull setters/getters out of RPC structures into a base class to which we just need to store a pointer; this reduces the memory footprint of these structures.
  o fix a bug with event_rpcgen for integers
  o move EV_PERSIST handling out of the event backends
+ o support for 32-bit tag numbers in rpc structures; this is wire compatible, but changes the API slightly.
+ o prefix {encode,decode}_tag functions with evtag to avoid collisions
 
 
 Changes in 1.4.0-beta:

event.h

@@ -1062,7 +1062,7 @@ void evbuffer_setcb(struct evbuffer *, void (*)(struct evbuffer *, size_t, size_
 
 void evtag_init(void);
 
-void evtag_marshal(struct evbuffer *evbuf, ev_uint8_t tag, const void *data,
+void evtag_marshal(struct evbuffer *evbuf, ev_uint32_t tag, const void *data,
     ev_uint32_t len);
 
 /**
@@ -1077,32 +1077,34 @@ void evtag_marshal(struct evbuffer *evbuf, ev_uint8_t tag, const void *data,
  */
 void encode_int(struct evbuffer *evbuf, ev_uint32_t number);
 
-void evtag_marshal_int(struct evbuffer *evbuf, ev_uint8_t tag, ev_uint32_t integer);
+void evtag_marshal_int(struct evbuffer *evbuf, ev_uint32_t tag,
+    ev_uint32_t integer);
 
-void evtag_marshal_string(struct evbuffer *buf, ev_uint8_t tag,
+void evtag_marshal_string(struct evbuffer *buf, ev_uint32_t tag,
     const char *string);
 
-void evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint8_t tag,
+void evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint32_t tag,
     struct timeval *tv);
 
 void evtag_test(void);
 
-int evtag_unmarshal(struct evbuffer *src, ev_uint8_t *ptag, struct evbuffer *dst);
+int evtag_unmarshal(struct evbuffer *src, ev_uint32_t *ptag,
-int evtag_peek(struct evbuffer *evbuf, ev_uint8_t *ptag);
+    struct evbuffer *dst);
+int evtag_peek(struct evbuffer *evbuf, ev_uint32_t *ptag);
 int evtag_peek_length(struct evbuffer *evbuf, ev_uint32_t *plength);
 int evtag_payload_length(struct evbuffer *evbuf, ev_uint32_t *plength);
 int evtag_consume(struct evbuffer *evbuf);
 
-int evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint8_t need_tag,
+int evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint32_t need_tag,
     ev_uint32_t *pinteger);
 
-int evtag_unmarshal_fixed(struct evbuffer *src, ev_uint8_t need_tag, void *data,
+int evtag_unmarshal_fixed(struct evbuffer *src, ev_uint32_t need_tag,
-    size_t len);
+    void *data, size_t len);
 
-int evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint8_t need_tag,
+int evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint32_t need_tag,
     char **pstring);
 
-int evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint8_t need_tag,
+int evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint32_t need_tag,
     struct timeval *ptv);
 
 #ifdef __cplusplus

event_rpcgen.py

@@ -97,9 +97,9 @@ void %(name)s_clear(struct %(name)s *);
 void %(name)s_marshal(struct evbuffer *, const struct %(name)s *);
 int %(name)s_unmarshal(struct %(name)s *, struct evbuffer *);
 int %(name)s_complete(struct %(name)s *);
-void evtag_marshal_%(name)s(struct evbuffer *, uint8_t, 
+void evtag_marshal_%(name)s(struct evbuffer *, uint32_t, 
     const struct %(name)s *);
-int evtag_unmarshal_%(name)s(struct evbuffer *, uint8_t,
+int evtag_unmarshal_%(name)s(struct evbuffer *, uint32_t,
     struct %(name)s *);""" % { 'name' : self._name }
 
 
@@ -209,7 +209,7 @@ int evtag_unmarshal_%(name)s(struct evbuffer *, uint8_t,
                        '%(name)s_unmarshal(struct %(name)s *tmp, '
                        ' struct evbuffer *evbuf)\n'
                        '{\n'
-                       '  uint8_t tag;\n'
+                       '  uint32_t tag;\n'
                        '  while (EVBUFFER_LENGTH(evbuf) > 0) {\n'
                        '    if (evtag_peek(evbuf, &tag) == -1)\n'
                        '      return (-1);\n'
@@ -260,9 +260,9 @@ int evtag_unmarshal_%(name)s(struct evbuffer *, uint8_t,
         print >>file, (
             'int\n'
             'evtag_unmarshal_%(name)s(struct evbuffer *evbuf, '
-            'uint8_t need_tag, struct %(name)s *msg)\n'
+            'uint32_t need_tag, struct %(name)s *msg)\n'
             '{\n'
-            '  uint8_t tag;\n'
+            '  uint32_t tag;\n'
             '  int res = -1;\n'
             '\n'
             '  struct evbuffer *tmp = evbuffer_new();\n'
@@ -284,7 +284,7 @@ int evtag_unmarshal_%(name)s(struct evbuffer *, uint8_t,
         # Complete message marshaling
         print >>file, (
             'void\n'
-            'evtag_marshal_%(name)s(struct evbuffer *evbuf, uint8_t tag, '
+            'evtag_marshal_%(name)s(struct evbuffer *evbuf, uint32_t tag, '
             'const struct %(name)s *msg)\n'
             '{\n'
             '  struct evbuffer *_buf = evbuffer_new();\n'
@@ -446,7 +446,7 @@ class EntryBytes(Entry):
         Entry.__init__(self, type, name, tag)
 
         self._length = length
-        self._ctype = 'uint8_t'
+        self._ctype = 'uint32_t'
 
     def GetDeclaration(self, funcname):
         code = [ 'int %s(struct %s *, %s **);' % (
@@ -1109,10 +1109,10 @@ def ProcessOneEntry(newstruct, entry):
 
         if not tag_set:
             tag_set = 1
-            if not re.match(r'^[0-9]+$', token):
+            if not re.match(r'^(0x)?[0-9]+$', token):
                 print >>sys.stderr, 'Expected tag number: \"%s\"' % entry
                 sys.exit(1)
-            tag = int(token)
+            tag = int(token, 0)
             continue
 
         print >>sys.stderr, 'Cannot parse \"%s\"' % entry

event_tagging.c

@@ -64,7 +64,7 @@
 #include "event.h"
 #include "log.h"
 
-int decode_int(uint32_t *pnumber, struct evbuffer *evbuf);
+int evtag_decode_int(uint32_t *pnumber, struct evbuffer *evbuf);
 
 static struct evbuffer *_buf;	/* not thread safe */
 
@@ -110,6 +110,71 @@ encode_int(struct evbuffer *evbuf, uint32_t number)
 	evbuffer_add(evbuf, data, (off + 1) / 2);
 }
 
+/*
+ * Support variable length encoding of tags; we use the high bit in each
+ * octet as a continuation signal.
+ */
+
+int
+evtag_encode_tag(struct evbuffer *evbuf, uint32_t tag)
+{
+	int bytes = 0;
+	uint8_t data[5];
+
+	memset(data, 0, sizeof(data));
+	do {
+		uint8_t lower = tag & 0x7f;
+		tag >>= 7;
+
+		if (tag)
+			lower |= 0x80;
+
+		data[bytes++] = lower;
+	} while (tag);
+
+	if (evbuf != NULL)
+		evbuffer_add(evbuf, data, bytes);
+
+	return (bytes);
+}
+
+static int
+decode_tag_internal(uint32_t *ptag, struct evbuffer *evbuf, int dodrain)
+{
+	uint32_t number = 0;
+	uint8_t *data = EVBUFFER_DATA(evbuf);
+	int len = EVBUFFER_LENGTH(evbuf);
+	int count = 0, shift = 0, done = 0;
+
+	while (count++ < len) {
+		uint8_t lower = *data++;
+		number |= (lower & 0x7f) << shift;
+		shift += 7;
+
+		if (!(lower & 0x80)) {
+			done = 1;
+			break;
+		}
+	}
+
+	if (!done)
+		return (-1);
+
+	if (dodrain)
+		evbuffer_drain(evbuf, count);
+
+	if (ptag != NULL)
+		*ptag = number;
+
+	return (count);
+}
+
+int
+evtag_decode_tag(uint32_t *ptag, struct evbuffer *evbuf)
+{
+	return (decode_tag_internal(ptag, evbuf, 1 /* dodrain */));
+}
+
 /*
  * Marshal a data type, the general format is as follows:
  *
@@ -117,34 +182,34 @@ encode_int(struct evbuffer *evbuf, uint32_t number)
  */
 
 void
-evtag_marshal(struct evbuffer *evbuf, uint8_t tag,
+evtag_marshal(struct evbuffer *evbuf, uint32_t tag,
     const void *data, uint32_t len)
 {
-	evbuffer_add(evbuf, &tag, sizeof(tag));
+	evtag_encode_tag(evbuf, tag);
 	encode_int(evbuf, len);
 	evbuffer_add(evbuf, (void *)data, len);
 }
 
 /* Marshaling for integers */
 void
-evtag_marshal_int(struct evbuffer *evbuf, uint8_t tag, uint32_t integer)
+evtag_marshal_int(struct evbuffer *evbuf, uint32_t tag, uint32_t integer)
 {
 	evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf));
 	encode_int(_buf, integer);
 
-	evbuffer_add(evbuf, &tag, sizeof(tag));
+	evtag_encode_tag(evbuf, tag);
 	encode_int(evbuf, EVBUFFER_LENGTH(_buf));
 	evbuffer_add_buffer(evbuf, _buf);
 }
 
 void
-evtag_marshal_string(struct evbuffer *buf, uint8_t tag, const char *string)
+evtag_marshal_string(struct evbuffer *buf, uint32_t tag, const char *string)
 {
 	evtag_marshal(buf, tag, string, strlen(string));
 }
 
 void
-evtag_marshal_timeval(struct evbuffer *evbuf, uint8_t tag, struct timeval *tv)
+evtag_marshal_timeval(struct evbuffer *evbuf, uint32_t tag, struct timeval *tv)
 {
 	evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf));
 
@@ -189,39 +254,36 @@ decode_int_internal(uint32_t *pnumber, struct evbuffer *evbuf, int dodrain)
 }
 
 int
-decode_int(uint32_t *pnumber, struct evbuffer *evbuf)
+evtag_decode_int(uint32_t *pnumber, struct evbuffer *evbuf)
 {
 	return (decode_int_internal(pnumber, evbuf, 1) == -1 ? -1 : 0);
 }
 
 int
-evtag_peek(struct evbuffer *evbuf, uint8_t *ptag)
+evtag_peek(struct evbuffer *evbuf, uint32_t *ptag)
 {
-	if (EVBUFFER_LENGTH(evbuf) < 2)
+	return (decode_tag_internal(ptag, evbuf, 0 /* dodrain */));
-		return (-1);
-	*ptag = EVBUFFER_DATA(evbuf)[0];
-
-	return (0);
 }
 
 int
 evtag_peek_length(struct evbuffer *evbuf, uint32_t *plength)
 {
 	struct evbuffer tmp;
-	int res;
+	int res, len;
 
-	if (EVBUFFER_LENGTH(evbuf) < 2)
+	len = decode_tag_internal(NULL, evbuf, 0 /* dodrain */);
+	if (len == -1)
 		return (-1);
 
 	tmp = *evbuf;
-	tmp.buffer += 1;
+	tmp.buffer += len;
-	tmp.off -= 1;
+	tmp.off -= len;
 
 	res = decode_int_internal(plength, &tmp, 0);
 	if (res == -1)
 		return (-1);
 
-	*plength += res + 1;
+	*plength += res + len;
 
 	return (0);
 }
@@ -230,14 +292,15 @@ int
 evtag_payload_length(struct evbuffer *evbuf, uint32_t *plength)
 {
 	struct evbuffer tmp;
-	int res;
+	int res, len;
 
-	if (EVBUFFER_LENGTH(evbuf) < 2)
+	len = decode_tag_internal(NULL, evbuf, 0 /* dodrain */);
+	if (len == -1)
 		return (-1);
 
 	tmp = *evbuf;
-	tmp.buffer += 1;
+	tmp.buffer += len;
-	tmp.off -= 1;
+	tmp.off -= len;
 
 	res = decode_int_internal(plength, &tmp, 0);
 	if (res == -1)
@@ -250,8 +313,9 @@ int
 evtag_consume(struct evbuffer *evbuf)
 {
 	uint32_t len;
-	evbuffer_drain(evbuf, 1);
+	if (decode_tag_internal(NULL, evbuf, 1 /* dodrain */) == -1)
-	if (decode_int(&len, evbuf) == -1)
+		return (-1);
+	if (evtag_decode_int(&len, evbuf) == -1)
 		return (-1);
 	evbuffer_drain(evbuf, len);
 
@@ -261,15 +325,14 @@ evtag_consume(struct evbuffer *evbuf)
 /* Reads the data type from an event buffer */
 
 int
-evtag_unmarshal(struct evbuffer *src, uint8_t *ptag, struct evbuffer *dst)
+evtag_unmarshal(struct evbuffer *src, uint32_t *ptag, struct evbuffer *dst)
 {
-	uint8_t tag;
 	uint32_t len;
 	uint32_t integer;
 
-	if (evbuffer_remove(src, &tag, sizeof(tag)) != sizeof(tag))
+	if (decode_tag_internal(ptag, src, 1 /* dodrain */) == -1)
 		return (-1);
-	if (decode_int(&integer, src) == -1)
+	if (evtag_decode_int(&integer, src) == -1)
 		return (-1);
 	len = integer;
 
@@ -281,24 +344,24 @@ evtag_unmarshal(struct evbuffer *src, uint8_t *ptag, struct evbuffer *dst)
 
 	evbuffer_drain(src, len);
 
-	*ptag = tag;
 	return (len);
 }
 
 /* Marshaling for integers */
 
 int
-evtag_unmarshal_int(struct evbuffer *evbuf, uint8_t need_tag,
+evtag_unmarshal_int(struct evbuffer *evbuf, uint32_t need_tag,
     uint32_t *pinteger)
 {
-	uint8_t tag;
+	uint32_t tag;
 	uint32_t len;
 	uint32_t integer;
 
-	if (evbuffer_remove(evbuf, &tag, sizeof(tag)) != sizeof(tag) ||
+	if (decode_tag_internal(&tag, evbuf, 1 /* dodrain */) == -1)
-	    tag != need_tag)
 		return (-1);
-	if (decode_int(&integer, evbuf) == -1)
+	if (need_tag != tag)
+		return (-1);
+	if (evtag_decode_int(&integer, evbuf) == -1)
 		return (-1);
 	len = integer;
 
@@ -311,16 +374,16 @@ evtag_unmarshal_int(struct evbuffer *evbuf, uint8_t need_tag,
 
 	evbuffer_drain(evbuf, len);
 
-	return (decode_int(pinteger, _buf));
+	return (evtag_decode_int(pinteger, _buf));
 }
 
 /* Unmarshal a fixed length tag */
 
 int
-evtag_unmarshal_fixed(struct evbuffer *src, uint8_t need_tag, void *data,
+evtag_unmarshal_fixed(struct evbuffer *src, uint32_t need_tag, void *data,
     size_t len)
 {
-	uint8_t tag;
+	uint32_t tag;
 
 	/* Initialize this event buffer so that we can read into it */
 	evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf));
@@ -337,10 +400,10 @@ evtag_unmarshal_fixed(struct evbuffer *src, uint8_t need_tag, void *data,
 }
 
 int
-evtag_unmarshal_string(struct evbuffer *evbuf, uint8_t need_tag,
+evtag_unmarshal_string(struct evbuffer *evbuf, uint32_t need_tag,
     char **pstring)
 {
-	uint8_t tag;
+	uint32_t tag;
 
 	evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf));
 
@@ -356,20 +419,20 @@ evtag_unmarshal_string(struct evbuffer *evbuf, uint8_t need_tag,
 }
 
 int
-evtag_unmarshal_timeval(struct evbuffer *evbuf, uint8_t need_tag,
+evtag_unmarshal_timeval(struct evbuffer *evbuf, uint32_t need_tag,
     struct timeval *ptv)
 {
-	uint8_t tag;
+	uint32_t tag;
 	uint32_t integer;
 
 	evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf));
 	if (evtag_unmarshal(evbuf, &tag, _buf) == -1 || tag != need_tag)
 		return (-1);
 
-	if (decode_int(&integer, _buf) == -1)
+	if (evtag_decode_int(&integer, _buf) == -1)
 		return (-1);
 	ptv->tv_sec = integer;
-	if (decode_int(&integer, _buf) == -1)
+	if (evtag_decode_int(&integer, _buf) == -1)
 		return (-1);
 	ptv->tv_usec = integer;
 

test/regress.c

@@ -1038,7 +1038,9 @@ test_multiple_events_for_same_fd(void)
    cleanup_test();
 }
 
-int decode_int(uint32_t *pnumber, struct evbuffer *evbuf);
+int evtag_decode_int(uint32_t *pnumber, struct evbuffer *evbuf);
+int evtag_encode_tag(struct evbuffer *evbuf, uint32_t number);
+int evtag_decode_tag(uint32_t *pnumber, struct evbuffer *evbuf);
 
 void
 read_once_cb(int fd, short event, void *arg)
@@ -1105,7 +1107,7 @@ evtag_int_test(void)
 	}
 
 	for (i = 0; i < TEST_MAX_INT; i++) {
-		if (decode_int(&integer, tmp) == -1) {
+		if (evtag_decode_int(&integer, tmp) == -1) {
 			fprintf(stderr, "decode %d failed", i);
 			exit(1);
 		}
@@ -1168,6 +1170,46 @@ evtag_fuzz(void)
 	fprintf(stdout, "\t%s: OK\n", __func__);
 }
 
+static void
+evtag_tag_encoding(void)
+{
+	struct evbuffer *tmp = evbuffer_new();
+	uint32_t integers[TEST_MAX_INT] = {
+		0xaf0, 0x1000, 0x1, 0xdeadbeef, 0x00, 0xbef000
+	};
+	uint32_t integer;
+	int i;
+
+	for (i = 0; i < TEST_MAX_INT; i++) {
+		int oldlen, newlen;
+		oldlen = EVBUFFER_LENGTH(tmp);
+		evtag_encode_tag(tmp, integers[i]);
+		newlen = EVBUFFER_LENGTH(tmp);
+		fprintf(stdout, "\t\tencoded 0x%08x with %d bytes\n",
+		    integers[i], newlen - oldlen);
+	}
+
+	for (i = 0; i < TEST_MAX_INT; i++) {
+		if (evtag_decode_tag(&integer, tmp) == -1) {
+			fprintf(stderr, "decode %d failed", i);
+			exit(1);
+		}
+		if (integer != integers[i]) {
+			fprintf(stderr, "got %x, wanted %x",
+			    integer, integers[i]);
+			exit(1);
+		}
+	}
+
+	if (EVBUFFER_LENGTH(tmp) != 0) {
+		fprintf(stderr, "trailing data");
+		exit(1);
+	}
+	evbuffer_free(tmp);
+
+	fprintf(stdout, "\t%s: OK\n", __func__);
+}
+
 void
 evtag_test(void)
 {
@@ -1177,6 +1219,8 @@ evtag_test(void)
 	evtag_int_test();
 	evtag_fuzz();
 
+	evtag_tag_encoding();
+
 	fprintf(stdout, "OK\n");
 }
 
@@ -1188,6 +1232,7 @@ rpc_test(void)
 	struct run *run;
 	struct evbuffer *tmp = evbuffer_new();
 	struct timeval tv_start, tv_end;
+	uint32_t tag;
 	int i;
 
 	fprintf(stdout, "Testing RPC: ");
@@ -1219,10 +1264,20 @@ rpc_test(void)
 		exit(1);
 	}
 
-	evtag_marshal_msg(tmp, 0, msg);
+	evtag_marshal_msg(tmp, 0xdeaf, msg);
+
+	if (evtag_peek(tmp, &tag) == -1) {
+		fprintf(stderr, "Failed to peak tag.\n");
+		exit (1);
+	}
+
+	if (tag != 0xdeaf) {
+		fprintf(stderr, "Got incorrect tag: %0x.\n", tag);
+		exit (1);
+	}
 
 	msg2 = msg_new();
-	if (evtag_unmarshal_msg(tmp, 0, msg2) == -1) {
+	if (evtag_unmarshal_msg(tmp, 0xdeaf, msg2) == -1) {
 		fprintf(stderr, "Failed to unmarshal message.\n");
 		exit(1);
 	}

test/regress.rpc

@@ -8,7 +8,7 @@ struct msg {
 }
 
 struct kill {
-	string weapon = 1;
+	string weapon = 0x10121;
 	string action = 2;
 	optional int how_often = 3;
 }