From acec2559a5da3390dcf2728eb37e69ecedf33267 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Sun, 7 Feb 2010 14:08:54 +0200 Subject: [PATCH 1/6] C++: Make proxies safer If a user happens to store an ElementProxy or a PropertyProxy instance, we need to take a reference to the JSON value they point to. With PropertyProxy, the key needs to be copied as well. --- src/jansson.hpp | 12 +++++++----- src/jansson.ipp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/jansson.hpp b/src/jansson.hpp index bf723bd..ad6d177 100644 --- a/src/jansson.hpp +++ b/src/jansson.hpp @@ -166,8 +166,9 @@ namespace json { // proxies an array element class ElementProxy { public: - // constructor - ElementProxy(json_t* array, unsigned int index) : _array(array), _index(index) {} + ElementProxy(json_t* array, unsigned int index); + ElementProxy(const ElementProxy& other); + ~ElementProxy(); // assign to the proxied element inline ElementProxy& operator=(const Value& value); @@ -186,8 +187,9 @@ namespace json { // proxies an object property class PropertyProxy { public: - // constructor - PropertyProxy(json_t* array, const char* key) : _object(array), _key(key) {} + PropertyProxy(json_t* object, const char *key); + PropertyProxy(const PropertyProxy& other); + ~PropertyProxy(); // assign to the proxied element inline PropertyProxy& operator=(const Value& value); @@ -200,7 +202,7 @@ namespace json { json_t* _object; // key of property - const char* _key; + char* _key; }; } // namespace json::detail diff --git a/src/jansson.ipp b/src/jansson.ipp index 5938f0f..fd3a1bf 100644 --- a/src/jansson.ipp +++ b/src/jansson.ipp @@ -8,6 +8,8 @@ #error "jansson.ipp may only be included from jansson.hpp" #endif +#include + namespace json { namespace detail { // assignment operator @@ -310,6 +312,20 @@ namespace json { return v; } + ElementProxy::ElementProxy(json_t* array, unsigned int index) + : _array(array), _index(index) { + json_incref(_array); + } + + ElementProxy::ElementProxy(const ElementProxy& other) + : _array(other._array), _index(other._index) { + json_incref(_array); + } + + ElementProxy::~ElementProxy() { + json_decref(_array); + } + // assign value to proxied array element ElementProxy& ElementProxy::operator=(const Value& value) { json_array_set(_array, _index, value.as_json()); @@ -321,6 +337,23 @@ namespace json { return json_array_get(_array, _index); } + PropertyProxy::PropertyProxy(json_t* object, const char* key) + : _object(object) { + _key = strdup(key); + json_incref(_object); + } + + PropertyProxy::PropertyProxy(const PropertyProxy& other) + : _object(other._object) { + _key = strdup(other._key); + json_incref(_object); + } + + PropertyProxy::~PropertyProxy() { + free(_key); + json_decref(_object); + } + // assign value to proxied object property PropertyProxy& PropertyProxy::operator=(const Value& value) { json_object_set(_object, _key, value.as_json()); From 7e8b128740336bad50e32bbe9dc86f47b406ce6a Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Mon, 8 Feb 2010 20:51:09 +0200 Subject: [PATCH 2/6] C++: Optimize PropertyProxy When the property already exists in the object, we can store an iterator pointing to that property, instead of duplicating the key. When the property (key) is not present in the object, we still have to duplicate the key. --- src/jansson.hpp | 3 +++ src/jansson.ipp | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/jansson.hpp b/src/jansson.hpp index ad6d177..efb4b52 100644 --- a/src/jansson.hpp +++ b/src/jansson.hpp @@ -201,6 +201,9 @@ namespace json { // array object we wrap json_t* _object; + // iterator pointing to property + void* _iter; + // key of property char* _key; }; diff --git a/src/jansson.ipp b/src/jansson.ipp index fd3a1bf..e965ef7 100644 --- a/src/jansson.ipp +++ b/src/jansson.ipp @@ -338,14 +338,17 @@ namespace json { } PropertyProxy::PropertyProxy(json_t* object, const char* key) - : _object(object) { - _key = strdup(key); + : _object(object), _key(0) { + _iter = json_object_iter_at(object, key); + if(!_iter) + _key = strdup(key); json_incref(_object); } PropertyProxy::PropertyProxy(const PropertyProxy& other) - : _object(other._object) { - _key = strdup(other._key); + : _object(other._object), _iter(other._iter), _key(0) { + if(other._key) + _key = strdup(other._key); json_incref(_object); } @@ -356,12 +359,18 @@ namespace json { // assign value to proxied object property PropertyProxy& PropertyProxy::operator=(const Value& value) { - json_object_set(_object, _key, value.as_json()); + if(_iter) + json_object_iter_set(_object, _iter, value.as_json()); + else + json_object_set(_object, _key, value.as_json()); return *this; } json_t* PropertyProxy::as_json() const { - return json_object_get(_object, _key); + if(_iter) + return json_object_iter_value(_iter); + else + return json_object_get(_object, _key); } } // namespace json::detail From 307167fb66e83946dca6d1b22bc43ae9fd406a16 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Tue, 9 Feb 2010 20:51:25 +0200 Subject: [PATCH 3/6] Optimize hashtable_set() If a key already exists in the hashtable, use the existing pair changing its value instead of removing the old one and allocating a new pair. --- src/hashtable.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/hashtable.c b/src/hashtable.c index 0d4a9f2..4b58b26 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -247,31 +247,39 @@ int hashtable_set(hashtable_t *hashtable, void *key, void *value) bucket_t *bucket; unsigned int hash, index; - hash = hashtable->hash_key(key); - - /* if the key already exists, delete it */ - hashtable_do_del(hashtable, key, hash); - /* rehash if the load ratio exceeds 1 */ if(hashtable->size >= num_buckets(hashtable)) if(hashtable_do_rehash(hashtable)) return -1; - pair = malloc(sizeof(pair_t)); - if(!pair) - return -1; - - pair->key = key; - pair->value = value; - pair->hash = hash; - list_init(&pair->list); - + hash = hashtable->hash_key(key); index = hash % num_buckets(hashtable); bucket = &hashtable->buckets[index]; + pair = hashtable_find_pair(hashtable, bucket, key, hash); - insert_to_bucket(hashtable, bucket, &pair->list); + if(pair) + { + if(hashtable->free_key) + hashtable->free_key(key); + if(hashtable->free_value) + hashtable->free_value(pair->value); + pair->value = value; + } + else + { + pair = malloc(sizeof(pair_t)); + if(!pair) + return -1; - hashtable->size++; + pair->key = key; + pair->value = value; + pair->hash = hash; + list_init(&pair->list); + + insert_to_bucket(hashtable, bucket, &pair->list); + + hashtable->size++; + } return 0; } From f18ef5144a77ebdbe7285711884b217b86e2c3b6 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Tue, 9 Feb 2010 21:29:33 +0200 Subject: [PATCH 4/6] Implement JSON_PRESERVE_ORDER encoding flag With this encoding flag, the object key-value pairs in output are in the same order in which they were first inserted into the object. To make this possible, a key of an object is now a serial number plus a string. An object keeps an increasing counter which is used to assign serial number to the keys. Hashing, comparison and public API functions were changed to act only on the string part, i.e. the serial number is ignored everywhere else but in the encoder, where it's used to order object keys if JSON_PRESERVE_ORDER flag is used. --- doc/apiref.rst | 8 ++++ src/dump.c | 31 ++++++++----- src/jansson.h | 1 + src/jansson_private.h | 8 ++++ src/value.c | 45 ++++++++++++++----- test/bin/json_process.c | 3 ++ test/suites/api/test_object.c | 36 +++++++++++++++ test/suites/encoding-flags/preserve-order/env | 1 + .../encoding-flags/preserve-order/input | 1 + .../encoding-flags/preserve-order/output | 1 + 10 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 test/suites/encoding-flags/preserve-order/env create mode 100644 test/suites/encoding-flags/preserve-order/input create mode 100644 test/suites/encoding-flags/preserve-order/output diff --git a/doc/apiref.rst b/doc/apiref.rst index 0a96999..ebe00ea 100644 --- a/doc/apiref.rst +++ b/doc/apiref.rst @@ -595,6 +595,14 @@ can be ORed together to obtain *flags*. .. versionadded:: 1.2 +``JSON_PRESERVE_ORDER`` + If this flag is used, object keys in the output are sorted into the + same order in which they were first inserted to the object. For + example, decoding a JSON text and then encoding with this flag + preserves the order of object keys. + + .. versionadded:: 1.3 + The following functions perform the actual JSON encoding. The result is in UTF-8. diff --git a/src/dump.c b/src/dump.c index e8ae440..a08c2e1 100644 --- a/src/dump.c +++ b/src/dump.c @@ -154,9 +154,16 @@ static int dump_string(const char *str, int ascii, dump_func dump, void *data) return dump("\"", 1, data); } -static int object_key_cmp(const void *key1, const void *key2) +static int object_key_compare_keys(const void *key1, const void *key2) { - return strcmp(*(const char **)key1, *(const char **)key2); + return strcmp((*(const object_key_t **)key1)->key, + (*(const object_key_t **)key2)->key); +} + +static int object_key_compare_serials(const void *key1, const void *key2) +{ + return (*(const object_key_t **)key1)->serial - + (*(const object_key_t **)key2)->serial; } static int do_dump(const json_t *json, unsigned long flags, int depth, @@ -290,36 +297,40 @@ static int do_dump(const json_t *json, unsigned long flags, int depth, if(dump_indent(flags, depth + 1, 0, dump, data)) return -1; - if(flags & JSON_SORT_KEYS) + if(flags & JSON_SORT_KEYS || flags & JSON_PRESERVE_ORDER) { - /* Sort keys */ - - const char **keys; + const object_key_t **keys; unsigned int size; unsigned int i; + int (*cmp_func)(const void *, const void *); size = json_object_size(json); - keys = malloc(size * sizeof(const char *)); + keys = malloc(size * sizeof(object_key_t *)); if(!keys) return -1; i = 0; while(iter) { - keys[i] = json_object_iter_key(iter); + keys[i] = jsonp_object_iter_fullkey(iter); iter = json_object_iter_next((json_t *)json, iter); i++; } assert(i == size); - qsort(keys, size, sizeof(const char *), object_key_cmp); + if(flags & JSON_SORT_KEYS) + cmp_func = object_key_compare_keys; + else + cmp_func = object_key_compare_serials; + + qsort(keys, size, sizeof(object_key_t *), cmp_func); for(i = 0; i < size; i++) { const char *key; json_t *value; - key = keys[i]; + key = keys[i]->key; value = json_object_get(json, key); assert(value); diff --git a/src/jansson.h b/src/jansson.h index 55dce0f..24b4949 100644 --- a/src/jansson.h +++ b/src/jansson.h @@ -173,6 +173,7 @@ json_t *json_load_file(const char *path, json_error_t *error); #define JSON_COMPACT 0x100 #define JSON_ENSURE_ASCII 0x200 #define JSON_SORT_KEYS 0x400 +#define JSON_PRESERVE_ORDER 0x800 char *json_dumps(const json_t *json, unsigned long flags); int json_dumpf(const json_t *json, FILE *output, unsigned long flags); diff --git a/src/jansson_private.h b/src/jansson_private.h index 3045956..4490702 100644 --- a/src/jansson_private.h +++ b/src/jansson_private.h @@ -17,6 +17,7 @@ typedef struct { json_t json; hashtable_t hashtable; + unsigned long serial; int visited; } json_object_t; @@ -49,4 +50,11 @@ typedef struct { #define json_to_real(json_) container_of(json_, json_real_t, json) #define json_to_integer(json_) container_of(json_, json_integer_t, json) +typedef struct { + unsigned long serial; + char key[]; +} object_key_t; + +const object_key_t *jsonp_object_iter_fullkey(void *iter); + #endif diff --git a/src/value.c b/src/value.c index 35166f4..f74c684 100644 --- a/src/value.c +++ b/src/value.c @@ -25,9 +25,16 @@ static inline void json_init(json_t *json, json_type type) /*** object ***/ -static unsigned int hash_string(const void *key) +/* This macro just returns a pointer that's a few bytes backwards from + string. This makes it possible to pass a pointer to object_key_t + when only the string inside it is used, without actually creating + an object_key_t instance. */ +#define string_to_key(string) container_of(string, object_key_t, key) + +static unsigned int hash_key(const void *ptr) { - const char *str = (const char *)key; + const char *str = ((const object_key_t *)ptr)->key; + unsigned int hash = 5381; unsigned int c; @@ -40,9 +47,10 @@ static unsigned int hash_string(const void *key) return hash; } -static int string_equal(const void *key1, const void *key2) +static int key_equal(const void *ptr1, const void *ptr2) { - return strcmp((const char *)key1, (const char *)key2) == 0; + return strcmp(((const object_key_t *)ptr1)->key, + ((const object_key_t *)ptr2)->key) == 0; } static void value_decref(void *value) @@ -57,13 +65,14 @@ json_t *json_object(void) return NULL; json_init(&object->json, JSON_OBJECT); - if(hashtable_init(&object->hashtable, hash_string, string_equal, + if(hashtable_init(&object->hashtable, hash_key, key_equal, free, value_decref)) { free(object); return NULL; } + object->serial = 0; object->visited = 0; return &object->json; @@ -94,12 +103,13 @@ json_t *json_object_get(const json_t *json, const char *key) return NULL; object = json_to_object(json); - return hashtable_get(&object->hashtable, key); + return hashtable_get(&object->hashtable, string_to_key(key)); } int json_object_set_new_nocheck(json_t *json, const char *key, json_t *value) { json_object_t *object; + object_key_t *k; if(!key || !value) return -1; @@ -111,7 +121,14 @@ int json_object_set_new_nocheck(json_t *json, const char *key, json_t *value) } object = json_to_object(json); - if(hashtable_set(&object->hashtable, strdup(key), value)) + k = malloc(sizeof(object_key_t) + strlen(key) + 1); + if(!k) + return -1; + + k->serial = object->serial++; + strcpy(k->key, key); + + if(hashtable_set(&object->hashtable, k, value)) { json_decref(value); return -1; @@ -139,7 +156,7 @@ int json_object_del(json_t *json, const char *key) return -1; object = json_to_object(json); - return hashtable_del(&object->hashtable, key); + return hashtable_del(&object->hashtable, string_to_key(key)); } int json_object_clear(json_t *json) @@ -198,7 +215,7 @@ void *json_object_iter_at(json_t *json, const char *key) return NULL; object = json_to_object(json); - return hashtable_iter_at(&object->hashtable, key); + return hashtable_iter_at(&object->hashtable, string_to_key(key)); } void *json_object_iter_next(json_t *json, void *iter) @@ -212,12 +229,20 @@ void *json_object_iter_next(json_t *json, void *iter) return hashtable_iter_next(&object->hashtable, iter); } +const object_key_t *jsonp_object_iter_fullkey(void *iter) +{ + if(!iter) + return NULL; + + return hashtable_iter_key(iter); +} + const char *json_object_iter_key(void *iter) { if(!iter) return NULL; - return (const char *)hashtable_iter_key(iter); + return jsonp_object_iter_fullkey(iter)->key; } json_t *json_object_iter_value(void *iter) diff --git a/test/bin/json_process.c b/test/bin/json_process.c index 809c4d4..77407e5 100644 --- a/test/bin/json_process.c +++ b/test/bin/json_process.c @@ -53,6 +53,9 @@ int main(int argc, char *argv[]) if(getenv_int("JSON_ENSURE_ASCII")) flags |= JSON_ENSURE_ASCII; + if(getenv_int("JSON_PRESERVE_ORDER")) + flags |= JSON_PRESERVE_ORDER; + if(getenv_int("JSON_SORT_KEYS")) flags |= JSON_SORT_KEYS; diff --git a/test/suites/api/test_object.c b/test/suites/api/test_object.c index 4e730bb..90370e5 100644 --- a/test/suites/api/test_object.c +++ b/test/suites/api/test_object.c @@ -402,6 +402,41 @@ static void test_misc() json_decref(object); } +static void test_preserve_order() +{ + json_t *object; + char *result; + + const char *expected = "{\"foobar\": 1, \"bazquux\": 6, \"lorem ipsum\": 3, \"sit amet\": 5, \"helicopter\": 7}"; + + object = json_object(); + + json_object_set_new(object, "foobar", json_integer(1)); + json_object_set_new(object, "bazquux", json_integer(2)); + json_object_set_new(object, "lorem ipsum", json_integer(3)); + json_object_set_new(object, "dolor", json_integer(4)); + json_object_set_new(object, "sit amet", json_integer(5)); + + /* changing a value should preserve the order */ + json_object_set_new(object, "bazquux", json_integer(6)); + + /* deletion shouldn't change the order of others */ + json_object_del(object, "dolor"); + + /* add a new item just to make sure */ + json_object_set_new(object, "helicopter", json_integer(7)); + + result = json_dumps(object, JSON_PRESERVE_ORDER); + + if(strcmp(expected, result) != 0) { + fprintf(stderr, "%s != %s", expected, result); + fail("JSON_PRESERVE_ORDER doesn't work"); + } + + free(result); + json_decref(object); +} + int main() { test_misc(); @@ -410,6 +445,7 @@ int main() test_circular(); test_set_nocheck(); test_iterators(); + test_preserve_order(); return 0; } diff --git a/test/suites/encoding-flags/preserve-order/env b/test/suites/encoding-flags/preserve-order/env new file mode 100644 index 0000000..ce3582d --- /dev/null +++ b/test/suites/encoding-flags/preserve-order/env @@ -0,0 +1 @@ +export JSON_PRESERVE_ORDER=1 diff --git a/test/suites/encoding-flags/preserve-order/input b/test/suites/encoding-flags/preserve-order/input new file mode 100644 index 0000000..27bcf18 --- /dev/null +++ b/test/suites/encoding-flags/preserve-order/input @@ -0,0 +1 @@ +{"foo": 1, "bar": 2, "asdf": 3, "deadbeef": 4, "badc0ffee": 5, "qwerty": 6} diff --git a/test/suites/encoding-flags/preserve-order/output b/test/suites/encoding-flags/preserve-order/output new file mode 100644 index 0000000..7a443f6 --- /dev/null +++ b/test/suites/encoding-flags/preserve-order/output @@ -0,0 +1 @@ +{"foo": 1, "bar": 2, "asdf": 3, "deadbeef": 4, "badc0ffee": 5, "qwerty": 6} \ No newline at end of file From 35ddd2de2058efdb3443760c85a8aa3ce8078584 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Thu, 11 Feb 2010 20:55:56 +0200 Subject: [PATCH 5/6] Update CHANGES, change version to 1.2+ --- CHANGES | 9 +++++++++ configure.ac | 2 +- doc/conf.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 02f28c4..dcbf6a0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,12 @@ +Version 1.3 (in development) +============================ + +* New encoding flags: + + - ``JSON_PRESERVE_ORDER``: Preserve the insertion order of object + keys. + + Version 1.2 =========== diff --git a/configure.ac b/configure.ac index 5290793..17f1e01 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([jansson], [1.2], [petri@digip.org]) +AC_INIT([jansson], [1.2+], [petri@digip.org]) AM_INIT_AUTOMAKE([1.10 foreign]) diff --git a/doc/conf.py b/doc/conf.py index f4ec126..2a95aad 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -52,7 +52,7 @@ copyright = u'2009, 2010 Petri Lehtinen' # The short X.Y version. version = '1.2' # The full version, including alpha/beta/rc tags. -release = '1.2' +release = '1.2+' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 42621370c3b562dff33cd8a8a043cb7bd519d4a6 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Thu, 11 Feb 2010 21:17:19 +0200 Subject: [PATCH 6/6] hashtable: Fix typo in comment --- src/hashtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hashtable.h b/src/hashtable.h index 6d3c736..f03a769 100644 --- a/src/hashtable.h +++ b/src/hashtable.h @@ -161,7 +161,7 @@ void hashtable_clear(hashtable_t *hashtable); void *hashtable_iter(hashtable_t *hashtable); /** - * hashtable_iter - Return an iterator at a specific key + * hashtable_iter_at - Return an iterator at a specific key * * @hashtable: The hashtable object * @key: The key that the iterator should point to