Skip to content

Commit c444094

Browse files
moticlessmoticless
committed
integrate python json parser service
1 parent 581221f commit c444094

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+580
-302
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ To see cool internal state printouts of the parser, set env-var `LIBRDB_DEBUG_DA
2020

2121
% LIBRDB_DEBUG_DATA=1 make example
2222

23-
To build and run tests, you need to have cmocka unit testing framework installed:
23+
To build and run tests, you need to have cmocka unit testing and python3 installed:
2424

2525
% make test
2626

‎src/ext/handlersToJson.c

+15-8
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint6
153153
}
154154

155155
/* output json part */
156-
fprintf(ctx->outfile, " \"%sdbSize\": {\n", jsonMetaPrefix);
156+
fprintf(ctx->outfile, " \"__dbsize__\": {\n");
157157
fprintf(ctx->outfile, " \"size\": %" PRIu64 ",\n", db_size);
158158
fprintf(ctx->outfile, " \"expires\": %" PRIu64 "\n", exp_size);
159159
fprintf(ctx->outfile, " }%s\n", (db_size) ? "," : "");
@@ -171,7 +171,7 @@ static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) {
171171
}
172172

173173
/* output json part */
174-
fprintf(ctx->outfile, " \"%sSlotInfo\": {\n", jsonMetaPrefix);
174+
fprintf(ctx->outfile, " \"__slotinfo__\": {\n");
175175
fprintf(ctx->outfile, " \"slotId\": %lu,\n", info->slot_id);
176176
fprintf(ctx->outfile, " \"slotSize\": %lu,\n", info->slot_size);
177177
fprintf(ctx->outfile, " \"slotSExpiresSize\": %lu\n", info->expires_slot_size);
@@ -184,7 +184,7 @@ static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBu
184184

185185
if (ctx->state == R2J_IDLE) {
186186
ctx->state = R2J_AUX_FIELDS;
187-
fprintf(ctx->outfile, "{\n "); /* group aux-fields with { ... } */
187+
fprintf(ctx->outfile, "\"__aux__\" : {\n "); /* group aux-fields with { ... } */
188188
} else if (ctx->state == R2J_AUX_FIELDS) {
189189
fprintf(ctx->outfile, ",\n ");
190190
} else {
@@ -432,14 +432,20 @@ static RdbRes toJsonSet(RdbParser *p, void *userData, RdbBulk member) {
432432
static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double score) {
433433
RdbxToJson *ctx = userData;
434434

435-
char score_str[MAX_D2STRING_CHARS];
436-
int len = d2string(score_str, sizeof(score_str), score);
435+
char scoreStr[MAX_D2STRING_CHARS];
436+
int len = d2string(scoreStr, sizeof(scoreStr), score);
437+
438+
/* -0 is a valid double, but we want to output it as 0 */
439+
if ((len == 2) && (scoreStr[0] == '-') && (scoreStr[1] == '0')) {
440+
scoreStr[0] = '0';
441+
scoreStr[1] = '\0';
442+
}
437443

438444
if (ctx->state == R2J_IN_KEY) {
439445
/* output json part */
440446
fprintf(ctx->outfile, "{");
441447
outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member));
442-
fprintf(ctx->outfile, ":\"%.*s\"", len, score_str);
448+
fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr);
443449

444450
/* update new state */
445451
ctx->state = R2J_IN_ZSET;
@@ -448,7 +454,7 @@ static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double sc
448454
/* output json part */
449455
fprintf(ctx->outfile, ",");
450456
outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member));
451-
fprintf(ctx->outfile, ":\"%.*s\"", len, score_str);
457+
fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr);
452458

453459
/* state unchanged */
454460

@@ -494,8 +500,9 @@ static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) {
494500

495501
if (ctx->state == R2J_IDLE) {
496502
ctx->state = R2J_FUNCTIONS;
503+
fprintf(ctx->outfile, "\"__func__\": {\n");
497504
} else if (ctx->state == R2J_AUX_FIELDS) {
498-
fprintf(ctx->outfile, "\n},{\n");
505+
fprintf(ctx->outfile, "\n},\n \"__func__\": {\n");
499506
ctx->state = R2J_FUNCTIONS;
500507
} else if (ctx->state == R2J_FUNCTIONS) {
501508
fprintf(ctx->outfile, ",\n");

‎src/lib/defines.h

+13-11
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,22 @@
1919
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
2020

2121
/* Object types for encoded objects. */
22-
#define RDB_TYPE_HASH_ZIPMAP 9
23-
#define RDB_TYPE_LIST_ZIPLIST 10
24-
#define RDB_TYPE_SET_INTSET 11
25-
#define RDB_TYPE_ZSET_ZIPLIST 12
26-
#define RDB_TYPE_HASH_ZIPLIST 13
27-
#define RDB_TYPE_LIST_QUICKLIST 14
28-
#define RDB_TYPE_STREAM_LISTPACKS 15
29-
#define RDB_TYPE_HASH_LISTPACK 16
30-
#define RDB_TYPE_ZSET_LISTPACK 17
22+
#define RDB_TYPE_HASH_ZIPMAP 9
23+
#define RDB_TYPE_LIST_ZIPLIST 10
24+
#define RDB_TYPE_SET_INTSET 11
25+
#define RDB_TYPE_ZSET_ZIPLIST 12
26+
#define RDB_TYPE_HASH_ZIPLIST 13
27+
#define RDB_TYPE_LIST_QUICKLIST 14
28+
#define RDB_TYPE_STREAM_LISTPACKS 15
29+
#define RDB_TYPE_HASH_LISTPACK 16
30+
#define RDB_TYPE_ZSET_LISTPACK 17
3131
#define RDB_TYPE_LIST_QUICKLIST_2 18
3232
#define RDB_TYPE_STREAM_LISTPACKS_2 19
33-
#define RDB_TYPE_SET_LISTPACK 20
33+
#define RDB_TYPE_SET_LISTPACK 20
3434
#define RDB_TYPE_STREAM_LISTPACKS_3 21
35-
#define RDB_TYPE_MAX 22
35+
#define RDB_TYPE_HASH_METADATA 22
36+
#define RDB_TYPE_HASH_LISTPACK_EX 23
37+
#define RDB_TYPE_MAX 24
3638
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
3739

3840
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */

‎test/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ STD = -std=c99
1717
STACK = -fstack-protector-all -Wstack-protector
1818
WARNS = -Wall -Wextra -pedantic -Werror -Wno-unused-function
1919

20-
OPTIMIZATION?=-O3
20+
OPTIMIZATION?=-O0
2121

2222
CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS)
2323
DEBUG = -g3 -DDEBUG=1

‎test/dumps/cluster_slot_info.json

+14-13
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
[{
1+
"__aux__" : {
22
"redis-ver":"255.255.255",
33
"redis-bits":"64",
44
"ctime":"1713005699",
@@ -8,15 +8,16 @@
88
"repl-offset":"390",
99
"aof-base":"0"
1010
},
11-
{
12-
"__dbSize": {
13-
"size": 1,
14-
"expires": 0
15-
},
16-
"__SlotInfo": {
17-
"slotId": 7638,
18-
"slotSize": 1,
19-
"slotSExpiresSize": 0
20-
},
21-
"abc":"abc"
22-
}]
11+
12+
"__dbsize__": {
13+
"size": 1,
14+
"expires": 0
15+
},
16+
17+
"__slotinfo__": {
18+
"slotId": 7638,
19+
"slotSize": 1,
20+
"slotSExpiresSize": 0
21+
},
22+
23+
"abc":"abc"

‎test/dumps/function.json

-6
This file was deleted.

‎test/dumps/function2.json

+15-14
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
1-
[{
1+
"__aux__" : {
22
"redis-ver":"255.255.255",
33
"redis-bits":"64",
44
"ctime":"1713086666",
55
"used-mem":"966312",
66
"aof-base":"0"
7-
},{
7+
},
8+
9+
"__func__": {
810
"__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n",
911
"__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n"
1012
},
11-
{
12-
"key_97":"value_97",
13-
"key_90":"value_90",
14-
"key_93":"value_93",
15-
"key_99":"value_99",
16-
"key_96":"value_96",
17-
"key_92":"value_92",
18-
"key_95":"value_95",
19-
"key_91":"value_91",
20-
"key_94":"value_94",
21-
"key_98":"value_98"
22-
}]
13+
14+
"key_97":"value_97",
15+
"key_90":"value_90",
16+
"key_93":"value_93",
17+
"key_99":"value_99",
18+
"key_96":"value_96",
19+
"key_92":"value_92",
20+
"key_95":"value_95",
21+
"key_91":"value_91",
22+
"key_94":"value_94",
23+
"key_98":"value_98"

‎test/dumps/hash_lp_v11_data.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1695280472",
4-
"used-mem":"1062024",
5-
"aof-base":"0",
1+
"__aux__" : {
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1695280472",
5+
"used-mem":"1062024",
6+
"aof-base":"0"
7+
},
8+
69
"hash2":{"field7":"value7","field8":"value8","9":"value9","field10":"value10","field11":"11","12":"12.0"},
710
"hash1":{"field2":"2","field3":"3","field4":"value4.0","field5":"5.0","6":"6"}

‎test/dumps/hash_lp_v11_raw.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1695280472",
4-
"used-mem":"1062024",
5-
"aof-base":"0",
1+
"__aux__" : {
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1695280472",
5+
"used-mem":"1062024",
6+
"aof-base":"0"
7+
},
8+
69
"hash2":"\u0010\u00c3@C@V\u0013V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value \u0007`\u000f\u00008\u00a0\u000f\u00038\u0007\t\u0001\u0080\u0019\u00029\u0007\u0087`\u0019\u000310\b\u0087`*@\b\u0080\u0011\f1\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff",
710
"hash1":"\u0010\u00c35>\u000f>\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0080\t\u00023\u0007\u0003\u00a0\t\u000b4\u0007\u0088value4.0\t\u0080\u001b\u000b5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"

‎test/dumps/hash_lp_v11_struct.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1695280472",
4-
"used-mem":"1062024",
5-
"aof-base":"0",
1+
"__aux__" : {
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1695280472",
5+
"used-mem":"1062024",
6+
"aof-base":"0"
7+
},
8+
69
"hash2":["V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value7\u0007\u0086field8\u0007\u0086value8\u0007\t\u0001\u0086value9\u0007\u0087field10\b\u0087value10\b\u0087field11\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff"],
710
"hash1":[">\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0086field3\u0007\u0003\u0001\u0086field4\u0007\u0088value4.0\t\u0086field5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"]

‎test/dumps/hash_zl_v6_data.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
"redis-ver":"6.0.16",
2-
"redis-bits":"64",
3-
"ctime":"1690533464",
4-
"used-mem":"865216",
5-
"aof-preamble":"0",
1+
"__aux__" : {
2+
"redis-ver":"6.0.16",
3+
"redis-bits":"64",
4+
"ctime":"1690533464",
5+
"used-mem":"865216",
6+
"aof-preamble":"0"
7+
},
8+
69
"myhash":{"1":"2","3":"4","5":"6","7.0":"8.0","str1":"str2","str3":"str4"}

‎test/dumps/hash_zl_v6_raw.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
"redis-ver":"6.0.16",
2-
"redis-bits":"64",
3-
"ctime":"1690533464",
4-
"used-mem":"865216",
5-
"aof-preamble":"0",
1+
"__aux__" : {
2+
"redis-ver":"6.0.16",
3+
"redis-bits":"64",
4+
"ctime":"1690533464",
5+
"used-mem":"865216",
6+
"aof-preamble":"0"
7+
},
8+
69
"myhash":"\r99\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"

‎test/dumps/hash_zl_v6_struct.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
"redis-ver":"6.0.16",
2-
"redis-bits":"64",
3-
"ctime":"1690533464",
4-
"used-mem":"865216",
5-
"aof-preamble":"0",
1+
"__aux__" : {
2+
"redis-ver":"6.0.16",
3+
"redis-bits":"64",
4+
"ctime":"1690533464",
5+
"used-mem":"865216",
6+
"aof-preamble":"0"
7+
},
8+
69
"myhash":["9\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"]

‎test/dumps/multiple_dbs_data.json

+2-9
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,6 @@
11
[{
2-
"redis-ver":"255.255.255",
3-
"redis-bits":"64",
4-
"ctime":"1683103535",
5-
"used-mem":"967040",
6-
"aof-base":"0"
7-
},
8-
{
9-
"x":"0"
10-
},{
2+
"x":"0"
3+
},{
114
"y":"1"
125
},{
136
"z":"2"

‎test/dumps/multiple_lists_strings_data.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1677580558",
4-
"used-mem":"937464",
5-
"aof-base":"0",
1+
"__aux__" : {
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1677580558",
5+
"used-mem":"937464",
6+
"aof-base":"0"
7+
},
8+
69
"string2":"Hi there!",
710
"mylist1":["v1"],
811
"mylist3":["v3","v2","v1"],

‎test/dumps/multiple_lists_strings_raw.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1677580558",
4-
"used-mem":"937464",
5-
"aof-base":"0",
1+
"__aux__" : {
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1677580558",
5+
"used-mem":"937464",
6+
"aof-base":"0"
7+
},
8+
69
"string2":"\u0000\tHi there!",
710
"mylist1":"\u0012\u0001\u0002\u000b\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff",
811
"mylist3":"\u0012\u0001\u0002\u0013\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff",

‎test/dumps/multiple_lists_strings_struct.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1677580558",
4-
"used-mem":"937464",
5-
"aof-base":"0",
1+
"__aux__" : {
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1677580558",
5+
"used-mem":"937464",
6+
"aof-base":"0"
7+
},
8+
69
"string2":"Hi there!",
710
"mylist1":["\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff"],
811
"mylist3":["\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff"],

‎test/dumps/plain_hash_data.json

+7-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
"redis-ver":"3.2.1",
2-
"redis-bits":"64",
3-
"ctime":"1689605183",
4-
"used-mem":"821904",
1+
"__aux__" : {
2+
"redis-ver":"3.2.1",
3+
"redis-bits":"64",
4+
"ctime":"1689605183",
5+
"used-mem":"821904"
6+
},
7+
58
"myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"}

‎test/dumps/plain_hash_raw.json

+7-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
"redis-ver":"3.2.1",
2-
"redis-bits":"64",
3-
"ctime":"1689605183",
4-
"used-mem":"821904",
1+
"__aux__" : {
2+
"redis-ver": "3.2.1",
3+
"redis-bits": "64",
4+
"ctime": "1689605183",
5+
"used-mem": "821904"
6+
},
7+
58
"myhash1":"\u0004\u000b\u00c0\u0005\u00c0\u0005\u00c0\u0006\u00c0\u0006\u00c0\n\u00c0\n\u00c0\u0001\u00c0\u0001\u00c0\u0002\u00c0\u0002\u00c0\b\u00c0\b\u00c0\u0004\u00c0\u0004\u00c0\t\u00c0\t\u00c0\u0003\u00c0\u0003\u00c0\u0007\u00c0\u0007\u00c0\u000b\u00c0\u000b"

0 commit comments

Comments
 (0)