Skip to content

Commit 3d473b9

Browse files
moticlessmoticless
committed
Support RDB v12
1 parent c182078 commit 3d473b9

14 files changed

+186
-62
lines changed

‎api/librdb-ext-api.h

+4
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,13 @@ typedef enum RdbxToJsonEnc {
7979
typedef struct RdbxToJsonConf {
8080
RdbHandlersLevel level; /* Parsing depth (raw, structures or data-types) */
8181
RdbxToJsonEnc encoding; /* Encoding format for the resulting JSON */
82+
83+
/* Additional metadata to include in json output */
84+
int includeDbInfo; /* Set to include DB and SLOT info in JSON output */
8285
int includeAuxField; /* Set to include auxiliary fields in JSON output */
8386
int includeFunc; /* Set to include functions in JSON output */
8487
int includeStreamMeta; /* Set to include Stream metadata in JSON output */
88+
8589
int flatten; /* Set to create a flattened JSON structure */
8690
} RdbxToJsonConf;
8791

‎src/cli/rdb-cli.c

+7-2
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ static void printUsage(int shortUsage) {
9797
printf("\t-D, --no-dbnum <DBNUM> Exclude DB number\n\n");
9898

9999
printf("FORMAT_OPTIONS ('json'):\n");
100-
printf("\t-i, --include <EXTRAS> To include: {aux-val|func|stream-meta}\n");
100+
printf("\t-i, --include <EXTRAS> To include: {aux-val|func|stream-meta|db-info}\n");
101+
printf("\t-m, --meta-prefix <PREFIX> To distinct EXTRAS from actual data, Prefix it (Default:\"__\")\n");
101102
printf("\t-f, --flatten Print flatten json, without DBs Parenthesis\n");
102103
printf("\t-o, --output <FILE> Specify the output file. If not specified, output to stdout\n\n");
103104

@@ -126,20 +127,23 @@ static void printUsage(int shortUsage) {
126127
}
127128

128129
static RdbRes formatJson(RdbParser *parser, int argc, char **argv) {
130+
extern const char *jsonMetaPrefix;
129131
const char *includeArg;
130132
const char *output = NULL;/*default:stdout*/
131-
int includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/
133+
int includeDbInfo=0, includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/
132134

133135
/* parse specific command options */
134136
for (int at = 1; at < argc; ++at) {
135137
char *opt = argv[at];
136138
if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue;
137139
if (getOptArg(argc, argv, &at, "-f", "--flatten", &flatten, NULL)) continue;
140+
if (getOptArg(argc, argv, &at, "-m", "--meta-prefix", NULL, &jsonMetaPrefix)) continue;
138141

139142
if (getOptArg(argc, argv, &at, "-i", "--include", NULL, &includeArg)) {
140143
if (strcmp(includeArg, "aux-val") == 0) { includeAuxField = 1; continue; }
141144
if (strcmp(includeArg, "func") == 0) { includeFunc = 1; continue; }
142145
if (strcmp(includeArg, "stream-meta") == 0) { includeStreamMeta = 1; continue; }
146+
if (strcmp(includeArg, "db-info") == 0) { includeDbInfo = 1; continue; }
143147
loggerWrap(RDB_LOG_ERR, "Invalid argument for '--include': %s\n", includeArg);
144148
return RDB_ERR_GENERAL;
145149
}
@@ -156,6 +160,7 @@ static RdbRes formatJson(RdbParser *parser, int argc, char **argv) {
156160
.includeAuxField = includeAuxField,
157161
.includeFunc = includeFunc,
158162
.includeStreamMeta = includeStreamMeta,
163+
.includeDbInfo = includeDbInfo,
159164
};
160165

161166
if (RDBX_createHandlersToJson(parser, output, &conf) == NULL)

‎src/ext/handlersToJson.c

+95-16
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ struct RdbxToJson;
1313
typedef enum
1414
{
1515
R2J_IDLE = 0,
16+
R2J_AUX_FIELDS,
17+
R2J_FUNCTIONS,
18+
1619
R2J_IN_DB,
1720
R2J_IN_KEY,
1821

@@ -53,6 +56,8 @@ struct RdbxToJson {
5356
unsigned int count_db;
5457
};
5558

59+
const char *jsonMetaPrefix = "__"; /* Distinct meta from data with prefix string. */
60+
5661
static void outputPlainEscaping(RdbxToJson *ctx, char *p, size_t len) {
5762
while(len--) {
5863
switch(*p) {
@@ -121,6 +126,7 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT
121126
ctx->conf.includeAuxField = 0;
122127
ctx->conf.includeFunc = 0;
123128
ctx->conf.includeStreamMeta = 0;
129+
ctx->conf.includeDbInfo = 0;
124130

125131
/* override configuration if provided */
126132
if (conf) ctx->conf = *conf;
@@ -136,15 +142,63 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT
136142

137143
/*** Handling common ***/
138144

145+
static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint64_t exp_size) {
146+
RdbxToJson *ctx = userData;
147+
UNUSED(p);
148+
149+
if (ctx->state != R2J_IN_DB) {
150+
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
151+
"toJsonDbSize(): Invalid state value: %d", ctx->state);
152+
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
153+
}
154+
155+
/* output json part */
156+
fprintf(ctx->outfile, " \"%sdbSize\": {\n", jsonMetaPrefix);
157+
fprintf(ctx->outfile, " \"size\": %lu,\n", db_size);
158+
fprintf(ctx->outfile, " \"expires\": %lu\n", exp_size);
159+
fprintf(ctx->outfile, " }%s\n", (db_size) ? "," : "");
160+
161+
return RDB_OK;
162+
}
163+
164+
static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) {
165+
RdbxToJson *ctx = userData;
166+
UNUSED(p, info);
167+
168+
if (ctx->state != R2J_IN_DB) {
169+
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
170+
"toJsonSlotInfo(): Invalid state value: %d", ctx->state);
171+
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
172+
}
173+
174+
/* output json part */
175+
fprintf(ctx->outfile, " \"%sSlotInfo\": {\n", jsonMetaPrefix);
176+
fprintf(ctx->outfile, " \"slotId\": %lu,\n", info->slot_id);
177+
fprintf(ctx->outfile, " \"slotSize\": %lu,\n", info->slot_size);
178+
fprintf(ctx->outfile, " \"slotSExpiresSize\": %lu\n", info->expires_slot_size);
179+
fprintf(ctx->outfile, " },\n");
180+
return RDB_OK;
181+
}
182+
139183
static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBulk auxval) {
140184
RdbxToJson *ctx = userData;
141185
UNUSED(p);
142186

187+
if (ctx->state == R2J_IDLE) {
188+
ctx->state = R2J_AUX_FIELDS;
189+
fprintf(ctx->outfile, "{\n "); /* group aux-fields with { ... } */
190+
} else if (ctx->state == R2J_AUX_FIELDS) {
191+
fprintf(ctx->outfile, ",\n ");
192+
} else {
193+
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
194+
"toJsonAuxField(): Invalid state value: %d", ctx->state);
195+
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
196+
}
197+
143198
/* output json part */
144199
outputQuotedEscaping(ctx, auxkey, RDB_bulkLen(p, auxkey));
145200
fprintf(ctx->outfile, ":");
146201
outputQuotedEscaping(ctx, auxval, RDB_bulkLen(p, auxval));
147-
fprintf(ctx->outfile, ",\n");
148202

149203
return RDB_OK;
150204
}
@@ -227,13 +281,17 @@ static RdbRes toJsonNewDb(RdbParser *p, void *userData, int db) {
227281
RdbxToJson *ctx = userData;
228282

229283
if (ctx->state == R2J_IDLE) {
284+
/* old RDBs might not have aux-fields */
285+
if (!ctx->conf.flatten) fprintf(ctx->outfile, "{\n");
286+
} else if (ctx->state == R2J_AUX_FIELDS || ctx->state == R2J_FUNCTIONS) {
287+
fprintf(ctx->outfile, "\n},\n");
230288
if (!ctx->conf.flatten) fprintf(ctx->outfile, "{\n");
231289
} else if (ctx->state == R2J_IN_DB) {
232290
/* output json part */
233-
if (!ctx->conf.flatten) {
234-
fprintf(ctx->outfile, "\n},{\n");
235-
} else {
291+
if (ctx->conf.flatten) {
236292
fprintf(ctx->outfile, ",\n");
293+
} else {
294+
fprintf(ctx->outfile, "\n},{\n");
237295
}
238296
} else {
239297
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
@@ -266,8 +324,10 @@ static RdbRes toJsonNewRdb(RdbParser *p, void *userData, int rdbVersion) {
266324
static RdbRes toJsonEndRdb(RdbParser *p, void *userData) {
267325
RdbxToJson *ctx = userData;
268326

269-
if (ctx->state == R2J_IDLE) {
327+
if ((ctx->state == R2J_IDLE)) {
270328
RDB_log(p, RDB_LOG_WRN, "RDB is empty.");
329+
} else if (ctx->state == R2J_AUX_FIELDS || ctx->state == R2J_FUNCTIONS) {
330+
fprintf(ctx->outfile, "\n},\n");
271331
} else if (ctx->state == R2J_IN_DB) {
272332
if (!ctx->conf.flatten) fprintf(ctx->outfile, "\n}");
273333
} else {
@@ -433,10 +493,24 @@ static RdbRes toJsonHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va
433493

434494
static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) {
435495
RdbxToJson *ctx = userData;
496+
497+
if (ctx->state == R2J_IDLE) {
498+
ctx->state = R2J_FUNCTIONS;
499+
} else if (ctx->state == R2J_AUX_FIELDS) {
500+
fprintf(ctx->outfile, "\n},{\n");
501+
ctx->state = R2J_FUNCTIONS;
502+
} else if (ctx->state == R2J_FUNCTIONS) {
503+
fprintf(ctx->outfile, ",\n");
504+
} else {
505+
RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE,
506+
"toJsonFunction(): Invalid state value: %d", ctx->state);
507+
return (RdbRes) RDBX_ERR_R2J_INVALID_STATE;
508+
}
509+
436510
/* output json part */
437-
fprintf(ctx->outfile, " \"Function_%d\":", ++ctx->count_functions);
511+
fprintf(ctx->outfile, " \"%sFunction_%d\":", jsonMetaPrefix, ++ctx->count_functions);
438512
outputQuotedEscaping( (RdbxToJson *) userData, func, RDB_bulkLen(p, func));
439-
fprintf(ctx->outfile, ",\n");
513+
ctx->count_functions++;
440514
return RDB_OK;
441515
}
442516

@@ -639,9 +713,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
639713
toJsonNewRdb,
640714
toJsonEndRdb,
641715
toJsonNewDb,
642-
NULL, /* handleResizeDb */
643-
NULL,
644-
NULL,
716+
NULL, /*handleDbSize*/
717+
NULL, /*handleSlotInfo*/
718+
NULL, /*handleAuxField*/
645719
toJsonNewKey,
646720
toJsonEndKey,
647721
toJsonString,
@@ -673,16 +747,21 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
673747
dataCb.handleStreamConsumerPendingEntry = toJsonStreamConsumerPendingEntry;
674748
}
675749

750+
if (ctx->conf.includeDbInfo) {
751+
dataCb.handleDbSize = toJsonDbSize;
752+
dataCb.handleSlotInfo = toJsonSlotInfo;
753+
}
754+
676755
RDB_createHandlersData(p, &dataCb, ctx, deleteRdbToJsonCtx);
677756

678757
} else if (ctx->conf.level == RDB_LEVEL_STRUCT) {
679758
RdbHandlersStructCallbacks structCb = {
680759
toJsonNewRdb,
681760
toJsonEndRdb,
682761
toJsonNewDb,
683-
NULL, /* handleResizeDb */
684-
NULL,
685-
NULL,
762+
NULL, /*handleDbSize*/
763+
NULL, /*handleSlotInfo*/
764+
NULL, /*handleAuxField*/
686765
toJsonNewKey,
687766
toJsonEndKey,
688767
toJsonString,
@@ -724,9 +803,9 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo
724803
toJsonNewRdb,
725804
toJsonEndRdb,
726805
toJsonNewDb,
727-
NULL, /* handleResizeDb */
728-
NULL,
729-
NULL,
806+
NULL, /*handleDbSize*/
807+
NULL, /*handleSlotInfo*/
808+
NULL, /*handleAuxField*/
730809
toJsonNewKey,
731810
toJsonEndKey,
732811
NULL, /*handleBeginModuleAux*/

‎src/ext/handlersToResp.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ typedef struct RedisToRdbVersion {
2525
} RedisToRdbVersion;
2626

2727
const RedisToRdbVersion redisToRdbVersion[] = {
28-
{"99.99", VER_VAL(99,99), 12}, // TODO: Update released version
28+
{"7.4", VER_VAL(7,4), 12},
2929
{"7.2", VER_VAL(7,2), 11},
3030
{"7.0", VER_VAL(7,0), 10},
3131
{"5.0", VER_VAL(5,0), 9}, //6 and 6.2 had v9 too

‎src/lib/defines.h

-7
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,6 @@
9090
#define RDB_ENC_LZF 3 /* string compressed with FASTLZ */
9191
/*#define RDB_ENC_GD 4 string is a gdcompressed entry */
9292

93-
/* rdbLoad...() functions flags. */
94-
#define RDB_LOAD_NONE 0
95-
#define RDB_LOAD_ENC (1<<0)
96-
#define RDB_LOAD_PLAIN (1<<1)
97-
#define RDB_LOAD_SDS (1<<2)
98-
99-
10093
/* quicklist node container formats */
10194
#define QUICKLIST_NODE_CONTAINER_PLAIN 1
10295
#define QUICKLIST_NODE_CONTAINER_PACKED 2

‎test/dumps/cluster_slot_info.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[{
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1713005699",
5+
"used-mem":"2550192",
6+
"repl-stream-db":"0",
7+
"repl-id":"734638bff92ee423e11e46e417b47acbd2d9c896",
8+
"repl-offset":"390",
9+
"aof-base":"0"
10+
},
11+
{
12+
"__dbSize": {
13+
"size": 1,
14+
"expires": 0
15+
},
16+
"__SlotInfo": {
17+
"slotId": 7638,
18+
"slotSize": 1,
19+
"slotSExpiresSize": 0
20+
},
21+
"abc":"abc"
22+
}]

‎test/dumps/cluster_slot_info.rdb

197 Bytes
Binary file not shown.

‎test/dumps/function2.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[{
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1713086666",
5+
"used-mem":"966312",
6+
"aof-base":"0"
7+
},{
8+
"__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n",
9+
"__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n"
10+
},
11+
{
12+
"key_97":"value_97",
13+
"key_90":"value_90",
14+
"key_93":"value_93",
15+
"key_99":"value_99",
16+
"key_96":"value_96",
17+
"key_92":"value_92",
18+
"key_95":"value_95",
19+
"key_91":"value_91",
20+
"key_94":"value_94",
21+
"key_98":"value_98"
22+
}]

‎test/dumps/function2.rdb

919 Bytes
Binary file not shown.

‎test/dumps/multiple_dbs_data.json

+14-8
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
1-
"redis-ver":"255.255.255",
2-
"redis-bits":"64",
3-
"ctime":"1683103535",
4-
"used-mem":"967040",
5-
"aof-base":"0",
6-
"x":"0",
7-
"y":"1",
8-
"z":"2"
1+
[{
2+
"redis-ver":"255.255.255",
3+
"redis-bits":"64",
4+
"ctime":"1683103535",
5+
"used-mem":"967040",
6+
"aof-base":"0"
7+
},
8+
{
9+
"x":"0"
10+
},{
11+
"y":"1"
12+
},{
13+
"z":"2"
14+
}]

0 commit comments

Comments
 (0)