Add 'stat' command and per-key memory estimation to rdb-cli

README.md

@@ -52,11 +52,36 @@ To run CLI extension of this library and let it parse RDB file to json:
 
 To generate formatted print:
 
-    rdb-cli dump.rdb print --key "db%d,%k,%v"
-    db0,key1,value1
-    db0,key2,value2
+    rdb-cli dump.rdb print --key "db=%d,key=%k,type=%t,enc=%n,bytes=%z,largest=%g,count=%i"
+    db=0,key=mylist,type=list,enc=quicklist,bytes=116 largest=2,count=2
+    db=0,key=myhash,type=hash,enc=listpack,bytes=240 largest=7,count=6
+    db=0,key=myset,type=set,enc=intset,bytes=96,largest=10,count=6
     ...
 
+To print a formatted memory-statistics report instead of per-key lines:
+
+    rdb-cli dump.rdb stat
+    Statistics (Memory is estimated):
+    type             keys          items  items/key   volatile   expired       memory        avg    mem%
+    hash          1560278       81456780         52          0         0        6.8 G      4.6 K   74.8%
+    set           7651482       27952535          3          3         3        1.5 G      216 B   16.9%
+    list          6017862        7906342          1          0         0      719.9 M      125 B    7.7%
+    string         106018              -          -          2         2       63.5 M      628 B    0.7%
+    stream              1            948        948          0         0       22.3 K     22.3 K    0.0%
+    TOTAL        15335641      117316605                     5         5        9.1 G
+    volatile keys hold 2.0 K (0.0% of memory)
+
+    Top 10 keys by memory:
+    memory type        db        items   avg item      ttl  key
+    61.2 M set          0      1369515       46 B     none  App:AccessGroup:001
+    57.4 M set          0      1290404       46 B     none  App:EntityIndex:All
+    38.2 M hash         0       858814       46 B     none  App:EntityMetadata:002
+    ...
+
+Modify the number of top keys shown with `--top`:
+
+    rdb-cli dump.rdb stat --top 500
+
 To generate RESP commands:
 
     % rdb-cli multiple_lists_strings.rdb resp
@@ -207,7 +232,7 @@ destruction, or when newer block replacing old one.
 
 ### rdb-cli usage
 
-    Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {print|json|resp|redis} [FORMAT_OPTIONS]
+    Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {print|json|resp|redis|stat} [FORMAT_OPTIONS]
     OPTIONS:
             -l, --log-file <PATH>         Path to the log file or stdout (Default: './rdb-cli.log')
             -i, --ignore-checksum         Ignore RDB file checksum verification
@@ -224,9 +249,14 @@ destruction, or when newer block replacing old one.
     FORMAT_OPTIONS ('print'):
             -a, --aux-val <FMT>           %f=Auxiliary-Field, %v=Auxiliary-Value (Default: "")
             -k, --key <FMT>               %d=Db %k=Key %v=Value %t=Type %e=Expiry %r=LRU %f=LFU
-                                          %i=Items %m=NumMeta (Default: "%d,%k,%v,%t,%e,%i")
+                                          %i=Items %m=NumMeta %n=Encoding %z=MemBytes(estimate) %g=LargestElement
+                                          Accept optional width, e.g. %-20k %10z (Default: "%d,%k,%v,%t,%e,%i")
             -o, --output <FILE>           Specify the output file. If not specified, output to stdout
 
+    FORMAT_OPTIONS ('stat'):
+            -t, --top <N>                 Show the top N keys by estimated memory (Default: 10)
+            -n, --now <UNIX-TIME-SEC>     For expiry evaluation (Default: now)
+
     FORMAT_OPTIONS ('json'):
             -i, --include <EXTRAS>        To include: {aux-val|func|stream-meta}
             -f, --flatten                 Print flatten json, without DBs Parenthesis

api/librdb-ext-api.h

@@ -16,6 +16,7 @@ typedef struct RdbxFilter RdbxFilter;
 typedef struct RdbxToJson RdbxToJson;
 typedef struct RdbxToResp RdbxToResp;
 typedef struct RdbxToPrint RdbxToPrint;
+typedef struct RdbxToStat RdbxToStat;
 typedef struct RdbxRespToRedisLoader RdbxRespToRedisLoader;
 
 /****************************************************************
@@ -122,13 +123,35 @@ _LIBRDB_API RdbxToJson *RDBX_createHandlersToJson(RdbParser *p,
  *          %r = LRU
  *          %f = LFU
  *          %i = Items
+ *          %n = Encoding
+ *          %z = Estimated in-memory size in bytes
+ *          %g = Size in bytes of the largest element
  *
  ****************************************************************/
 _LIBRDB_API RdbxToPrint *RDBX_createHandlersToPrint(RdbParser *p,
                                                     const char *auxFmt,
                                                     const char *keyFmt,
                                                     const char *outFilename);
 
+/****************************************************************
+ * Create STAT Handlers
+ *
+ * Aggregates while parsing and, on end-of-RDB, prints a built-in, human-formatted
+ * memory-statistics report (estimated):
+ *   - a by-type table (keys, items, volatile, expired, memory, avg, mem%) + TOTAL row
+ *   - the top `topN` keys by estimated memory (topN <= 0 => default 10)
+ *
+ * `nowSecs` is the reference Unix time in seconds used to decide whether a key with
+ * an expiry is already expired; pass 0 to use the current wall-clock time.
+ *
+ * Memory use is bounded (per-type aggregates + the top-N buffer), independent of
+ * the number of keys in the dump. Output goes to outFilename (NULL => stdout).
+ ****************************************************************/
+_LIBRDB_API RdbxToStat *RDBX_createHandlersToStat(RdbParser *p,
+                                                  int topN,
+                                                  long long nowSecs,
+                                                  const char *outFilename);
+
 /****************************************************************
  * Create Filter Handlers
  ****************************************************************/

src/cli/rdb-cli.c

@@ -108,7 +108,7 @@ static void printUsage(int shortUsage) {
         return;
     }
     printf("[v%s] ", RDB_getLibVersion(NULL,NULL,NULL));
-    printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {print|json|resp|redis} [FORMAT_OPTIONS]\n");
+    printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {print|json|resp|redis|stat} [FORMAT_OPTIONS]\n");
     printf("OPTIONS:\n");
     printf("\t-l, --log-file <PATH>         Path to the log file or stdout (Default: './rdb-cli.log')\n");
     printf("\t-i, --ignore-checksum         Ignore RDB file checksum verification\n");
@@ -125,9 +125,14 @@ static void printUsage(int shortUsage) {
     printf("FORMAT_OPTIONS ('print'):\n");
     printf("\t-a, --aux-val <FMT>           %%f=Auxiliary-Field, %%v=Auxiliary-Value (Default: \"\") \n");
     printf("\t-k, --key <FMT>               %%d=Db %%k=Key %%v=Value %%t=Type %%e=Expiry %%r=LRU %%f=LFU\n");
-    printf("\t                              %%i=Items %%m=NumMeta (Default: \"%%d,%%k,%%v,%%t,%%e,%%i\")\n");
+    printf("\t                              %%i=Items %%m=NumMeta %%n=Encoding %%z=MemBytes(estimate) %%g=LargestElement\n");
+    printf("\t                              Specifiers accept optional width, e.g. %%-20k %%10z (Default: \"%%d,%%k,%%v,%%t,%%e,%%i\")\n");
     printf("\t-o, --output <FILE>           Specify the output file. If not specified, output to stdout\n\n");
 
+    printf("FORMAT_OPTIONS ('stat'):\n");
+    printf("\t-t, --top <N>                 Show the top N keys by estimated memory (Default: 10)\n");
+    printf("\t-n, --now <UNIX-TIME-SEC>     For expiry evaluation (Default: now)\n\n");
+
     printf("FORMAT_OPTIONS ('json'):\n");
     printf("\t-i, --include <EXTRAS>        To include: {aux-val|func|stream-meta|db-info}\n");
     printf("\t-m, --meta-prefix <PREFIX>    To distinct EXTRAS from actual data, Prefix it (Default:\"__\")\n");
@@ -253,6 +258,27 @@ static RdbRes formatPrint(RdbParser *parser, int argc, char **argv) {
     return RDB_OK;
 }
 
+static RdbRes formatStat(RdbParser *parser, int argc, char **argv) {
+    const char *topArg, *nowArg;
+    int topN = 0;          /* 0 => handler default (10)         */
+    long long nowSecs = 0; /* 0 => handler uses wall-clock time */
+
+    /* parse specific command options */
+    for (int at = 1; at < argc; ++at) {
+        char *opt = argv[at];
+        if (getOptArg(argc, argv, &at, "-t", "--top", NULL, &topArg)) { topN = atoi(topArg); continue; }
+        if (getOptArg(argc, argv, &at, "-n", "--now", NULL, &nowArg)) { nowSecs = atoll(nowArg); continue; }
+        loggerWrap(RDB_LOG_ERR, "Invalid 'stat' [FORMAT_OPTIONS] argument: %s\n", opt);
+        printUsage(1);
+        return RDB_ERR_GENERAL;
+    }
+
+    if (RDBX_createHandlersToStat(parser, topN, nowSecs, NULL) == NULL)
+        return RDB_ERR_GENERAL;
+
+    return RDB_OK;
+}
+
 static RdbRes formatRedis(RdbParser *parser, int argc, char **argv) {
     const char *output = NULL;
     RdbxRedisAuth auth = {0};
@@ -542,6 +568,7 @@ int readCommonOptions(RdbParser *p, int argc, char* argv[], Options *options, in
         else if (strcmp(opt, "resp") == 0) { options->formatFunc = formatResp; break; }
         else if (strcmp(opt, "redis") == 0) { options->formatFunc = formatRedis; break; }
         else if (strcmp(opt, "print") == 0) { options->formatFunc = formatPrint; break; }
+        else if (strcmp(opt, "stat") == 0) { options->formatFunc = formatStat; break; }
 
         loggerWrap(RDB_LOG_ERR, "At argv[%d], unexpected OPTIONS argument: %s\n", at, opt);
         printUsage(1);

src/ext/extCommon.c

@@ -1,6 +1,25 @@
+#include <ctype.h>
 #include "extCommon.h"
 #include "../../deps/redis/util.h"
 
+void rdbxOutputPlainEscaping(FILE *out, char *p, size_t len) {
+    while (len--) {
+        switch (*p) {
+            case '\\':
+            case '"':
+                fprintf(out, "\\%c", *p); break;
+            case '\n': fprintf(out, "\\n"); break;
+            case '\f': fprintf(out, "\\f"); break;
+            case '\r': fprintf(out, "\\r"); break;
+            case '\t': fprintf(out, "\\t"); break;
+            case '\b': fprintf(out, "\\b"); break;
+            default:
+                fprintf(out, (isprint((unsigned char)*p)) ? "%c" : "\\u%04x", (unsigned char)*p);
+        }
+        p++;
+    }
+}
+
 /* Example:: Input: length=123  return: buf="\r\n$123\r\n" */
 void iov_length(struct iovec *iov, long long length, char *buf, int bufsize) {
     int len = 0;

src/ext/extCommon.h

@@ -53,4 +53,8 @@ static inline void iov_plain(struct iovec *iov, const char *s, size_t l) {
 /*** hidden LIB API function (not declared in librdb-api.h) ***/
 _LIBRDB_API char *__RDB_key(RdbParser *p, char *key, char buf[9]);
 
+/* Escapes `len` bytes of `p` to `out` (JSON/string rules: \\ \" \n \f \r \t \b,
+ * printable bytes verbatim, others as \uXXXX). Shared by the print/json handlers. */
+void rdbxOutputPlainEscaping(FILE *out, char *p, size_t len);
+
 #endif /*define RDBX_COMMON_H*/