fix crash in parsing module data types containing floats

* modules save binary float, and not the old string based floats.
* fix crash in the memory profiler when handling rdb files with modules or streams
* improve memory report to contains the total estimated memory, and used_mem aux field, etc.

Author: oranagra

rdbtools/memprofiler.py

@@ -25,9 +25,11 @@ def __init__(self, key_groupings = None):
         self.aggregates = {}
         self.scatters = {}
         self.histograms = {}
+        self.metadata = {}
 
     def next_record(self, record):
         self.add_aggregate('database_memory', record.database, record.bytes)
+        self.add_aggregate('database_memory', 'all', record.bytes)
         self.add_aggregate('type_memory', record.type, record.bytes)
         self.add_aggregate('encoding_memory', record.encoding, record.bytes)
 
@@ -47,7 +49,7 @@ def next_record(self, record):
             self.add_scatter('sortedset_memory_by_length', record.bytes, record.size)
         elif record.type == 'string':
             self.add_scatter('string_memory_by_length', record.bytes, record.size)
-        elif record.type == 'dict':
+        elif record.type in ['dict', 'module', 'stream']:
             pass
         else:
             raise Exception('Invalid data type %s' % record.type)
@@ -74,9 +76,12 @@ def add_scatter(self, heading, x, y):
         if not heading in self.scatters:
             self.scatters[heading] = []
         self.scatters[heading].append([x, y])
+
+    def set_metadata(self, key, val):
+        self.metadata[key] = val
 
     def get_json(self):
-        return json.dumps({"aggregates":self.aggregates, "scatters":self.scatters, "histograms":self.histograms})
+        return json.dumps({"aggregates": self.aggregates, "scatters": self.scatters, "histograms": self.histograms, "metadata": self.metadata})
 
 class PrintAllKeys(object):
     def __init__(self, out, bytes, largest):
@@ -159,7 +164,6 @@ def start_rdb(self):
         pass
 
     def aux_field(self, key, value):
-        #print('aux: %s %s' % (key, value))
         if key == 'used-mem':
             self._aux_used_mem = int(value)
         if key == 'redis-ver':
@@ -179,9 +183,13 @@ def end_database(self, db_number):
             self._stream.end_database(db_number)
 
     def end_rdb(self):
-        #print('internal fragmentation: %s' % self._total_internal_frag)
         if hasattr(self._stream, 'end_rdb'):
             self._stream.end_rdb()
+        if hasattr(self._stream, 'set_metadata'):
+            self._stream.set_metadata('used_mem', self._aux_used_mem)
+            self._stream.set_metadata('redis_ver', self._aux_redis_ver)
+            self._stream.set_metadata('redis_bits', self._aux_redis_bits)
+            self._stream.set_metadata('internal_frag', self._total_internal_frag)
 
     def set(self, key, value, expiry, info):
         self._current_encoding = info['encoding']

rdbtools/parser.py

@@ -558,7 +558,7 @@ def read_object(self, f, enc_type) :
             self._callback.start_sorted_set(self._key, length, self._expiry, info={'encoding':'skiplist','idle':self._idle,'freq':self._freq})
             for count in range(0, length):
                 val = self.read_string(f)
-                score = read_double(f) if enc_type == REDIS_RDB_TYPE_ZSET_2 else self.read_float(f)
+                score = read_binary_double(f) if enc_type == REDIS_RDB_TYPE_ZSET_2 else self.read_float(f)
                 self._callback.zadd(self._key, score, val)
             self._callback.end_sorted_set(self._key)
         elif enc_type == REDIS_RDB_TYPE_HASH:
@@ -825,9 +825,9 @@ def skip_module(self, f):
             if opcode == REDIS_RDB_MODULE_OPCODE_SINT or opcode == REDIS_RDB_MODULE_OPCODE_UINT:
                 self.read_length(f)
             elif opcode == REDIS_RDB_MODULE_OPCODE_FLOAT:
-                self.skip_float(f)
+                read_binary_float(f)
             elif opcode == REDIS_RDB_MODULE_OPCODE_DOUBLE:
-                read_double(f)
+                read_binary_double(f)
             elif opcode == REDIS_RDB_MODULE_OPCODE_STRING:
                 self.skip_string(f)
             else:
@@ -851,9 +851,9 @@ def read_module(self, f):
             if opcode == REDIS_RDB_MODULE_OPCODE_SINT or opcode == REDIS_RDB_MODULE_OPCODE_UINT:
                 data = self.read_length(iowrapper)
             elif opcode == REDIS_RDB_MODULE_OPCODE_FLOAT:
-                data = self.read_float(iowrapper)
+                data = read_binary_float(iowrapper)
             elif opcode == REDIS_RDB_MODULE_OPCODE_DOUBLE:
-                data = read_double(iowrapper)
+                data = read_binary_double(iowrapper)
             elif opcode == REDIS_RDB_MODULE_OPCODE_STRING:
                 data = self.read_string(iowrapper)
             else:
@@ -1104,9 +1104,12 @@ def read_milliseconds_time(f) :
 def read_unsigned_long_be(f) :
     return struct.unpack('>Q', f.read(8))[0]
 
-def read_double(f) :
+def read_binary_double(f) :
     return struct.unpack('d', f.read(8))[0]
 
+def read_binary_float(f) :
+    return struct.unpack('f', f.read(4))[0]
+
 def string_as_hexcode(string) :
     for s in string :
         if isinstance(s, int) :

rdbtools/templates/report.html.template

@@ -15,7 +15,10 @@
         google.setOnLoadCallback(draw_charts);
 
         function draw_charts() {
-            //draw_pie_chart('database_memory', chart_data.aggregates.database_memory, 'Database Number', 'Size in Bytes', 'Memory Usage by Database')
+            document.getElementById('database_memory').innerHTML = chart_data.aggregates.database_memory['all']
+            document.getElementById('aux_used_mem').innerHTML = chart_data.metadata.used_mem
+            document.getElementById('aux_redis_ver_and_bits').innerHTML = chart_data.metadata.redis_ver + ' - ' + chart_data.metadata.redis_bits + 'bit'
+            document.getElementById('internal_frag').innerHTML = chart_data.metadata.internal_frag
 
             draw_pie_chart('type_memory', chart_data.aggregates.type_memory, 'Data Type', 'Total Size in Bytes', 'Memory Usage by Data Type')
             draw_column_chart('type_count', chart_data.aggregates.type_count, 'Data Type', 'Keys', 'Number of Keys by Data Type')
@@ -105,14 +108,26 @@
   <body>
     <h1>Redis Memory Distribution for dump.rdb</h1>
     <div class="container">
-        <!--
+        <h2>Total database computed memory:</h2>
         <div class="row">
             <div class="span6" id="database_memory">
             </div>
-            <div class="span6" id="">
+        </div>
+        <h2>Total database memory from RDB aux field:</h2>
+        <div class="row">
+            <div class="span6" id="aux_used_mem">
+            </div>
+        </div>
+        <h2>Database redis version and bits:</h2>
+        <div class="row">
+            <div class="span6" id="aux_redis_ver_and_bits">
+            </div>
+        </div>
+        <h2>Total estimated internal fragmentation:</h2>
+        <div class="row">
+            <div class="span6" id="internal_frag">
             </div>
         </div>
-        -->
         <h2>Memory Usage By Data Type and Data Encoding</h2>
         <div class="row">
             <div class="span6" id="type_memory">