add history subcommand

iavlread

@@ -34,25 +34,25 @@ def decode_output(format: str, data: bytes) -> str:
     elif format == 'u32':
         return struct.unpack('>I', data[:4])[0]
     elif format == 'u16':
-        return struct.unpack('>H', data[:4])[0]
+        return struct.unpack('>H', data[:2])[0]
     elif format == 'u8':
-        return struct.unpack('>B', data[:4])[0]
+        return struct.unpack('>B', data[:1])[0]
     elif format == 'i64':
         return struct.unpack('>q', data[:8])[0]
     elif format == 'i32':
         return struct.unpack('>i', data[:4])[0]
     elif format == 'i16':
-        return struct.unpack('>h', data[:4])[0]
+        return struct.unpack('>h', data[:2])[0]
     elif format == 'i8':
-        return struct.unpack('>b', data[:4])[0]
+        return struct.unpack('>b', data[:1])[0]
     elif format == 'i64ord':
         return struct.unpack('>Q', data[:8])[0] - (1<<63)
     elif format == 'i32ord':
         return struct.unpack('>I', data[:4])[0] - (1<<31)
     elif format == 'i16ord':
-        return struct.unpack('>H', data[:4])[0] - (1<<15)
+        return struct.unpack('>H', data[:2])[0] - (1<<15)
     elif format == 'i8ord':
-        return struct.unpack('>B', data[:4])[0] - (1<<7)
+        return struct.unpack('>B', data[:1])[0] - (1<<7)
     elif format.startswith('pbdict'):
         subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)}
         return dict(decode_protobuf(subformats, '', data))
@@ -78,6 +78,9 @@ def get_args():
     p_get = subparsers.add_parser('get', help = 'Retrieve a single item')
     p_get.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
     p_get.add_argument('key', nargs='+', help = 'Key parts')
+    p_hist = subparsers.add_parser('history', help = 'Get all stored past values of the item')
+    p_hist.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
+    p_hist.add_argument('key', nargs='+', help = 'Key parts')
     p_count = subparsers.add_parser('count', help = 'Count number of items with a prefix')
     p_count.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
     p_count.add_argument('key', nargs='*', help = 'Key parts')
@@ -126,6 +129,14 @@ def run(args):
 
             if result is not None:
                 print(decode_output(valueformat, result))
+        elif args.cmd == 'history':
+            it = iavltree.history(db, args.prefix, keyformat, key, height)
+
+            try:
+                for h, v in it:
+                    print(f'{h} {decode_output(valueformat, v)}')
+            except BrokenPipeError:
+                pass
         elif args.cmd == 'count':
             result = iavltree.count(db, args.prefix, height, keyformat, key = key)
 

iavltree.py

@@ -86,29 +86,32 @@ def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int,
 
         return (height, length, key, (left_version, left_nonce), (right_version, right_nonce))
 
-def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
+def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | tuple[int, int, int, int, bytes, bytes]:
-    root = db.get(prefix + write_key((version, 1)))
+    key = db.get(prefix + write_key((version, 1)))
-    if root is None:
+    if key is None:
         return None
 
-    node = read_node(root)
+    node = read_node(key)
 
     if len(node) == 2: # root copy?
-        node = read_node(db.get(prefix + write_key(node)))
+        key = node
+        node = read_node(db.get(prefix + write_key(key)))
 
     while node[0] > 0:
         # print(node)
 
         nodekey = node[2]
         if searchkey < nodekey:
-            next = node[3]
+            key = node[3]
         else:
-            next = node[4]
+            key = node[4]
 
-        node = read_node(db.get(prefix + write_key(next)))
+        node = read_node(db.get(prefix + write_key(key)))
 
     if node[2] == searchkey:
-        return node[3]
+        (version, nonce) = key
+        (height, length, itemkey, value) = node
+        return (version, nonce, height, length, itemkey, value)
     else:
         return None
 
@@ -139,7 +142,8 @@ def get_next_key_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: byte
     return lowest_geq_key
 
 def get(db: plyvel.DB, prefix: str, version: int, format: str, searchkey: list) -> None | bytes:
-    return get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))
+    x = get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))
+    return x[5] if x is not None else None
 
 def parse_pb(data):
     n = 0
@@ -172,26 +176,32 @@ def next_key(db, k: bytes) -> bytes | None:
     finally:
         it.close()
 
-def max_height(db: plyvel.DB, prefix: bytes) -> int:
+def max_height(db: plyvel.DB, prefix: bytes | str) -> int:
+    if isinstance(prefix, str):
+        prefix = prefix.encode('utf-8')
+
     testnr = 1<<63
 
     for i in range(62, -1, -1):
         n = next_key(db, prefix + b's' + struct.pack('>Q', testnr) + struct.pack('>I', 1))
 
         if n is not None and n.startswith(prefix):
-            # print(f'{testnr:16x} is low')
+            # print(f'{testnr} is low')
             testnr += 1 << i
         else:
-            # print(f'{testnr:16x} is high')
+            # print(f'{testnr} is high')
             testnr -= 1 << i
 
-    n = next_key(db, prefix + struct.pack('>Q', testnr))
+    n = db.get(prefix + struct.pack('>Q', testnr))
     if n is not None and n.startswith(prefix):
         return testnr
     else:
         return testnr - 1
 
 def min_max_height(db: plyvel.DB, prefix: bytes) -> tuple[int, int]:
+    if isinstance(prefix, str):
+        prefix = prefix.encode('utf-8')
+
     hmax = max_height(db, prefix)
 
     h = 1<<hmax.bit_length()
@@ -429,3 +439,29 @@ def indexof_raw(db: plyvel.DB, prefix: bytes, version: int, key: bytes) -> int:
 
 def indexof(db: plyvel.DB, prefix: str, version: int, format: str, key: list) -> int:
     return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key))
+
+class IAVLTreeHistoryIterator:
+    def __init__(self, db: plyvel.DB, prefix: bytes, key: bytes, max_height: int, min_height: int = 0):
+        self.db = db
+        self.prefix = prefix
+        self.key = key
+        self.height = max_height
+        self.min_height = min_height
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> tuple[int, bytes]:
+        if self.height < self.min_height:
+            raise StopIteration
+        result = get_raw(self.db, self.prefix, self.height, self.key)
+        if result is None:
+            raise StopIteration
+        (h, _, _, _, _, v) = result
+        self.height = h - 1
+        return (h, v)
+
+def history(db: plyvel.DB, prefix: str, format: str, key: list, max_height: int, min_height: int = 0) -> IAVLTreeHistoryIterator:
+    prefix_enc = prefix.encode('utf-8')
+    key_enc = encode_key(format, key)
+    return IAVLTreeHistoryIterator(db, prefix_enc, key_enc, max_height, min_height)