From 3fb5868dd4a14edaeb91b2a09d930218b0b7e014 Mon Sep 17 00:00:00 2001 From: Florian Stecker Date: Mon, 13 Oct 2025 13:04:33 -0400 Subject: [PATCH] estimate min height --- iavlread | 18 ++++++++++++------ iavltree.py | 44 +++++++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/iavlread b/iavlread index 3a50b6d..ea7ab1d 100755 --- a/iavlread +++ b/iavlread @@ -11,9 +11,9 @@ def decode_protobuf(subformats: dict, format_prefix: str, data: bytes): idx = f'{format_prefix}.{k}' if idx in subformats: f = subformats[idx] - if f == 'proto': + if f == 'pb': decoded_value = decode_protobuf(subformats, idx, v) - elif f == 'protodict': + elif f == 'pbdict': decoded_value = dict(decode_protobuf(subformats, idx, v)) else: decoded_value = decode_output(f, v) @@ -53,10 +53,10 @@ def decode_output(format: str, data: bytes) -> str: return struct.unpack('>H', data[:4])[0] - (1<<15) elif format == 'i8ord': return struct.unpack('>B', data[:4])[0] - (1<<7) - elif format.startswith('protodict'): + elif format.startswith('pbdict'): subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)} return dict(decode_protobuf(subformats, '', data)) - elif format.startswith('proto'): + elif format.startswith('pb'): subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)} return decode_protobuf(subformats, '', data) else: @@ -72,6 +72,9 @@ def get_args(): subparsers = parser.add_subparsers(required=True, dest='cmd') p_max_height = subparsers.add_parser('max_height', help = 'Get the max block height in the snapshot') + p_max_height.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")') + p_min_height = subparsers.add_parser('min_height', help = 'Get the min block height in the snapshot') + p_min_height.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")') p_get = subparsers.add_parser('get', help = 'Retrieve a single item') p_get.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")') p_get.add_argument('key', nargs='+', help = 'Key parts') @@ -95,7 +98,7 @@ def run(args): keyformat = args.keyformat if args.keyformat is not None else '' valueformat = args.valueformat if args.valueformat is not None else 'b' - if args.cmd == 'max_height' or args.key is None or len(args.key) == 0: + if args.cmd == 'max_height' or args.cmd == 'min_height' or args.key is None or len(args.key) == 0: key = None else: if len(args.key) > len(keyformat) + 1: @@ -109,12 +112,15 @@ def run(args): with plyvel.DB(dbpath) as db: if args.height is None or args.cmd == 'max_height': - height = iavltree.max_height(db) + height = iavltree.max_height(db, args.prefix.encode('utf-8')) else: height = args.height if args.cmd == 'max_height': print(height) + elif args.cmd == 'min_height': + hmin, _ = iavltree.min_max_height(db, args.prefix.encode('utf-8')) + print(hmin) elif args.cmd == 'get': result = iavltree.get(db, args.prefix, height, keyformat, key) diff --git a/iavltree.py b/iavltree.py index f0760f1..9cba4ea 100644 --- a/iavltree.py +++ b/iavltree.py @@ -1,6 +1,5 @@ import plyvel import struct -import numpy as np # functions for reading IAVL tree def read_varint(x: bytes, offset: int = 0) -> tuple[int, int]: @@ -87,7 +86,7 @@ def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int, return (height, length, key, (left_version, left_nonce), (right_version, right_nonce)) -def get_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: +def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: root = db.get(prefix + write_key((version, 1))) if root is None: return None @@ -113,7 +112,7 @@ def get_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: else: return None -def get_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: +def get_next_key_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: root = db.get(prefix + write_key((version, 1))) if root is None: return None @@ -139,7 +138,7 @@ def get_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None return lowest_geq_key -def get(db, prefix: str, version: int, format: str, searchkey: list) -> None | bytes: +def get(db: plyvel.DB, prefix: str, version: int, format: str, searchkey: list) -> None | bytes: return get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey)) def parse_pb(data): @@ -173,12 +172,11 @@ def next_key(db, k: bytes) -> bytes | None: finally: it.close() -def max_height(db) -> int: +def max_height(db: plyvel.DB, prefix: bytes) -> int: testnr = 1<<63 for i in range(62, -1, -1): - prefix = b's/k:emissions/s' - n = next_key(db, prefix + struct.pack('>Q', testnr)) + n = next_key(db, prefix + b's' + struct.pack('>Q', testnr) + struct.pack('>I', 1)) if n is not None and n.startswith(prefix): # print(f'{testnr:16x} is low') @@ -193,6 +191,26 @@ def max_height(db) -> int: else: return testnr - 1 +def min_max_height(db: plyvel.DB, prefix: bytes) -> tuple[int, int]: + hmax = max_height(db, prefix) + + h = 1<>1 + + for _ in range(25): + if h > hmax: + highenough = True + else: + root = db.get(prefix + write_key((h, 1))) + highenough = root is not None + # print(h, highenough, inc) + (h, inc) = (h + (1 - 2*highenough) * inc, inc >> 1) + if not highenough: + h += 1 + + return (h, hmax) + + # encode and decode keys def encode_key(format: str, key: list) -> bytes: result_bytes = [] @@ -254,7 +272,7 @@ def decode_key(format: str, key: bytes) -> list: # iteration class IAVLTreeIteratorRaw: - def __init__(self, db, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None): + def __init__(self, db: plyvel.DB, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None): self.db = db self.prefix = prefix self.version = version @@ -331,7 +349,7 @@ class IAVLTreeIteratorRaw: return (node[2], node[3]) class IAVLTreeIterator: - def __init__(self, db, prefix: bytes, version: int, format: str, start: bytes | None = None, end: bytes | None = None): + def __init__(self, db: plyvel.DB, prefix: bytes, version: int, format: str, start: bytes | None = None, end: bytes | None = None): self.format = format self.inner = IAVLTreeIteratorRaw(db, prefix, version, start, end) @@ -354,7 +372,7 @@ def next_bs(x: bytes) -> bytes | None: return x_enc -def iterate(db, prefix, version, format = '', key = None, start = None, end = None): +def iterate(db: plyvel.DB, prefix, version, format = '', key = None, start = None, end = None): prefix_enc = prefix.encode('utf-8') if key is not None: @@ -366,7 +384,7 @@ def iterate(db, prefix, version, format = '', key = None, start = None, end = No return IAVLTreeIterator(db, prefix_enc, version, format, start = start_enc, end = end_enc) -def count(db, prefix, version, format = '', key = None, start = None, end = None): +def count(db: plyvel.DB, prefix, version, format = '', key = None, start = None, end = None): prefix_enc = prefix.encode('utf-8') if key is not None: @@ -391,7 +409,7 @@ def count(db, prefix, version, format = '', key = None, start = None, end = None return endidx - startidx -def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int: +def indexof_raw(db: plyvel.DB, prefix: bytes, version: int, key: bytes) -> int: """ Find how many items come before `key` in the tree. If `key` doesn't exist, how many items come before the slot it would get inserted at @@ -409,5 +427,5 @@ def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int: return count -def indexof(db, prefix: str, version: int, format: str, key: list) -> int: +def indexof(db: plyvel.DB, prefix: str, version: int, format: str, key: list) -> int: return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key))