florian/kvstore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

estimate min height

Browse Source
This commit is contained in:
Florian Stecker 2025-10-13 13:04:33 -04:00
parent f2ebb049e4
commit 3fb5868dd4
2 changed files with 43 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
iavlread
View File

@ -11,9 +11,9 @@ def decode_protobuf(subformats: dict, format_prefix: str, data: bytes):
idx = f'{format_prefix}.{k}' idx = f'{format_prefix}.{k}'
if idx in subformats: if idx in subformats:
f = subformats[idx] f = subformats[idx]
if f == 'proto': if f == 'pb':
decoded_value = decode_protobuf(subformats, idx, v) decoded_value = decode_protobuf(subformats, idx, v)
elif f == 'protodict': elif f == 'pbdict':
decoded_value = dict(decode_protobuf(subformats, idx, v)) decoded_value = dict(decode_protobuf(subformats, idx, v))
else: else:
decoded_value = decode_output(f, v) decoded_value = decode_output(f, v)
@ -53,10 +53,10 @@ def decode_output(format: str, data: bytes) -> str:
return struct.unpack('>H', data[:4])[0] - (1<<15) return struct.unpack('>H', data[:4])[0] - (1<<15)
elif format == 'i8ord': elif format == 'i8ord':
return struct.unpack('>B', data[:4])[0] - (1<<7) return struct.unpack('>B', data[:4])[0] - (1<<7)
elif format.startswith('protodict'): elif format.startswith('pbdict'):
subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)} subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)}
return dict(decode_protobuf(subformats, '', data)) return dict(decode_protobuf(subformats, '', data))
elif format.startswith('proto'): elif format.startswith('pb'):
subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)} subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)}
return decode_protobuf(subformats, '', data) return decode_protobuf(subformats, '', data)
else: else:
@ -72,6 +72,9 @@ def get_args():
subparsers = parser.add_subparsers(required=True, dest='cmd') subparsers = parser.add_subparsers(required=True, dest='cmd')
p_max_height = subparsers.add_parser('max_height', help = 'Get the max block height in the snapshot') p_max_height = subparsers.add_parser('max_height', help = 'Get the max block height in the snapshot')
p_max_height.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
p_min_height = subparsers.add_parser('min_height', help = 'Get the min block height in the snapshot')
p_min_height.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
p_get = subparsers.add_parser('get', help = 'Retrieve a single item') p_get = subparsers.add_parser('get', help = 'Retrieve a single item')
p_get.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")') p_get.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
p_get.add_argument('key', nargs='+', help = 'Key parts') p_get.add_argument('key', nargs='+', help = 'Key parts')
@ -95,7 +98,7 @@ def run(args):
keyformat = args.keyformat if args.keyformat is not None else '' keyformat = args.keyformat if args.keyformat is not None else ''
valueformat = args.valueformat if args.valueformat is not None else 'b' valueformat = args.valueformat if args.valueformat is not None else 'b'
if args.cmd == 'max_height' or args.key is None or len(args.key) == 0: if args.cmd == 'max_height' or args.cmd == 'min_height' or args.key is None or len(args.key) == 0:
key = None key = None
else: else:
if len(args.key) > len(keyformat) + 1: if len(args.key) > len(keyformat) + 1:
@ -109,12 +112,15 @@ def run(args):
with plyvel.DB(dbpath) as db: with plyvel.DB(dbpath) as db:
if args.height is None or args.cmd == 'max_height': if args.height is None or args.cmd == 'max_height':
height = iavltree.max_height(db) height = iavltree.max_height(db, args.prefix.encode('utf-8'))
else: else:
height = args.height height = args.height
if args.cmd == 'max_height': if args.cmd == 'max_height':
print(height) print(height)
elif args.cmd == 'min_height':
hmin, _ = iavltree.min_max_height(db, args.prefix.encode('utf-8'))
print(hmin)
elif args.cmd == 'get': elif args.cmd == 'get':
result = iavltree.get(db, args.prefix, height, keyformat, key) result = iavltree.get(db, args.prefix, height, keyformat, key)

44
iavltree.py
View File

@ -1,6 +1,5 @@
import plyvel import plyvel
import struct import struct
import numpy as np
# functions for reading IAVL tree # functions for reading IAVL tree
def read_varint(x: bytes, offset: int = 0) -> tuple[int, int]: def read_varint(x: bytes, offset: int = 0) -> tuple[int, int]:
@ -87,7 +86,7 @@ def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int,
return (height, length, key, (left_version, left_nonce), (right_version, right_nonce)) return (height, length, key, (left_version, left_nonce), (right_version, right_nonce))
def get_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
root = db.get(prefix + write_key((version, 1))) root = db.get(prefix + write_key((version, 1)))
if root is None: if root is None:
return None return None
@ -113,7 +112,7 @@ def get_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
else: else:
return None return None
def get_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: def get_next_key_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
root = db.get(prefix + write_key((version, 1))) root = db.get(prefix + write_key((version, 1)))
if root is None: if root is None:
return None return None
@ -139,7 +138,7 @@ def get_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None
return lowest_geq_key return lowest_geq_key
def get(db, prefix: str, version: int, format: str, searchkey: list) -> None | bytes: def get(db: plyvel.DB, prefix: str, version: int, format: str, searchkey: list) -> None | bytes:
return get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey)) return get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))
def parse_pb(data): def parse_pb(data):
@ -173,12 +172,11 @@ def next_key(db, k: bytes) -> bytes | None:
finally: finally:
it.close() it.close()
def max_height(db) -> int: def max_height(db: plyvel.DB, prefix: bytes) -> int:
testnr = 1<<63 testnr = 1<<63
for i in range(62, -1, -1): for i in range(62, -1, -1):
prefix = b's/k:emissions/s' n = next_key(db, prefix + b's' + struct.pack('>Q', testnr) + struct.pack('>I', 1))
n = next_key(db, prefix + struct.pack('>Q', testnr))
if n is not None and n.startswith(prefix): if n is not None and n.startswith(prefix):
# print(f'{testnr:16x} is low') # print(f'{testnr:16x} is low')
@ -193,6 +191,26 @@ def max_height(db) -> int:
else: else:
return testnr - 1 return testnr - 1
def min_max_height(db: plyvel.DB, prefix: bytes) -> tuple[int, int]:
hmax = max_height(db, prefix)
h = 1<<hmax.bit_length()
inc = h>>1
for _ in range(25):
if h > hmax:
highenough = True
else:
root = db.get(prefix + write_key((h, 1)))
highenough = root is not None
# print(h, highenough, inc)
(h, inc) = (h + (1 - 2*highenough) * inc, inc >> 1)
if not highenough:
h += 1
return (h, hmax)
# encode and decode keys # encode and decode keys
def encode_key(format: str, key: list) -> bytes: def encode_key(format: str, key: list) -> bytes:
result_bytes = [] result_bytes = []
@ -254,7 +272,7 @@ def decode_key(format: str, key: bytes) -> list:
# iteration # iteration
class IAVLTreeIteratorRaw: class IAVLTreeIteratorRaw:
def __init__(self, db, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None): def __init__(self, db: plyvel.DB, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None):
self.db = db self.db = db
self.prefix = prefix self.prefix = prefix
self.version = version self.version = version
@ -331,7 +349,7 @@ class IAVLTreeIteratorRaw:
return (node[2], node[3]) return (node[2], node[3])
class IAVLTreeIterator: class IAVLTreeIterator:
def __init__(self, db, prefix: bytes, version: int, format: str, start: bytes | None = None, end: bytes | None = None): def __init__(self, db: plyvel.DB, prefix: bytes, version: int, format: str, start: bytes | None = None, end: bytes | None = None):
self.format = format self.format = format
self.inner = IAVLTreeIteratorRaw(db, prefix, version, start, end) self.inner = IAVLTreeIteratorRaw(db, prefix, version, start, end)
@ -354,7 +372,7 @@ def next_bs(x: bytes) -> bytes | None:
return x_enc return x_enc
def iterate(db, prefix, version, format = '', key = None, start = None, end = None): def iterate(db: plyvel.DB, prefix, version, format = '', key = None, start = None, end = None):
prefix_enc = prefix.encode('utf-8') prefix_enc = prefix.encode('utf-8')
if key is not None: if key is not None:
@ -366,7 +384,7 @@ def iterate(db, prefix, version, format = '', key = None, start = None, end = No
return IAVLTreeIterator(db, prefix_enc, version, format, start = start_enc, end = end_enc) return IAVLTreeIterator(db, prefix_enc, version, format, start = start_enc, end = end_enc)
def count(db, prefix, version, format = '', key = None, start = None, end = None): def count(db: plyvel.DB, prefix, version, format = '', key = None, start = None, end = None):
prefix_enc = prefix.encode('utf-8') prefix_enc = prefix.encode('utf-8')
if key is not None: if key is not None:
@ -391,7 +409,7 @@ def count(db, prefix, version, format = '', key = None, start = None, end = None
return endidx - startidx return endidx - startidx
def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int: def indexof_raw(db: plyvel.DB, prefix: bytes, version: int, key: bytes) -> int:
""" """
Find how many items come before `key` in the tree. If `key` doesn't exist, how many Find how many items come before `key` in the tree. If `key` doesn't exist, how many
items come before the slot it would get inserted at items come before the slot it would get inserted at
@ -409,5 +427,5 @@ def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int:
return count return count
def indexof(db, prefix: str, version: int, format: str, key: list) -> int: def indexof(db: plyvel.DB, prefix: str, version: int, format: str, key: list) -> int:
return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key)) return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key))