estimate min height
This commit is contained in:
Florian Stecker
parent
f2ebb049e4
commit
3fb5868dd4
18
iavlread
18
iavlread
@ -11,9 +11,9 @@ def decode_protobuf(subformats: dict, format_prefix: str, data: bytes):
|
||||
idx = f'{format_prefix}.{k}'
|
||||
if idx in subformats:
|
||||
f = subformats[idx]
|
||||
if f == 'proto':
|
||||
if f == 'pb':
|
||||
decoded_value = decode_protobuf(subformats, idx, v)
|
||||
elif f == 'protodict':
|
||||
elif f == 'pbdict':
|
||||
decoded_value = dict(decode_protobuf(subformats, idx, v))
|
||||
else:
|
||||
decoded_value = decode_output(f, v)
|
||||
@ -53,10 +53,10 @@ def decode_output(format: str, data: bytes) -> str:
|
||||
return struct.unpack('>H', data[:4])[0] - (1<<15)
|
||||
elif format == 'i8ord':
|
||||
return struct.unpack('>B', data[:4])[0] - (1<<7)
|
||||
elif format.startswith('protodict'):
|
||||
elif format.startswith('pbdict'):
|
||||
subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)}
|
||||
return dict(decode_protobuf(subformats, '', data))
|
||||
elif format.startswith('proto'):
|
||||
elif format.startswith('pb'):
|
||||
subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)}
|
||||
return decode_protobuf(subformats, '', data)
|
||||
else:
|
||||
@ -72,6 +72,9 @@ def get_args():
|
||||
|
||||
subparsers = parser.add_subparsers(required=True, dest='cmd')
|
||||
p_max_height = subparsers.add_parser('max_height', help = 'Get the max block height in the snapshot')
|
||||
p_max_height.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
|
||||
p_min_height = subparsers.add_parser('min_height', help = 'Get the min block height in the snapshot')
|
||||
p_min_height.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
|
||||
p_get = subparsers.add_parser('get', help = 'Retrieve a single item')
|
||||
p_get.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
|
||||
p_get.add_argument('key', nargs='+', help = 'Key parts')
|
||||
@ -95,7 +98,7 @@ def run(args):
|
||||
keyformat = args.keyformat if args.keyformat is not None else ''
|
||||
valueformat = args.valueformat if args.valueformat is not None else 'b'
|
||||
|
||||
if args.cmd == 'max_height' or args.key is None or len(args.key) == 0:
|
||||
if args.cmd == 'max_height' or args.cmd == 'min_height' or args.key is None or len(args.key) == 0:
|
||||
key = None
|
||||
else:
|
||||
if len(args.key) > len(keyformat) + 1:
|
||||
@ -109,12 +112,15 @@ def run(args):
|
||||
|
||||
with plyvel.DB(dbpath) as db:
|
||||
if args.height is None or args.cmd == 'max_height':
|
||||
height = iavltree.max_height(db)
|
||||
height = iavltree.max_height(db, args.prefix.encode('utf-8'))
|
||||
else:
|
||||
height = args.height
|
||||
|
||||
if args.cmd == 'max_height':
|
||||
print(height)
|
||||
elif args.cmd == 'min_height':
|
||||
hmin, _ = iavltree.min_max_height(db, args.prefix.encode('utf-8'))
|
||||
print(hmin)
|
||||
elif args.cmd == 'get':
|
||||
result = iavltree.get(db, args.prefix, height, keyformat, key)
|
||||
|
||||
|
||||
44
iavltree.py
44
iavltree.py
@ -1,6 +1,5 @@
|
||||
import plyvel
|
||||
import struct
|
||||
import numpy as np
|
||||
|
||||
# functions for reading IAVL tree
|
||||
def read_varint(x: bytes, offset: int = 0) -> tuple[int, int]:
|
||||
@ -87,7 +86,7 @@ def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int,
|
||||
|
||||
return (height, length, key, (left_version, left_nonce), (right_version, right_nonce))
|
||||
|
||||
def get_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
|
||||
def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
|
||||
root = db.get(prefix + write_key((version, 1)))
|
||||
if root is None:
|
||||
return None
|
||||
@ -113,7 +112,7 @@ def get_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
|
||||
def get_next_key_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
|
||||
root = db.get(prefix + write_key((version, 1)))
|
||||
if root is None:
|
||||
return None
|
||||
@ -139,7 +138,7 @@ def get_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None
|
||||
|
||||
return lowest_geq_key
|
||||
|
||||
def get(db, prefix: str, version: int, format: str, searchkey: list) -> None | bytes:
|
||||
def get(db: plyvel.DB, prefix: str, version: int, format: str, searchkey: list) -> None | bytes:
|
||||
return get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))
|
||||
|
||||
def parse_pb(data):
|
||||
@ -173,12 +172,11 @@ def next_key(db, k: bytes) -> bytes | None:
|
||||
finally:
|
||||
it.close()
|
||||
|
||||
def max_height(db) -> int:
|
||||
def max_height(db: plyvel.DB, prefix: bytes) -> int:
|
||||
testnr = 1<<63
|
||||
|
||||
for i in range(62, -1, -1):
|
||||
prefix = b's/k:emissions/s'
|
||||
n = next_key(db, prefix + struct.pack('>Q', testnr))
|
||||
n = next_key(db, prefix + b's' + struct.pack('>Q', testnr) + struct.pack('>I', 1))
|
||||
|
||||
if n is not None and n.startswith(prefix):
|
||||
# print(f'{testnr:16x} is low')
|
||||
@ -193,6 +191,26 @@ def max_height(db) -> int:
|
||||
else:
|
||||
return testnr - 1
|
||||
|
||||
def min_max_height(db: plyvel.DB, prefix: bytes) -> tuple[int, int]:
|
||||
hmax = max_height(db, prefix)
|
||||
|
||||
h = 1<<hmax.bit_length()
|
||||
inc = h>>1
|
||||
|
||||
for _ in range(25):
|
||||
if h > hmax:
|
||||
highenough = True
|
||||
else:
|
||||
root = db.get(prefix + write_key((h, 1)))
|
||||
highenough = root is not None
|
||||
# print(h, highenough, inc)
|
||||
(h, inc) = (h + (1 - 2*highenough) * inc, inc >> 1)
|
||||
if not highenough:
|
||||
h += 1
|
||||
|
||||
return (h, hmax)
|
||||
|
||||
|
||||
# encode and decode keys
|
||||
def encode_key(format: str, key: list) -> bytes:
|
||||
result_bytes = []
|
||||
@ -254,7 +272,7 @@ def decode_key(format: str, key: bytes) -> list:
|
||||
|
||||
# iteration
|
||||
class IAVLTreeIteratorRaw:
|
||||
def __init__(self, db, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None):
|
||||
def __init__(self, db: plyvel.DB, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None):
|
||||
self.db = db
|
||||
self.prefix = prefix
|
||||
self.version = version
|
||||
@ -331,7 +349,7 @@ class IAVLTreeIteratorRaw:
|
||||
return (node[2], node[3])
|
||||
|
||||
class IAVLTreeIterator:
|
||||
def __init__(self, db, prefix: bytes, version: int, format: str, start: bytes | None = None, end: bytes | None = None):
|
||||
def __init__(self, db: plyvel.DB, prefix: bytes, version: int, format: str, start: bytes | None = None, end: bytes | None = None):
|
||||
self.format = format
|
||||
self.inner = IAVLTreeIteratorRaw(db, prefix, version, start, end)
|
||||
|
||||
@ -354,7 +372,7 @@ def next_bs(x: bytes) -> bytes | None:
|
||||
|
||||
return x_enc
|
||||
|
||||
def iterate(db, prefix, version, format = '', key = None, start = None, end = None):
|
||||
def iterate(db: plyvel.DB, prefix, version, format = '', key = None, start = None, end = None):
|
||||
prefix_enc = prefix.encode('utf-8')
|
||||
|
||||
if key is not None:
|
||||
@ -366,7 +384,7 @@ def iterate(db, prefix, version, format = '', key = None, start = None, end = No
|
||||
|
||||
return IAVLTreeIterator(db, prefix_enc, version, format, start = start_enc, end = end_enc)
|
||||
|
||||
def count(db, prefix, version, format = '', key = None, start = None, end = None):
|
||||
def count(db: plyvel.DB, prefix, version, format = '', key = None, start = None, end = None):
|
||||
prefix_enc = prefix.encode('utf-8')
|
||||
|
||||
if key is not None:
|
||||
@ -391,7 +409,7 @@ def count(db, prefix, version, format = '', key = None, start = None, end = None
|
||||
|
||||
return endidx - startidx
|
||||
|
||||
def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int:
|
||||
def indexof_raw(db: plyvel.DB, prefix: bytes, version: int, key: bytes) -> int:
|
||||
"""
|
||||
Find how many items come before `key` in the tree. If `key` doesn't exist, how many
|
||||
items come before the slot it would get inserted at
|
||||
@ -409,5 +427,5 @@ def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int:
|
||||
|
||||
return count
|
||||
|
||||
def indexof(db, prefix: str, version: int, format: str, key: list) -> int:
|
||||
def indexof(db: plyvel.DB, prefix: str, version: int, format: str, key: list) -> int:
|
||||
return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user