add history subcommand
This commit is contained in:
Florian Stecker
parent
3fb5868dd4
commit
2753a5d2d9
23
iavlread
23
iavlread
@ -34,25 +34,25 @@ def decode_output(format: str, data: bytes) -> str:
|
||||
elif format == 'u32':
|
||||
return struct.unpack('>I', data[:4])[0]
|
||||
elif format == 'u16':
|
||||
return struct.unpack('>H', data[:4])[0]
|
||||
return struct.unpack('>H', data[:2])[0]
|
||||
elif format == 'u8':
|
||||
return struct.unpack('>B', data[:4])[0]
|
||||
return struct.unpack('>B', data[:1])[0]
|
||||
elif format == 'i64':
|
||||
return struct.unpack('>q', data[:8])[0]
|
||||
elif format == 'i32':
|
||||
return struct.unpack('>i', data[:4])[0]
|
||||
elif format == 'i16':
|
||||
return struct.unpack('>h', data[:4])[0]
|
||||
return struct.unpack('>h', data[:2])[0]
|
||||
elif format == 'i8':
|
||||
return struct.unpack('>b', data[:4])[0]
|
||||
return struct.unpack('>b', data[:1])[0]
|
||||
elif format == 'i64ord':
|
||||
return struct.unpack('>Q', data[:8])[0] - (1<<63)
|
||||
elif format == 'i32ord':
|
||||
return struct.unpack('>I', data[:4])[0] - (1<<31)
|
||||
elif format == 'i16ord':
|
||||
return struct.unpack('>H', data[:4])[0] - (1<<15)
|
||||
return struct.unpack('>H', data[:2])[0] - (1<<15)
|
||||
elif format == 'i8ord':
|
||||
return struct.unpack('>B', data[:4])[0] - (1<<7)
|
||||
return struct.unpack('>B', data[:1])[0] - (1<<7)
|
||||
elif format.startswith('pbdict'):
|
||||
subformats = {'.' + id: subformat for x in format.split(',')[1:] for id, subformat in (x.split('='),)}
|
||||
return dict(decode_protobuf(subformats, '', data))
|
||||
@ -78,6 +78,9 @@ def get_args():
|
||||
p_get = subparsers.add_parser('get', help = 'Retrieve a single item')
|
||||
p_get.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
|
||||
p_get.add_argument('key', nargs='+', help = 'Key parts')
|
||||
p_hist = subparsers.add_parser('history', help = 'Get all stored past values of the item')
|
||||
p_hist.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
|
||||
p_hist.add_argument('key', nargs='+', help = 'Key parts')
|
||||
p_count = subparsers.add_parser('count', help = 'Count number of items with a prefix')
|
||||
p_count.add_argument('prefix', help = 'Prefix (e.g. "s/k:emissions/")')
|
||||
p_count.add_argument('key', nargs='*', help = 'Key parts')
|
||||
@ -126,6 +129,14 @@ def run(args):
|
||||
|
||||
if result is not None:
|
||||
print(decode_output(valueformat, result))
|
||||
elif args.cmd == 'history':
|
||||
it = iavltree.history(db, args.prefix, keyformat, key, height)
|
||||
|
||||
try:
|
||||
for h, v in it:
|
||||
print(f'{h} {decode_output(valueformat, v)}')
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
elif args.cmd == 'count':
|
||||
result = iavltree.count(db, args.prefix, height, keyformat, key = key)
|
||||
|
||||
|
||||
64
iavltree.py
64
iavltree.py
@ -86,29 +86,32 @@ def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int,
|
||||
|
||||
return (height, length, key, (left_version, left_nonce), (right_version, right_nonce))
|
||||
|
||||
def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:
|
||||
root = db.get(prefix + write_key((version, 1)))
|
||||
if root is None:
|
||||
def get_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: bytes) -> None | tuple[int, int, int, int, bytes, bytes]:
|
||||
key = db.get(prefix + write_key((version, 1)))
|
||||
if key is None:
|
||||
return None
|
||||
|
||||
node = read_node(root)
|
||||
node = read_node(key)
|
||||
|
||||
if len(node) == 2: # root copy?
|
||||
node = read_node(db.get(prefix + write_key(node)))
|
||||
key = node
|
||||
node = read_node(db.get(prefix + write_key(key)))
|
||||
|
||||
while node[0] > 0:
|
||||
# print(node)
|
||||
|
||||
nodekey = node[2]
|
||||
if searchkey < nodekey:
|
||||
next = node[3]
|
||||
key = node[3]
|
||||
else:
|
||||
next = node[4]
|
||||
key = node[4]
|
||||
|
||||
node = read_node(db.get(prefix + write_key(next)))
|
||||
node = read_node(db.get(prefix + write_key(key)))
|
||||
|
||||
if node[2] == searchkey:
|
||||
return node[3]
|
||||
(version, nonce) = key
|
||||
(height, length, itemkey, value) = node
|
||||
return (version, nonce, height, length, itemkey, value)
|
||||
else:
|
||||
return None
|
||||
|
||||
@ -139,7 +142,8 @@ def get_next_key_raw(db: plyvel.DB, prefix: bytes, version: int, searchkey: byte
|
||||
return lowest_geq_key
|
||||
|
||||
def get(db: plyvel.DB, prefix: str, version: int, format: str, searchkey: list) -> None | bytes:
|
||||
return get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))
|
||||
x = get_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))
|
||||
return x[5] if x is not None else None
|
||||
|
||||
def parse_pb(data):
|
||||
n = 0
|
||||
@ -172,26 +176,32 @@ def next_key(db, k: bytes) -> bytes | None:
|
||||
finally:
|
||||
it.close()
|
||||
|
||||
def max_height(db: plyvel.DB, prefix: bytes) -> int:
|
||||
def max_height(db: plyvel.DB, prefix: bytes | str) -> int:
|
||||
if isinstance(prefix, str):
|
||||
prefix = prefix.encode('utf-8')
|
||||
|
||||
testnr = 1<<63
|
||||
|
||||
for i in range(62, -1, -1):
|
||||
n = next_key(db, prefix + b's' + struct.pack('>Q', testnr) + struct.pack('>I', 1))
|
||||
|
||||
if n is not None and n.startswith(prefix):
|
||||
# print(f'{testnr:16x} is low')
|
||||
# print(f'{testnr} is low')
|
||||
testnr += 1 << i
|
||||
else:
|
||||
# print(f'{testnr:16x} is high')
|
||||
# print(f'{testnr} is high')
|
||||
testnr -= 1 << i
|
||||
|
||||
n = next_key(db, prefix + struct.pack('>Q', testnr))
|
||||
n = db.get(prefix + struct.pack('>Q', testnr))
|
||||
if n is not None and n.startswith(prefix):
|
||||
return testnr
|
||||
else:
|
||||
return testnr - 1
|
||||
|
||||
def min_max_height(db: plyvel.DB, prefix: bytes) -> tuple[int, int]:
|
||||
if isinstance(prefix, str):
|
||||
prefix = prefix.encode('utf-8')
|
||||
|
||||
hmax = max_height(db, prefix)
|
||||
|
||||
h = 1<<hmax.bit_length()
|
||||
@ -429,3 +439,29 @@ def indexof_raw(db: plyvel.DB, prefix: bytes, version: int, key: bytes) -> int:
|
||||
|
||||
def indexof(db: plyvel.DB, prefix: str, version: int, format: str, key: list) -> int:
|
||||
return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key))
|
||||
|
||||
class IAVLTreeHistoryIterator:
|
||||
def __init__(self, db: plyvel.DB, prefix: bytes, key: bytes, max_height: int, min_height: int = 0):
|
||||
self.db = db
|
||||
self.prefix = prefix
|
||||
self.key = key
|
||||
self.height = max_height
|
||||
self.min_height = min_height
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self) -> tuple[int, bytes]:
|
||||
if self.height < self.min_height:
|
||||
raise StopIteration
|
||||
result = get_raw(self.db, self.prefix, self.height, self.key)
|
||||
if result is None:
|
||||
raise StopIteration
|
||||
(h, _, _, _, _, v) = result
|
||||
self.height = h - 1
|
||||
return (h, v)
|
||||
|
||||
def history(db: plyvel.DB, prefix: str, format: str, key: list, max_height: int, min_height: int = 0) -> IAVLTreeHistoryIterator:
|
||||
prefix_enc = prefix.encode('utf-8')
|
||||
key_enc = encode_key(format, key)
|
||||
return IAVLTreeHistoryIterator(db, prefix_enc, key_enc, max_height, min_height)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user