diff --git a/read_tree.py b/read_tree.py new file mode 100644 index 0000000..cc66eab --- /dev/null +++ b/read_tree.py @@ -0,0 +1,364 @@ +import plyvel +import struct +import numpy as np + +# functions for reading IAVL tree +def read_varint(x: bytes, offset: int = 0) -> int: + result = 0 + factor = 1 + + for i, b in enumerate(x[offset:]): + if b >= 128: + result = result + (b - 128) * factor + else: + result = result + b * factor + return result // 2, offset+i+1 + factor *= 128 + +def read_uvarint(x: bytes, offset: int = 0) -> int: + result = 0 + factor = 1 + + for i, b in enumerate(x[offset:]): + if b >= 128: + result = result + (b - 128) * factor + else: + result = result + b * factor + return result, offset+i+1 + factor *= 128 + +def read_key(key: bytes) -> tuple[int, int] | None: + if not key.startswith(b's'): + return None + + version = struct.unpack_from('>Q', key[1:9])[0] + nonce = struct.unpack_from('>I', key[9:13])[0] + + return (version, nonce) + +def write_key(key: tuple[int, int]) -> bytes: + version = struct.pack('>Q', key[0]) + nonce = struct.pack('>I', key[1]) + + return b's' + version + nonce + +def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int, int]] | tuple[int, int, list[int], bytes] | tuple[int, int]: + + if node.startswith(b's'): + return read_key(node) + + n = 0 + height, n = read_varint(node, n) + + if height == 0: + length, n = read_varint(node, n) + size, n = read_uvarint(node, n) + key = node[n:n+size] + n += size + valuesize, n = read_uvarint(node, n) + value = node[n:n+valuesize] + + return (height, length, key, value) + else: + length, n = read_varint(node, n) + size, n = read_uvarint(node, n) + key = node[n:n+size] + n += size + hashsize, n = read_uvarint(node, n) + n += hashsize + mode, n = read_uvarint(node, n) + left_version, n = read_varint(node, n) + left_nonce, n = read_varint(node, n) + right_version, n = read_varint(node, n) + right_nonce, n = read_varint(node, n) + + return (height, length, key, (left_version, left_nonce), (right_version, right_nonce)) + +def walk(tree, version, searchkey): + if (version, 1) not in tree: + return None + + node = tree[(version, 1)] + if len(node) == 2: # root copy? + node = tree[node] + + while node[0] > 0: + nodekey = node[2] + if searchkey < nodekey: + next = node[3] + else: + next = node[4] + + node = tree[next] + + return node[3] + +def walk_disk_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: + + root = db.get(prefix + write_key((version, 1))) + if root is None: + return None + + node = read_node(root) + + if len(node) == 2: # root copy? + node = read_node(db.get(prefix + write_key(node))) + + while node[0] > 0: + # print(node) + + nodekey = node[2] + if searchkey < nodekey: + next = node[3] + else: + next = node[4] + + node = read_node(db.get(prefix + write_key(next))) + + if node[2] == searchkey: + return node[3] + else: + return None + +def walk_disk_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes: + root = db.get(prefix + write_key((version, 1))) + if root is None: + return None + + node = read_node(root) + lowest_geq_key = node[2] if node[2] >= searchkey else None + + if len(node) == 2: # root copy? + node = read_node(db.get(prefix + write_key(node))) + + while node[0] > 0: + # print(node) + + nodekey = node[2] + if searchkey < nodekey: + next = node[3] + else: + next = node[4] + + node = read_node(db.get(prefix + write_key(next))) + if node[2] >= searchkey and (lowest_geq_key is None or node[2] < lowest_geq_key): + lowest_geq_key = node[2] + + return lowest_geq_key + +def walk_disk(db, prefix: str, version: int, format: str, searchkey: list) -> None | bytes: + return walk_disk_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey)) + +def parse_struct(data): + n = 0 + results = [] + + while n < len(data): + key, n = read_uvarint(data, n) + ty = key & 7 + key >>= 3 + if ty == 2: + l, n = read_uvarint(data, n) + val = data[n:n+l] + n += l + elif ty == 0: + val, n = read_uvarint(data, n) + else: + raise Exception(f'unknown type {ty}, {data[n:]}') + results.append((key, val)) + + return results + +# find max height +def next_key(db, k: bytes) -> bytes | None: + it = db.iterator(start = k) + try: + nk, _ = next(it) + return nk + except StopIteration: + return None + finally: + it.close() + +def max_height(db) -> int: + testnr = 1<<63 + + for i in range(62, -1, -1): + prefix = b's/k:emissions/s' + n = next_key(db, prefix + struct.pack('>Q', testnr)) + + if n is not None and n.startswith(prefix): + # print(f'{testnr:16x} is low') + testnr += 1 << i + else: + # print(f'{testnr:16x} is high') + testnr -= 1 << i + + n = next_key(db, prefix + struct.pack('>Q', testnr)) + if n is not None and n.startswith(prefix): + return testnr + else: + return testnr - 1 + +# encode and decode keys +def encode_key(format: str, key: list) -> bytes: + result_bytes = [] + + result_bytes.append(key[0]) + + for i, f in enumerate(format): + if i >= len(key) - 1: + break + if f == 's': + result_bytes += list(key[i+1].encode('utf-8')) + if i < len(format) - 1: + result_bytes += [0] + elif f == 'Q': + result_bytes += list(struct.pack('>Q', key[i+1])) + elif f == 'q': + result_bytes += list(struct.pack('>Q', key[i+1] + (1<<63))) + + return bytes(result_bytes) + +def decode_key(format: str, key: bytes) -> list: + result = [] + + result.append(key[0]) + idx = 1 + + for f in format: + if f == 's': + end = key[idx:].find(b'\x00') + if end < 0: + result.append(key[idx:].decode('utf-8')) + idx = len(key) + break + else: + result.append(key[idx:idx+end].decode('utf-8')) + idx += end + 1 + elif f == 'Q': + v = struct.unpack('>Q', key[idx:idx+8])[0] + result.append(v) + idx += 8 + elif f == 'q': + v = struct.unpack('>Q', key[idx:idx+8])[0] + result.append(v - (1<<63)) + idx += 8 + + if idx < len(key): + result.append(key[idx:]) + + return result + +# iteration +class IAVLTreeIteratorRaw: + def __init__(self, db, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None): + self.db = db + self.prefix = prefix + self.version = version + self.start = start + self.end = end + self.stack = [] + + def __iter__(self): + return self + + def __next__(self): + if len(self.stack) == 0: + # get root node + root = db.get(self.prefix + write_key((self.version, 1))) + if root is None: + raise StopIteration + node = read_node(root) + if len(node) == 2: # link to other root node + node = read_node(db.get(self.prefix + write_key(node))) + self.stack.append(((self.version, 1), node)) + + # walk tree to either last before start or first after start + while node[0] > 0: + # print(node) + nodekey = node[2] + if self.start is None or self.start < nodekey: + next = node[3] + else: + next = node[4] + node = read_node(db.get(self.prefix + write_key(next))) + self.stack.append((next, node)) + + # return early if we ended up at first item after start + if self.start is None or node[2] >= self.start: + return (node[2], node[3]) + + # print('Stack:', [x[0] for x in self.stack]) + + # go up to first parent which we're a left child of + key = None + for i in range(len(self.stack)-1, 0, -1): + current_key = self.stack[i][0] + parent_node = self.stack[i-1][1] + self.stack.pop() + left = parent_node[3] + right = parent_node[4] + if current_key == left: + key = right + break + + # are we at the right end of the tree? + if key is None: + raise StopIteration + + # go right + node = read_node(db.get(self.prefix + write_key(key))) + self.stack.append((key, node)) + + # go left until at a leaf + while node[0] > 0: + key = node[3] + node = read_node(db.get(self.prefix + write_key(key))) + self.stack.append((key, node)) + + if self.end is not None and node[2] >= self.end: + raise StopIteration + + return (node[2], node[3]) + +class IAVLTreeIterator: + def __init__(self, db, prefix: str, version: int, format: str, start: list | None = None, end: list | None = None): + self.format = format + start_enc = encode_key(format, start) if start is not None else None + end_enc = encode_key(format, end) if end is not None else None + self.inner = IAVLTreeIteratorRaw(db, prefix.encode('utf-8'), version, start = start_enc, end = end_enc) + + def __iter__(self): + return self + + def __next__(self): + (k, v) = next(self.inner) + return (decode_key(self.format, k), v) + +def iterate(db, prefix, version, format = '', field = None, start = None, end = None): + if field is not None: + return IAVLTreeIterator(db, prefix, version, format, start = [field], end = [field+1] if field < 255 else None) + else: + return IAVLTreeIterator(db, prefix, version, format, start = start, end = end) + +def indexof_raw(db, prefix: bytes, version: int, key: bytes) -> int: + """ + Find how many items come before `key` in the tree. If `key` doesn't exist, how many + items come before the slot it would get inserted at + """ + it = IAVLTreeIteratorRaw(db, prefix, version, start=key) + try: + next(it) + except StopIteration: + # get root count + return read_node(db.get(prefix + write_key(it.stack[0][0])))[1] + + keys = [p[1][3] for p, c in zip(it.stack, it.stack[1:]) if c[0] == p[1][4]] + keys_encoded = [prefix + write_key(k) for k in keys] + count = sum([read_node(db.get(k))[1] for k in keys_encoded]) + + return count + +def indexof(db, prefix: str, version: int, format: str, key: list) -> int: + return indexof_raw(db, prefix.encode('utf-8'), version, encode_key(format, key)) diff --git a/store.ipynb b/store.ipynb index 6603ebe..16ea945 100644 --- a/store.ipynb +++ b/store.ipynb @@ -2,281 +2,13 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import plyvel\n", - "import struct\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# functions for reading IAVL tree\n", "\n", - "def read_varint(x: bytes, offset: int = 0) -> int:\n", - " result = 0\n", - " factor = 1\n", - "\n", - " for i, b in enumerate(x[offset:]):\n", - " if b >= 128:\n", - " result = result + (b - 128) * factor\n", - " else:\n", - " result = result + b * factor\n", - " return result // 2, offset+i+1\n", - " factor *= 128\n", - "\n", - "def read_uvarint(x: bytes, offset: int = 0) -> int:\n", - " result = 0\n", - " factor = 1\n", - "\n", - " for i, b in enumerate(x[offset:]):\n", - " if b >= 128:\n", - " result = result + (b - 128) * factor\n", - " else:\n", - " result = result + b * factor\n", - " return result, offset+i+1\n", - " factor *= 128\n", - "\n", - "def read_key(key: bytes) -> tuple[int, int] | None:\n", - " if not key.startswith(b's'):\n", - " return None\n", - "\n", - " version = struct.unpack_from('>Q', key[1:9])[0]\n", - " nonce = struct.unpack_from('>I', key[9:13])[0]\n", - "\n", - " return (version, nonce)\n", - "\n", - "def write_key(key: tuple[int, int]) -> bytes:\n", - " version = struct.pack('>Q', key[0])\n", - " nonce = struct.pack('>I', key[1])\n", - "\n", - " return b's' + version + nonce\n", - "\n", - "def read_node(node: bytes) -> tuple[int, int, bytes, tuple[int, int], tuple[int, int]] | tuple[int, int, list[int], bytes] | tuple[int, int]:\n", - "\n", - " if node.startswith(b's'):\n", - " return read_key(node)\n", - "\n", - " n = 0\n", - " height, n = read_varint(node, n)\n", - "\n", - " if height == 0:\n", - " length, n = read_varint(node, n)\n", - " size, n = read_uvarint(node, n)\n", - " key = node[n:n+size]\n", - " n += size\n", - " valuesize, n = read_uvarint(node, n)\n", - " value = node[n:n+valuesize]\n", - "\n", - " return (height, length, key, value)\n", - " else:\n", - " length, n = read_varint(node, n)\n", - " size, n = read_uvarint(node, n)\n", - " key = node[n:n+size]\n", - " n += size\n", - " hashsize, n = read_uvarint(node, n)\n", - " n += hashsize\n", - " mode, n = read_uvarint(node, n)\n", - " left_version, n = read_varint(node, n)\n", - " left_nonce, n = read_varint(node, n)\n", - " right_version, n = read_varint(node, n)\n", - " right_nonce, n = read_varint(node, n)\n", - "\n", - " return (height, length, key, (left_version, left_nonce), (right_version, right_nonce))\n", - "\n", - "def walk(tree, version, searchkey):\n", - " if (version, 1) not in tree:\n", - " return None\n", - "\n", - " node = tree[(version, 1)]\n", - " if len(node) == 2: # root copy?\n", - " node = tree[node]\n", - "\n", - " while node[0] > 0:\n", - " nodekey = node[2]\n", - " if searchkey < nodekey:\n", - " next = node[3]\n", - " else:\n", - " next = node[4]\n", - "\n", - " node = tree[next]\n", - "\n", - " return node[3]\n", - "\n", - "def walk_disk_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:\n", - "\n", - " root = db.get(prefix + write_key((version, 1)))\n", - " if root is None:\n", - " return None\n", - "\n", - " node = read_node(root)\n", - "\n", - " if len(node) == 2: # root copy?\n", - " node = read_node(db.get(prefix + write_key(node)))\n", - "\n", - " while node[0] > 0:\n", - " # print(node)\n", - "\n", - " nodekey = node[2]\n", - " if searchkey < nodekey:\n", - " next = node[3]\n", - " else:\n", - " next = node[4]\n", - "\n", - " node = read_node(db.get(prefix + write_key(next)))\n", - "\n", - " if node[2] == searchkey:\n", - " return node[3]\n", - " else:\n", - " return None\n", - "\n", - "def walk_disk_next_key_raw(db, prefix: bytes, version: int, searchkey: bytes) -> None | bytes:\n", - " root = db.get(prefix + write_key((version, 1)))\n", - " if root is None:\n", - " return None\n", - "\n", - " node = read_node(root)\n", - " lowest_geq_key = node[2] if node[2] >= searchkey else None\n", - "\n", - " if len(node) == 2: # root copy?\n", - " node = read_node(db.get(prefix + write_key(node)))\n", - "\n", - " while node[0] > 0:\n", - " # print(node)\n", - "\n", - " nodekey = node[2]\n", - " if searchkey < nodekey:\n", - " next = node[3]\n", - " else:\n", - " next = node[4]\n", - "\n", - " node = read_node(db.get(prefix + write_key(next)))\n", - " if node[2] >= searchkey and (lowest_geq_key is None or node[2] < lowest_geq_key):\n", - " lowest_geq_key = node[2]\n", - "\n", - " return lowest_geq_key\n", - "\n", - "def walk_disk(db, prefix: str, version: int, format: str, searchkey: list) -> None | bytes:\n", - " return walk_disk_raw(db, prefix.encode('utf-8'), version, encode_key(format, searchkey))\n", - "\n", - "def parse_struct(data):\n", - " n = 0\n", - " results = []\n", - "\n", - " while n < len(data):\n", - " key, n = read_uvarint(data, n)\n", - " ty = key & 7\n", - " key >>= 3\n", - " if ty == 2:\n", - " l, n = read_uvarint(data, n)\n", - " val = data[n:n+l]\n", - " n += l\n", - " elif ty == 0:\n", - " val, n = read_uvarint(data, n)\n", - " else:\n", - " raise Exception(f'unknown type {ty}, {data[n:]}')\n", - " results.append((key, val))\n", - "\n", - " return results" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# find max height\n", - "\n", - "def next_key(db, k: bytes) -> bytes | None:\n", - " it = db.iterator(start = k)\n", - " try:\n", - " nk, _ = next(it)\n", - " return nk\n", - " except StopIteration:\n", - " return None\n", - " finally:\n", - " it.close()\n", - "\n", - "def max_height(db) -> int:\n", - " testnr = 1<<63\n", - "\n", - " for i in range(62, -1, -1):\n", - " prefix = b's/k:emissions/s'\n", - " n = next_key(db, prefix + struct.pack('>Q', testnr))\n", - "\n", - " if n is not None and n.startswith(prefix):\n", - " # print(f'{testnr:16x} is low')\n", - " testnr += 1 << i\n", - " else:\n", - " # print(f'{testnr:16x} is high')\n", - " testnr -= 1 << i\n", - "\n", - " n = next_key(db, prefix + struct.pack('>Q', testnr))\n", - " if n is not None and n.startswith(prefix):\n", - " return testnr\n", - " else:\n", - " return testnr - 1" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# encode and decode keys\n", - "\n", - "def encode_key(format: str, key: list) -> bytes:\n", - " result_bytes = []\n", - "\n", - " result_bytes.append(key[0])\n", - "\n", - " for i, f in enumerate(format):\n", - " if i >= len(key) - 1:\n", - " break\n", - " if f == 's':\n", - " result_bytes += list(key[i+1].encode('utf-8'))\n", - " if i < len(format) - 1:\n", - " result_bytes += [0]\n", - " elif f == 'Q':\n", - " result_bytes += list(struct.pack('>Q', key[i+1]))\n", - " elif f == 'q':\n", - " result_bytes += list(struct.pack('>Q', key[i+1] + (1<<63)))\n", - "\n", - " return bytes(result_bytes)\n", - "\n", - "def decode_key(format: str, key: bytes) -> list:\n", - " result = []\n", - "\n", - " result.append(key[0])\n", - " idx = 1\n", - "\n", - " for f in format:\n", - " if f == 's':\n", - " end = key[idx:].find(b'\\x00')\n", - " if end < 0:\n", - " result.append(key[idx:].decode('utf-8'))\n", - " break\n", - " else:\n", - " result.append(key[idx:idx+end].decode('utf-8'))\n", - " idx += end + 1\n", - " elif f == 'Q':\n", - " v = struct.unpack('>Q', key[idx:idx+8])[0]\n", - " result.append(v)\n", - " idx += 8\n", - " elif f == 'q':\n", - " v = struct.unpack('>Q', key[idx:idx+8])[0]\n", - " result.append(v - (1<<63))\n", - " idx += 8\n", - "\n", - " return result" + "%run -i read_tree.py" ] }, { @@ -285,8 +17,8 @@ "metadata": {}, "outputs": [], "source": [ - "db = plyvel.DB('../node/nodedir/data/application.db')\n", - "height = max_height(db)" + "# db = plyvel.DB('../testnode/nodedir/data/application.db')\n", + "max_height(db)" ] }, { @@ -295,106 +27,7 @@ "metadata": {}, "outputs": [], "source": [ - "# get\n", - "v = walk_disk(db, 's/k:emissions/', height, 'Qs', [11, 37, 'allo1mn4d32hwyn6grp89akek52arsw2vcdqezr0dc7'])\n", - "parse_struct(v)" - ] - }, - { - "cell_type": "code", - "execution_count": 183, - "metadata": {}, - "outputs": [], - "source": [ - "class IAVLTreeIteratorRaw:\n", - " def __init__(self, db, prefix: bytes, version: int, start: bytes | None = None, end: bytes | None = None):\n", - " self.db = db\n", - " self.prefix = prefix\n", - " self.version = version\n", - " self.start = start\n", - " self.end = end\n", - " self.stack = []\n", - "\n", - " def __iter__(self):\n", - " return self\n", - "\n", - " def __next__(self):\n", - " if len(self.stack) == 0:\n", - " # get root node\n", - " root = db.get(self.prefix + write_key((self.version, 1)))\n", - " if root is None:\n", - " raise StopIteration\n", - " node = read_node(root)\n", - " if len(node) == 2: # link to other root node\n", - " node = read_node(db.get(self.prefix + write_key(node)))\n", - " self.stack.append(((self.version, 1), node))\n", - "\n", - " # walk tree to either last before start or first after start\n", - " while node[0] > 0:\n", - " # print(node)\n", - " nodekey = node[2]\n", - " if self.start is None or self.start < nodekey:\n", - " next = node[3]\n", - " else:\n", - " next = node[4]\n", - " node = read_node(db.get(self.prefix + write_key(next)))\n", - " self.stack.append((next, node))\n", - "\n", - " # return early if we ended up at first item after start\n", - " if self.start is None or node[2] >= self.start:\n", - " return (node[2], node[3])\n", - "\n", - " # print('Stack:', [x[0] for x in self.stack])\n", - "\n", - " # go up to first parent which we're a left child of\n", - " key = None\n", - " for i in range(len(self.stack)-1, 0, -1):\n", - " current_key = self.stack[i][0]\n", - " parent_node = self.stack[i-1][1]\n", - " self.stack.pop()\n", - " left = parent_node[3]\n", - " right = parent_node[4]\n", - " if current_key == left:\n", - " key = right\n", - " break\n", - "\n", - " # are we at the right end of the tree?\n", - " if key is None:\n", - " raise StopIteration\n", - "\n", - " # go right\n", - " node = read_node(db.get(self.prefix + write_key(key)))\n", - " self.stack.append((key, node))\n", - "\n", - " # go left until at a leaf\n", - " while node[0] > 0:\n", - " key = node[3]\n", - " node = read_node(db.get(self.prefix + write_key(key)))\n", - " self.stack.append((key, node))\n", - "\n", - " if self.end is not None and node[2] >= self.end:\n", - " raise StopIteration\n", - "\n", - " return (node[2], node[3])\n", - "\n", - "class IAVLTreeIterator:\n", - " def __init__(self, db, prefix: str, version: int, format: str, start: list | None = None, end: list | None = None):\n", - " self.format = format\n", - " start_enc = encode_key(format, start) if start is not None else None\n", - " end_enc = encode_key(format, end) if end is not None else None\n", - " self.inner = IAVLTreeIteratorRaw(db, prefix.encode('utf-8'), version, start = start_enc, end = end_enc)\n", - "\n", - " def __iter__(self):\n", - " return self\n", - "\n", - " def __next__(self):\n", - " (k, v) = next(self.inner)\n", - " return (decode_key(self.format, k), v)\n", - "\n", - "def iterate(db, prefix, version, format, field):\n", - " return IAVLTreeIterator(db, prefix, version, format, start = [field], end = [field+1] if field < 255 else None)\n", - "\n", - "# [k for k, v in IAVLTreeIterator(db, 's/k:mint/', height, '')]" + "[k for k, v in iterate(db, 's/k:mint/', 5224815)]" ] }, { @@ -403,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "[(k, parse_struct(v)) for k, v in iterate(db, 's/k:emissions/', height, 'Q', 5)]" + "parse_struct(next(iterate(db, 's/k:mint/', 5224815, field = 138))[1])" ] }, { @@ -412,7 +45,7 @@ "metadata": {}, "outputs": [], "source": [ - "[k for k, v in IAVLTreeIterator(db, 's/k:emissions/', height, 'Q', start = [5], end = [6])]" + "[k for k,v in iterate(db, 's/k:emissions/', 5224815, start = [62, 60], end = [62, 61], format = 'Qss')]" ] }, { @@ -421,83 +54,16 @@ "metadata": {}, "outputs": [], "source": [ - "encode_key('Q', [5])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "[decode_key('Q', k) for k,v in IAVLTreeIteratorRaw(db, b's/k:emissions/', height, start = bytes([5]), end = bytes([6]))]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "encode_key('', [1]), decode_key('', b'\\x8a')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "decode_key()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "list(range(5, 0, -1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "read_node(db.get(b's/k:emissions/' + write_key((height, 1))))" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [], - "source": [ - "heights = np.arange(5776000, height+1)\n", - "sizes = np.empty(len(heights), dtype = int)\n", + "lens = np.zeros(256, dtype = int)\n", "\n", - "for i in range(len(heights)):\n", - " h = heights[i]\n", - " n = db.get(b's/k:emissions/s' + struct.pack('>Q', h) + struct.pack('>I', 1))\n", - " nd = read_node(n)\n", - " sizes[i] = nd[1]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", + "with plyvel.DB('../testnode/nodedir/data/application.db') as db:\n", + " height = max_height(db)\n", + " for field in range(255):\n", + " count1 = indexof(db, 's/k:emissions/', height, '', [field])\n", + " count2 = indexof(db, 's/k:emissions/', height, '', [field+1])\n", + " lens[field] = count2 - count1\n", "\n", - "plt.plot(sizes)\n", - "# plt.xlim(0, 1000)\n", - "# plt.ylim(19906000, 19908000)\n", - "plt.grid()\n", - "plt.show()" + "np.argsort(lens)" ] }, { @@ -505,18 +71,6 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "n = db.get(b's/k:emissions/s' + struct.pack('>Q', 5776000) + struct.pack('>I', 1))\n", - "nd = read_node(n)\n", - "# decode_key('Qss', nd[2])\n", - "nd" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], "source": [ "# def has_prefix(db, prefix: bytes) -> bool:\n", "# it = db.iterator(start = prefix)\n",