From 5075fd9f7d752f718d79a86b9ba06b249ad1e05c Mon Sep 17 00:00:00 2001 From: Holger Nahrstaedt Date: Sat, 7 Nov 2020 00:01:38 +0100 Subject: [PATCH] Implement filter operations when using https://api.hive.blog * Implement _get_operation_filter and use filter operations in history and history_reverse on the https://api.hive.blog api node --- CHANGELOG.rst | 1 + beem/account.py | 96 +++++++++++++++++++++++++++++++------- beemapi/exceptions.py | 4 ++ beemapi/noderpc.py | 2 + tests/beem/test_account.py | 7 +++ 5 files changed, 94 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8e9c9a88..ffaf6399 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,7 @@ Changelog * Speed up history call, when limit is below 1000 * Improve unit tests for account history * Fix estimate_virtual_op_num, when get_account_history returns an empty entry for an index +* Implement _get_operation_filter and use filter operations in history and history_reverse on the https://api.hive.blog api node 0.24.17 ------- diff --git a/beem/account.py b/beem/account.py index e981923f..3e76ab87 100644 --- a/beem/account.py +++ b/beem/account.py @@ -8,7 +8,7 @@ from prettytable import PrettyTable from beem.instance import shared_blockchain_instance from .exceptions import AccountDoesNotExistsException, OfflineHasNoRPCException -from beemapi.exceptions import ApiNotSupported, MissingRequiredActiveAuthority, SupportedByHivemind +from beemapi.exceptions import ApiNotSupported, MissingRequiredActiveAuthority, SupportedByHivemind, FilteredItemNotFound from .blockchainobject import BlockchainObject from .blockchain import Blockchain from .utils import formatTimeString, formatTimedelta, remove_from_dict, reputation_to_score, addTzInfo @@ -1865,7 +1865,7 @@ def _get_account_history(self, account=None, start=-1, limit=1, operation_filter try: ret = self.blockchain.rpc.get_account_history({'account': account, 'start': start, 'limit': limit, 'operation_filter_low': operation_filter_low, - 'operation_filter_high': operation_filter_low}, api="account_history") + 'operation_filter_high': operation_filter_high}, api="account_history") if ret is not None: ret = ret["history"] except ApiNotSupported: @@ -2041,6 +2041,35 @@ def curation_stats(self): "7d": self.get_curation_reward(days=7), "avg": self.get_curation_reward(days=7) / 7} + def _get_operation_filter(self, only_ops=[], exclude_ops=[]): + from beembase.operationids import operations + operation_filter_low = 0 + operation_filter_high = 0 + if len(only_ops) == 0 and len(exclude_ops) == 0: + return None, None + if len(only_ops) > 0: + for op in only_ops: + op_id = operations[op] + if op_id <= 64: + operation_filter_low += 2**op_id + else: + operation_filter_high += 2 ** (op_id - 64 - 1) + else: + for op in operations: + op_id = operations[op] + if op_id <= 64: + operation_filter_low += 2**op_id + else: + operation_filter_high += 2 ** (op_id - 64 - 1) + for op in exclude_ops: + op_id = operations[op] + if op_id <= 64: + operation_filter_low -= 2**op_id + else: + operation_filter_high -= 2 ** (op_id - 64 - 1) + return operation_filter_low, operation_filter_high + + def get_account_history(self, index, limit, order=-1, start=None, stop=None, use_block_num=True, only_ops=[], exclude_ops=[], raw_output=False): """ Returns a generator for individual account transactions. This call can be used in a ``for`` loop. @@ -2074,7 +2103,14 @@ def get_account_history(self, index, limit, order=-1, start=None, stop=None, use if order != -1 and order != 1: raise ValueError("order must be -1 or 1!") # self.blockchain.rpc.set_next_node_on_empty_reply(True) - txs = self._get_account_history(start=index, limit=limit) + operation_filter_low = None + operation_filter_high = None + if self.blockchain.rpc.url == 'https://api.hive.blog': + operation_filter_low, operation_filter_high = self._get_operation_filter(only_ops=only_ops, exclude_ops=exclude_ops) + try: + txs = self._get_account_history(start=index, limit=limit, operation_filter_low=operation_filter_low, operation_filter_high=operation_filter_high) + except FilteredItemNotFound: + txs = [] if txs is None: return start = addTzInfo(start) @@ -2272,11 +2308,19 @@ def history( if _limit < 0: return last_item_index = -1 + + if self.blockchain.rpc.url == 'https://api.hive.blog' and (len(only_ops) > 0 or len(exclude_ops) > 0): + operation_filter = True + else: + operation_filter = False + while True: # RPC call if first < _limit: - first = _limit - for item in self.get_account_history(first, _limit, start=None, stop=None, order=1, raw_output=raw_output): + first = _limit + batch_count = 0 + for item in self.get_account_history(first, _limit, start=None, stop=None, order=1, only_ops=only_ops, exclude_ops=exclude_ops, raw_output=raw_output): + batch_count += 1 if raw_output: item_index, event = item op_type, op = event['op'] @@ -2295,7 +2339,7 @@ def history( continue elif start is not None and not use_block_num and item_index < start: continue - elif last_item_index == item_index: + elif last_item_index >= item_index: continue if stop is not None and isinstance(stop, (datetime, date, time)): timediff = stop - formatTimeString(timestamp) @@ -2306,17 +2350,23 @@ def history( return elif stop is not None and not use_block_num and item_index > stop: return - if exclude_ops and op_type in exclude_ops: - continue - if not only_ops or op_type in only_ops: + if operation_filter: yield item + else: + if exclude_ops and op_type in exclude_ops: + continue + if not only_ops or op_type in only_ops: + yield item last_item_index = item_index if first < max_index and first + _limit >= max_index and not last_round: _limit = max_index - first first = max_index last_round = True else: - first += (_limit) + if operation_filter and batch_count < _limit and first + 2000 < max_index and _limit == 1000: + first += 2000 + else: + first += _limit if stop is not None and not use_block_num and isinstance(stop, int) and first >= stop + _limit + 1: break elif first > max_index or last_round: @@ -2437,12 +2487,20 @@ def history_reverse( first = op_est + est_diff if stop is not None and isinstance(stop, int) and stop < 0 and not use_block_num: stop += first - last_item_index = -1 + + if self.blockchain.rpc.url == 'https://api.hive.blog' and (len(only_ops) > 0 or len(exclude_ops) > 0): + operation_filter = True + else: + operation_filter = False + + last_item_index = first + 1 while True: # RPC call if first - _limit < 0: _limit = first + batch_count = 0 for item in self.get_account_history(first, _limit, start=None, stop=None, order=-1, only_ops=only_ops, exclude_ops=exclude_ops, raw_output=raw_output): + batch_count += 1 if raw_output: item_index, event = item op_type, op = event['op'] @@ -2461,7 +2519,7 @@ def history_reverse( continue elif start is not None and not use_block_num and item_index > start: continue - elif item_index == last_item_index: + elif last_item_index <= item_index: continue if stop is not None and isinstance(stop, (datetime, date, time)): timediff = stop - formatTimeString(timestamp) @@ -2474,12 +2532,18 @@ def history_reverse( elif stop is not None and not use_block_num and item_index < stop: first = 0 return - if exclude_ops and op_type in exclude_ops: - continue - if not only_ops or op_type in only_ops: + if operation_filter: yield item + else: + if exclude_ops and op_type in exclude_ops: + continue + if not only_ops or op_type in only_ops: + yield item last_item_index = item_index - first -= (_limit) + if operation_filter and batch_count < _limit and _limit == 1000: + first -= 2000 + else: + first -= (_limit) if first < 1: break diff --git a/beemapi/exceptions.py b/beemapi/exceptions.py index 8a44b3d4..a75a6127 100644 --- a/beemapi/exceptions.py +++ b/beemapi/exceptions.py @@ -92,6 +92,10 @@ class NoAccessApi(RPCError): pass +class FilteredItemNotFound(RPCError): + pass + + class InvalidEndpointUrl(Exception): pass diff --git a/beemapi/noderpc.py b/beemapi/noderpc.py index 5a893348..a87cccf0 100644 --- a/beemapi/noderpc.py +++ b/beemapi/noderpc.py @@ -139,6 +139,8 @@ def _check_error_message(self, e, cnt): raise exceptions.InvalidParameters() elif re.search("Supported by Hivemind", msg): raise exceptions.SupportedByHivemind() + elif re.search("Could not find filtered operation", msg): + raise exceptions.FilteredItemNotFound(msg) elif re.search("Unable to acquire database lock", msg): self.nodes.sleep_and_check_retries(str(msg), call_retry=True) doRetry = True diff --git a/tests/beem/test_account.py b/tests/beem/test_account.py index 7dc2c468..aa24a87e 100644 --- a/tests/beem/test_account.py +++ b/tests/beem/test_account.py @@ -535,6 +535,13 @@ def test_history_votes(self): votes_list2.append(v) self.assertTrue(abs(len(votes_list) - len(votes_list2)) < 2) + account = Account("beembot", blockchain_instance=stm) + votes_list = list(account.history(only_ops=["vote"])) + votes_list2 = list(account.history_reverse(only_ops=["vote"])) + self.assertEqual(len(votes_list), len(votes_list2)) + self.assertEqual(votes_list[0]["voter"], votes_list2[-1]["voter"]) + self.assertEqual(votes_list[-1]["voter"], votes_list2[0]["voter"]) + def test_comment_history(self): account = self.account comments = []