From 421a10024aed44e0fe19bbabf95037b28d497564 Mon Sep 17 00:00:00 2001 From: Karol Chojnowski Date: Wed, 23 Feb 2022 14:23:27 +0100 Subject: [PATCH 1/3] Set config as global, update cache, refactor, add core --- Pipfile | 4 +- ethtx/__init__.py | 3 +- ethtx/core/__init__.py | 0 ethtx/core/config.py | 24 +++++++ ethtx/decoders/abi/calls.py | 2 +- ethtx/decoders/abi/transfers.py | 2 +- ethtx/decoders/semantic/helpers/utils.py | 1 - ethtx/decoders/semantic/metadata.py | 2 +- ethtx/ethtx.py | 37 +++-------- ethtx/models/semantics_model.py | 7 +- .../semantic_providers/repository.py | 65 +++++++++---------- ethtx/semantics/protocols_router.py | 2 +- ethtx/utils/{decorators.py => cache.py} | 6 +- 13 files changed, 81 insertions(+), 74 deletions(-) create mode 100644 ethtx/core/__init__.py create mode 100644 ethtx/core/config.py rename ethtx/utils/{decorators.py => cache.py} (87%) diff --git a/Pipfile b/Pipfile index 1dd9b64f..d94eb6c1 100644 --- a/Pipfile +++ b/Pipfile @@ -9,8 +9,8 @@ pymongo = ">=3.12.0" dnspython = ">=2.2.0" mongoengine = "==0.23.1" mongomock = "==3.23.0" -web3 = "==5.26.0" -eth-account = "==0.5.6" +web3 = "==5.28.0" +eth-account = "==0.5.7" eth-utils = "==1.10.0" eth-rlp = "==0.2.1" pydantic = ">=1.9.0" diff --git a/ethtx/__init__.py b/ethtx/__init__.py index 439df78c..5373fee2 100644 --- a/ethtx/__init__.py +++ b/ethtx/__init__.py @@ -10,4 +10,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .ethtx import EthTx, EthTxConfig +from .core.config import EthTxConfig +from .ethtx import EthTx diff --git a/ethtx/core/__init__.py b/ethtx/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ethtx/core/config.py b/ethtx/core/config.py new file mode 100644 index 00000000..f9815864 --- /dev/null +++ b/ethtx/core/config.py @@ -0,0 +1,24 @@ +from pydantic import BaseSettings + + +class Settings(BaseSettings): + DEFAULT_CHAIN: str = "mainnet" + CACHE_SIZE: int = 128 + + MONGO_CONNECTION_STRING: str + + WEB3_NODES: dict + + ETHERSCAN_API_KEY: str + ETHERSCAN_URLS: dict = { + "mainnet": "https://api.etherscan.io/api", + "goerli": "https://api-goerli.etherscan.io/api", + "rinkeby": "https://api-rinkeby.etherscan.io/api", + } + + class Config: + case_sensitive = True + env_file = "../../.env" + + +EthTxConfig = Settings() diff --git a/ethtx/decoders/abi/calls.py b/ethtx/decoders/abi/calls.py index 86d165cb..9c50e623 100644 --- a/ethtx/decoders/abi/calls.py +++ b/ethtx/decoders/abi/calls.py @@ -191,7 +191,7 @@ def decode_call( call_type=call.call_type, from_address=AddressInfo(address=call.from_address, name=from_name), to_address=AddressInfo(address=call.to_address, name=to_name), - value=call.call_value / 10 ** 18, + value=call.call_value / 10**18, function_signature=function_signature, function_name=function_name, arguments=function_input, diff --git a/ethtx/decoders/abi/transfers.py b/ethtx/decoders/abi/transfers.py index 8140100e..22eec9b8 100644 --- a/ethtx/decoders/abi/transfers.py +++ b/ethtx/decoders/abi/transfers.py @@ -74,7 +74,7 @@ def _transfers_calls(decoded_call): ) = self._repository.get_token_data( event.chain_id, event.contract.address, proxies ) - value = event.parameters[2].value / 10 ** token_decimals + value = event.parameters[2].value / 10**token_decimals transfers.append( DecodedTransfer( from_address=AddressInfo( diff --git a/ethtx/decoders/semantic/helpers/utils.py b/ethtx/decoders/semantic/helpers/utils.py index 82acc48a..9fb6a784 100644 --- a/ethtx/decoders/semantic/helpers/utils.py +++ b/ethtx/decoders/semantic/helpers/utils.py @@ -26,7 +26,6 @@ log = logging.getLogger(__name__) - def get_badge(address, sender, receiver): sender_address = sender.address if isinstance(sender, AddressInfo) else sender receiver_address = ( diff --git a/ethtx/decoders/semantic/metadata.py b/ethtx/decoders/semantic/metadata.py index bc28c5ac..114dbae5 100644 --- a/ethtx/decoders/semantic/metadata.py +++ b/ethtx/decoders/semantic/metadata.py @@ -32,7 +32,7 @@ def decode( block_number=block_metadata.block_number, block_hash=block_metadata.block_hash, timestamp=block_metadata.timestamp, - gas_price=tx_metadata.gas_price / 10 ** 9, + gas_price=tx_metadata.gas_price / 10**9, sender=AddressInfo( address=tx_metadata.from_address, name=self.repository.get_address_label( diff --git a/ethtx/ethtx.py b/ethtx/ethtx.py index b1b15a67..986a5e4e 100644 --- a/ethtx/ethtx.py +++ b/ethtx/ethtx.py @@ -15,6 +15,7 @@ from mongoengine import connect from pymongo import MongoClient +from .core.config import Settings from .decoders.abi.decoder import ABIDecoder from .decoders.decoder_service import DecoderService from .decoders.semantic.decoder import SemanticDecoder @@ -29,28 +30,6 @@ from .utils.validators import assert_tx_hash -class EthTxConfig: - mongo_connection_string: str - etherscan_api_key: str - web3nodes: Dict[str, dict] - etherscan_urls: Dict[str, str] - default_chain: str - - def __init__( - self, - mongo_connection_string: str, - web3nodes: Dict[str, dict], - etherscan_api_key: str, - etherscan_urls: Dict[str, str], - default_chain: str = "mainnet", - ): - self.mongo_connection_string = mongo_connection_string - self.etherscan_api_key = etherscan_api_key - self.web3nodes = web3nodes - self.default_chain = default_chain - self.etherscan_urls = etherscan_urls - - class EthTxDecoders: semantic_decoder: SemanticDecoder abi_decoder: ABIDecoder @@ -117,23 +96,23 @@ def __init__( ) @staticmethod - def initialize(config: EthTxConfig): - mongo_client: MongoClient = connect(host=config.mongo_connection_string) + def initialize(config: Settings): + mongo_client: MongoClient = connect(host=config.MONGO_CONNECTION_STRING) repository = MongoSemanticsDatabase(db=mongo_client.get_database()) web3provider = Web3Provider( - nodes=config.web3nodes, default_chain=config.default_chain + nodes=config.WEB3_NODES, default_chain=config.DEFAULT_CHAIN ) etherscan_provider = EtherscanProvider( - api_key=config.etherscan_api_key, - nodes=config.etherscan_urls, - default_chain_id=config.default_chain, + api_key=config.ETHERSCAN_API_KEY, + nodes=config.ETHERSCAN_URLS, + default_chain_id=config.DEFAULT_CHAIN, ) ens_provider = ENSProvider return EthTx( - config.default_chain, + config.DEFAULT_CHAIN, repository, web3provider, etherscan_provider, diff --git a/ethtx/models/semantics_model.py b/ethtx/models/semantics_model.py index a571d8d5..6a6fdb76 100644 --- a/ethtx/models/semantics_model.py +++ b/ethtx/models/semantics_model.py @@ -13,6 +13,7 @@ from typing import List, Dict, Optional, TYPE_CHECKING from ethtx.models.base_model import BaseModel + if TYPE_CHECKING: from ethtx.providers.semantic_providers import ISemanticsDatabase @@ -85,9 +86,10 @@ class AddressSemantics(BaseModel): class Config: allow_mutation = True - @staticmethod - def from_mongo_record(raw_address_semantics: Dict, database: 'ISemanticsDatabase') -> 'AddressSemantics': + def from_mongo_record( + raw_address_semantics: Dict, database: "ISemanticsDatabase" + ) -> "AddressSemantics": ZERO_HASH = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" @@ -182,7 +184,6 @@ def decode_parameter(_parameter): chain_id = raw_address_semantics.get("chain_id") name = raw_address_semantics.get("name", address) - address_semantics = AddressSemantics( chain_id=chain_id, address=address, diff --git a/ethtx/providers/semantic_providers/repository.py b/ethtx/providers/semantic_providers/repository.py index c9bfe6cb..c9c3e0e8 100644 --- a/ethtx/providers/semantic_providers/repository.py +++ b/ethtx/providers/semantic_providers/repository.py @@ -36,12 +36,12 @@ class SemanticsRepository: def __init__( - self, - database_connection: ISemanticsDatabase, - etherscan_provider: EtherscanProvider, - web3provider: NodeDataProvider, - ens_provider: ENSProvider, - refresh_ens: bool = True + self, + database_connection: ISemanticsDatabase, + etherscan_provider: EtherscanProvider, + web3provider: NodeDataProvider, + ens_provider: ENSProvider, + refresh_ens: bool = True, ): self.database = database_connection self.etherscan = etherscan_provider @@ -62,7 +62,7 @@ def end_record(self) -> List: return tmp_records def _read_stored_semantics( - self, address: str, chain_id: str + self, address: str, chain_id: str ) -> Optional[AddressSemantics]: if not address: @@ -73,9 +73,15 @@ def _read_stored_semantics( if not raw_address_semantics: return None - address_semantics = AddressSemantics.from_mongo_record(raw_address_semantics, self.database) + address_semantics = AddressSemantics.from_mongo_record( + raw_address_semantics, self.database + ) - if self.refresh_ens and address_semantics.name == address_semantics.address and not raw_address_semantics["is_contract"]: + if ( + self.refresh_ens + and address_semantics.name == address_semantics.address + and not raw_address_semantics["is_contract"] + ): address_semantics.name = self._ens_provider.name( provider=self._web3provider._get_node_connection(chain_id), address=address, @@ -84,7 +90,9 @@ def _read_stored_semantics( return address_semantics - def _create_address_semantics(self, chain_id: str, address: str) -> AddressSemantics: + def _create_address_semantics( + self, chain_id: str, address: str + ) -> AddressSemantics: ZERO_HASH = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" # try to read the semantics form the Etherscan provider @@ -98,9 +106,7 @@ def _create_address_semantics(self, chain_id: str, address: str) -> AddressSeman ) if decoded and raw_semantics: # raw semantics received from Etherscan - events, functions = decode_events_and_functions( - raw_semantics["abi"] - ) + events, functions = decode_events_and_functions(raw_semantics["abi"]) standard, standard_semantics = self._decode_standard_semantics( address, raw_semantics["name"], events, functions ) @@ -195,7 +201,7 @@ def get_semantics(self, chain_id: str, address: str) -> Optional[AddressSemantic return address_semantics def _decode_standard_semantics( - self, address, name, events, functions: Dict[str, FunctionSemantics] + self, address, name, events, functions: Dict[str, FunctionSemantics] ) -> Tuple[Optional[str], Optional[ERC20Semantics]]: standard = None standard_semantics = None @@ -204,7 +210,7 @@ def _decode_standard_semantics( return standard, standard_semantics if all(erc20_event in events for erc20_event in ERC20_EVENTS) and all( - erc20_function in functions for erc20_function in ERC20_FUNCTIONS + erc20_function in functions for erc20_function in ERC20_FUNCTIONS ): standard = "ERC20" try: @@ -218,7 +224,7 @@ def _decode_standard_semantics( except Exception: standard_semantics = ERC20Semantics(name=name, symbol=name, decimals=18) elif all(erc721_event in events for erc721_event in ERC721_EVENTS) and all( - erc721_function in functions for erc721_function in ERC721_FUNCTIONS + erc721_function in functions for erc721_function in ERC721_FUNCTIONS ): standard = "ERC721" standard_semantics = None @@ -234,12 +240,11 @@ def get_event_abi(self, chain_id, address, signature) -> Optional[EventSemantics semantics = self.get_semantics(chain_id, address) event_semantics = semantics.contract.events.get(signature) - return event_semantics @lru_cache(maxsize=1024) def get_transformations( - self, chain_id, address, signature + self, chain_id, address, signature ) -> Optional[Dict[str, TransformationSemantics]]: if not address: @@ -272,7 +277,7 @@ def get_anonymous_event_abi(self, chain_id, address) -> Optional[EventSemantics] @lru_cache(maxsize=1024) def get_function_abi( - self, chain_id, address, signature + self, chain_id, address, signature ) -> Optional[FunctionSemantics]: if not address: @@ -346,7 +351,7 @@ def get_standard(self, chain_id, address) -> Optional[str]: return semantics.standard def get_token_data( - self, chain_id, address, proxies=None + self, chain_id, address, proxies=None ) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[str]]: if not address: @@ -354,15 +359,9 @@ def get_token_data( semantics = self.get_semantics(chain_id, address) if semantics.erc20: - token_name = ( - semantics.erc20.name if semantics.erc20 else address - ) - token_symbol = ( - semantics.erc20.symbol if semantics.erc20 else "Unknown" - ) - token_decimals = ( - semantics.erc20.decimals if semantics.erc20 else 18 - ) + token_name = semantics.erc20.name if semantics.erc20 else address + token_symbol = semantics.erc20.symbol if semantics.erc20 else "Unknown" + token_decimals = semantics.erc20.decimals if semantics.erc20 else 18 elif proxies and address in proxies and proxies[address].token: token_name = proxies[address].token.name token_symbol = proxies[address].token.symbol @@ -457,12 +456,12 @@ def update_or_insert_signature(self, signature: Signature) -> None: ) for sig in signatures: if ( - signature.name == sig["name"] - and signature.signature_hash == sig["signature_hash"] - and len(signature.args) == len(sig["args"]) + signature.name == sig["name"] + and signature.signature_hash == sig["signature_hash"] + and len(signature.args) == len(sig["args"]) ): if signature.args and any( - arg for arg in list(sig["args"][0].values()) if "arg" in arg + arg for arg in list(sig["args"][0].values()) if "arg" in arg ): for index, argument in enumerate(sig["args"]): argument["name"] = signature.args[index].name diff --git a/ethtx/semantics/protocols_router.py b/ethtx/semantics/protocols_router.py index 72de88b6..b81c31e7 100644 --- a/ethtx/semantics/protocols_router.py +++ b/ethtx/semantics/protocols_router.py @@ -15,7 +15,7 @@ from ..models.semantics_model import ContractSemantics from ..semantics.router import Router -from ..utils.decorators import ignore_unhashable +from ..utils.cache import ignore_unhashable log = logging.getLogger(__name__) diff --git a/ethtx/utils/decorators.py b/ethtx/utils/cache.py similarity index 87% rename from ethtx/utils/decorators.py rename to ethtx/utils/cache.py index 10f80fb2..6cbc47eb 100644 --- a/ethtx/utils/decorators.py +++ b/ethtx/utils/cache.py @@ -9,7 +9,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from functools import WRAPPER_ASSIGNMENTS, wraps +from functools import WRAPPER_ASSIGNMENTS, wraps, lru_cache + +from ethtx.core.config import EthTxConfig + +cache = lru_cache(maxsize=EthTxConfig.CACHE_SIZE) def ignore_unhashable(func): From 3236bcf372350a092089718fcc641d9cffec9acd Mon Sep 17 00:00:00 2001 From: Karol Chojnowski Date: Wed, 23 Feb 2022 15:00:54 +0100 Subject: [PATCH 2/3] update doc and add .env_sample --- .env_sample | 37 +++++++++++++++ README.md | 104 ++++++++++++++++++++++++++++--------------- ethtx/core/config.py | 2 +- 3 files changed, 107 insertions(+), 36 deletions(-) create mode 100644 .env_sample diff --git a/.env_sample b/.env_sample new file mode 100644 index 00000000..18e74b23 --- /dev/null +++ b/.env_sample @@ -0,0 +1,37 @@ +# REQUIRED: + +# WEB3 NODES: +# Proper nodes are required to run ethtx. +# Values are populated from the environment by treating the environment variable's value as a JSON-encoded string. +# EXAMPLE: WEB3_NODES='{"mainnet": {"hook": "https://geth", "poa": false}, "rinkeby": {"hook": "https://eth-rinkeby", "poa": true}}' +# EthTx supports multiple nodes, if one is unavailable, it will use others. You only need to specify them with a comma. +WEB3_NODES= + +# ETHERSCAN: +# Etherscan API is used to get contract source code, required for decoding process +# You can get free key here https://etherscan.io/apis +ETHERSCAN_API_KEY= + + +# OPTIONAL: + +# DEFAULT_CHAIN: +# Default chain to use when no chain is specified. +DEFAULT_CHAIN=mainnet + +# CACHE_SIZE: +# lru_cache size. +CACHE_SIZE=128 + +# MONGO_CONNECTION_STRING: +# Those represent data required for connecting to mongoDB. It's used for caching semantics +# used in decoding process. But, it's not neccessary for running, If you don't want to use permanent +# db or setup mongo, leave those values, mongomock package is used to simulate in-memory mongo. +MONGO_CONNECTION_STRING=mongomock://localhost/ethtx + +# ETHERSCAN_URLS: +# URLs for etherscan APIs. +# Values are populated from the environment by treating the environment variable's value as a JSON-encoded string. +ETHERSCAN_URLS='{"mainnet": "https://api.etherscan.io/api", "rinkeby": "https://api-rinkeby.etherscan.io/api", "goerli": "https://api-goerli.etherscan.io/api"}' + + diff --git a/README.md b/README.md index 76f1df28..7fca1cd7 100644 --- a/README.md +++ b/README.md @@ -38,29 +38,58 @@ The package needs a few external resources, defined in `EthTxConfig` object: the `debug` option ON 2. **Etherscan API key** - required to get the source code and ABI for smart contracts used in transaction 3. (Optional) **MongoDB database** - required to store smart contracts' ABI and semantics used in the decoding process. - If you don't want to setup permanent database, you can enter `mongomock://localhost`, then in-memory mongo will be + If you don't want to setup permanent database, you can enter `mongomock://localhost/ethtx`, then in-memory mongo will be set up that discards all data with every run. +4. Copy `.env_sample` to `.env` and fill required field according to description +```dotenv +# REQUIRED: + +# WEB3 NODES: +# Proper nodes are required to run ethtx. +# Values are populated from the environment by treating the environment variable's value as a JSON-encoded string. +# EXAMPLE: WEB3_NODES='{"mainnet": {"hook": "https://geth", "poa": false}, "rinkeby": {"hook": "https://eth-rinkeby", "poa": true}}' +# EthTx supports multiple nodes, if one is unavailable, it will use others. You only need to specify them with a comma. +WEB3_NODES= + +# ETHERSCAN: +# Etherscan API is used to get contract source code, required for decoding process +# You can get free key here https://etherscan.io/apis +ETHERSCAN_API_KEY= + + +# OPTIONAL: + +# DEFAULT_CHAIN: +# Default chain to use when no chain is specified. +DEFAULT_CHAIN=mainnet + +# CACHE_SIZE: +# lru_cache size. +CACHE_SIZE=128 + +# MONGO_CONNECTION_STRING: +# Those represent data required for connecting to mongoDB. It's used for caching semantics +# used in decoding process. But, it's not neccessary for running, If you don't want to use permanent +# db or setup mongo, leave those values, mongomock package is used to simulate in-memory mongo. +MONGO_CONNECTION_STRING=mongomock://localhost/ethtx + +# ETHERSCAN_URLS: +# URLs for etherscan APIs. +# Values are populated from the environment by treating the environment variable's value as a JSON-encoded string. +ETHERSCAN_URLS='{"mainnet": "https://api.etherscan.io/api", "rinkeby": "https://api-rinkeby.etherscan.io/api", "goerli": "https://api-goerli.etherscan.io/api"}' +``` ## Getting started ```python +from dotenv import load_dotenv + +load_dotenv("") + from ethtx import EthTx, EthTxConfig from ethtx.models.decoded_model import DecodedTransaction -ethtx_config = EthTxConfig( - mongo_connection_string="mongomock://localhost/ethtx", ##MongoDB connection string, - etherscan_api_key="", ##Etherscan API key, - web3nodes={ - "mainnet": { - "hook": "_Geth_archive_node_URL_", # multiple nodes supported, separate them with comma - "poa": _POA_chain_indicator_ # represented by bool value - } - }, - default_chain="mainnet", - etherscan_urls={"mainnet": "https://api.etherscan.io/api", }, -) - -ethtx = EthTx.initialize(ethtx_config) +ethtx = EthTx.initialize(EthTxConfig) transaction: DecodedTransaction = ethtx.decoders.decode_transaction( '0x50051e0a6f216ab9484c2080001c7e12d5138250acee1f4b7c725b8fb6bb922d') ``` @@ -72,7 +101,6 @@ EthTx most important functions: 1. Raw node data access: ```python -ethtx = EthTx.initialize(ethtx_config) web3provider = ethtx.providers.web3provider from ethtx.models.w3_model import W3Transaction, W3Block, W3Receipt, W3CallTree @@ -96,25 +124,25 @@ from ethtx.models.decoded_model import ( from ethtx.models.objects_model import Transaction, Event, Block, Call # read the raw transaction from the node -transaction: Transaction = web3provider.get_full_transaction( - '0x50051e0a6f216ab9484c2080001c7e12d5138250acee1f4b7c725b8fb6bb922d') +transaction = Transaction.from_raw( + w3transaction=w3transaction, w3receipt=w3receipt, w3calltree=w3calls +) # get proxies used in the transaction -proxies = ethtx.decoders.get_proxies(transaction.root_call, 'mainnet') +proxies = ethtx.decoders.get_proxies(transaction.root_call, "mainnet") block: Block = Block.from_raw( - w3block=web3provider.get_block( - transaction.metadata.block_number - ), - chain_id='mainnet', + w3block=web3provider.get_block(transaction.metadata.block_number), + chain_id="mainnet", ) # decode transaction components abi_decoded_events: List[Event] = ethtx.decoders.abi_decoder.decode_events( transaction.events, block.metadata, transaction.metadata ) -abi_decoded_calls: DecodedCall = ethtx.decoders.abi_decoder.decode_calls(transaction.root_call, block.metadata, - transaction.metadata, proxies) +abi_decoded_calls: DecodedCall = ethtx.decoders.abi_decoder.decode_calls( + transaction.root_call, block.metadata, transaction.metadata, proxies +) abi_decoded_transfers: List[ DecodedTransfer ] = ethtx.decoders.abi_decoder.decode_transfers(abi_decoded_calls, abi_decoded_events) @@ -124,13 +152,15 @@ abi_decoded_balances: List[DecodedBalance] = ethtx.decoders.abi_decoder.decode_b # decode a single event raw_event: Event = transaction.events[3] -abi_decoded_event: DecodedEvent = ethtx.decoders.abi_decoder.decode_event(raw_event, block.metadata, - transaction.metadata) +abi_decoded_event: DecodedEvent = ethtx.decoders.abi_decoder.decode_event( + raw_event, block.metadata, transaction.metadata +) # decode a single call raw_call: Call = transaction.root_call.subcalls[0] -abi_decoded_call: DecodedCall = ethtx.decoders.abi_decoder.decode_call(raw_call, block.metadata, transaction.metadata, - proxies) +abi_decoded_call: DecodedCall = ethtx.decoders.abi_decoder.decode_call( + raw_call, block.metadata, transaction.metadata, proxies +) ``` 3. Semantic decoding: @@ -141,14 +171,16 @@ from ethtx.models.decoded_model import DecodedTransactionMetadata # semantically decode transaction components decoded_metadata: DecodedTransactionMetadata = ( ethtx.decoders.semantic_decoder.decode_metadata( - block.metadata, transaction.metadata, 'mainnet' + block.metadata, transaction.metadata, "mainnet" ) ) decoded_events: List[DecodedEvent] = ethtx.decoders.semantic_decoder.decode_events( abi_decoded_events, decoded_metadata, proxies ) -decoded_calls: Call = ethtx.decoders.semantic_decoder.decode_calls(abi_decoded_calls, decoded_metadata, proxies) +decoded_calls: Call = ethtx.decoders.semantic_decoder.decode_calls( + abi_decoded_calls, decoded_metadata, proxies +) decoded_transfers: List[ DecodedTransfer ] = ethtx.decoders.semantic_decoder.decode_transfers( @@ -161,9 +193,11 @@ decoded_balances: List[ ) # semantically decode a single event -decoded_event: DecodedEvent = ethtx.decoders.semantic_decoder.decode_event(abi_decoded_events[0], decoded_metadata, - proxies) +decoded_event: DecodedEvent = ethtx.decoders.semantic_decoder.decode_event( + abi_decoded_events[0], decoded_metadata, proxies +) # semantically decode a single call -decoded_call: Call = ethtx.decoders.semantic_decoder.decode_call(abi_decoded_calls.subcalls[0], - decoded_metadata, proxies) +decoded_call: Call = ethtx.decoders.semantic_decoder.decode_call( + abi_decoded_calls.subcalls[0], decoded_metadata, proxies +) ``` \ No newline at end of file diff --git a/ethtx/core/config.py b/ethtx/core/config.py index f9815864..7c54a15d 100644 --- a/ethtx/core/config.py +++ b/ethtx/core/config.py @@ -5,7 +5,7 @@ class Settings(BaseSettings): DEFAULT_CHAIN: str = "mainnet" CACHE_SIZE: int = 128 - MONGO_CONNECTION_STRING: str + MONGO_CONNECTION_STRING: str = "mongomock://localhost/ethtx" WEB3_NODES: dict From 1745b90255e2c36947cfd23eab9a36803248a1bb Mon Sep 17 00:00:00 2001 From: Karol Chojnowski Date: Wed, 23 Feb 2022 15:06:44 +0100 Subject: [PATCH 3/3] replace lru_cache with own cache --- ethtx/providers/etherscan/contracts.py | 4 ++-- .../providers/semantic_providers/database.py | 8 +++---- .../semantic_providers/repository.py | 22 +++++++++---------- ethtx/providers/web3_provider.py | 22 +++++++++---------- ethtx/semantics/protocols_router.py | 5 ++--- ethtx/semantics/standards/eip1969.py | 8 +++---- 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/ethtx/providers/etherscan/contracts.py b/ethtx/providers/etherscan/contracts.py index 3a552644..06f1d583 100644 --- a/ethtx/providers/etherscan/contracts.py +++ b/ethtx/providers/etherscan/contracts.py @@ -12,13 +12,13 @@ import json import logging -from functools import lru_cache from typing import Dict, Tuple, Union, Any, Optional from web3 import Web3 from ethtx.exceptions import InvalidEtherscanReturnCodeException from .client import EtherscanClient +from ...utils.cache import cache log = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def get_contract_abi( return dict(name=contract_name, abi=abi), decoded - @lru_cache(maxsize=1024) + @cache def _get_contract_abi(self, chain_id, contract_name) -> Dict: url_dict = self.contract_dict.copy() url_dict[self.ACTION] = "getsourcecode" diff --git a/ethtx/providers/semantic_providers/database.py b/ethtx/providers/semantic_providers/database.py index 189eb589..95897c6d 100644 --- a/ethtx/providers/semantic_providers/database.py +++ b/ethtx/providers/semantic_providers/database.py @@ -11,7 +11,6 @@ # limitations under the License. import logging -from functools import lru_cache from typing import Dict, Optional import bson @@ -20,6 +19,7 @@ from .base import ISemanticsDatabase from .const import MongoCollections +from ...utils.cache import cache log = logging.getLogger(__name__) @@ -39,12 +39,12 @@ def __init__(self, db: MongoDatabase): def get_collection_count(self) -> int: return len(self._db.list_collection_names()) - @lru_cache(maxsize=1024) + @cache def get_address_semantics(self, chain_id, address) -> Optional[Dict]: _id = f"{chain_id}-{address}" return self._addresses.find_one({"_id": _id}, {"_id": 0}) - @lru_cache(maxsize=1024) + @cache def get_signature_semantics(self, signature_hash: str) -> Cursor: return self._signatures.find({"signature_hash": signature_hash}) @@ -64,7 +64,7 @@ def insert_signature( inserted_signature = self._signatures.insert_one(signature) return inserted_signature.inserted_id - @lru_cache(maxsize=1024) + @cache def get_contract_semantics(self, code_hash): """Contract hashes are always the same, no mather what chain we use, so there is no need to use chain_id""" diff --git a/ethtx/providers/semantic_providers/repository.py b/ethtx/providers/semantic_providers/repository.py index c9c3e0e8..c4ea001a 100644 --- a/ethtx/providers/semantic_providers/repository.py +++ b/ethtx/providers/semantic_providers/repository.py @@ -10,7 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from functools import lru_cache from typing import Optional, List, Dict, Tuple from ethtx.decoders.decoders.semantics import decode_events_and_functions @@ -25,13 +24,14 @@ Signature, SignatureArg, ) -from ethtx.providers import EtherscanProvider, Web3Provider, ENSProvider +from ethtx.providers import EtherscanProvider, ENSProvider from ethtx.providers.semantic_providers.database import ISemanticsDatabase from ethtx.providers.web3_provider import NodeDataProvider from ethtx.semantics.protocols_router import amend_contract_semantics from ethtx.semantics.solidity.precompiles import precompiles from ethtx.semantics.standards.erc20 import ERC20_FUNCTIONS, ERC20_EVENTS from ethtx.semantics.standards.erc721 import ERC721_FUNCTIONS, ERC721_EVENTS +from ethtx.utils.cache import cache class SemanticsRepository: @@ -181,7 +181,7 @@ def _create_address_semantics( return address_semantics - @lru_cache(maxsize=1024) + @cache def get_semantics(self, chain_id: str, address: str) -> Optional[AddressSemantics]: if not address: @@ -231,7 +231,7 @@ def _decode_standard_semantics( return standard, standard_semantics - @lru_cache(maxsize=1024) + @cache def get_event_abi(self, chain_id, address, signature) -> Optional[EventSemantics]: if not address: @@ -242,7 +242,7 @@ def get_event_abi(self, chain_id, address, signature) -> Optional[EventSemantics return event_semantics - @lru_cache(maxsize=1024) + @cache def get_transformations( self, chain_id, address, signature ) -> Optional[Dict[str, TransformationSemantics]]: @@ -255,7 +255,7 @@ def get_transformations( return transformations - @lru_cache(maxsize=1024) + @cache def get_anonymous_event_abi(self, chain_id, address) -> Optional[EventSemantics]: if not address: @@ -275,7 +275,7 @@ def get_anonymous_event_abi(self, chain_id, address) -> Optional[EventSemantics] return event_semantics - @lru_cache(maxsize=1024) + @cache def get_function_abi( self, chain_id, address, signature ) -> Optional[FunctionSemantics]: @@ -288,7 +288,7 @@ def get_function_abi( return function_semantics - @lru_cache(maxsize=1024) + @cache def get_constructor_abi(self, chain_id, address) -> Optional[FunctionSemantics]: if not address: @@ -330,7 +330,7 @@ def get_address_label(self, chain_id, address, proxies=None) -> str: return contract_label - @lru_cache(maxsize=1024) + @cache def check_is_contract(self, chain_id, address) -> bool: if not address: @@ -341,7 +341,7 @@ def check_is_contract(self, chain_id, address) -> bool: return is_contract - @lru_cache(maxsize=1024) + @cache def get_standard(self, chain_id, address) -> Optional[str]: if not address: @@ -429,7 +429,7 @@ def insert_contract_signatures(self, contract_semantics: ContractSemantics) -> N self.update_or_insert_signature(new_signature) - @lru_cache(maxsize=1024) + @cache def get_most_used_signature(self, signature_hash: str) -> Optional[Signature]: signatures = list( self.database.get_signature_semantics(signature_hash=signature_hash) diff --git a/ethtx/providers/web3_provider.py b/ethtx/providers/web3_provider.py index 77b889c3..21ea7bd6 100644 --- a/ethtx/providers/web3_provider.py +++ b/ethtx/providers/web3_provider.py @@ -12,7 +12,6 @@ import logging import os -from functools import lru_cache from typing import List, Dict, Optional from web3 import Web3 @@ -26,6 +25,7 @@ from ..models.semantics_model import FunctionSemantics from ..models.w3_model import W3Block, W3Transaction, W3Receipt, W3CallTree, W3Log from ..semantics.standards import erc20 +from ..utils.cache import cache log = logging.getLogger(__name__) @@ -141,7 +141,7 @@ def _get_node_connection(self, chain_id: Optional[str] = None) -> Web3: raise NodeConnectionException # get the raw block data from the node - @lru_cache(maxsize=1024) + @cache def get_block(self, block_number: int, chain_id: Optional[str] = None) -> W3Block: chain = self._get_node_connection(chain_id) raw_block: BlockData = chain.eth.get_block(block_number) @@ -171,7 +171,7 @@ def get_block(self, block_number: int, chain_id: Optional[str] = None) -> W3Bloc return block # get the raw transaction data from the node - @lru_cache(maxsize=1024) + @cache def get_transaction( self, tx_hash: str, chain_id: Optional[str] = None ) -> W3Transaction: @@ -197,7 +197,7 @@ def get_transaction( return transaction - @lru_cache(maxsize=1024) + @cache def get_receipt(self, tx_hash: str, chain_id: Optional[str] = None) -> W3Receipt: chain = self._get_node_connection(chain_id) raw_receipt: TxReceipt = chain.eth.get_transaction_receipt(tx_hash) @@ -244,7 +244,7 @@ def get_receipt(self, tx_hash: str, chain_id: Optional[str] = None) -> W3Receipt def _get_custom_calls_tracer(): return open(os.path.join(os.path.dirname(__file__), "static/tracer.js")).read() - @lru_cache(maxsize=1024) + @cache def get_calls(self, tx_hash: str, chain_id: Optional[str] = None) -> W3CallTree: # tracer is a temporary fixed implementation of geth tracer chain = self._get_node_connection(chain_id) @@ -258,7 +258,7 @@ def get_calls(self, tx_hash: str, chain_id: Optional[str] = None) -> W3CallTree: ) # get the contract bytecode hash from the node - @lru_cache(maxsize=1024) + @cache def get_code_hash( self, contract_address: str, chain_id: Optional[str] = None ) -> str: @@ -268,7 +268,7 @@ def get_code_hash( return code_hash # get the erc20 token data from the node - @lru_cache(maxsize=1024) + @cache def get_erc20_token( self, token_address: str, @@ -339,7 +339,7 @@ def get_erc20_token( return dict(address=token_address, symbol=symbol, name=name, decimals=decimals) # guess if the contract is and erc20 token and get the data - @lru_cache(maxsize=1024) + @cache def guess_erc20_token(self, contract_address, chain_id: Optional[str] = None): chain = self._get_node_connection(chain_id) @@ -396,7 +396,7 @@ def guess_erc20_token(self, contract_address, chain_id: Optional[str] = None): return None # guess if the contract is and erc20 token proxy and get the data - @lru_cache(maxsize=1024) + @cache def guess_erc20_proxy(self, contract_address, chain_id: Optional[str] = None): chain = self._get_node_connection(chain_id) @@ -431,7 +431,7 @@ def guess_erc20_proxy(self, contract_address, chain_id: Optional[str] = None): return None # guess if the contract is and erc721 token proxy and get the data - @lru_cache(maxsize=1024) + @cache def guess_erc721_proxy(self, contract_address, chain_id: Optional[str] = None): chain = self._get_node_connection(chain_id) @@ -460,7 +460,7 @@ def guess_erc721_proxy(self, contract_address, chain_id: Optional[str] = None): return None - @lru_cache(maxsize=1024) + @cache def get_full_transaction(self, tx_hash: str, chain_id: Optional[str] = None): w3transaction = self.get_transaction(tx_hash, chain_id) diff --git a/ethtx/semantics/protocols_router.py b/ethtx/semantics/protocols_router.py index b81c31e7..ee1e2714 100644 --- a/ethtx/semantics/protocols_router.py +++ b/ethtx/semantics/protocols_router.py @@ -11,17 +11,16 @@ # limitations under the License. import logging -from functools import lru_cache from ..models.semantics_model import ContractSemantics from ..semantics.router import Router -from ..utils.cache import ignore_unhashable +from ..utils.cache import ignore_unhashable, cache log = logging.getLogger(__name__) @ignore_unhashable -@lru_cache(maxsize=1024) +@cache def amend_contract_semantics(semantics: ContractSemantics, router_=Router()): if semantics.code_hash in router_: try: diff --git a/ethtx/semantics/standards/eip1969.py b/ethtx/semantics/standards/eip1969.py index d0e91aee..1f0b83d4 100644 --- a/ethtx/semantics/standards/eip1969.py +++ b/ethtx/semantics/standards/eip1969.py @@ -1,9 +1,9 @@ -from functools import lru_cache - from web3 import Web3 +from ethtx.utils.cache import cache + -@lru_cache(maxsize=1024) +@cache def is_eip1969_proxy(chain, delegator, delegate): implementation_slot = hex( int(Web3.keccak(text="eip1967.proxy.implementation").hex(), 16) - 1 @@ -20,7 +20,7 @@ def is_eip1969_proxy(chain, delegator, delegate): return False -@lru_cache(maxsize=1024) +@cache def is_eip1969_beacon_proxy(chain, delegator, delegate): ibeacon_abi = """[ {