From ba41ecdc276fb4696205aa6525c743b7ee3ace7f Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Thu, 13 Feb 2025 14:09:25 -0300 Subject: [PATCH 01/14] Universal config generator --- src/dipdup/_survey.py | 503 ------------------ src/dipdup/config/__init__.py | 9 +- src/dipdup/config/coinbase.py | 2 +- src/dipdup/config/evm.py | 2 +- src/dipdup/config/evm_etherscan.py | 2 +- src/dipdup/config/evm_events.py | 2 +- src/dipdup/config/evm_node.py | 2 +- src/dipdup/config/evm_subsquid.py | 2 +- src/dipdup/config/evm_transactions.py | 2 +- src/dipdup/config/http.py | 2 +- src/dipdup/config/ipfs.py | 2 +- src/dipdup/config/starknet.py | 2 +- src/dipdup/config/starknet_events.py | 2 +- src/dipdup/config/starknet_node.py | 2 +- src/dipdup/config/starknet_subsquid.py | 2 +- src/dipdup/config/substrate_events.py | 2 +- src/dipdup/config/substrate_node.py | 2 +- src/dipdup/config/substrate_subscan.py | 2 +- src/dipdup/config/substrate_subsquid.py | 2 +- src/dipdup/config/tezos.py | 2 +- src/dipdup/config/tezos_big_maps.py | 2 +- src/dipdup/config/tezos_events.py | 2 +- src/dipdup/config/tezos_head.py | 2 +- src/dipdup/config/tezos_operations.py | 4 +- src/dipdup/config/tezos_token_balances.py | 2 +- src/dipdup/config/tezos_token_transfers.py | 2 +- src/dipdup/config/tezos_tzkt.py | 2 +- src/dipdup/config/tzip_metadata.py | 2 +- src/dipdup/datasources/starknet_node.py | 8 +- src/dipdup/indexes/starknet_events/fetcher.py | 10 +- src/dipdup/project.py | 154 +++++- 31 files changed, 173 insertions(+), 565 deletions(-) delete mode 100644 src/dipdup/_survey.py diff --git a/src/dipdup/_survey.py b/src/dipdup/_survey.py deleted file mode 100644 index 5d88c16d6..000000000 --- a/src/dipdup/_survey.py +++ /dev/null @@ -1,503 +0,0 @@ -from typing import Any -from typing import TypedDict - -import survey # type: ignore[import-untyped] - -from dipdup.cli import echo -from dipdup.install import ask - - -class Pattern(TypedDict): - destination: str - entrypoint: str - - -class Handler(TypedDict): - name: str | None - callback: str | None - contract: str | None - path: str | None - tag: str | None - pattern: Pattern | None - to: str | None - method: str | None - - -class Datasource(TypedDict): - name: str - kind: str - url: str - ws_url: str | None - api_key: str | None - - -class Contract(TypedDict): - name: str - kind: str - address: str - typename: str - - -class Index(TypedDict): - name: str - kind: str - datasources: list[str] - handlers: list[Handler] | None - skip_history: str | None - first_level: int | None - last_level: int | None - callback: str | None - types: list[str] | None - contracts: list[str] | None - - -class DipDupSurveyConfig(TypedDict): - datasources: list[Datasource] - contracts: list[Contract] | None - indexes: list[Index] - - -class DatasourceConfig(TypedDict): - kind: str - requires_api_key: bool - default_url: str - name: str - - -class IndexerConfig(TypedDict): - handler_fields: list[str] - optional_fields: dict[str, str | int] - kind: str - - -class BlockchainConfig(TypedDict): - datasources: list[DatasourceConfig] - contract_kind: str - indexes: dict[str, IndexerConfig] - - -CONFIG_STRUCTURE: dict[str, BlockchainConfig] = { - 'evm': { - 'datasources': [ - { - 'kind': 'evm.subsquid', - 'requires_api_key': False, - 'default_url': 'https://v2.archive.subsquid.io/network/ethereum-mainnet', - 'name': 'subsquid', - }, - { - 'kind': 'evm.node', - 'requires_api_key': True, - 'default_url': 'https://eth-mainnet.g.alchemy.com/v2', - 'name': 'node', - }, - { - 'kind': 'evm.etherscan', - 'requires_api_key': True, - 'default_url': 'https://api.etherscan.io/api', - 'name': 'etherscan', - }, - ], - 'contract_kind': 'evm', - 'indexes': { - 'events': { - 'handler_fields': ['callback', 'contract', 'name'], - 'optional_fields': {}, - 'kind': 'evm.events', - }, - 'transactions': { - 'handler_fields': ['callback', 'to', 'method'], - 'optional_fields': { - 'first_level': 'integer', - }, - 'kind': 'evm.transactions', - }, - }, - }, - 'tezos': { - 'datasources': [ - { - 'kind': 'tezos.tzkt', - 'requires_api_key': False, - 'default_url': 'https://api.ghostnet.tzkt.io', - 'name': 'tzkt', - } - ], - 'contract_kind': 'tezos', - 'indexes': { - 'big_maps': { - 'handler_fields': ['callback', 'contract', 'path'], - 'optional_fields': { - 'skip_history': 'select', - }, - 'kind': 'tezos.big_maps', - }, - 'events': { - 'handler_fields': ['callback', 'contract', 'tag'], - 'optional_fields': {}, - 'kind': 'tezos.events', - }, - 'head': { - 'handler_fields': [], - 'optional_fields': { - 'callback': 'string', - }, - 'kind': 'tezos.head', - }, - 'operations': { - 'handler_fields': ['callback', 'pattern'], - 'optional_fields': {}, - 'kind': 'tezos.operations', - }, - 'operations_unfiltered': { - 'handler_fields': [], - 'optional_fields': { - 'types': 'select', - 'callback': 'string', - 'first_level': 'integer', - 'last_level': 'integer', - }, - 'kind': 'tezos.operations_unfiltered', - }, - 'token_balances': { - 'handler_fields': ['callback', 'contract'], - 'optional_fields': {}, - 'kind': 'tezos.token_balances', - }, - 'token_transfers': { - 'handler_fields': ['callback', 'contract'], - 'optional_fields': {}, - 'kind': 'tezos.token_transfers', - }, - }, - }, - 'starknet': { - 'datasources': [ - { - 'kind': 'starknet.subsquid', - 'requires_api_key': False, - 'default_url': 'https://v2.archive.subsquid.io/network/starknet-mainnet', - 'name': 'subsquid', - }, - { - 'kind': 'starknet.node', - 'requires_api_key': True, - 'default_url': 'https://starknet-mainnet.g.alchemy.com/v2', - 'name': 'node', - }, - ], - 'contract_kind': 'starknet', - 'indexes': { - 'events': { - 'handler_fields': ['callback', 'contract', 'name'], - 'optional_fields': {}, - 'kind': 'starknet.events', - } - }, - }, -} - - -def prompt_anyof( - question: str, - options: tuple[str, ...], - comments: tuple[str, ...], - default: int, -) -> tuple[int, str]: - """Ask user to choose one of the options; returns index and value""" - from tabulate import tabulate - - table = tabulate( - zip(options, comments, strict=False), - tablefmt='plain', - ) - index = survey.routines.select( - question + '\n', - options=table.split('\n'), - index=default, - ) - return index, options[index] - - -# Helper function to generate comments for datasource options -def get_datasource_comments(datasources: list[str]) -> tuple[str, ...]: - default_comments = { - 'evm.subsquid': 'Use Subsquid as your datasource for EVM.', - 'evm.node': 'Connect to an EVM node.', - 'evm.etherscan': 'Fetch ABI from Etherscan.', - 'tezos.tzkt': 'Use TzKT API for Tezos.', - 'starknet.subsquid': 'Use Subsquid for Starknet.', - 'starknet.node': 'Connect to a Starknet node.', - } - return tuple(default_comments.get(option, 'No description available') for option in datasources) - - -# Helper function to generate comments for indexes options -def get_index_comments(indexes: dict[str, IndexerConfig]) -> tuple[str, ...]: - default_comments: dict[str, str] = { - 'evm.events': 'Listen to EVM blockchain events.', - 'evm.transactions': 'Track EVM blockchain transactions.', - 'tezos.big_maps': 'Monitor changes in Tezos big maps.', - 'tezos.events': 'Track specific events in Tezos.', - 'tezos.head': 'Monitor Tezos chain head updates.', - 'tezos.operations': 'Track operations on Tezos blockchain.', - 'tezos.operations_unfiltered': 'Monitor unfiltered Tezos operations.', - 'tezos.token_balances': 'Track Tezos token balances.', - 'tezos.token_transfers': 'Monitor token transfers on Tezos.', - 'starknet.events': 'Listen to Starknet blockchain events.', - } - comments = [] - for _, config in indexes.items(): - kind = config.get('kind') - comment = default_comments.get(kind, 'No description available') # type: ignore[arg-type] - comments.append(comment) - return tuple(comments) - - -def query_handlers( - contract_names: list[str], - additional_fields: list[str] | None, -) -> list[Handler] | None: - - handlers: list[Handler] = [] - - if not additional_fields: - return None - - first = True - while first or ask('Add another handler?', False): - first = False - - handler: Handler = { - 'callback': '', - 'path': None, - 'tag': None, - 'pattern': None, - 'name': None, - 'contract': None, - 'to': None, - 'method': None, - } - - # Prompt for additional fields - for field in additional_fields: - if field in ['contract', 'to']: - handler[field] = prompt_anyof( # type: ignore[literal-required] - 'Choose contract for the handler', - tuple(contract_names), - ('Contract to listen to',) * len(contract_names), - 0, - )[1] - elif field == 'pattern': - handler['pattern'] = { - 'destination': prompt_anyof( - 'Choose contract for the handler', - tuple(contract_names), - ('Contract to listen to',) * len(contract_names), - 0, - )[1], - 'entrypoint': validate_non_empty_input( - survey.routines.input('Enter pattern entrypoint: ', value=''), 'entrypoint' - ), - } - else: - handler[field] = validate_non_empty_input( # type: ignore[literal-required] - survey.routines.input(f'Enter handler {field}: ', value=''), - field, - ) - - handlers.append(handler) - - return handlers - - -def query_optional_fields(optional_fields: dict[str, str | int]) -> dict[str, Any]: - field_values: dict[str, Any] = {} - for field, field_type in optional_fields.items(): - if field_type == 'select': - if field == 'types': - types: list[str] = [] - while True: - _, type = prompt_anyof( - 'Select operation type', - ('origination', 'transaction', 'migration'), - ('origination', 'transaction', 'migration'), - 0, - ) - types.append(type) - if not ask('Add another type?', False): - break - field_values[field] = types - else: - _, field_values[field] = prompt_anyof( - f'Select {field}', - ('never', 'always', 'auto'), - ('Never', 'Always', 'Auto'), - 0, - ) - else: - field_values[field] = survey.routines.input(f'Enter {field}: ') - - return field_values - - -def validate_non_empty_input(input_value: str, field_name: str) -> str: - while not input_value.strip(): - echo(f'{field_name} cannot be empty. Please enter a valid value.', fg='red') - input_value = survey.routines.input(f'Enter {field_name}: ') - return input_value - - -def filter_index_options( - indexes: tuple[str, ...], - index_comments: tuple[str, ...], - contracts: list[Contract], - blockchain: str, -) -> tuple[tuple[str, ...], tuple[str, ...]]: - blockchain_config = CONFIG_STRUCTURE[blockchain] - index_dict = blockchain_config['indexes'] - required_contract_fields = {'contract', 'to', 'pattern'} - if not contracts: - valid_indexes = [ - idx - for idx, index in enumerate(indexes) - if not required_contract_fields.intersection(set(index_dict[index].get('handler_fields', []))) or contracts - ] - - indexes = tuple(indexes[idx] for idx in valid_indexes) - index_comments = tuple(index_comments[idx] for idx in valid_indexes) - - return indexes, index_comments - - -def query_survey_config(blockchain: str) -> DipDupSurveyConfig: - blockchain_config = CONFIG_STRUCTURE[blockchain] - datasources: list[Datasource] = [] - contracts: list[Contract] = [] - indexes: list[Index] = [] - survey_config = DipDupSurveyConfig( - datasources=datasources, - contracts=contracts, - indexes=indexes, - ) - - first = True - while ask(f'Add {'' if first else 'another '}datasource?', first): - first = False - - datasource_options = blockchain_config['datasources'] - datasource_comments = get_datasource_comments([ds['kind'] for ds in datasource_options]) - - selected_index, _ = prompt_anyof( - f'Select the datasource kind for {blockchain}:', - tuple(ds['name'] for ds in datasource_options), - datasource_comments, - 0, - ) - selected_datasource = datasource_options[selected_index] - datasource_kind = selected_datasource['kind'] - - final_url = validate_non_empty_input( - survey.routines.input('Enter Datasource URL: ', value=selected_datasource.get('default_url', '')), - 'Datasource URL', - ) - api_key = None - ws_url = None - - if selected_datasource['requires_api_key']: - api_key = validate_non_empty_input( - survey.routines.input('Enter API key: ', value=''), - 'API key', - ) - if 'node' in datasource_kind: - final_url = '${NODE_URL:-' + final_url + '}/${NODE_API_KEY:-' + api_key + '}' - if datasource_kind == 'evm.node': - ws_url = survey.routines.input( - 'Enter WebSocket (wss) URL: ', value='wss://eth-mainnet.g.alchemy.com/v2' - ) - ws_url = '${NODE_WS_URL:-' + ws_url + '}/${NODE_API_KEY:-' + api_key + '}' - api_key = None - else: - api_key = '${ETHERSCAN_API_KEY:-' + api_key + '}' - - if datasource_kind != 'evm.etherscan': - api_key = None - - if 'subsquid' in datasource_kind: - final_url = '${SUBSQUID_URL:-' + final_url + '}' - - datasource: Datasource = { - 'kind': datasource_kind, - 'url': final_url, - 'ws_url': ws_url, - 'api_key': api_key, - 'name': selected_datasource['name'], - } - - datasources.append(datasource) - - first = True - while ask(f'Add {'' if first else 'another '}contract?', first): - first = False - - contract_name = validate_non_empty_input( - survey.routines.input('Enter contract name: '), - 'Contract name', - ) - contract_address = validate_non_empty_input( - survey.routines.input('Enter contract address: '), - 'Contract address', - ) - - contract: Contract = { - 'name': contract_name, - 'kind': blockchain_config['contract_kind'], - 'address': contract_address, - 'typename': contract_name, - } - contracts.append(contract) - - if not datasources: - return survey_config - - index_options = tuple(blockchain_config['indexes'].keys()) - index_comments = get_index_comments(blockchain_config['indexes']) - index_options, index_comments = filter_index_options(index_options, index_comments, contracts, blockchain) - - if not index_options: - return survey_config - - first = True - while ask(f'Add {'' if first else 'another '}index?', first): - first = False - - _, kind = prompt_anyof(f'Select the type of index for {blockchain}:', index_options, index_comments, 0) - - index_config = blockchain_config['indexes'][kind] - index_name = validate_non_empty_input( - survey.routines.input('Enter the index name: '), - 'Indexer name', - ) - - index: Index = { - 'name': index_name, - 'kind': index_config['kind'], - 'datasources': [ds['name'] for ds in datasources], - 'handlers': None, - 'skip_history': None, - 'first_level': None, - 'last_level': None, - 'callback': None, - 'types': None, - 'contracts': None, - } - - handlers = query_handlers([c['name'] for c in contracts], index_config.get('handler_fields')) - index['handlers'] = handlers - - if 'optional_fields' in index_config: - index.update(query_optional_fields(index_config['optional_fields'])) # type: ignore[typeddict-item] - - indexes.append(index) - - return survey_config diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index a0cb103c9..30aa1234a 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -100,7 +100,7 @@ class SqliteDatabaseConfig: :param immune_tables: List of tables to preserve during reindexing """ - kind: Literal['sqlite'] + kind: Literal['sqlite'] = 'sqlite' path: str = DEFAULT_SQLITE_PATH immune_tables: set[str] = Field(default_factory=set) @@ -139,7 +139,7 @@ class PostgresDatabaseConfig: :param connection_timeout: Connection timeout """ - kind: Literal['postgres'] + kind: Literal['postgres'] = 'postgres' host: str user: str = DEFAULT_POSTGRES_USER database: str = DEFAULT_POSTGRES_DATABASE @@ -586,9 +586,7 @@ class DipDupConfig: spec_version: ToStr package: str datasources: dict[str, DatasourceConfigU] = Field(default_factory=dict) - database: SqliteDatabaseConfig | PostgresDatabaseConfig = Field( - default_factory=lambda *a, **kw: SqliteDatabaseConfig(kind='sqlite') - ) + database: DatabaseConfigU = Field(default_factory=lambda *a, **kw: SqliteDatabaseConfig(kind='sqlite')) runtimes: dict[str, RuntimeConfigU] = Field(default_factory=dict) contracts: dict[str, ContractConfigU] = Field(default_factory=dict) indexes: dict[str, IndexConfigU] = Field(default_factory=dict) @@ -1162,6 +1160,7 @@ def _set_names(self) -> None: from dipdup.config.tzip_metadata import TzipMetadataDatasourceConfig # NOTE: Unions for Pydantic config deserialization +DatabaseConfigU = SqliteDatabaseConfig | PostgresDatabaseConfig RuntimeConfigU = SubstrateRuntimeConfig ContractConfigU = EvmContractConfig | TezosContractConfig | StarknetContractConfig DatasourceConfigU = ( diff --git a/src/dipdup/config/coinbase.py b/src/dipdup/config/coinbase.py index fe194af3f..41e3d0cfe 100644 --- a/src/dipdup/config/coinbase.py +++ b/src/dipdup/config/coinbase.py @@ -21,7 +21,7 @@ class CoinbaseDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['coinbase'] + kind: Literal['coinbase'] = 'coinbase' api_key: str | None = None secret_key: str | None = Field(default=None, repr=False) passphrase: str | None = Field(default=None, repr=False) diff --git a/src/dipdup/config/evm.py b/src/dipdup/config/evm.py index d903937a1..9b0e4714b 100644 --- a/src/dipdup/config/evm.py +++ b/src/dipdup/config/evm.py @@ -55,7 +55,7 @@ class EvmContractConfig(ContractConfig): :param typename: Alias for the contract script """ - kind: Literal['evm'] + kind: Literal['evm'] = 'evm' address: EvmAddress | None = None abi: EvmAddress | None = None typename: str | None = None diff --git a/src/dipdup/config/evm_etherscan.py b/src/dipdup/config/evm_etherscan.py index 2ff8ba722..8e24df9d9 100644 --- a/src/dipdup/config/evm_etherscan.py +++ b/src/dipdup/config/evm_etherscan.py @@ -23,7 +23,7 @@ class EvmEtherscanDatasourceConfig(DatasourceConfig): """ # NOTE: Alias, remove in 9.0 - kind: Literal['evm.etherscan'] | Literal['abi.etherscan'] + kind: Literal['evm.etherscan'] | Literal['abi.etherscan'] = 'evm.etherscan' url: str api_key: str | None = None diff --git a/src/dipdup/config/evm_events.py b/src/dipdup/config/evm_events.py index aedb92fe3..f277227c1 100644 --- a/src/dipdup/config/evm_events.py +++ b/src/dipdup/config/evm_events.py @@ -61,7 +61,7 @@ class EvmEventsIndexConfig(EvmIndexConfig): :param last_level: Level to stop indexing and disable this index """ - kind: Literal['evm.events'] + kind: Literal['evm.events'] = 'evm.events' datasources: tuple[Alias[EvmDatasourceConfigU], ...] handlers: tuple[EvmEventsHandlerConfig, ...] diff --git a/src/dipdup/config/evm_node.py b/src/dipdup/config/evm_node.py index 9ee1eb170..fe7a0ec95 100644 --- a/src/dipdup/config/evm_node.py +++ b/src/dipdup/config/evm_node.py @@ -22,7 +22,7 @@ class EvmNodeDatasourceConfig(DatasourceConfig): :param rollback_depth: A number of blocks to store in database for rollback """ - kind: Literal['evm.node'] + kind: Literal['evm.node'] = 'evm.node' url: Url ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/evm_subsquid.py b/src/dipdup/config/evm_subsquid.py index 9ce1ac9a4..c8848480e 100644 --- a/src/dipdup/config/evm_subsquid.py +++ b/src/dipdup/config/evm_subsquid.py @@ -19,7 +19,7 @@ class EvmSubsquidDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['evm.subsquid'] + kind: Literal['evm.subsquid'] = 'evm.subsquid' url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/evm_transactions.py b/src/dipdup/config/evm_transactions.py index 956685206..3fdeb06c7 100644 --- a/src/dipdup/config/evm_transactions.py +++ b/src/dipdup/config/evm_transactions.py @@ -93,7 +93,7 @@ class EvmTransactionsIndexConfig(EvmIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['evm.transactions'] + kind: Literal['evm.transactions'] = 'evm.transactions' datasources: tuple[Alias[EvmDatasourceConfigU], ...] handlers: tuple[EvmTransactionsHandlerConfig, ...] diff --git a/src/dipdup/config/http.py b/src/dipdup/config/http.py index 6b3cfcf0c..b4341e90e 100644 --- a/src/dipdup/config/http.py +++ b/src/dipdup/config/http.py @@ -18,6 +18,6 @@ class HttpDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['http'] + kind: Literal['http'] = 'http' url: str http: HttpConfig | None = None diff --git a/src/dipdup/config/ipfs.py b/src/dipdup/config/ipfs.py index 148ce6df1..90993b647 100644 --- a/src/dipdup/config/ipfs.py +++ b/src/dipdup/config/ipfs.py @@ -20,6 +20,6 @@ class IpfsDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['ipfs'] + kind: Literal['ipfs'] = 'ipfs' url: str = DEFAULT_IPFS_URL http: HttpConfig | None = None diff --git a/src/dipdup/config/starknet.py b/src/dipdup/config/starknet.py index 52469d5a9..8e026c70d 100644 --- a/src/dipdup/config/starknet.py +++ b/src/dipdup/config/starknet.py @@ -65,7 +65,7 @@ class StarknetContractConfig(ContractConfig): :param typename: Alias for the contract script """ - kind: Literal['starknet'] + kind: Literal['starknet'] = 'starknet' address: StarknetAddress | None = None abi: StarknetAddress | None = None typename: str | None = None diff --git a/src/dipdup/config/starknet_events.py b/src/dipdup/config/starknet_events.py index e09b69392..14e7cf901 100644 --- a/src/dipdup/config/starknet_events.py +++ b/src/dipdup/config/starknet_events.py @@ -61,7 +61,7 @@ class StarknetEventsIndexConfig(StarknetIndexConfig): """ - kind: Literal['starknet.events'] + kind: Literal['starknet.events'] = 'starknet.events' datasources: tuple[Alias[StarknetDatasourceConfigU], ...] handlers: tuple[StarknetEventsHandlerConfig, ...] diff --git a/src/dipdup/config/starknet_node.py b/src/dipdup/config/starknet_node.py index 5e2b72251..54318a36a 100644 --- a/src/dipdup/config/starknet_node.py +++ b/src/dipdup/config/starknet_node.py @@ -22,7 +22,7 @@ class StarknetNodeDatasourceConfig(DatasourceConfig): :param rollback_depth: A number of blocks to store in database for rollback """ - kind: Literal['starknet.node'] + kind: Literal['starknet.node'] = 'starknet.node' url: Url ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/starknet_subsquid.py b/src/dipdup/config/starknet_subsquid.py index 581f8400d..6b270ec7a 100644 --- a/src/dipdup/config/starknet_subsquid.py +++ b/src/dipdup/config/starknet_subsquid.py @@ -19,7 +19,7 @@ class StarknetSubsquidDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['starknet.subsquid'] + kind: Literal['starknet.subsquid'] = 'starknet.subsquid' url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/substrate_events.py b/src/dipdup/config/substrate_events.py index c814fe712..5e008b656 100644 --- a/src/dipdup/config/substrate_events.py +++ b/src/dipdup/config/substrate_events.py @@ -62,7 +62,7 @@ class SubstrateEventsIndexConfig(SubstrateIndexConfig): :param runtime: Substrate runtime """ - kind: Literal['substrate.events'] + kind: Literal['substrate.events'] = 'substrate.events' datasources: tuple[Alias[SubstrateDatasourceConfigU], ...] handlers: tuple[SubstrateEventsHandlerConfig, ...] runtime: Alias[SubstrateRuntimeConfig] diff --git a/src/dipdup/config/substrate_node.py b/src/dipdup/config/substrate_node.py index a7eeb402c..a0bd09f4f 100644 --- a/src/dipdup/config/substrate_node.py +++ b/src/dipdup/config/substrate_node.py @@ -21,7 +21,7 @@ class SubstrateNodeDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['substrate.node'] + kind: Literal['substrate.node'] = 'substrate.node' url: Url ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/substrate_subscan.py b/src/dipdup/config/substrate_subscan.py index 30dc2d178..8167da3b4 100644 --- a/src/dipdup/config/substrate_subscan.py +++ b/src/dipdup/config/substrate_subscan.py @@ -19,7 +19,7 @@ class SubstrateSubscanDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['substrate.subscan'] + kind: Literal['substrate.subscan'] = 'substrate.subscan' url: str api_key: str | None = None diff --git a/src/dipdup/config/substrate_subsquid.py b/src/dipdup/config/substrate_subsquid.py index b2de5eb3e..777ae1880 100644 --- a/src/dipdup/config/substrate_subsquid.py +++ b/src/dipdup/config/substrate_subsquid.py @@ -19,7 +19,7 @@ class SubstrateSubsquidDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['substrate.subsquid'] + kind: Literal['substrate.subsquid'] = 'substrate.subsquid' url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/tezos.py b/src/dipdup/config/tezos.py index 707cfe5a5..83e3490cf 100644 --- a/src/dipdup/config/tezos.py +++ b/src/dipdup/config/tezos.py @@ -59,7 +59,7 @@ class TezosContractConfig(ContractConfig): :param typename: Alias for the contract script """ - kind: Literal['tezos'] + kind: Literal['tezos'] = 'tezos' address: TezosAddress | None = None code_hash: int | TezosAddress | None = None typename: str | None = None diff --git a/src/dipdup/config/tezos_big_maps.py b/src/dipdup/config/tezos_big_maps.py index 0777a1ea3..d09e664d6 100644 --- a/src/dipdup/config/tezos_big_maps.py +++ b/src/dipdup/config/tezos_big_maps.py @@ -79,7 +79,7 @@ class TezosBigMapsIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.big_maps'] + kind: Literal['tezos.big_maps'] = 'tezos.big_maps' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosBigMapsHandlerConfig, ...] diff --git a/src/dipdup/config/tezos_events.py b/src/dipdup/config/tezos_events.py index 1c39d4d3d..a38bb394c 100644 --- a/src/dipdup/config/tezos_events.py +++ b/src/dipdup/config/tezos_events.py @@ -83,7 +83,7 @@ class TezosEventsIndexConfig(TezosIndexConfig): :param last_level: Last block level to index """ - kind: Literal['tezos.events'] + kind: Literal['tezos.events'] = 'tezos.events' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosEventsHandlerConfigU, ...] diff --git a/src/dipdup/config/tezos_head.py b/src/dipdup/config/tezos_head.py index f727a2d2d..c76b57ecc 100644 --- a/src/dipdup/config/tezos_head.py +++ b/src/dipdup/config/tezos_head.py @@ -41,7 +41,7 @@ class TezosHeadIndexConfig(TezosIndexConfig): :param datasources: `tezos` datasources to use """ - kind: Literal['tezos.head'] + kind: Literal['tezos.head'] = 'tezos.head' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] callback: str diff --git a/src/dipdup/config/tezos_operations.py b/src/dipdup/config/tezos_operations.py index 702282d28..e6fa68bfe 100644 --- a/src/dipdup/config/tezos_operations.py +++ b/src/dipdup/config/tezos_operations.py @@ -306,7 +306,7 @@ class TezosOperationsIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.operations'] + kind: Literal['tezos.operations'] = 'tezos.operations' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosOperationsHandlerConfig, ...] contracts: list[Alias[TezosContractConfig]] = Field(default_factory=list) @@ -431,7 +431,7 @@ class TezosOperationsUnfilteredIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.operations_unfiltered'] + kind: Literal['tezos.operations_unfiltered'] = 'tezos.operations_unfiltered' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] callback: str types: tuple[TezosOperationType, ...] = (TezosOperationType.transaction,) diff --git a/src/dipdup/config/tezos_token_balances.py b/src/dipdup/config/tezos_token_balances.py index 3dbdbe0f4..ecd7667ed 100644 --- a/src/dipdup/config/tezos_token_balances.py +++ b/src/dipdup/config/tezos_token_balances.py @@ -56,7 +56,7 @@ class TezosTokenBalancesIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.token_balances'] + kind: Literal['tezos.token_balances'] = 'tezos.token_balances' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosTokenBalancesHandlerConfig, ...] diff --git a/src/dipdup/config/tezos_token_transfers.py b/src/dipdup/config/tezos_token_transfers.py index bd76a29b7..caf00e227 100644 --- a/src/dipdup/config/tezos_token_transfers.py +++ b/src/dipdup/config/tezos_token_transfers.py @@ -59,7 +59,7 @@ class TezosTokenTransfersIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.token_transfers'] + kind: Literal['tezos.token_transfers'] = 'tezos.token_transfers' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosTokenTransfersHandlerConfig, ...] diff --git a/src/dipdup/config/tezos_tzkt.py b/src/dipdup/config/tezos_tzkt.py index 9eba47d18..564fe414a 100644 --- a/src/dipdup/config/tezos_tzkt.py +++ b/src/dipdup/config/tezos_tzkt.py @@ -34,7 +34,7 @@ class TezosTzktDatasourceConfig(DatasourceConfig): :param rollback_depth: Number of blocks to keep in the database to handle reorgs """ - kind: Literal['tezos.tzkt'] + kind: Literal['tezos.tzkt'] = 'tezos.tzkt' url: Url = DEFAULT_TZKT_URL http: HttpConfig | None = None buffer_size: int = 0 diff --git a/src/dipdup/config/tzip_metadata.py b/src/dipdup/config/tzip_metadata.py index 1a26ab5e7..b787fde43 100644 --- a/src/dipdup/config/tzip_metadata.py +++ b/src/dipdup/config/tzip_metadata.py @@ -22,7 +22,7 @@ class TzipMetadataDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['tzip_metadata'] + kind: Literal['tzip_metadata'] = 'tzip_metadata' network: TzipMetadataNetwork url: str = DEFAULT_TZIP_METADATA_URL http: HttpConfig | None = None diff --git a/src/dipdup/datasources/starknet_node.py b/src/dipdup/datasources/starknet_node.py index f05d53108..d210ea430 100644 --- a/src/dipdup/datasources/starknet_node.py +++ b/src/dipdup/datasources/starknet_node.py @@ -31,7 +31,9 @@ class StarknetNodeDatasource(IndexDatasource[StarknetNodeDatasourceConfig]): def __init__(self, config: StarknetNodeDatasourceConfig, merge_subscriptions: bool = False) -> None: super().__init__(config, merge_subscriptions) self._starknetpy: StarknetpyClient | None = None - self._block_cache: LRU[int, StarknetBlockWithTxHashes | PendingStarknetBlockWithTxHashes] = LRU(BLOCK_CACHE_SIZE) + self._block_cache: LRU[int, StarknetBlockWithTxHashes | PendingStarknetBlockWithTxHashes] = LRU( + BLOCK_CACHE_SIZE + ) @property def starknetpy(self) -> 'StarknetpyClient': @@ -82,7 +84,9 @@ async def get_events( continuation_token=continuation_token, ) - async def get_block_with_tx_hashes(self, block_hash: int) -> Union['StarknetBlockWithTxHashes', 'PendingStarknetBlockWithTxHashes']: + async def get_block_with_tx_hashes( + self, block_hash: int + ) -> Union['StarknetBlockWithTxHashes', 'PendingStarknetBlockWithTxHashes']: if block := self._block_cache.get(block_hash, None): return block diff --git a/src/dipdup/indexes/starknet_events/fetcher.py b/src/dipdup/indexes/starknet_events/fetcher.py index ddc7e432b..d23e8893b 100644 --- a/src/dipdup/indexes/starknet_events/fetcher.py +++ b/src/dipdup/indexes/starknet_events/fetcher.py @@ -55,7 +55,7 @@ class EventFetcherChannel(FetcherChannel[StarknetEventData, StarknetNodeDatasour async def fetch(self) -> None: address, key0s = next(iter(self._filter)) - + datasource = self._datasources[0] events_chunk = await datasource.get_events( @@ -67,9 +67,9 @@ async def fetch(self) -> None: ) for event in events_chunk.events: - # NOTE: + # NOTE: if event.block_hash is None or event.transaction_hash is None: - _logger.info("Skipping event. No block_hash or transaction_hash found in %s", event) + _logger.info('Skipping event. No block_hash or transaction_hash found in %s', event) continue block = await datasource.get_block_with_tx_hashes( @@ -77,7 +77,7 @@ async def fetch(self) -> None: ) if block is None: - _logger.info("Skipping event. No block exists for block_hash. BlackHash=%s", event.block_hash) + _logger.info('Skipping event. No block exists for block_hash. BlackHash=%s', event.block_hash) continue timestamp = block.timestamp @@ -85,7 +85,7 @@ async def fetch(self) -> None: # NOTE: This event is corrupt, possibly due to old age. if transaction_idx < 0: - _logger.info("Skipping event. No transaction_hash exists in block. TxHash=%s", event.transaction_hash) + _logger.info('Skipping event. No transaction_hash exists in block. TxHash=%s', event.transaction_hash) continue self._buffer[event.block_number].append( # type: ignore[index] diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 5b983f668..91b68e9d3 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -3,21 +3,26 @@ Ask user some question with Click; render Jinja2 templates with answers. """ +import dataclasses import logging import re +from itertools import chain from pathlib import Path +from typing import Any +from typing import get_args from pydantic import ConfigDict from pydantic import TypeAdapter from pydantic.dataclasses import dataclass +from pydantic.fields import FieldInfo from typing_extensions import TypedDict from dipdup import __version__ -from dipdup._survey import DipDupSurveyConfig -from dipdup._survey import prompt_anyof -from dipdup._survey import query_survey_config from dipdup.cli import big_yellow_echo from dipdup.cli import echo +from dipdup.config import DatabaseConfigU +from dipdup.config import DatasourceConfigU +from dipdup.config import RuntimeConfigU from dipdup.config import ToStr from dipdup.env import get_package_path from dipdup.env import get_pyproject_name @@ -80,7 +85,7 @@ class Answers(TypedDict): hasura_image: str line_length: ToStr package_manager: str - _survey_config: DipDupSurveyConfig | None + _survey_config: dict[str, Any] | None def get_default_answers() -> Answers: @@ -125,21 +130,22 @@ def get_replay_path(name: str) -> Path: return Path(__file__).parent / 'projects' / name / 'replay.yaml' -def template_from_terminal() -> tuple[str | None, DipDupSurveyConfig | None]: - _, mode = prompt_anyof( - question='How would you like to set up your new DipDup project?', - options=( - 'From template', - 'Interactively', - 'Blank', - ), - comments=( - 'Use one of demo projects', - 'Guided setup with prompts', - 'Begin with an empty project', - ), - default=0, - ) +def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: + # _, mode = prompt_anyof( + # question='How would you like to set up your new DipDup project?', + # options=( + # 'From template', + # 'Interactively', + # 'Blank', + # ), + # comments=( + # 'Use one of demo projects', + # 'Guided setup with prompts', + # 'Begin with an empty project', + # ), + # default=0, + # ) + mode = 'Interactively' if mode == 'Blank': return ('demo_blank', None) @@ -151,25 +157,28 @@ def template_from_terminal() -> tuple[str | None, DipDupSurveyConfig | None]: 'Starknet', 'Substrate', 'Tezos', + '[multiple]', ), comments=( - 'EVM-compatible blockchains', + 'EVM-compatible', 'Starknet', 'Substrate', 'Tezos', + 'Disable filtering', ), default=0, ) - blockchain = res[1].lower() + namespace = res[1].lower() if res[1] != '[multiple]' else None if mode == 'Interactively': replay_path = get_replay_path('demo_blank') - survey_config = query_survey_config(blockchain) + survey_config = query_survey_config(namespace) return ('demo_blank', survey_config) if mode == 'From template': options, comments = [], [] - for name in TEMPLATES[blockchain]: + templates = TEMPLATES[namespace] if namespace else chain(v for v in TEMPLATES.values()) + for name in templates: replay_path = get_replay_path(name) _answers = answers_from_replay(replay_path) options.append(_answers['template']) @@ -409,3 +418,102 @@ def _render(answers: Answers, template_path: Path, output_path: Path, force: boo ) write(output_path, content, overwrite=force) + + +def fill_config_from_input( + config: dict[str, Any], + section: str, + section_kinds: tuple[type], + filter: str, +) -> None: + section_dict = config.get(section, {}) + another = False + + print(f'Configuring `{section}` section') + + while True: + add_entity = input(f'Do you want to add{' another' if another else ''} one? (yes/no): ').strip().lower() + if add_entity != 'yes': + break + + # Ask for the entity name + if section == 'database': + name = '' + else: + name = input('Enter a name for the entity: ').strip() + + # Ask for the type of the entity + print('Available entity types:') + for i, entity_type in enumerate(section_kinds, start=1): + kind = entity_type.__dataclass_fields__['kind'].default + assert kind + + if filter and '.' in kind and not kind.startswith(filter): + continue + + print(f'{i}. {kind}') + + type_choice = int(input('Choose an entity type (number): ').strip()) - 1 + # entity_type_name = list(section_kinds)[type_choice] + entity_model = section_kinds[type_choice] + + # Gather input for the fields of the chosen type + entity_data = {} + for field_name, field in entity_model.__dataclass_fields__.items(): + if field_name in {'kind', 'http'}: + continue + + default = field.default + if default is dataclasses.MISSING: + default = None + elif isinstance(default, FieldInfo): + # else: + # print(default.__dict__) + default = default.default_factory() + + field_value = input(f'Enter value for `{field_name}` [{field.type}] ({default}): ').strip() + entity_data[field_name] = field_value or default + + # Validate and add the entity + try: + entity_instance = entity_model(**entity_data) + section_dict[name] = entity_instance.__dict__ + another = True + except Exception as e: + print(f'Error: {e}. Please try again.') + + +def prompt_anyof( + question: str, + options: tuple[str, ...], + comments: tuple[str, ...], + default: int, +) -> tuple[int, str]: + """Ask user to choose one of the options; returns index and value""" + import survey + from tabulate import tabulate + + table = tabulate( + zip(options, comments, strict=False), + tablefmt='plain', + ) + index = survey.routines.select( + question + '\n', + options=table.split('\n'), + index=default, + ) + return index, options[index] + + +def query_survey_config(namespace: str | None) -> dict[str, Any]: + config_dict = {} + + # NOTE: It's not a config section; we'll handle it later + fill_config_from_input(config_dict, 'database', get_args(DatabaseConfigU), namespace) + + if namespace == 'substrate': + fill_config_from_input(config_dict, 'runtimes', get_args(RuntimeConfigU), namespace) + + fill_config_from_input(config_dict, 'datasources', get_args(DatasourceConfigU), namespace) + + return config_dict From e38a71e6bd66818f549ab40e9b74239318c26bd4 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Thu, 13 Feb 2025 19:58:22 -0300 Subject: [PATCH 02/14] almost there --- src/dipdup/cli.py | 21 +++++- src/dipdup/config/__init__.py | 2 +- src/dipdup/project.py | 116 ++++++++++++++++++++-------------- src/dipdup/yaml.py | 2 +- 4 files changed, 91 insertions(+), 50 deletions(-) diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 6eccc226a..3b8bd4761 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -27,6 +27,7 @@ from dipdup.install import EPILOG from dipdup.install import WELCOME_ASCII from dipdup.sys import set_up_process +from dipdup.yaml import DipDupYAMLConfig if TYPE_CHECKING: from dipdup.config import DipDupConfig @@ -849,6 +850,7 @@ async def new( from dipdup.project import answers_from_terminal from dipdup.project import get_default_answers from dipdup.project import render_project + from dipdup.project import template_from_terminal if quiet: answers = get_default_answers() @@ -860,13 +862,30 @@ async def new( answers['template'] = template else: try: - answers = answers_from_terminal(template) + answers = answers_from_terminal() + answers['template'] = template or 'demo_blank' + + if template: + echo(f'Using template `{template}`\n') + config_dict = {} + else: + template, config_dict = template_from_terminal() + except Escape: return _logger.info('Rendering project') render_project(answers, force) + if config_dict: + config_dict['package'] = answers['package'] + config_dict['spec_version'] = '3.0' + + config = DipDupYAMLConfig(**config_dict) + + path = env.get_package_path(config['package']) / ROOT_CONFIG + path.write_text(config.dump()) + _logger.info('Initializing project') config = DipDupConfig.load([Path(answers['package'])]) config.initialize() diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 30aa1234a..0020e5b8c 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -316,7 +316,7 @@ class IndexTemplateConfig(NameMixin): """ - kind = 'template' + kind: Literal['template'] = 'template' template: str values: dict[str, Any] first_level: int = 0 diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 91b68e9d3..b97342268 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -20,8 +20,10 @@ from dipdup import __version__ from dipdup.cli import big_yellow_echo from dipdup.cli import echo -from dipdup.config import DatabaseConfigU from dipdup.config import DatasourceConfigU +from dipdup.config import HookConfig +from dipdup.config import IndexConfigU +from dipdup.config import JobConfig from dipdup.config import RuntimeConfigU from dipdup.config import ToStr from dipdup.env import get_package_path @@ -34,6 +36,12 @@ CODEGEN_HEADER = f'# generated by DipDup {__version__.split("+")[0]}' +SINGULAR_FORMS = { + 'datasources': 'datasource', + 'indexes': 'index', + 'hooks': 'hook', + 'jobs': 'job', +} # NOTE: All templates are stored in src/dipdup/projects TEMPLATES: dict[str, tuple[str, ...]] = { @@ -85,7 +93,6 @@ class Answers(TypedDict): hasura_image: str line_length: ToStr package_manager: str - _survey_config: dict[str, Any] | None def get_default_answers() -> Answers: @@ -103,7 +110,6 @@ def get_default_answers() -> Answers: hasura_image='hasura/graphql-engine:latest', line_length='120', package_manager='uv', - _survey_config=None, ) @@ -131,21 +137,20 @@ def get_replay_path(name: str) -> Path: def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: - # _, mode = prompt_anyof( - # question='How would you like to set up your new DipDup project?', - # options=( - # 'From template', - # 'Interactively', - # 'Blank', - # ), - # comments=( - # 'Use one of demo projects', - # 'Guided setup with prompts', - # 'Begin with an empty project', - # ), - # default=0, - # ) - mode = 'Interactively' + _, mode = prompt_anyof( + question='How would you like to set up your new DipDup project?', + options=( + 'Interactively', + 'From template', + 'Blank', + ), + comments=( + 'Guided setup with prompts', + 'Use one of demo projects', + 'Begin with an empty project', + ), + default=0, + ) if mode == 'Blank': return ('demo_blank', None) @@ -164,7 +169,7 @@ def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: 'Starknet', 'Substrate', 'Tezos', - 'Disable filtering', + '(disable filtering)', ), default=0, ) @@ -172,8 +177,8 @@ def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: if mode == 'Interactively': replay_path = get_replay_path('demo_blank') - survey_config = query_survey_config(namespace) - return ('demo_blank', survey_config) + config_dict = config_from_terminal(namespace) + return ('demo_blank', config_dict) if mode == 'From template': options, comments = [], [] @@ -195,7 +200,7 @@ def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: raise NotImplementedError -def answers_from_terminal(template: str | None) -> Answers: +def answers_from_terminal() -> Answers: """Script running on dipdup new command and will create a new project base from interactive survey""" import survey # type: ignore[import-untyped] @@ -206,14 +211,6 @@ def answers_from_terminal(template: str | None) -> Answers: answers = get_default_answers() - if template: - echo(f'Using template `{template}`\n') - else: - template, survey_config = template_from_terminal() - answers['_survey_config'] = survey_config - - answers['template'] = template or 'demo_blank' - big_yellow_echo('Set up project') while True: @@ -301,6 +298,7 @@ def answers_from_terminal(template: str | None) -> Answers: 'Enter maximum line length for linters: ', value=answers['line_length'], ) + return answers @@ -424,28 +422,31 @@ def fill_config_from_input( config: dict[str, Any], section: str, section_kinds: tuple[type], - filter: str, + filter: str | None, ) -> None: section_dict = config.get(section, {}) another = False - print(f'Configuring `{section}` section') + print(f'Configuring `{section}` section \n') while True: - add_entity = input(f'Do you want to add{' another' if another else ''} one? (yes/no): ').strip().lower() + entity_singular = SINGULAR_FORMS[section] + another_str = 'another' if another else 'the first' + add_entity = input(f'Do you want to add {another_str} {entity_singular}? (yes/no): ').strip().lower() if add_entity != 'yes': break # Ask for the entity name - if section == 'database': - name = '' - else: - name = input('Enter a name for the entity: ').strip() + name = input(f'Enter {entity_singular} name: ').strip() # Ask for the type of the entity - print('Available entity types:') + print(f'Available {entity_singular} kinds:') for i, entity_type in enumerate(section_kinds, start=1): - kind = entity_type.__dataclass_fields__['kind'].default + try: + kind = entity_type.__dataclass_fields__['kind'].default + except KeyError: + kind = entity_type.__name__ + assert kind if filter and '.' in kind and not kind.startswith(filter): @@ -453,8 +454,11 @@ def fill_config_from_input( print(f'{i}. {kind}') - type_choice = int(input('Choose an entity type (number): ').strip()) - 1 - # entity_type_name = list(section_kinds)[type_choice] + if i == 1: + type_choice = 0 + else: + type_choice = int(input(f'Choose {entity_singular} type (number): ').strip()) - 1 + entity_model = section_kinds[type_choice] # Gather input for the fields of the chosen type @@ -462,9 +466,14 @@ def fill_config_from_input( for field_name, field in entity_model.__dataclass_fields__.items(): if field_name in {'kind', 'http'}: continue + if field_name.startswith('_'): + continue + if field_name == 'handlers': + entity_data[field_name] = () + continue default = field.default - if default is dataclasses.MISSING: + if default == dataclasses.MISSING: default = None elif isinstance(default, FieldInfo): # else: @@ -472,6 +481,14 @@ def fill_config_from_input( default = default.default_factory() field_value = input(f'Enter value for `{field_name}` [{field.type}] ({default}): ').strip() + if field_value == '': + field_value = default + + # NOTE: Basic transformations: string lists + if isinstance(field_value, str): + if ',' in field_value: + field_value = field_value.split(',') + entity_data[field_name] = field_value or default # Validate and add the entity @@ -482,6 +499,10 @@ def fill_config_from_input( except Exception as e: print(f'Error: {e}. Please try again.') + print(section_dict) + + config[section] = section_dict + def prompt_anyof( question: str, @@ -505,15 +526,16 @@ def prompt_anyof( return index, options[index] -def query_survey_config(namespace: str | None) -> dict[str, Any]: +def config_from_terminal(namespace: str | None) -> dict[str, Any]: config_dict = {} - # NOTE: It's not a config section; we'll handle it later - fill_config_from_input(config_dict, 'database', get_args(DatabaseConfigU), namespace) - - if namespace == 'substrate': + # NOTE: Substrate or multichain + if namespace in {'substrate', None}: fill_config_from_input(config_dict, 'runtimes', get_args(RuntimeConfigU), namespace) fill_config_from_input(config_dict, 'datasources', get_args(DatasourceConfigU), namespace) + fill_config_from_input(config_dict, 'indexes', get_args(IndexConfigU), namespace) + fill_config_from_input(config_dict, 'hooks', (HookConfig,), None) + fill_config_from_input(config_dict, 'jobs', (JobConfig,), None) return config_dict diff --git a/src/dipdup/yaml.py b/src/dipdup/yaml.py index 771c20747..3f30fb710 100644 --- a/src/dipdup/yaml.py +++ b/src/dipdup/yaml.py @@ -45,7 +45,7 @@ def exclude_none(config_json: Any) -> Any: if isinstance(config_json, list | tuple): return [exclude_none(i) for i in config_json if i is not None] if isinstance(config_json, dict): - return {k: exclude_none(v) for k, v in config_json.items() if v is not None} + return {k: exclude_none(v) for k, v in config_json.items() if v is not None and not k.startswith('_')} return config_json From 35efc9c5b7111792dda8a995d2de610bc77992a2 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sat, 15 Feb 2025 15:12:24 -0300 Subject: [PATCH 03/14] another iteration --- CHANGELOG.md | 10 +++++ src/dipdup/cli.py | 8 +++- src/dipdup/config/coinbase.py | 4 ++ src/dipdup/datasources/coinbase.py | 1 - src/dipdup/project.py | 63 +++++++++++++++++++++++------- 5 files changed, 69 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4b01189..2c9973223 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic Releases prior to 7.0 has been removed from this file to declutter search results; see the [archived copy](https://github.com/dipdup-io/dipdup/blob/8.0.0b5/CHANGELOG.md) for the full list. +## [Unreleased] + +### Added + +- cli: Rewritten interactive mode for `new` command. + +### Fixed + +- coinbase: Fixed crash when using coinbase datasource. + ## [8.2.0] - 2025-02-10 ### Added diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 3b8bd4761..c1459b81f 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -878,8 +878,12 @@ async def new( render_project(answers, force) if config_dict: - config_dict['package'] = answers['package'] - config_dict['spec_version'] = '3.0' + # NOTE: Preserve the header at the top of the file + config_dict = { + 'package': answers['package'], + 'spec_version': '3.0', + **config_dict, + } config = DipDupYAMLConfig(**config_dict) diff --git a/src/dipdup/config/coinbase.py b/src/dipdup/config/coinbase.py index 41e3d0cfe..c6b8a8f88 100644 --- a/src/dipdup/config/coinbase.py +++ b/src/dipdup/config/coinbase.py @@ -27,3 +27,7 @@ class CoinbaseDatasourceConfig(DatasourceConfig): passphrase: str | None = Field(default=None, repr=False) http: HttpConfig | None = None + + @property + def url(self) -> str: + return 'https://api.pro.coinbase.com' diff --git a/src/dipdup/datasources/coinbase.py b/src/dipdup/datasources/coinbase.py index 2871fd23e..346fe1087 100644 --- a/src/dipdup/datasources/coinbase.py +++ b/src/dipdup/datasources/coinbase.py @@ -11,7 +11,6 @@ from dipdup.models.coinbase import CoinbaseCandleInterval CANDLES_REQUEST_LIMIT = 300 -API_URL = 'https://api.pro.coinbase.com' class CoinbaseDatasource(Datasource[CoinbaseDatasourceConfig]): diff --git a/src/dipdup/project.py b/src/dipdup/project.py index b97342268..adc891660 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -38,9 +38,11 @@ SINGULAR_FORMS = { 'datasources': 'datasource', + 'contracts': 'contract', 'indexes': 'index', 'hooks': 'hook', 'jobs': 'job', + 'runtimes': 'runtime', } # NOTE: All templates are stored in src/dipdup/projects @@ -424,6 +426,8 @@ def fill_config_from_input( section_kinds: tuple[type], filter: str | None, ) -> None: + import survey + section_dict = config.get(section, {}) another = False @@ -431,17 +435,37 @@ def fill_config_from_input( while True: entity_singular = SINGULAR_FORMS[section] - another_str = 'another' if another else 'the first' - add_entity = input(f'Do you want to add {another_str} {entity_singular}? (yes/no): ').strip().lower() + + if another: + another_str = 'another' + choices = ('no', 'yes') + else: + another_str = 'the first' + choices = ( + 'yes', + 'no', + ) + + _, add_entity = prompt_anyof( + f'Do you want to add {another_str} {entity_singular}?', + choices, + ('', ''), + default=0, + ) if add_entity != 'yes': break - # Ask for the entity name - name = input(f'Enter {entity_singular} name: ').strip() + # NOTE: All sections are mappings alias to dict + name = survey.routines.input( + f'Enter {entity_singular} name: ', + ) + + # NOTE: Prepare a filtered (or not) list of entity types # Ask for the type of the entity print(f'Available {entity_singular} kinds:') - for i, entity_type in enumerate(section_kinds, start=1): + matched = {} + for entity_type in section_kinds: try: kind = entity_type.__dataclass_fields__['kind'].default except KeyError: @@ -452,14 +476,23 @@ def fill_config_from_input( if filter and '.' in kind and not kind.startswith(filter): continue - print(f'{i}. {kind}') + matched[kind] = entity_type + + if len(matched) != 1: + kinds = sorted(matched.keys()) + comments = tuple('' for _ in kinds) + print(f'Available {entity_singular} kinds: {kinds} {comments}') - if i == 1: - type_choice = 0 + _, kind = prompt_anyof( + f'Choose {entity_singular} kind: ', + kinds, + comments, + default=0, + ) else: - type_choice = int(input(f'Choose {entity_singular} type (number): ').strip()) - 1 + kind = next(iter(matched)) - entity_model = section_kinds[type_choice] + entity_model = matched[kind] # Gather input for the fields of the chosen type entity_data = {} @@ -476,11 +509,13 @@ def fill_config_from_input( if default == dataclasses.MISSING: default = None elif isinstance(default, FieldInfo): - # else: - # print(default.__dict__) - default = default.default_factory() + default = default.default_factory() if default.default_factory else default.default - field_value = input(f'Enter value for `{field_name}` [{field.type}] ({default}): ').strip() + # field_value = input(f'Enter value for `{field_name}` [{field.type}] ({default}): ').strip() + field_value = survey.routines.input( + f'Enter value for `{field_name}` [{field.type}]: ', + value=default or '', + ) if field_value == '': field_value = default From a4c7da3c39a02b677e558bbdd7cae44b7ceaa82a Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sat, 15 Feb 2025 16:15:59 -0300 Subject: [PATCH 04/14] wip --- src/dipdup/cli.py | 2 +- src/dipdup/config/__init__.py | 29 +++++++++++- src/dipdup/config/_mixin.py | 15 +++++++ src/dipdup/config/coinbase.py | 2 +- src/dipdup/project.py | 84 ++++++++++++++--------------------- 5 files changed, 79 insertions(+), 53 deletions(-) diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index c1459b81f..dcf6bb102 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -869,7 +869,7 @@ async def new( echo(f'Using template `{template}`\n') config_dict = {} else: - template, config_dict = template_from_terminal() + template, config_dict = template_from_terminal(answers['package']) except Escape: return diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 0020e5b8c..fd9b159b7 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -30,8 +30,10 @@ from typing import Annotated from typing import Any from typing import Literal +from typing import Self from typing import TypeVar from typing import cast +from typing import get_args from urllib.parse import quote_plus import orjson @@ -47,8 +49,10 @@ from dipdup import __spec_version__ from dipdup import env from dipdup.config._mixin import CallbackMixin +from dipdup.config._mixin import InteractiveMixin from dipdup.config._mixin import NameMixin from dipdup.config._mixin import ParentMixin +from dipdup.config._mixin import TerminalOptions from dipdup.exceptions import ConfigInitializationException from dipdup.exceptions import ConfigurationError from dipdup.exceptions import IndexAlreadyExistsError @@ -561,7 +565,7 @@ class AdvancedConfig: @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) -class DipDupConfig: +class DipDupConfig(InteractiveMixin): """DipDup project configuration file :param spec_version: Version of config specification, currently always `3.0` @@ -617,6 +621,29 @@ def schema_name(self) -> str: def package_path(self) -> Path: return env.get_package_path(self.package) + @classmethod + def from_terminal(cls, opts: TerminalOptions) -> Self: + from dipdup.project import fill_config_from_input + + config_dict = {} + + # NOTE: Substrate or multichain + if opts.namespace in {'substrate', None}: + fill_config_from_input(config_dict, 'runtimes', get_args(RuntimeConfigU), opts.namespace) + + fill_config_from_input(config_dict, 'datasources', get_args(DatasourceConfigU), opts.namespace) + fill_config_from_input(config_dict, 'indexes', get_args(IndexConfigU), opts.namespace) + fill_config_from_input(config_dict, 'hooks', (HookConfig,), None) + fill_config_from_input(config_dict, 'jobs', (JobConfig,), None) + + config_dict = { + 'package': opts.package, + 'spec_version': '3.0', + **config_dict, + } + + return cls(**config_dict) + @classmethod def load( cls, diff --git a/src/dipdup/config/_mixin.py b/src/dipdup/config/_mixin.py index 270e88ca4..98890924d 100644 --- a/src/dipdup/config/_mixin.py +++ b/src/dipdup/config/_mixin.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from typing import Any from typing import Generic +from typing import Self from typing import TypeVar from typing import cast @@ -121,3 +122,17 @@ def subgroup_index(self) -> int: @subgroup_index.setter def subgroup_index(self, value: int) -> None: self._subgroup_index = value + + +@dataclass +class TerminalOptions: + package: str + namespace: str | None = None + + +@dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) +class InteractiveMixin: + + @classmethod + @abstractmethod + def from_terminal(cls, opts: TerminalOptions) -> Self: ... diff --git a/src/dipdup/config/coinbase.py b/src/dipdup/config/coinbase.py index c6b8a8f88..a1dc34918 100644 --- a/src/dipdup/config/coinbase.py +++ b/src/dipdup/config/coinbase.py @@ -29,5 +29,5 @@ class CoinbaseDatasourceConfig(DatasourceConfig): http: HttpConfig | None = None @property - def url(self) -> str: + def url(self) -> str: # type: ignore[override] return 'https://api.pro.coinbase.com' diff --git a/src/dipdup/project.py b/src/dipdup/project.py index adc891660..5355393f4 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -9,7 +9,6 @@ from itertools import chain from pathlib import Path from typing import Any -from typing import get_args from pydantic import ConfigDict from pydantic import TypeAdapter @@ -20,12 +19,9 @@ from dipdup import __version__ from dipdup.cli import big_yellow_echo from dipdup.cli import echo -from dipdup.config import DatasourceConfigU -from dipdup.config import HookConfig -from dipdup.config import IndexConfigU -from dipdup.config import JobConfig -from dipdup.config import RuntimeConfigU +from dipdup.config import DipDupConfig from dipdup.config import ToStr +from dipdup.config._mixin import TerminalOptions from dipdup.env import get_package_path from dipdup.env import get_pyproject_name from dipdup.utils import load_template @@ -138,25 +134,7 @@ def get_replay_path(name: str) -> Path: return Path(__file__).parent / 'projects' / name / 'replay.yaml' -def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: - _, mode = prompt_anyof( - question='How would you like to set up your new DipDup project?', - options=( - 'Interactively', - 'From template', - 'Blank', - ), - comments=( - 'Guided setup with prompts', - 'Use one of demo projects', - 'Begin with an empty project', - ), - default=0, - ) - - if mode == 'Blank': - return ('demo_blank', None) - +def namespace_from_terminal() -> str | None: res = prompt_anyof( question='What blockchain are you going to index?', options=( @@ -175,12 +153,38 @@ def template_from_terminal() -> tuple[str | None, dict[str, Any] | None]: ), default=0, ) - namespace = res[1].lower() if res[1] != '[multiple]' else None + return res[1].lower() if res[1] != '[multiple]' else None + + +def template_from_terminal(package: str) -> tuple[str | None, dict[str, Any] | None]: + _, mode = prompt_anyof( + question='How would you like to set up your new DipDup project?', + options=( + 'Interactively', + 'From template', + 'Blank', + ), + comments=( + 'Guided setup with prompts', + 'Use one of demo projects', + 'Begin with an empty project', + ), + default=0, + ) + + if mode == 'Blank': + return ('demo_blank', None) + + namespace = namespace_from_terminal() if mode == 'Interactively': replay_path = get_replay_path('demo_blank') - config_dict = config_from_terminal(namespace) - return ('demo_blank', config_dict) + opts = TerminalOptions( + package=package, + namespace=namespace, + ) + config = DipDupConfig.from_terminal(opts) + return ('demo_blank', config._json) if mode == 'From template': options, comments = [], [] @@ -431,8 +435,6 @@ def fill_config_from_input( section_dict = config.get(section, {}) another = False - print(f'Configuring `{section}` section \n') - while True: entity_singular = SINGULAR_FORMS[section] @@ -463,7 +465,6 @@ def fill_config_from_input( # NOTE: Prepare a filtered (or not) list of entity types # Ask for the type of the entity - print(f'Available {entity_singular} kinds:') matched = {} for entity_type in section_kinds: try: @@ -481,8 +482,6 @@ def fill_config_from_input( if len(matched) != 1: kinds = sorted(matched.keys()) comments = tuple('' for _ in kinds) - print(f'Available {entity_singular} kinds: {kinds} {comments}') - _, kind = prompt_anyof( f'Choose {entity_singular} kind: ', kinds, @@ -514,7 +513,7 @@ def fill_config_from_input( # field_value = input(f'Enter value for `{field_name}` [{field.type}] ({default}): ').strip() field_value = survey.routines.input( f'Enter value for `{field_name}` [{field.type}]: ', - value=default or '', + value=str(default) if default is not None else '', ) if field_value == '': field_value = default @@ -529,7 +528,7 @@ def fill_config_from_input( # Validate and add the entity try: entity_instance = entity_model(**entity_data) - section_dict[name] = entity_instance.__dict__ + section_dict[name] = {k: v for k, v in entity_instance.__dict__.items() if not k.startswith('_')} another = True except Exception as e: print(f'Error: {e}. Please try again.') @@ -559,18 +558,3 @@ def prompt_anyof( index=default, ) return index, options[index] - - -def config_from_terminal(namespace: str | None) -> dict[str, Any]: - config_dict = {} - - # NOTE: Substrate or multichain - if namespace in {'substrate', None}: - fill_config_from_input(config_dict, 'runtimes', get_args(RuntimeConfigU), namespace) - - fill_config_from_input(config_dict, 'datasources', get_args(DatasourceConfigU), namespace) - fill_config_from_input(config_dict, 'indexes', get_args(IndexConfigU), namespace) - fill_config_from_input(config_dict, 'hooks', (HookConfig,), None) - fill_config_from_input(config_dict, 'jobs', (JobConfig,), None) - - return config_dict From bf647d583ccc53b444033c6a186921f3285d5b7a Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sat, 15 Feb 2025 17:12:20 -0300 Subject: [PATCH 05/14] working defaults and bools --- src/dipdup/config/__init__.py | 4 +++ src/dipdup/project.py | 56 +++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index fd9b159b7..891878632 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -295,6 +295,10 @@ class DatasourceConfig(ABC, NameMixin): ws_url: WsUrl | None = None http: HttpConfig | None = None + # @classmethod + # def from_terminal(cls, opts): + # return super().from_terminal(opts) + @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) class HandlerConfig(CallbackMixin, ParentMixin['IndexConfig']): diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 5355393f4..3eccdc216 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -424,6 +424,20 @@ def _render(answers: Answers, template_path: Path, output_path: Path, force: boo write(output_path, content, overwrite=force) +def prompt_bool( + question: str, + default: bool = False, +) -> bool: + choices = ('yes', 'no') if default else ('no', 'yes') + _, value = prompt_anyof( + question, + choices, + ('', ''), + default=0, + ) + return value == 'yes' + + def fill_config_from_input( config: dict[str, Any], section: str, @@ -437,24 +451,12 @@ def fill_config_from_input( while True: entity_singular = SINGULAR_FORMS[section] + another_str = 'another' if another else 'the first' - if another: - another_str = 'another' - choices = ('no', 'yes') - else: - another_str = 'the first' - choices = ( - 'yes', - 'no', - ) - - _, add_entity = prompt_anyof( + if not prompt_bool( f'Do you want to add {another_str} {entity_singular}?', - choices, - ('', ''), - default=0, - ) - if add_entity != 'yes': + default=not another, + ): break # NOTE: All sections are mappings alias to dict @@ -496,6 +498,8 @@ def fill_config_from_input( # Gather input for the fields of the chosen type entity_data = {} for field_name, field in entity_model.__dataclass_fields__.items(): + # print(field) + if field_name in {'kind', 'http'}: continue if field_name.startswith('_'): @@ -505,26 +509,34 @@ def fill_config_from_input( continue default = field.default + if default == dataclasses.MISSING: default = None elif isinstance(default, FieldInfo): default = default.default_factory() if default.default_factory else default.default - # field_value = input(f'Enter value for `{field_name}` [{field.type}] ({default}): ').strip() field_value = survey.routines.input( f'Enter value for `{field_name}` [{field.type}]: ', value=str(default) if default is not None else '', ) - if field_value == '': - field_value = default + + if field.default_factory == dataclasses.MISSING: + default = None # NOTE: Basic transformations: string lists - if isinstance(field_value, str): - if ',' in field_value: + if 'tuple' in field.type or 'list' in field.type: + if not field_value: + field_value = default + elif ',' in field_value: field_value = field_value.split(',') + else: + field_value = [field_value] entity_data[field_name] = field_value or default + # print(default, 'default') + # print(entity_data[field_name], 'field_value') + # Validate and add the entity try: entity_instance = entity_model(**entity_data) @@ -533,7 +545,7 @@ def fill_config_from_input( except Exception as e: print(f'Error: {e}. Please try again.') - print(section_dict) + # print(section_dict) config[section] = section_dict From dd41b86899ccea11f2d07f8378cb4582f3910edd Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sat, 15 Feb 2025 18:33:21 -0300 Subject: [PATCH 06/14] ref --- src/dipdup/project.py | 131 ++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 61 deletions(-) diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 3eccdc216..d01c10b49 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -281,29 +281,29 @@ def answers_from_terminal() -> Answers: fg='yellow', ) - big_yellow_echo('Miscellaneous tunables; leave default values if unsure') - - _, answers['package_manager'] = prompt_anyof( - question='Choose package manager', - options=( - 'uv', - 'poetry', - 'pdm', - 'none', - ), - comments=( - 'uv (recommended)', - 'Poetry', - 'PDM', - '[none]', - ), - default=0, - ) - - answers['line_length'] = survey.routines.input( - 'Enter maximum line length for linters: ', - value=answers['line_length'], - ) + # big_yellow_echo('Miscellaneous tunables; leave default values if unsure') + + # _, answers['package_manager'] = prompt_anyof( + # question='Choose package manager', + # options=( + # 'uv', + # 'poetry', + # 'pdm', + # 'none', + # ), + # comments=( + # 'uv (recommended)', + # 'Poetry', + # 'PDM', + # '[none]', + # ), + # default=0, + # ) + + # answers['line_length'] = survey.routines.input( + # 'Enter maximum line length for linters: ', + # value=answers['line_length'], + # ) return answers @@ -438,10 +438,45 @@ def prompt_bool( return value == 'yes' +def prompt_kind( + entity: str, + types: tuple[type, ...], + filter: str | None, +) -> type: + + matched = {} + for entity_type in types: + try: + kind = entity_type.__dataclass_fields__['kind'].default + except KeyError: + kind = entity_type.__name__ + + assert kind + + if filter and '.' in kind and not kind.startswith(filter): + continue + + matched[kind] = entity_type + + if len(matched) == 1: + return next(iter(matched)) + + kinds = sorted(matched.keys()) + comments = tuple('' for _ in kinds) + _, kind = prompt_anyof( + f'Choose {entity} kind: ', + kinds, + comments, + default=0, + ) + return matched[kind] + + + def fill_config_from_input( config: dict[str, Any], section: str, - section_kinds: tuple[type], + types: tuple[type, ...], filter: str | None, ) -> None: import survey @@ -450,54 +485,28 @@ def fill_config_from_input( another = False while True: - entity_singular = SINGULAR_FORMS[section] - another_str = 'another' if another else 'the first' + section_singular = SINGULAR_FORMS[section] if not prompt_bool( - f'Do you want to add {another_str} {entity_singular}?', + f'Do you want to add {'another' if another else 'the first'} {section_singular}?', default=not another, ): break # NOTE: All sections are mappings alias to dict name = survey.routines.input( - f'Enter {entity_singular} name: ', + f'Enter {section_singular} name: ', ) - # NOTE: Prepare a filtered (or not) list of entity types - - # Ask for the type of the entity - matched = {} - for entity_type in section_kinds: - try: - kind = entity_type.__dataclass_fields__['kind'].default - except KeyError: - kind = entity_type.__name__ - - assert kind - - if filter and '.' in kind and not kind.startswith(filter): - continue - - matched[kind] = entity_type - - if len(matched) != 1: - kinds = sorted(matched.keys()) - comments = tuple('' for _ in kinds) - _, kind = prompt_anyof( - f'Choose {entity_singular} kind: ', - kinds, - comments, - default=0, - ) - else: - kind = next(iter(matched)) - - entity_model = matched[kind] + type_ = prompt_kind( + section_singular, + types, + filter, + ) # Gather input for the fields of the chosen type entity_data = {} - for field_name, field in entity_model.__dataclass_fields__.items(): + for field_name, field in type_.__dataclass_fields__.items(): # print(field) if field_name in {'kind', 'http'}: @@ -539,8 +548,8 @@ def fill_config_from_input( # Validate and add the entity try: - entity_instance = entity_model(**entity_data) - section_dict[name] = {k: v for k, v in entity_instance.__dict__.items() if not k.startswith('_')} + obj = type_(**entity_data) + section_dict[name] = {k: v for k, v in obj.__dict__.items() if not k.startswith('_')} another = True except Exception as e: print(f'Error: {e}. Please try again.') From b49200ad9168d4be58f65bf0ffc2d0315039fe88 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sat, 15 Feb 2025 19:00:13 -0300 Subject: [PATCH 07/14] from_terminal with failover --- src/dipdup/config/__init__.py | 55 +++++++++++++-- src/dipdup/project.py | 122 +++++++++++++--------------------- 2 files changed, 93 insertions(+), 84 deletions(-) diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 891878632..74c32262f 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -627,18 +627,59 @@ def package_path(self) -> Path: @classmethod def from_terminal(cls, opts: TerminalOptions) -> Self: - from dipdup.project import fill_config_from_input + import survey - config_dict = {} + from dipdup.project import SINGULAR_FORMS + from dipdup.project import fill_type_from_input + from dipdup.project import prompt_bool + from dipdup.project import prompt_kind + config_dict = defaultdict(dict) + + sections = { + 'datasources': get_args(DatasourceConfigU), + 'runtimes': get_args(RuntimeConfigU), + 'contracts': get_args(ContractConfigU), + # NOTE: Skip the `template` kind + 'indexes': get_args(ResolvedIndexConfigU), + } # NOTE: Substrate or multichain if opts.namespace in {'substrate', None}: - fill_config_from_input(config_dict, 'runtimes', get_args(RuntimeConfigU), opts.namespace) + sections['runtimes'] = get_args(RuntimeConfigU) + + for section, types in sections.items(): + another = False + + while True: + section_singular = SINGULAR_FORMS[section] + + if not prompt_bool( + f'Do you want to add {'another' if another else 'the first'} {section_singular}?', + default=not another, + ): + break + + # NOTE: All sections are mappings alias to dict + name = survey.routines.input( + f'Enter {section_singular} name: ', + ) + + type_ = prompt_kind( + section_singular, + types, + opts.namespace, + ) + + if issubclass(type_, InteractiveMixin): + + res = type_.from_terminal(opts) + else: + _logger.debug('Not an `InteractiveMixin`; falling back to field inspection', type_.__name__) + res = fill_type_from_input(type_) - fill_config_from_input(config_dict, 'datasources', get_args(DatasourceConfigU), opts.namespace) - fill_config_from_input(config_dict, 'indexes', get_args(IndexConfigU), opts.namespace) - fill_config_from_input(config_dict, 'hooks', (HookConfig,), None) - fill_config_from_input(config_dict, 'jobs', (JobConfig,), None) + if res is not None: + config_dict[section][name] = res + another = True config_dict = { 'package': opts.package, diff --git a/src/dipdup/project.py b/src/dipdup/project.py index d01c10b49..592156bcc 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -471,92 +471,60 @@ def prompt_kind( ) return matched[kind] - -def fill_config_from_input( - config: dict[str, Any], - section: str, - types: tuple[type, ...], - filter: str | None, -) -> None: +def fill_type_from_input( + type_: type, +) -> Any: import survey - section_dict = config.get(section, {}) - another = False + # Gather input for the fields of the chosen type + entity_data = {} + for field_name, field in type_.__dataclass_fields__.items(): + # print(field) - while True: - section_singular = SINGULAR_FORMS[section] + if field_name in {'kind', 'http'}: + continue + if field_name.startswith('_'): + continue + if field_name == 'handlers': + entity_data[field_name] = () + continue - if not prompt_bool( - f'Do you want to add {'another' if another else 'the first'} {section_singular}?', - default=not another, - ): - break + default = field.default - # NOTE: All sections are mappings alias to dict - name = survey.routines.input( - f'Enter {section_singular} name: ', - ) + if default == dataclasses.MISSING: + default = None + elif isinstance(default, FieldInfo): + default = default.default_factory() if default.default_factory else default.default - type_ = prompt_kind( - section_singular, - types, - filter, + field_value = survey.routines.input( + f'Enter value for `{field_name}` [{field.type}]: ', + value=str(default) if default is not None else '', ) - # Gather input for the fields of the chosen type - entity_data = {} - for field_name, field in type_.__dataclass_fields__.items(): - # print(field) - - if field_name in {'kind', 'http'}: - continue - if field_name.startswith('_'): - continue - if field_name == 'handlers': - entity_data[field_name] = () - continue - - default = field.default - - if default == dataclasses.MISSING: - default = None - elif isinstance(default, FieldInfo): - default = default.default_factory() if default.default_factory else default.default - - field_value = survey.routines.input( - f'Enter value for `{field_name}` [{field.type}]: ', - value=str(default) if default is not None else '', - ) - - if field.default_factory == dataclasses.MISSING: - default = None - - # NOTE: Basic transformations: string lists - if 'tuple' in field.type or 'list' in field.type: - if not field_value: - field_value = default - elif ',' in field_value: - field_value = field_value.split(',') - else: - field_value = [field_value] - - entity_data[field_name] = field_value or default - - # print(default, 'default') - # print(entity_data[field_name], 'field_value') - - # Validate and add the entity - try: - obj = type_(**entity_data) - section_dict[name] = {k: v for k, v in obj.__dict__.items() if not k.startswith('_')} - another = True - except Exception as e: - print(f'Error: {e}. Please try again.') - - # print(section_dict) - - config[section] = section_dict + if field.default_factory == dataclasses.MISSING: + default = None + + # NOTE: Basic transformations: string lists + if 'tuple' in field.type or 'list' in field.type: + if not field_value: + field_value = default + elif ',' in field_value: + field_value = field_value.split(',') + else: + field_value = [field_value] + + entity_data[field_name] = field_value or default + + # print(default, 'default') + # print(entity_data[field_name], 'field_value') + + # Validate and add the entity + try: + obj = type_(**entity_data) + return {k: v for k, v in obj.__dict__.items() if not k.startswith('_')} + except Exception as e: + print(f'Error: {e}. Please try again.') def prompt_anyof( From 2eb80f534cc55994f7301b58d926ab6e9526f7b2 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Sun, 16 Feb 2025 19:38:30 -0300 Subject: [PATCH 08/14] misc fixes --- docs/9.release-notes/1.v8.2.md | 1 - docs/9.release-notes/_8.0_changelog.md | 2 + docs/9.release-notes/_8.2_changelog.md | 2 +- schemas/dipdup-3.0.json | 99 ++++++++++++-------------- src/dipdup/cli.py | 9 +-- src/dipdup/config/__init__.py | 25 +++++-- src/dipdup/config/evm_etherscan.py | 12 ++-- src/dipdup/config/http.py | 3 +- src/dipdup/config/ipfs.py | 3 +- src/dipdup/config/substrate_subscan.py | 3 +- src/dipdup/config/tzip_metadata.py | 3 +- src/dipdup/project.py | 27 +++---- 12 files changed, 101 insertions(+), 88 deletions(-) diff --git a/docs/9.release-notes/1.v8.2.md b/docs/9.release-notes/1.v8.2.md index 8b8bee710..081b6a02e 100644 --- a/docs/9.release-notes/1.v8.2.md +++ b/docs/9.release-notes/1.v8.2.md @@ -44,6 +44,5 @@ DipDup 7.5, our previous major release, has reached end-of-life. We recommend up Going forward, we'll focus on supporting only the latest major version to reduce maintenance overhead. Any breaking changes will be introduced gradually and can be enabled using the `DIPDUP_NEXT` environment variable. - {{ #include 9.release-notes/_8.2_changelog.md }} {{ #include 9.release-notes/_footer.md }} diff --git a/docs/9.release-notes/_8.0_changelog.md b/docs/9.release-notes/_8.0_changelog.md index e0555feed..afde020c6 100644 --- a/docs/9.release-notes/_8.0_changelog.md +++ b/docs/9.release-notes/_8.0_changelog.md @@ -9,6 +9,7 @@ - cli: Added `package verify` command to check the package consistency. - cli: Added full project migration support for 3.0 spec. - cli: Added spec_version 3.0 support to `migrate` command. +- cli: Rewritten interactive mode for `new` command. - config: Publish JSON schemas for config validation and autocompletion. - database: Added `dipdup_status` view to the schema. - env: Added `DIPDUP_JSON_LOG` environment variable to enable JSON logging. @@ -30,6 +31,7 @@ - cli: Fixed progress estimation when there are indexes with `last_level` option set. - cli: Import some dependencies on demand to reduce memory footprint. - cli: Improved logging of indexer status. +- coinbase: Fixed crash when using coinbase datasource. - config: Allow `sentry.dsn` to be empty string. - config: Fixed (de)serialization of hex strings in config. - config: Fixed setting logging levels according to the config. diff --git a/docs/9.release-notes/_8.2_changelog.md b/docs/9.release-notes/_8.2_changelog.md index fa1ece0a8..66e0d7082 100644 --- a/docs/9.release-notes/_8.2_changelog.md +++ b/docs/9.release-notes/_8.2_changelog.md @@ -15,8 +15,8 @@ - cli: Fixed help message on `CallbackError` reporting `batch` handler instead of actual one. - database: Don't process internal models twice if imported from the project. - evm.subsquid: Fixed event/transaction model deserialization. -- starknet: Process all data types correctly. - starknet.node: Fetch missing block timestamp and txn id when synching with node. +- starknet: Process all data types correctly. - substrate.subsquid: Fixed parsing for `__kind` junctions with multiple keys. - substrate.subsquid: Fixed parsing nested structures in response. diff --git a/schemas/dipdup-3.0.json b/schemas/dipdup-3.0.json index 04dc08524..b2df1ad24 100644 --- a/schemas/dipdup-3.0.json +++ b/schemas/dipdup-3.0.json @@ -134,6 +134,7 @@ }, "kind": { "const": "coinbase", + "default": "coinbase", "description": "always 'coinbase'", "title": "kind", "type": "string" @@ -165,9 +166,6 @@ "title": "secret_key" } }, - "required": [ - "kind" - ], "title": "CoinbaseDatasourceConfig", "type": "object" }, @@ -212,6 +210,7 @@ }, "kind": { "const": "evm", + "default": "evm", "description": "Always `evm`", "title": "kind", "type": "string" @@ -230,9 +229,6 @@ "title": "typename" } }, - "required": [ - "kind" - ], "title": "EvmContractConfig", "type": "object" }, @@ -277,17 +273,17 @@ "type": "string" } ], + "default": "evm.etherscan", "description": "always 'evm.etherscan'", "title": "kind" }, "url": { + "$ref": "#/$defs/Url", "description": "API URL", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "url" ], "title": "EvmEtherscanDatasourceConfig", @@ -369,6 +365,7 @@ }, "kind": { "const": "evm.events", + "default": "evm.events", "description": "Always 'evm.events'", "title": "kind", "type": "string" @@ -381,7 +378,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -407,6 +403,7 @@ }, "kind": { "const": "evm.node", + "default": "evm.node", "description": "Always 'evm.node'", "title": "kind", "type": "string" @@ -437,7 +434,6 @@ } }, "required": [ - "kind", "url" ], "title": "EvmNodeDatasourceConfig", @@ -462,6 +458,7 @@ }, "kind": { "const": "evm.subsquid", + "default": "evm.subsquid", "description": "always 'evm.subsquid'", "title": "kind", "type": "string" @@ -473,7 +470,6 @@ } }, "required": [ - "kind", "url" ], "title": "EvmSubsquidDatasourceConfig", @@ -594,6 +590,7 @@ }, "kind": { "const": "evm.transactions", + "default": "evm.transactions", "description": "always 'evm.transactions'", "title": "kind", "type": "string" @@ -606,7 +603,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -957,18 +953,18 @@ }, "kind": { "const": "http", + "default": "http", "description": "always 'http'", "title": "kind", "type": "string" }, "url": { + "$ref": "#/$defs/Url", "description": "URL to fetch data from", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "url" ], "title": "HttpDatasourceConfig", @@ -984,6 +980,13 @@ "title": "first_level", "type": "integer" }, + "kind": { + "const": "template", + "default": "template", + "description": "always 'template'", + "title": "kind", + "type": "string" + }, "last_level": { "default": 0, "description": "Level to stop indexing at", @@ -1027,20 +1030,18 @@ }, "kind": { "const": "ipfs", + "default": "ipfs", "description": "always 'ipfs'", "title": "kind", "type": "string" }, "url": { + "$ref": "#/$defs/Url", "default": "https://ipfs.io/ipfs", "description": "IPFS node URL, e.g. https://ipfs.io/ipfs/", - "title": "url", - "type": "string" + "title": "url" } }, - "required": [ - "kind" - ], "title": "IpfsDatasourceConfig", "type": "object" }, @@ -1136,6 +1137,7 @@ }, "kind": { "const": "postgres", + "default": "postgres", "description": "always 'postgres'", "title": "kind", "type": "string" @@ -1166,7 +1168,6 @@ } }, "required": [ - "kind", "host" ], "title": "PostgresDatabaseConfig", @@ -1324,6 +1325,7 @@ }, "kind": { "const": "sqlite", + "default": "sqlite", "description": "always 'sqlite'", "title": "kind", "type": "string" @@ -1335,9 +1337,6 @@ "type": "string" } }, - "required": [ - "kind" - ], "title": "SqliteDatabaseConfig", "type": "object" }, @@ -1382,6 +1381,7 @@ }, "kind": { "const": "starknet", + "default": "starknet", "description": "Always `starknet`", "title": "kind", "type": "string" @@ -1400,9 +1400,6 @@ "title": "typename" } }, - "required": [ - "kind" - ], "title": "StarknetContractConfig", "type": "object" }, @@ -1479,6 +1476,7 @@ }, "kind": { "const": "starknet.events", + "default": "starknet.events", "description": "Always 'starknet.events'", "title": "kind", "type": "string" @@ -1491,7 +1489,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -1517,6 +1514,7 @@ }, "kind": { "const": "starknet.node", + "default": "starknet.node", "description": "Always 'starknet.node'", "title": "kind", "type": "string" @@ -1547,7 +1545,6 @@ } }, "required": [ - "kind", "url" ], "title": "StarknetNodeDatasourceConfig", @@ -1572,6 +1569,7 @@ }, "kind": { "const": "starknet.subsquid", + "default": "starknet.subsquid", "description": "always 'starknet.subsquid'", "title": "kind", "type": "string" @@ -1583,7 +1581,6 @@ } }, "required": [ - "kind", "url" ], "title": "StarknetSubsquidDatasourceConfig", @@ -1652,6 +1649,7 @@ }, "kind": { "const": "substrate.events", + "default": "substrate.events", "description": "Always 'substrate.events'", "title": "kind", "type": "string" @@ -1676,7 +1674,6 @@ } }, "required": [ - "kind", "datasources", "runtime", "handlers" @@ -1703,6 +1700,7 @@ }, "kind": { "const": "substrate.node", + "default": "substrate.node", "description": "Always 'substrate.node'", "title": "kind", "type": "string" @@ -1727,7 +1725,6 @@ } }, "required": [ - "kind", "url" ], "title": "SubstrateNodeDatasourceConfig", @@ -1793,18 +1790,18 @@ }, "kind": { "const": "substrate.subscan", + "default": "substrate.subscan", "description": "always 'substrate.subscan'", "title": "kind", "type": "string" }, "url": { + "$ref": "#/$defs/Url", "description": "API URL", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "url" ], "title": "SubstrateSubscanDatasourceConfig", @@ -1829,6 +1826,7 @@ }, "kind": { "const": "substrate.subsquid", + "default": "substrate.subsquid", "description": "always 'substrate.subsquid'", "title": "kind", "type": "string" @@ -1840,7 +1838,6 @@ } }, "required": [ - "kind", "url" ], "title": "SubstrateSubsquidDatasourceConfig", @@ -1919,6 +1916,7 @@ }, "kind": { "const": "tezos.big_maps", + "default": "tezos.big_maps", "description": "always 'tezos.big_maps'", "title": "kind", "type": "string" @@ -1937,7 +1935,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -1979,6 +1976,7 @@ }, "kind": { "const": "tezos", + "default": "tezos", "description": "Always `tezos`", "title": "kind", "type": "string" @@ -1997,9 +1995,6 @@ "title": "typename" } }, - "required": [ - "kind" - ], "title": "TezosContractConfig", "type": "object" }, @@ -2080,6 +2075,7 @@ }, "kind": { "const": "tezos.events", + "default": "tezos.events", "description": "always 'tezos.events'", "title": "kind", "type": "string" @@ -2092,7 +2088,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2154,13 +2149,13 @@ }, "kind": { "const": "tezos.head", + "default": "tezos.head", "description": "always 'tezos.head'", "title": "kind", "type": "string" } }, "required": [ - "kind", "datasources", "callback" ], @@ -2550,6 +2545,7 @@ }, "kind": { "const": "tezos.operations", + "default": "tezos.operations", "description": "always 'tezos.operations'", "title": "kind", "type": "string" @@ -2573,7 +2569,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2612,6 +2607,7 @@ }, "kind": { "const": "tezos.operations_unfiltered", + "default": "tezos.operations_unfiltered", "description": "always 'tezos.operations_unfiltered'", "title": "kind", "type": "string" @@ -2635,7 +2631,6 @@ } }, "required": [ - "kind", "datasources", "callback" ], @@ -2722,6 +2717,7 @@ }, "kind": { "const": "tezos.token_balances", + "default": "tezos.token_balances", "description": "always 'tezos.token_balances'", "title": "kind", "type": "string" @@ -2734,7 +2730,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2853,6 +2848,7 @@ }, "kind": { "const": "tezos.token_transfers", + "default": "tezos.token_transfers", "description": "always 'tezos.token_transfers'", "title": "kind", "type": "string" @@ -2865,7 +2861,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2897,6 +2892,7 @@ }, "kind": { "const": "tezos.tzkt", + "default": "tezos.tzkt", "description": "always 'tezos.tzkt'", "title": "kind", "type": "string" @@ -2920,9 +2916,6 @@ "title": "url" } }, - "required": [ - "kind" - ], "title": "TezosTzktDatasourceConfig", "type": "object" }, @@ -2955,6 +2948,7 @@ }, "kind": { "const": "tzip_metadata", + "default": "tzip_metadata", "description": "always 'tzip_metadata'", "title": "kind", "type": "string" @@ -2965,14 +2959,13 @@ "title": "network" }, "url": { + "$ref": "#/$defs/Url", "default": "https://metadata.dipdup.net", "description": "GraphQL API URL, e.g. https://metadata.dipdup.net", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "network" ], "title": "TzipMetadataDatasourceConfig", diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index dcf6bb102..5d054fee3 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -867,7 +867,7 @@ async def new( if template: echo(f'Using template `{template}`\n') - config_dict = {} + config_dict: dict[str, Any] | None = {} else: template, config_dict = template_from_terminal(answers['package']) @@ -884,11 +884,8 @@ async def new( 'spec_version': '3.0', **config_dict, } - - config = DipDupYAMLConfig(**config_dict) - - path = env.get_package_path(config['package']) / ROOT_CONFIG - path.write_text(config.dump()) + path = env.get_package_path(config_dict['package']) / ROOT_CONFIG + path.write_text(DipDupYAMLConfig(**config_dict).dump()) _logger.info('Initializing project') config = DipDupConfig.load([Path(answers['package'])]) diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 74c32262f..32afa0d11 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -75,6 +75,8 @@ def _valid_url(v: str, ws: bool) -> str: + if not v: + raise ConfigurationError('URL is required') if not ws and not v.startswith(('http://', 'https://')): raise ConfigurationError(f'`{v}` is not a valid HTTP URL') if ws and not v.startswith(('ws://', 'wss://')): @@ -627,14 +629,14 @@ def package_path(self) -> Path: @classmethod def from_terminal(cls, opts: TerminalOptions) -> Self: - import survey + import survey # type: ignore[import-untyped] from dipdup.project import SINGULAR_FORMS from dipdup.project import fill_type_from_input from dipdup.project import prompt_bool from dipdup.project import prompt_kind - config_dict = defaultdict(dict) + config_dict: defaultdict[str, dict[str, Any]] = defaultdict(dict) sections = { 'datasources': get_args(DatasourceConfigU), @@ -660,9 +662,18 @@ def from_terminal(cls, opts: TerminalOptions) -> Self: break # NOTE: All sections are mappings alias to dict - name = survey.routines.input( - f'Enter {section_singular} name: ', - ) + name = None + while True: + name = survey.routines.input( + f'Enter {section_singular} name: ', + ) + if not name: + print('Name is required') + continue + if name in config_dict[section]: + print(f'{section_singular.capitalize()} with name `{name}` already exists') + continue + break type_ = prompt_kind( section_singular, @@ -681,13 +692,13 @@ def from_terminal(cls, opts: TerminalOptions) -> Self: config_dict[section][name] = res another = True - config_dict = { + config_dict = { # type: ignore[assignment] 'package': opts.package, 'spec_version': '3.0', **config_dict, } - return cls(**config_dict) + return cls(**config_dict) # type: ignore[arg-type] @classmethod def load( diff --git a/src/dipdup/config/evm_etherscan.py b/src/dipdup/config/evm_etherscan.py index 8e24df9d9..9d68377a2 100644 --- a/src/dipdup/config/evm_etherscan.py +++ b/src/dipdup/config/evm_etherscan.py @@ -6,8 +6,11 @@ from pydantic import ConfigDict from pydantic.dataclasses import dataclass +from dipdup import env from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url +from dipdup.exceptions import ConfigurationError _logger = logging.getLogger(__name__) @@ -24,13 +27,14 @@ class EvmEtherscanDatasourceConfig(DatasourceConfig): # NOTE: Alias, remove in 9.0 kind: Literal['evm.etherscan'] | Literal['abi.etherscan'] = 'evm.etherscan' - url: str + url: Url api_key: str | None = None http: HttpConfig | None = None def __post_init__(self) -> None: if self.kind == 'abi.etherscan': - _logger.warning( - '`abi.etherscan` datasource has been renamed to `evm.etherscan`. Please, update your config.' - ) + msg = '`abi.etherscan` datasource has been renamed to `evm.etherscan`. Please, update your config.' + if env.NEXT: + raise ConfigurationError(msg) + _logger.warning(msg) diff --git a/src/dipdup/config/http.py b/src/dipdup/config/http.py index b4341e90e..772e7c918 100644 --- a/src/dipdup/config/http.py +++ b/src/dipdup/config/http.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) @@ -19,5 +20,5 @@ class HttpDatasourceConfig(DatasourceConfig): """ kind: Literal['http'] = 'http' - url: str + url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/ipfs.py b/src/dipdup/config/ipfs.py index 90993b647..df1bc962d 100644 --- a/src/dipdup/config/ipfs.py +++ b/src/dipdup/config/ipfs.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url DEFAULT_IPFS_URL = 'https://ipfs.io/ipfs' @@ -21,5 +22,5 @@ class IpfsDatasourceConfig(DatasourceConfig): """ kind: Literal['ipfs'] = 'ipfs' - url: str = DEFAULT_IPFS_URL + url: Url = DEFAULT_IPFS_URL http: HttpConfig | None = None diff --git a/src/dipdup/config/substrate_subscan.py b/src/dipdup/config/substrate_subscan.py index 8167da3b4..296ab629e 100644 --- a/src/dipdup/config/substrate_subscan.py +++ b/src/dipdup/config/substrate_subscan.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) @@ -20,7 +21,7 @@ class SubstrateSubscanDatasourceConfig(DatasourceConfig): """ kind: Literal['substrate.subscan'] = 'substrate.subscan' - url: str + url: Url api_key: str | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/tzip_metadata.py b/src/dipdup/config/tzip_metadata.py index b787fde43..52665c4db 100644 --- a/src/dipdup/config/tzip_metadata.py +++ b/src/dipdup/config/tzip_metadata.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url from dipdup.models.tzip_metadata import TzipMetadataNetwork DEFAULT_TZIP_METADATA_URL = 'https://metadata.dipdup.net' @@ -24,5 +25,5 @@ class TzipMetadataDatasourceConfig(DatasourceConfig): kind: Literal['tzip_metadata'] = 'tzip_metadata' network: TzipMetadataNetwork - url: str = DEFAULT_TZIP_METADATA_URL + url: Url = DEFAULT_TZIP_METADATA_URL http: HttpConfig | None = None diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 592156bcc..ae5618053 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -135,7 +135,7 @@ def get_replay_path(name: str) -> Path: def namespace_from_terminal() -> str | None: - res = prompt_anyof( + _, res = prompt_anyof( question='What blockchain are you going to index?', options=( 'EVM', @@ -153,7 +153,10 @@ def namespace_from_terminal() -> str | None: ), default=0, ) - return res[1].lower() if res[1] != '[multiple]' else None + print(res) + if res == '[multiple]': + return None + return res.lower() def template_from_terminal(package: str) -> tuple[str | None, dict[str, Any] | None]: @@ -190,7 +193,7 @@ def template_from_terminal(package: str) -> tuple[str | None, dict[str, Any] | N options, comments = [], [] templates = TEMPLATES[namespace] if namespace else chain(v for v in TEMPLATES.values()) for name in templates: - replay_path = get_replay_path(name) + replay_path = get_replay_path(name) # type: ignore[arg-type] _answers = answers_from_replay(replay_path) options.append(_answers['template']) comments.append(_answers['description']) @@ -233,10 +236,10 @@ def answers_from_terminal() -> Answers: ) answers['package'] = package - answers['version'] = survey.routines.input( - 'Enter project version: ', - value=answers['version'], - ) + # answers['version'] = survey.routines.input( + # 'Enter project version: ', + # value=answers['version'], + # ) # NOTE: Used in pyproject.toml, README.md and some other places answers['description'] = survey.routines.input( @@ -447,7 +450,7 @@ def prompt_kind( matched = {} for entity_type in types: try: - kind = entity_type.__dataclass_fields__['kind'].default + kind = entity_type.__dataclass_fields__['kind'].default # type: ignore[attr-defined] except KeyError: kind = entity_type.__name__ @@ -459,9 +462,9 @@ def prompt_kind( matched[kind] = entity_type if len(matched) == 1: - return next(iter(matched)) + return next(iter(matched)) # type: ignore[no-any-return] - kinds = sorted(matched.keys()) + kinds = tuple(sorted(matched.keys())) comments = tuple('' for _ in kinds) _, kind = prompt_anyof( f'Choose {entity} kind: ', @@ -479,7 +482,7 @@ def fill_type_from_input( # Gather input for the fields of the chosen type entity_data = {} - for field_name, field in type_.__dataclass_fields__.items(): + for field_name, field in type_.__dataclass_fields__.items(): # type: ignore[attr-defined] # print(field) if field_name in {'kind', 'http'}: @@ -495,7 +498,7 @@ def fill_type_from_input( if default == dataclasses.MISSING: default = None elif isinstance(default, FieldInfo): - default = default.default_factory() if default.default_factory else default.default + default = default.default_factory() if default.default_factory else default.default # type: ignore[call-arg] field_value = survey.routines.input( f'Enter value for `{field_name}` [{field.type}]: ', From afab55fda92a73adfdccaeae8a31ca8e6f7ff2ab Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Mon, 17 Feb 2025 08:52:56 -0300 Subject: [PATCH 09/14] bump dmcg --- requirements.txt | 30 ++++---- src/dipdup/config/__init__.py | 8 +- uv.lock | 138 +++++++++++++++++----------------- 3 files changed, 89 insertions(+), 87 deletions(-) diff --git a/requirements.txt b/requirements.txt index 97acc296b..8dd56fada 100644 --- a/requirements.txt +++ b/requirements.txt @@ -172,9 +172,9 @@ cytoolz==1.0.1 ; implementation_name == 'cpython' \ --hash=sha256:c8231b9abbd8e368e036f4cc2e16902c9482d4cf9e02a6147ed0e9a3cd4a9ab0 \ --hash=sha256:fb988c333f05ee30ad4693fe4da55d95ec0bb05775d2b60191236493ea2e01f9 \ --hash=sha256:fcb8f7d0d65db1269022e7e0428471edee8c937bc288ebdcb72f13eaa67c2fe4 -datamodel-code-generator==0.27.2 \ - --hash=sha256:1a7655f5fd3a61329b57534904f5c40dd850850e420696fd946ec7a4f59c32b8 \ - --hash=sha256:efcbfbe6a1488d3411fc588b1ce1af5f854f5107810b1cc9026a6d6333a7c4d8 +datamodel-code-generator==0.28.1 \ + --hash=sha256:1ff8a56f9550a82bcba3e1ad7ebdb89bc655eeabbc4bc6acfb05977cbdc6381c \ + --hash=sha256:37ef5f3b488f7d7a3f0b5b3ba0f2bc1ae01bab4dc7e0f6b99ff6c40713a6beb3 dictdiffer==0.9.0 \ --hash=sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578 \ --hash=sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595 @@ -190,9 +190,9 @@ eth-account==0.13.3 \ eth-hash==0.7.1 \ --hash=sha256:0fb1add2adf99ef28883fd6228eb447ef519ea72933535ad1a0b28c6f65f868a \ --hash=sha256:d2411a403a0b0a62e8247b4117932d900ffb4c8c64b15f92620547ca5ce46be5 -eth-keyfile==0.9.0 \ - --hash=sha256:45d3513b6433ad885370225ba0429ed26493ba23589c5b1ca5da024765020fef \ - --hash=sha256:8621c35e83cbc05909d2f23dbb8a87633918733caea553ae0e298f6a06291526 +eth-keyfile==0.9.1 \ + --hash=sha256:9789c3b4fa0bb6e2616cdc2bdd71b8755b42947d78ef1e900a0149480fabb5c2 \ + --hash=sha256:c7a8bc6af4527d1ab2eb1d1b949d59925252e17663eaf90087da121327b51df6 eth-keys==0.6.1 \ --hash=sha256:7deae4cd56e862e099ec58b78176232b931c4ea5ecded2f50c7b1ccbc10c24cf \ --hash=sha256:a43e263cbcabfd62fa769168efc6c27b1f5603040e4de22bb84d12567e4fd962 @@ -499,9 +499,9 @@ ruamel-yaml-clib==0.2.12 ; platform_python_implementation == 'CPython' \ scalecodec==1.2.11 \ --hash=sha256:99a2cdbfccdcaf22bd86b86da55a730a2855514ad2309faef4a4a93ac6cbeb8d \ --hash=sha256:d15c94965f617caa25096f83a45f5f73031d05e6ee08d6039969f0a64fc35de1 -sentry-sdk==2.20.0 \ - --hash=sha256:afa82713a92facf847df3c6f63cec71eb488d826a50965def3d7722aa6f0fdab \ - --hash=sha256:c359a1edf950eb5e80cffd7d9111f3dbeef57994cb4415df37d39fda2cf22364 +sentry-sdk==2.21.0 \ + --hash=sha256:7623cfa9e2c8150948a81ca253b8e2bfe4ce0b96ab12f8cd78e3ac9c490fd92f \ + --hash=sha256:a6d38e0fb35edda191acf80b188ec713c863aaa5ad8d5798decb8671d02077b6 six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 @@ -530,9 +530,9 @@ toolz==1.0.0 ; implementation_name == 'cpython' or implementation_name == 'pypy' tortoise-orm==0.24.0 \ --hash=sha256:ae0704a93ea27931724fc899e57268c8081afce3b32b110b00037ec206553e7d \ --hash=sha256:ee3b72b226767293b24c5c4906ae5f027d7cc84496cd503352c918564b4fd687 -typeguard==4.4.1 \ - --hash=sha256:0d22a89d00b453b47c49875f42b6601b961757541a2e1e0ef517b6e24213c21b \ - --hash=sha256:9324ec07a27ec67fc54a9c063020ca4c0ae6abad5e9f0f9804ca59aee68c6e21 +typeguard==4.4.2 \ + --hash=sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c \ + --hash=sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49 types-requests==2.32.0.20241016 \ --hash=sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95 \ --hash=sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747 @@ -545,9 +545,9 @@ typing-inspect==0.9.0 \ tzdata==2025.1 ; sys_platform == 'win32' \ --hash=sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694 \ --hash=sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639 -tzlocal==5.2 \ - --hash=sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8 \ - --hash=sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e +tzlocal==5.3 \ + --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ + --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c urllib3==2.3.0 \ --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 32afa0d11..48a5a34b7 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -692,13 +692,16 @@ def from_terminal(cls, opts: TerminalOptions) -> Self: config_dict[section][name] = res another = True + # NOTE: Make sure that header is above other sections config_dict = { # type: ignore[assignment] 'package': opts.package, 'spec_version': '3.0', **config_dict, } - return cls(**config_dict) # type: ignore[arg-type] + self = cls(**config_dict) # type: ignore[arg-type] + self._json = config_dict # type: ignore[assignment] + return self @classmethod def load( @@ -716,9 +719,6 @@ def load( ) try: - # from pydantic.dataclasses import rebuild_dataclass - # rebuild_dataclass(cls, force=True) - config = TypeAdapter(cls).validate_python(config_json) except ConfigurationError: raise diff --git a/uv.lock b/uv.lock index 922958893..4372a6db0 100644 --- a/uv.lock +++ b/uv.lock @@ -376,19 +376,21 @@ wheels = [ [[package]] name = "coverage" -version = "7.6.11" +version = "7.6.12" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/89/4e/38141d42af7452f4b7c5d3d7442a8018de34754ef52eb9a400768bc8d59e/coverage-7.6.11.tar.gz", hash = "sha256:e642e6a46a04e992ebfdabed79e46f478ec60e2c528e1e1a074d63800eda4286", size = 805460 } +sdist = { url = "https://files.pythonhosted.org/packages/0c/d6/2b53ab3ee99f2262e6f0b8369a43f6d66658eab45510331c0b3d5c8c4272/coverage-7.6.12.tar.gz", hash = "sha256:48cfc4641d95d34766ad41d9573cc0f22a48aa88d22657a1fe01dca0dbae4de2", size = 805941 } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/83/cf3d6ac06bd02e1fb7fc6609d7a3be799328a94938dd2a64cf091989b8ce/coverage-7.6.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbb1a822fd858d9853333a7c95d4e70dde9a79e65893138ce32c2ec6457d7a36", size = 208543 }, - { url = "https://files.pythonhosted.org/packages/e7/e1/b1448995072ab033898758179e208afa924f4625ea4524ec868fafbae77d/coverage-7.6.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61c834cbb80946d6ebfddd9b393a4c46bec92fcc0fa069321fcb8049117f76ea", size = 208805 }, - { url = "https://files.pythonhosted.org/packages/80/22/11ae7726086bf16ad35ecd1ebf31c0c709647b2618977bc088003bd38808/coverage-7.6.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a46d56e99a31d858d6912d31ffa4ede6a325c86af13139539beefca10a1234ce", size = 239768 }, - { url = "https://files.pythonhosted.org/packages/7d/68/717286bda6530f39f3ac16899dac1855a71921aca5ee565484269326c979/coverage-7.6.11-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b48db06f53d1864fea6dbd855e6d51d41c0f06c212c3004511c0bdc6847b297", size = 242023 }, - { url = "https://files.pythonhosted.org/packages/93/57/4b028c7c882411d9ca3f12cd4223ceeb5cb39f84bb91c4fb21a06440cbd9/coverage-7.6.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b6ff5be3b1853e0862da9d349fe87f869f68e63a25f7c37ce1130b321140f963", size = 239610 }, - { url = "https://files.pythonhosted.org/packages/44/88/720c9eba316406f243670237306bcdb8e269e4d0e12b191a697f66369404/coverage-7.6.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be05bde21d5e6eefbc3a6de6b9bee2b47894b8945342e8663192809c4d1f08ce", size = 241212 }, - { url = "https://files.pythonhosted.org/packages/1d/ae/a09edf77bd535d597de13679262845f5cb6ff1fab37a3065640fb3d5e6e8/coverage-7.6.11-cp312-cp312-win32.whl", hash = "sha256:e3b746fa0ffc5b6b8856529de487da8b9aeb4fb394bb58de6502ef45f3434f12", size = 211186 }, - { url = "https://files.pythonhosted.org/packages/80/5d/63ad5e3f1421504194da0228d259a3913884830999d1297b5e16b59bcb0f/coverage-7.6.11-cp312-cp312-win_amd64.whl", hash = "sha256:ac476e6d0128fb7919b3fae726de72b28b5c9644cb4b579e4a523d693187c551", size = 211974 }, - { url = "https://files.pythonhosted.org/packages/24/f3/63cd48409a519d4f6cf79abc6c89103a8eabc5c93e496f40779269dba0c0/coverage-7.6.11-py3-none-any.whl", hash = "sha256:f0f334ae844675420164175bf32b04e18a81fe57ad8eb7e0cfd4689d681ffed7", size = 200446 }, + { url = "https://files.pythonhosted.org/packages/e2/7f/4af2ed1d06ce6bee7eafc03b2ef748b14132b0bdae04388e451e4b2c529b/coverage-7.6.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b172f8e030e8ef247b3104902cc671e20df80163b60a203653150d2fc204d1ad", size = 208645 }, + { url = "https://files.pythonhosted.org/packages/dc/60/d19df912989117caa95123524d26fc973f56dc14aecdec5ccd7d0084e131/coverage-7.6.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:641dfe0ab73deb7069fb972d4d9725bf11c239c309ce694dd50b1473c0f641c3", size = 208898 }, + { url = "https://files.pythonhosted.org/packages/bd/10/fecabcf438ba676f706bf90186ccf6ff9f6158cc494286965c76e58742fa/coverage-7.6.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e549f54ac5f301e8e04c569dfdb907f7be71b06b88b5063ce9d6953d2d58574", size = 242987 }, + { url = "https://files.pythonhosted.org/packages/4c/53/4e208440389e8ea936f5f2b0762dcd4cb03281a7722def8e2bf9dc9c3d68/coverage-7.6.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959244a17184515f8c52dcb65fb662808767c0bd233c1d8a166e7cf74c9ea985", size = 239881 }, + { url = "https://files.pythonhosted.org/packages/c4/47/2ba744af8d2f0caa1f17e7746147e34dfc5f811fb65fc153153722d58835/coverage-7.6.12-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bda1c5f347550c359f841d6614fb8ca42ae5cb0b74d39f8a1e204815ebe25750", size = 242142 }, + { url = "https://files.pythonhosted.org/packages/e9/90/df726af8ee74d92ee7e3bf113bf101ea4315d71508952bd21abc3fae471e/coverage-7.6.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ceeb90c3eda1f2d8c4c578c14167dbd8c674ecd7d38e45647543f19839dd6ea", size = 241437 }, + { url = "https://files.pythonhosted.org/packages/f6/af/995263fd04ae5f9cf12521150295bf03b6ba940d0aea97953bb4a6db3e2b/coverage-7.6.12-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f16f44025c06792e0fb09571ae454bcc7a3ec75eeb3c36b025eccf501b1a4c3", size = 239724 }, + { url = "https://files.pythonhosted.org/packages/1c/8e/5bb04f0318805e190984c6ce106b4c3968a9562a400180e549855d8211bd/coverage-7.6.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b076e625396e787448d27a411aefff867db2bffac8ed04e8f7056b07024eed5a", size = 241329 }, + { url = "https://files.pythonhosted.org/packages/9e/9d/fa04d9e6c3f6459f4e0b231925277cfc33d72dfab7fa19c312c03e59da99/coverage-7.6.12-cp312-cp312-win32.whl", hash = "sha256:00b2086892cf06c7c2d74983c9595dc511acca00665480b3ddff749ec4fb2a95", size = 211289 }, + { url = "https://files.pythonhosted.org/packages/53/40/53c7ffe3c0c3fff4d708bc99e65f3d78c129110d6629736faf2dbd60ad57/coverage-7.6.12-cp312-cp312-win_amd64.whl", hash = "sha256:7ae6eabf519bc7871ce117fb18bf14e0e343eeb96c377667e3e5dd12095e0288", size = 212079 }, + { url = "https://files.pythonhosted.org/packages/fb/b2/f655700e1024dec98b10ebaafd0cedbc25e40e4abe62a3c8e2ceef4f8f0a/coverage-7.6.12-py3-none-any.whl", hash = "sha256:eb8668cfbc279a536c633137deeb9435d2962caec279c3f8cf8b91fff6ff8953", size = 200552 }, ] [[package]] @@ -444,7 +446,7 @@ wheels = [ [[package]] name = "datamodel-code-generator" -version = "0.27.2" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "argcomplete" }, @@ -457,9 +459,9 @@ dependencies = [ { name = "pydantic" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8c/49/9cb4f868856304dd4e2fc0795d848889a7c9c6f2539165ad24977cef0da3/datamodel_code_generator-0.27.2.tar.gz", hash = "sha256:1a7655f5fd3a61329b57534904f5c40dd850850e420696fd946ec7a4f59c32b8", size = 436345 } +sdist = { url = "https://files.pythonhosted.org/packages/cb/d3/80f6a2394bbf3b46b150fc75afa5b0050f91baa5771e9be87df148013d83/datamodel_code_generator-0.28.1.tar.gz", hash = "sha256:37ef5f3b488f7d7a3f0b5b3ba0f2bc1ae01bab4dc7e0f6b99ff6c40713a6beb3", size = 434901 } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/a0/678f10ecc40f1cce3c170246c3dd1b86735867d2844eb9f4596abf187dac/datamodel_code_generator-0.27.2-py3-none-any.whl", hash = "sha256:efcbfbe6a1488d3411fc588b1ce1af5f854f5107810b1cc9026a6d6333a7c4d8", size = 115483 }, + { url = "https://files.pythonhosted.org/packages/c8/17/2876ca0a4ac7dd7cb5f56a2f0f6d9ac910969f467e8142c847c45a76b897/datamodel_code_generator-0.28.1-py3-none-any.whl", hash = "sha256:1ff8a56f9550a82bcba3e1ad7ebdb89bc655eeabbc4bc6acfb05977cbdc6381c", size = 115601 }, ] [[package]] @@ -684,7 +686,7 @@ pycryptodome = [ [[package]] name = "eth-keyfile" -version = "0.9.0" +version = "0.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "eth-keys" }, @@ -692,9 +694,9 @@ dependencies = [ { name = "py-ecc" }, { name = "pycryptodome" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6b/fc/1b957ce3403158417b8e632c87ceffb0c2b0845505c280571e23ae57bff3/eth_keyfile-0.9.0.tar.gz", hash = "sha256:8621c35e83cbc05909d2f23dbb8a87633918733caea553ae0e298f6a06291526", size = 19793 } +sdist = { url = "https://files.pythonhosted.org/packages/08/e4/3f0c20b020786e1fa6e1ecd81806c54167fa2b0839e0020086b95a6e8faf/eth_keyfile-0.9.1.tar.gz", hash = "sha256:c7a8bc6af4527d1ab2eb1d1b949d59925252e17663eaf90087da121327b51df6", size = 19787 } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/5f/854a4c2bb7184b69947b77ed1337009ad3e57e085dc7b2010dc13d868d4a/eth_keyfile-0.9.0-py3-none-any.whl", hash = "sha256:45d3513b6433ad885370225ba0429ed26493ba23589c5b1ca5da024765020fef", size = 9869 }, + { url = "https://files.pythonhosted.org/packages/5f/08/9c8bf617b39e1dd56303593292e8b4eb66497a5f0f5b997a4b291e5343c0/eth_keyfile-0.9.1-py3-none-any.whl", hash = "sha256:9789c3b4fa0bb6e2616cdc2bdd71b8755b42947d78ef1e900a0149480fabb5c2", size = 9866 }, ] [[package]] @@ -1066,20 +1068,20 @@ wheels = [ [[package]] name = "numpy" -version = "2.2.2" +version = "2.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/d0/c12ddfd3a02274be06ffc71f3efc6d0e457b0409c4481596881e748cb264/numpy-2.2.2.tar.gz", hash = "sha256:ed6906f61834d687738d25988ae117683705636936cc605be0bb208b23df4d8f", size = 20233295 } +sdist = { url = "https://files.pythonhosted.org/packages/fb/90/8956572f5c4ae52201fdec7ba2044b2c882832dcec7d5d0922c9e9acf2de/numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020", size = 20262700 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/e6/847d15770ab7a01e807bdfcd4ead5bdae57c0092b7dc83878171b6af97bb/numpy-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ac9bea18d6d58a995fac1b2cb4488e17eceeac413af014b1dd26170b766d8467", size = 20912636 }, - { url = "https://files.pythonhosted.org/packages/d1/af/f83580891577b13bd7e261416120e036d0d8fb508c8a43a73e38928b794b/numpy-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23ae9f0c2d889b7b2d88a3791f6c09e2ef827c2446f1c4a3e3e76328ee4afd9a", size = 14098403 }, - { url = "https://files.pythonhosted.org/packages/2b/86/d019fb60a9d0f1d4cf04b014fe88a9135090adfadcc31c1fadbb071d7fa7/numpy-2.2.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3074634ea4d6df66be04f6728ee1d173cfded75d002c75fac79503a880bf3825", size = 5128938 }, - { url = "https://files.pythonhosted.org/packages/7a/1b/50985edb6f1ec495a1c36452e860476f5b7ecdc3fc59ea89ccad3c4926c5/numpy-2.2.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:8ec0636d3f7d68520afc6ac2dc4b8341ddb725039de042faf0e311599f54eb37", size = 6661937 }, - { url = "https://files.pythonhosted.org/packages/f4/1b/17efd94cad1b9d605c3f8907fb06bcffc4ce4d1d14d46b95316cccccf2b9/numpy-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ffbb1acd69fdf8e89dd60ef6182ca90a743620957afb7066385a7bbe88dc748", size = 14049518 }, - { url = "https://files.pythonhosted.org/packages/5b/73/65d2f0b698df1731e851e3295eb29a5ab8aa06f763f7e4188647a809578d/numpy-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0349b025e15ea9d05c3d63f9657707a4e1d471128a3b1d876c095f328f8ff7f0", size = 16099146 }, - { url = "https://files.pythonhosted.org/packages/d5/69/308f55c0e19d4b5057b5df286c5433822e3c8039ede06d4051d96f1c2c4e/numpy-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:463247edcee4a5537841d5350bc87fe8e92d7dd0e8c71c995d2c6eecb8208278", size = 15246336 }, - { url = "https://files.pythonhosted.org/packages/f0/d8/d8d333ad0d8518d077a21aeea7b7c826eff766a2b1ce1194dea95ca0bacf/numpy-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9dd47ff0cb2a656ad69c38da850df3454da88ee9a6fde0ba79acceee0e79daba", size = 17863507 }, - { url = "https://files.pythonhosted.org/packages/82/6e/0b84ad3103ffc16d6673e63b5acbe7901b2af96c2837174c6318c98e27ab/numpy-2.2.2-cp312-cp312-win32.whl", hash = "sha256:4525b88c11906d5ab1b0ec1f290996c0020dd318af8b49acaa46f198b1ffc283", size = 6276491 }, - { url = "https://files.pythonhosted.org/packages/fc/84/7f801a42a67b9772a883223a0a1e12069a14626c81a732bd70aac57aebc1/numpy-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:5acea83b801e98541619af398cc0109ff48016955cc0818f478ee9ef1c5c3dcb", size = 12616372 }, + { url = "https://files.pythonhosted.org/packages/43/ec/43628dcf98466e087812142eec6d1c1a6c6bdfdad30a0aa07b872dc01f6f/numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d", size = 20929458 }, + { url = "https://files.pythonhosted.org/packages/9b/c0/2f4225073e99a5c12350954949ed19b5d4a738f541d33e6f7439e33e98e4/numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95", size = 14115299 }, + { url = "https://files.pythonhosted.org/packages/ca/fa/d2c5575d9c734a7376cc1592fae50257ec95d061b27ee3dbdb0b3b551eb2/numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea", size = 5145723 }, + { url = "https://files.pythonhosted.org/packages/eb/dc/023dad5b268a7895e58e791f28dc1c60eb7b6c06fcbc2af8538ad069d5f3/numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532", size = 6678797 }, + { url = "https://files.pythonhosted.org/packages/3f/19/bcd641ccf19ac25abb6fb1dcd7744840c11f9d62519d7057b6ab2096eb60/numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e", size = 14067362 }, + { url = "https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe", size = 16116679 }, + { url = "https://files.pythonhosted.org/packages/d0/a1/e90f7aa66512be3150cb9d27f3d9995db330ad1b2046474a13b7040dfd92/numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021", size = 15264272 }, + { url = "https://files.pythonhosted.org/packages/dc/b6/50bd027cca494de4fa1fc7bf1662983d0ba5f256fa0ece2c376b5eb9b3f0/numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8", size = 17880549 }, + { url = "https://files.pythonhosted.org/packages/96/30/f7bf4acb5f8db10a96f73896bdeed7a63373137b131ca18bd3dab889db3b/numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe", size = 6293394 }, + { url = "https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d", size = 12626357 }, ] [[package]] @@ -1217,17 +1219,17 @@ wheels = [ [[package]] name = "psutil" -version = "6.1.1" +version = "7.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/5a/07871137bb752428aa4b659f910b399ba6f291156bdea939be3e96cae7cb/psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5", size = 508502 } +sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003 } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/99/ca79d302be46f7bdd8321089762dd4476ee725fce16fc2b2e1dbba8cac17/psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8", size = 247511 }, - { url = "https://files.pythonhosted.org/packages/0b/6b/73dbde0dd38f3782905d4587049b9be64d76671042fdcaf60e2430c6796d/psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377", size = 248985 }, - { url = "https://files.pythonhosted.org/packages/17/38/c319d31a1d3f88c5b79c68b3116c129e5133f1822157dd6da34043e32ed6/psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003", size = 284488 }, - { url = "https://files.pythonhosted.org/packages/9c/39/0f88a830a1c8a3aba27fededc642da37613c57cbff143412e3536f89784f/psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160", size = 287477 }, - { url = "https://files.pythonhosted.org/packages/47/da/99f4345d4ddf2845cb5b5bd0d93d554e84542d116934fde07a0c50bd4e9f/psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3", size = 289017 }, - { url = "https://files.pythonhosted.org/packages/38/53/bd755c2896f4461fd4f36fa6a6dcb66a88a9e4b9fd4e5b66a77cf9d4a584/psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53", size = 250602 }, - { url = "https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649", size = 254444 }, + { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051 }, + { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535 }, + { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004 }, + { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986 }, + { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544 }, + { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053 }, + { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885 }, ] [[package]] @@ -1569,27 +1571,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.9.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/74/6c359f6b9ed85b88df6ef31febce18faeb852f6c9855651dfb1184a46845/ruff-0.9.5.tar.gz", hash = "sha256:11aecd7a633932875ab3cb05a484c99970b9d52606ce9ea912b690b02653d56c", size = 3634177 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/4b/82b7c9ac874e72b82b19fd7eab57d122e2df44d2478d90825854f9232d02/ruff-0.9.5-py3-none-linux_armv6l.whl", hash = "sha256:d466d2abc05f39018d53f681fa1c0ffe9570e6d73cde1b65d23bb557c846f442", size = 11681264 }, - { url = "https://files.pythonhosted.org/packages/27/5c/f5ae0a9564e04108c132e1139d60491c0abc621397fe79a50b3dc0bd704b/ruff-0.9.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:38840dbcef63948657fa7605ca363194d2fe8c26ce8f9ae12eee7f098c85ac8a", size = 11657554 }, - { url = "https://files.pythonhosted.org/packages/2a/83/c6926fa3ccb97cdb3c438bb56a490b395770c750bf59f9bc1fe57ae88264/ruff-0.9.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d56ba06da53536b575fbd2b56517f6f95774ff7be0f62c80b9e67430391eeb36", size = 11088959 }, - { url = "https://files.pythonhosted.org/packages/af/a7/42d1832b752fe969ffdbfcb1b4cb477cb271bed5835110fb0a16ef31ab81/ruff-0.9.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7cb2a01da08244c50b20ccfaeb5972e4228c3c3a1989d3ece2bc4b1f996001", size = 11902041 }, - { url = "https://files.pythonhosted.org/packages/53/cf/1fffa09fb518d646f560ccfba59f91b23c731e461d6a4dedd21a393a1ff1/ruff-0.9.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:96d5c76358419bc63a671caac70c18732d4fd0341646ecd01641ddda5c39ca0b", size = 11421069 }, - { url = "https://files.pythonhosted.org/packages/09/27/bb8f1b7304e2a9431f631ae7eadc35550fe0cf620a2a6a0fc4aa3d736f94/ruff-0.9.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:deb8304636ed394211f3a6d46c0e7d9535b016f53adaa8340139859b2359a070", size = 12625095 }, - { url = "https://files.pythonhosted.org/packages/d7/ce/ab00bc9d3df35a5f1b64f5117458160a009f93ae5caf65894ebb63a1842d/ruff-0.9.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:df455000bf59e62b3e8c7ba5ed88a4a2bc64896f900f311dc23ff2dc38156440", size = 13257797 }, - { url = "https://files.pythonhosted.org/packages/88/81/c639a082ae6d8392bc52256058ec60f493c6a4d06d5505bccface3767e61/ruff-0.9.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de92170dfa50c32a2b8206a647949590e752aca8100a0f6b8cefa02ae29dce80", size = 12763793 }, - { url = "https://files.pythonhosted.org/packages/b3/d0/0a3d8f56d1e49af466dc770eeec5c125977ba9479af92e484b5b0251ce9c/ruff-0.9.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d28532d73b1f3f627ba88e1456f50748b37f3a345d2be76e4c653bec6c3e393", size = 14386234 }, - { url = "https://files.pythonhosted.org/packages/04/70/e59c192a3ad476355e7f45fb3a87326f5219cc7c472e6b040c6c6595c8f0/ruff-0.9.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c746d7d1df64f31d90503ece5cc34d7007c06751a7a3bbeee10e5f2463d52d2", size = 12437505 }, - { url = "https://files.pythonhosted.org/packages/55/4e/3abba60a259d79c391713e7a6ccabf7e2c96e5e0a19100bc4204f1a43a51/ruff-0.9.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11417521d6f2d121fda376f0d2169fb529976c544d653d1d6044f4c5562516ee", size = 11884799 }, - { url = "https://files.pythonhosted.org/packages/a3/db/b0183a01a9f25b4efcae919c18fb41d32f985676c917008620ad692b9d5f/ruff-0.9.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5b9d71c3879eb32de700f2f6fac3d46566f644a91d3130119a6378f9312a38e1", size = 11527411 }, - { url = "https://files.pythonhosted.org/packages/0a/e4/3ebfcebca3dff1559a74c6becff76e0b64689cea02b7aab15b8b32ea245d/ruff-0.9.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2e36c61145e70febcb78483903c43444c6b9d40f6d2f800b5552fec6e4a7bb9a", size = 12078868 }, - { url = "https://files.pythonhosted.org/packages/ec/b2/5ab808833e06c0a1b0d046a51c06ec5687b73c78b116e8d77687dc0cd515/ruff-0.9.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:2f71d09aeba026c922aa7aa19a08d7bd27c867aedb2f74285a2639644c1c12f5", size = 12524374 }, - { url = "https://files.pythonhosted.org/packages/e0/51/1432afcc3b7aa6586c480142caae5323d59750925c3559688f2a9867343f/ruff-0.9.5-py3-none-win32.whl", hash = "sha256:134f958d52aa6fdec3b294b8ebe2320a950d10c041473c4316d2e7d7c2544723", size = 9853682 }, - { url = "https://files.pythonhosted.org/packages/b7/ad/c7a900591bd152bb47fc4882a27654ea55c7973e6d5d6396298ad3fd6638/ruff-0.9.5-py3-none-win_amd64.whl", hash = "sha256:78cc6067f6d80b6745b67498fb84e87d32c6fc34992b52bffefbdae3442967d6", size = 10865744 }, - { url = "https://files.pythonhosted.org/packages/75/d9/fde7610abd53c0c76b6af72fc679cb377b27c617ba704e25da834e0a0608/ruff-0.9.5-py3-none-win_arm64.whl", hash = "sha256:18a29f1a005bddb229e580795627d297dfa99f16b30c7039e73278cf6b5f9fa9", size = 10064595 }, +version = "0.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/e1/e265aba384343dd8ddd3083f5e33536cd17e1566c41453a5517b5dd443be/ruff-0.9.6.tar.gz", hash = "sha256:81761592f72b620ec8fa1068a6fd00e98a5ebee342a3642efd84454f3031dca9", size = 3639454 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/e3/3d2c022e687e18cf5d93d6bfa2722d46afc64eaa438c7fbbdd603b3597be/ruff-0.9.6-py3-none-linux_armv6l.whl", hash = "sha256:2f218f356dd2d995839f1941322ff021c72a492c470f0b26a34f844c29cdf5ba", size = 11714128 }, + { url = "https://files.pythonhosted.org/packages/e1/22/aff073b70f95c052e5c58153cba735748c9e70107a77d03420d7850710a0/ruff-0.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b908ff4df65dad7b251c9968a2e4560836d8f5487c2f0cc238321ed951ea0504", size = 11682539 }, + { url = "https://files.pythonhosted.org/packages/75/a7/f5b7390afd98a7918582a3d256cd3e78ba0a26165a467c1820084587cbf9/ruff-0.9.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b109c0ad2ececf42e75fa99dc4043ff72a357436bb171900714a9ea581ddef83", size = 11132512 }, + { url = "https://files.pythonhosted.org/packages/a6/e3/45de13ef65047fea2e33f7e573d848206e15c715e5cd56095589a7733d04/ruff-0.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de4367cca3dac99bcbd15c161404e849bb0bfd543664db39232648dc00112dc", size = 11929275 }, + { url = "https://files.pythonhosted.org/packages/7d/f2/23d04cd6c43b2e641ab961ade8d0b5edb212ecebd112506188c91f2a6e6c/ruff-0.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3ee4d7c2c92ddfdaedf0bf31b2b176fa7aa8950efc454628d477394d35638b", size = 11466502 }, + { url = "https://files.pythonhosted.org/packages/b5/6f/3a8cf166f2d7f1627dd2201e6cbc4cb81f8b7d58099348f0c1ff7b733792/ruff-0.9.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc1edd1775270e6aa2386119aea692039781429f0be1e0949ea5884e011aa8e", size = 12676364 }, + { url = "https://files.pythonhosted.org/packages/f5/c4/db52e2189983c70114ff2b7e3997e48c8318af44fe83e1ce9517570a50c6/ruff-0.9.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4a091729086dffa4bd070aa5dab7e39cc6b9d62eb2bef8f3d91172d30d599666", size = 13335518 }, + { url = "https://files.pythonhosted.org/packages/66/44/545f8a4d136830f08f4d24324e7db957c5374bf3a3f7a6c0bc7be4623a37/ruff-0.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1bbc6808bf7b15796cef0815e1dfb796fbd383e7dbd4334709642649625e7c5", size = 12823287 }, + { url = "https://files.pythonhosted.org/packages/c5/26/8208ef9ee7431032c143649a9967c3ae1aae4257d95e6f8519f07309aa66/ruff-0.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589d1d9f25b5754ff230dce914a174a7c951a85a4e9270613a2b74231fdac2f5", size = 14592374 }, + { url = "https://files.pythonhosted.org/packages/31/70/e917781e55ff39c5b5208bda384fd397ffd76605e68544d71a7e40944945/ruff-0.9.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc61dd5131742e21103fbbdcad683a8813be0e3c204472d520d9a5021ca8b217", size = 12500173 }, + { url = "https://files.pythonhosted.org/packages/84/f5/e4ddee07660f5a9622a9c2b639afd8f3104988dc4f6ba0b73ffacffa9a8c/ruff-0.9.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5e2d9126161d0357e5c8f30b0bd6168d2c3872372f14481136d13de9937f79b6", size = 11906555 }, + { url = "https://files.pythonhosted.org/packages/f1/2b/6ff2fe383667075eef8656b9892e73dd9b119b5e3add51298628b87f6429/ruff-0.9.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:68660eab1a8e65babb5229a1f97b46e3120923757a68b5413d8561f8a85d4897", size = 11538958 }, + { url = "https://files.pythonhosted.org/packages/3c/db/98e59e90de45d1eb46649151c10a062d5707b5b7f76f64eb1e29edf6ebb1/ruff-0.9.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4cae6c4cc7b9b4017c71114115db0445b00a16de3bcde0946273e8392856f08", size = 12117247 }, + { url = "https://files.pythonhosted.org/packages/ec/bc/54e38f6d219013a9204a5a2015c09e7a8c36cedcd50a4b01ac69a550b9d9/ruff-0.9.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19f505b643228b417c1111a2a536424ddde0db4ef9023b9e04a46ed8a1cb4656", size = 12554647 }, + { url = "https://files.pythonhosted.org/packages/a5/7d/7b461ab0e2404293c0627125bb70ac642c2e8d55bf590f6fce85f508f1b2/ruff-0.9.6-py3-none-win32.whl", hash = "sha256:194d8402bceef1b31164909540a597e0d913c0e4952015a5b40e28c146121b5d", size = 9949214 }, + { url = "https://files.pythonhosted.org/packages/ee/30/c3cee10f915ed75a5c29c1e57311282d1a15855551a64795c1b2bbe5cf37/ruff-0.9.6-py3-none-win_amd64.whl", hash = "sha256:03482d5c09d90d4ee3f40d97578423698ad895c87314c4de39ed2af945633caa", size = 10999914 }, + { url = "https://files.pythonhosted.org/packages/e8/a8/d71f44b93e3aa86ae232af1f2126ca7b95c0f515ec135462b3e1f351441c/ruff-0.9.6-py3-none-win_arm64.whl", hash = "sha256:0e2bb706a2be7ddfea4a4af918562fdc1bcb16df255e5fa595bbd800ce322a5a", size = 10177499 }, ] [[package]] @@ -1630,15 +1632,15 @@ wheels = [ [[package]] name = "sentry-sdk" -version = "2.20.0" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/68/e8/6a366c0cd5e129dda6ecb20ff097f70b18182c248d4c27e813c21f98992a/sentry_sdk-2.20.0.tar.gz", hash = "sha256:afa82713a92facf847df3c6f63cec71eb488d826a50965def3d7722aa6f0fdab", size = 300125 } +sdist = { url = "https://files.pythonhosted.org/packages/08/63/3f0e88709cf4af992e2813c27d8ba628a891db0805e3fcc6dc834e142c5b/sentry_sdk-2.21.0.tar.gz", hash = "sha256:a6d38e0fb35edda191acf80b188ec713c863aaa5ad8d5798decb8671d02077b6", size = 301965 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/0f/6f7e6cd0f4a141752caef3f79300148422fdf2b8b68b531f30b2b0c0cbda/sentry_sdk-2.20.0-py2.py3-none-any.whl", hash = "sha256:c359a1edf950eb5e80cffd7d9111f3dbeef57994cb4415df37d39fda2cf22364", size = 322576 }, + { url = "https://files.pythonhosted.org/packages/a4/18/7587660cb5e4d07134913d8e74137efcd4903fda873bf612c30eb34c7ab4/sentry_sdk-2.21.0-py2.py3-none-any.whl", hash = "sha256:7623cfa9e2c8150948a81ca253b8e2bfe4ce0b96ab12f8cd78e3ac9c490fd92f", size = 324096 }, ] [[package]] @@ -1867,14 +1869,14 @@ wheels = [ [[package]] name = "typeguard" -version = "4.4.1" +version = "4.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/c3/400917dd37d7b8c07e9723f3046818530423e1e759a56a22133362adab00/typeguard-4.4.1.tar.gz", hash = "sha256:0d22a89d00b453b47c49875f42b6601b961757541a2e1e0ef517b6e24213c21b", size = 74959 } +sdist = { url = "https://files.pythonhosted.org/packages/70/60/8cd6a3d78d00ceeb2193c02b7ed08f063d5341ccdfb24df88e61f383048e/typeguard-4.4.2.tar.gz", hash = "sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49", size = 75746 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/53/9465dedf2d69fe26008e7732cf6e0a385e387c240869e7d54eed49782a3c/typeguard-4.4.1-py3-none-any.whl", hash = "sha256:9324ec07a27ec67fc54a9c063020ca4c0ae6abad5e9f0f9804ca59aee68c6e21", size = 35635 }, + { url = "https://files.pythonhosted.org/packages/cf/4b/9a77dc721aa0b7f74440a42e4ef6f9a4fae7324e17f64f88b96f4c25cc05/typeguard-4.4.2-py3-none-any.whl", hash = "sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c", size = 35801 }, ] [[package]] @@ -1940,14 +1942,14 @@ wheels = [ [[package]] name = "tzlocal" -version = "5.2" +version = "5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/d3/c19d65ae67636fe63953b20c2e4a8ced4497ea232c43ff8d01db16de8dc0/tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e", size = 30201 } +sdist = { url = "https://files.pythonhosted.org/packages/33/cc/11360404b20a6340b9b4ed39a3338c4af47bc63f87f6cea94dbcbde07029/tzlocal-5.3.tar.gz", hash = "sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2", size = 30480 } wheels = [ - { url = "https://files.pythonhosted.org/packages/97/3f/c4c51c55ff8487f2e6d0e618dba917e3c3ee2caae6cf0fbb59c9b1876f2e/tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8", size = 17859 }, + { url = "https://files.pythonhosted.org/packages/e9/9f/1c0b69d3abf4c65acac051ad696b8aea55afbb746dea8017baab53febb5e/tzlocal-5.3-py3-none-any.whl", hash = "sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c", size = 17920 }, ] [[package]] From d51f7e6f6572e64946bc0821fa9c6809b96c25d2 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Mon, 17 Feb 2025 10:53:45 -0300 Subject: [PATCH 10/14] fix CI --- src/dipdup/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 5d054fee3..de16b8048 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -852,6 +852,8 @@ async def new( from dipdup.project import render_project from dipdup.project import template_from_terminal + config_dict: dict[str, Any] = {} + if quiet: answers = get_default_answers() if template: @@ -867,7 +869,6 @@ async def new( if template: echo(f'Using template `{template}`\n') - config_dict: dict[str, Any] | None = {} else: template, config_dict = template_from_terminal(answers['package']) From 58e1c477b21905d9175d6b049d7b2ff199e71494 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 18 Feb 2025 07:12:40 -0300 Subject: [PATCH 11/14] default order --- src/dipdup/cli.py | 2 +- src/dipdup/project.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index de16b8048..9e210da44 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -852,7 +852,7 @@ async def new( from dipdup.project import render_project from dipdup.project import template_from_terminal - config_dict: dict[str, Any] = {} + config_dict: dict[str, Any] | None = None if quiet: answers = get_default_answers() diff --git a/src/dipdup/project.py b/src/dipdup/project.py index ae5618053..5fa5eba4c 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -163,13 +163,13 @@ def template_from_terminal(package: str) -> tuple[str | None, dict[str, Any] | N _, mode = prompt_anyof( question='How would you like to set up your new DipDup project?', options=( - 'Interactively', 'From template', + 'Interactively', 'Blank', ), comments=( - 'Guided setup with prompts', 'Use one of demo projects', + 'Guided setup with prompts', 'Begin with an empty project', ), default=0, From fcb2152d925718002cd95e910e7441c9b506606a Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 18 Feb 2025 15:07:00 -0300 Subject: [PATCH 12/14] Monad WIP --- CHANGELOG.md | 3 + docs/10.supported-networks/31.mode.md | 10 ++-- docs/10.supported-networks/32.monad.md | 23 ++++++++ .../{32.moonbeam.md => 33.moonbeam.md} | 0 .../{33.neon.md => 34.neon.md} | 0 .../{34.opbnb.md => 35.opbnb.md} | 0 .../{35.optimism.md => 36.optimism.md} | 0 .../{36.peaq.md => 37.peaq.md} | 0 .../{37.polygon.md => 38.polygon.md} | 0 .../{38.prom.md => 39.prom.md} | 0 .../{39.scroll.md => 40.scroll.md} | 0 .../{40.shibarium.md => 41.shibarium.md} | 0 .../{41.shibuya.md => 42.shibuya.md} | 0 .../{42.shiden.md => 43.shiden.md} | 0 .../{43.scale.md => 44.scale.md} | 0 .../{44.sonic.md => 45.sonic.md} | 0 .../{45.taiko.md => 46.taiko.md} | 0 .../{46.tanssi.md => 47.tanssi.md} | 0 .../{47.x1.md => 48.x1.md} | 0 .../{48.x-layer.md => 49.x-layer.md} | 0 .../{49.zksync.md => 50.zksync.md} | 0 .../{50.zora.md => 51.zora.md} | 0 docs/7.references/2.config.md | 2 +- schemas/dipdup-3.0.json | 58 +++++++++++++++++++ src/dipdup/codegen/evm.py | 7 ++- src/dipdup/config/__init__.py | 4 ++ src/dipdup/config/evm.py | 10 +++- src/dipdup/config/evm_blockvision.py | 30 ++++++++++ src/dipdup/config/evm_sourcify.py | 32 ++++++++++ src/dipdup/context.py | 14 ++++- src/dipdup/datasources/__init__.py | 6 ++ src/dipdup/datasources/evm_blockvision.py | 20 +++++++ src/dipdup/datasources/evm_sourcify.py | 21 +++++++ src/dipdup/indexes/evm_node.py | 2 +- 34 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 docs/10.supported-networks/32.monad.md rename docs/10.supported-networks/{32.moonbeam.md => 33.moonbeam.md} (100%) rename docs/10.supported-networks/{33.neon.md => 34.neon.md} (100%) rename docs/10.supported-networks/{34.opbnb.md => 35.opbnb.md} (100%) rename docs/10.supported-networks/{35.optimism.md => 36.optimism.md} (100%) rename docs/10.supported-networks/{36.peaq.md => 37.peaq.md} (100%) rename docs/10.supported-networks/{37.polygon.md => 38.polygon.md} (100%) rename docs/10.supported-networks/{38.prom.md => 39.prom.md} (100%) rename docs/10.supported-networks/{39.scroll.md => 40.scroll.md} (100%) rename docs/10.supported-networks/{40.shibarium.md => 41.shibarium.md} (100%) rename docs/10.supported-networks/{41.shibuya.md => 42.shibuya.md} (100%) rename docs/10.supported-networks/{42.shiden.md => 43.shiden.md} (100%) rename docs/10.supported-networks/{43.scale.md => 44.scale.md} (100%) rename docs/10.supported-networks/{44.sonic.md => 45.sonic.md} (100%) rename docs/10.supported-networks/{45.taiko.md => 46.taiko.md} (100%) rename docs/10.supported-networks/{46.tanssi.md => 47.tanssi.md} (100%) rename docs/10.supported-networks/{47.x1.md => 48.x1.md} (100%) rename docs/10.supported-networks/{48.x-layer.md => 49.x-layer.md} (100%) rename docs/10.supported-networks/{49.zksync.md => 50.zksync.md} (100%) rename docs/10.supported-networks/{50.zora.md => 51.zora.md} (100%) create mode 100644 src/dipdup/config/evm_blockvision.py create mode 100644 src/dipdup/config/evm_sourcify.py create mode 100644 src/dipdup/datasources/evm_blockvision.py create mode 100644 src/dipdup/datasources/evm_sourcify.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9973223..e8267741e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,13 @@ Releases prior to 7.0 has been removed from this file to declutter search result ### Added - cli: Rewritten interactive mode for `new` command. +- evm.blockvision: Added `evm.blockvision` datasource to fetch ABIs from Blockvision API. +- evm.sourcify: Added `evm.sourcify` datasource to fetch ABIs from Sourcify API. ### Fixed - coinbase: Fixed crash when using coinbase datasource. +- evm.node: Fixed crash when block range goes out of bounds. ## [8.2.0] - 2025-02-10 diff --git a/docs/10.supported-networks/31.mode.md b/docs/10.supported-networks/31.mode.md index 90b4c0f64..c01f14b6b 100644 --- a/docs/10.supported-networks/31.mode.md +++ b/docs/10.supported-networks/31.mode.md @@ -11,8 +11,8 @@ description: "Mode network support" Explorer: [Blockscout](https://explorer.mode.network/) -| datasource | status | URLs | -| -----------------:|:------------- | ------------------------------ | -| **evm.subsquid** | ๐Ÿค” not tested | `https://v2.archive.subsquid.io/network/mode-mainnet` | -| **evm.etherscan** | ๐Ÿค” not tested | | -| **evm.node** | ๐Ÿค” not tested | `https://mode.drpc.org`
`wss://mode.drpc.org` | +| datasource | status | URLs | +| -----------------:|:------------- | ----------------------------------------------------- | +| **evm.subsquid** | ๐Ÿค” not tested | `https://v2.archive.subsquid.io/network/mode-mainnet` | +| **evm.etherscan** | ๐Ÿค” not tested | | +| **evm.node** | ๐Ÿค” not tested | `https://mode.drpc.org`
`wss://mode.drpc.org` | diff --git a/docs/10.supported-networks/32.monad.md b/docs/10.supported-networks/32.monad.md new file mode 100644 index 000000000..ee6ae0d17 --- /dev/null +++ b/docs/10.supported-networks/32.monad.md @@ -0,0 +1,23 @@ +--- +title: "Monad" +description: "Monad network support" +--- + + + +# Monad + +{{ #include 10.supported-networks/_intro.md }} + +## Monad Testnet + +Chain ID: `10143` + +Explorer: [Blockvision](https://testnet.monadexplorer.com/) + +| datasource | status | URLs | +| -------------------:|:--------- | ------------------------------------------------------------------------- | +| **evm.subsquid** | ๐Ÿ”ด no API | N/A | +| **evm.sourcify** | ๐Ÿ”ด 500 | `https://sourcify-api-monad.blockvision.org` | +| **evm.blockvision** | ๐ŸŸข works | `https://monad-api.blockvision.org/testnet/api` | +| **evm.node** | ๐ŸŸข works | `https://testnet-rpc2.monad.xyz/52227f026fa8fac9e2014c58fbf5643369b3bfc6` | diff --git a/docs/10.supported-networks/32.moonbeam.md b/docs/10.supported-networks/33.moonbeam.md similarity index 100% rename from docs/10.supported-networks/32.moonbeam.md rename to docs/10.supported-networks/33.moonbeam.md diff --git a/docs/10.supported-networks/33.neon.md b/docs/10.supported-networks/34.neon.md similarity index 100% rename from docs/10.supported-networks/33.neon.md rename to docs/10.supported-networks/34.neon.md diff --git a/docs/10.supported-networks/34.opbnb.md b/docs/10.supported-networks/35.opbnb.md similarity index 100% rename from docs/10.supported-networks/34.opbnb.md rename to docs/10.supported-networks/35.opbnb.md diff --git a/docs/10.supported-networks/35.optimism.md b/docs/10.supported-networks/36.optimism.md similarity index 100% rename from docs/10.supported-networks/35.optimism.md rename to docs/10.supported-networks/36.optimism.md diff --git a/docs/10.supported-networks/36.peaq.md b/docs/10.supported-networks/37.peaq.md similarity index 100% rename from docs/10.supported-networks/36.peaq.md rename to docs/10.supported-networks/37.peaq.md diff --git a/docs/10.supported-networks/37.polygon.md b/docs/10.supported-networks/38.polygon.md similarity index 100% rename from docs/10.supported-networks/37.polygon.md rename to docs/10.supported-networks/38.polygon.md diff --git a/docs/10.supported-networks/38.prom.md b/docs/10.supported-networks/39.prom.md similarity index 100% rename from docs/10.supported-networks/38.prom.md rename to docs/10.supported-networks/39.prom.md diff --git a/docs/10.supported-networks/39.scroll.md b/docs/10.supported-networks/40.scroll.md similarity index 100% rename from docs/10.supported-networks/39.scroll.md rename to docs/10.supported-networks/40.scroll.md diff --git a/docs/10.supported-networks/40.shibarium.md b/docs/10.supported-networks/41.shibarium.md similarity index 100% rename from docs/10.supported-networks/40.shibarium.md rename to docs/10.supported-networks/41.shibarium.md diff --git a/docs/10.supported-networks/41.shibuya.md b/docs/10.supported-networks/42.shibuya.md similarity index 100% rename from docs/10.supported-networks/41.shibuya.md rename to docs/10.supported-networks/42.shibuya.md diff --git a/docs/10.supported-networks/42.shiden.md b/docs/10.supported-networks/43.shiden.md similarity index 100% rename from docs/10.supported-networks/42.shiden.md rename to docs/10.supported-networks/43.shiden.md diff --git a/docs/10.supported-networks/43.scale.md b/docs/10.supported-networks/44.scale.md similarity index 100% rename from docs/10.supported-networks/43.scale.md rename to docs/10.supported-networks/44.scale.md diff --git a/docs/10.supported-networks/44.sonic.md b/docs/10.supported-networks/45.sonic.md similarity index 100% rename from docs/10.supported-networks/44.sonic.md rename to docs/10.supported-networks/45.sonic.md diff --git a/docs/10.supported-networks/45.taiko.md b/docs/10.supported-networks/46.taiko.md similarity index 100% rename from docs/10.supported-networks/45.taiko.md rename to docs/10.supported-networks/46.taiko.md diff --git a/docs/10.supported-networks/46.tanssi.md b/docs/10.supported-networks/47.tanssi.md similarity index 100% rename from docs/10.supported-networks/46.tanssi.md rename to docs/10.supported-networks/47.tanssi.md diff --git a/docs/10.supported-networks/47.x1.md b/docs/10.supported-networks/48.x1.md similarity index 100% rename from docs/10.supported-networks/47.x1.md rename to docs/10.supported-networks/48.x1.md diff --git a/docs/10.supported-networks/48.x-layer.md b/docs/10.supported-networks/49.x-layer.md similarity index 100% rename from docs/10.supported-networks/48.x-layer.md rename to docs/10.supported-networks/49.x-layer.md diff --git a/docs/10.supported-networks/49.zksync.md b/docs/10.supported-networks/50.zksync.md similarity index 100% rename from docs/10.supported-networks/49.zksync.md rename to docs/10.supported-networks/50.zksync.md diff --git a/docs/10.supported-networks/50.zora.md b/docs/10.supported-networks/51.zora.md similarity index 100% rename from docs/10.supported-networks/50.zora.md rename to docs/10.supported-networks/51.zora.md diff --git a/docs/7.references/2.config.md b/docs/7.references/2.config.md index e41959e14..e453c14e9 100644 --- a/docs/7.references/2.config.md +++ b/docs/7.references/2.config.md @@ -17,7 +17,7 @@ description: "Config file reference"

  • spec_version (ToStr) โ€“ Version of config specification, currently always 3.0

  • package (str) โ€“ Name of indexerโ€™s Python package, existing or not

    • -

  • datasources (dict[str, CoinbaseDatasourceConfig | EvmEtherscanDatasourceConfig | HttpDatasourceConfig | IpfsDatasourceConfig | EvmSubsquidDatasourceConfig | EvmNodeDatasourceConfig | TzipMetadataDatasourceConfig | TezosTzktDatasourceConfig | StarknetSubsquidDatasourceConfig | StarknetNodeDatasourceConfig | SubstrateSubsquidDatasourceConfig | SubstrateSubscanDatasourceConfig | SubstrateNodeDatasourceConfig]) โ€“ Mapping of datasource aliases and datasource configs

    • +

  • datasources (dict[str, CoinbaseDatasourceConfig | EvmEtherscanDatasourceConfig | EvmBlockvisionDatasourceConfig | HttpDatasourceConfig | IpfsDatasourceConfig | EvmSubsquidDatasourceConfig | EvmNodeDatasourceConfig | TzipMetadataDatasourceConfig | TezosTzktDatasourceConfig | StarknetSubsquidDatasourceConfig | StarknetNodeDatasourceConfig | SubstrateSubsquidDatasourceConfig | SubstrateSubscanDatasourceConfig | SubstrateNodeDatasourceConfig]) โ€“ Mapping of datasource aliases and datasource configs

  • database (SqliteDatabaseConfig | PostgresDatabaseConfig) โ€“ Database config

  • runtimes (dict[str, SubstrateRuntimeConfig]) โ€“ Mapping of runtime aliases and runtime configs

  • contracts (dict[str, EvmContractConfig | TezosContractConfig | StarknetContractConfig]) โ€“ Mapping of contract aliases and contract configs

    • diff --git a/schemas/dipdup-3.0.json b/schemas/dipdup-3.0.json index b2df1ad24..5d48209c5 100644 --- a/schemas/dipdup-3.0.json +++ b/schemas/dipdup-3.0.json @@ -172,6 +172,55 @@ "EvmAddress": { "$ref": "#/$defs/Hex" }, + "EvmBlockvisionDatasourceConfig": { + "additionalProperties": false, + "description": "Blockvision datasource config", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "API key", + "title": "api_key" + }, + "http": { + "anyOf": [ + { + "$ref": "#/$defs/HttpConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "HTTP client configuration", + "title": "http" + }, + "kind": { + "const": "evm.blockvision", + "default": "evm.blockvision", + "description": "always 'evm.blockvision'", + "title": "kind", + "type": "string" + }, + "url": { + "$ref": "#/$defs/Url", + "description": "API URL", + "title": "url" + } + }, + "required": [ + "url" + ], + "title": "EvmBlockvisionDatasourceConfig", + "type": "object" + }, "EvmContractConfig": { "additionalProperties": false, "description": "EVM contract config", @@ -341,6 +390,9 @@ { "$ref": "#/$defs/EvmNodeDatasourceConfig" }, + { + "$ref": "#/$defs/EvmBlockvisionDatasourceConfig" + }, { "$ref": "#/$defs/EvmEtherscanDatasourceConfig" } @@ -566,6 +618,9 @@ { "$ref": "#/$defs/EvmNodeDatasourceConfig" }, + { + "$ref": "#/$defs/EvmBlockvisionDatasourceConfig" + }, { "$ref": "#/$defs/EvmEtherscanDatasourceConfig" } @@ -3057,6 +3112,9 @@ { "$ref": "#/$defs/EvmEtherscanDatasourceConfig" }, + { + "$ref": "#/$defs/EvmBlockvisionDatasourceConfig" + }, { "$ref": "#/$defs/HttpDatasourceConfig" }, diff --git a/src/dipdup/codegen/evm.py b/src/dipdup/codegen/evm.py index 1f0dbc8aa..410e56fbb 100644 --- a/src/dipdup/codegen/evm.py +++ b/src/dipdup/codegen/evm.py @@ -9,9 +9,11 @@ from dipdup.config import HandlerConfig from dipdup.config.evm import EvmContractConfig from dipdup.config.evm import EvmIndexConfig +from dipdup.config.evm_blockvision import EvmBlockvisionDatasourceConfig from dipdup.config.evm_etherscan import EvmEtherscanDatasourceConfig from dipdup.config.evm_events import EvmEventsHandlerConfig from dipdup.config.evm_events import EvmEventsIndexConfig +from dipdup.config.evm_sourcify import EvmSourcifyDatasourceConfig from dipdup.config.evm_transactions import EvmTransactionsHandlerConfig from dipdup.config.evm_transactions import EvmTransactionsIndexConfig from dipdup.datasources import AbiDatasource @@ -84,7 +86,10 @@ async def _fetch_abi(self, index_config: EvmIndexConfigU) -> None: { datasource_config.name: cast(AbiDatasource[Any], self._datasources[datasource_config.name]) for datasource_config in index_config.datasources - if isinstance(datasource_config, EvmEtherscanDatasourceConfig) + if isinstance( + datasource_config, + EvmEtherscanDatasourceConfig | EvmSourcifyDatasourceConfig | EvmBlockvisionDatasourceConfig, + ) }.values() ) diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 48a5a34b7..102fc8f14 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -1211,9 +1211,11 @@ def _set_names(self) -> None: # NOTE: Reimport to avoid circular imports from dipdup.config.coinbase import CoinbaseDatasourceConfig from dipdup.config.evm import EvmContractConfig +from dipdup.config.evm_blockvision import EvmBlockvisionDatasourceConfig from dipdup.config.evm_etherscan import EvmEtherscanDatasourceConfig from dipdup.config.evm_events import EvmEventsIndexConfig from dipdup.config.evm_node import EvmNodeDatasourceConfig +from dipdup.config.evm_sourcify import EvmSourcifyDatasourceConfig from dipdup.config.evm_subsquid import EvmSubsquidDatasourceConfig from dipdup.config.evm_transactions import EvmTransactionsIndexConfig from dipdup.config.http import HttpDatasourceConfig @@ -1249,6 +1251,8 @@ def _set_names(self) -> None: DatasourceConfigU = ( CoinbaseDatasourceConfig | EvmEtherscanDatasourceConfig + | EvmSourcifyDatasourceConfig + | EvmBlockvisionDatasourceConfig | HttpDatasourceConfig | IpfsDatasourceConfig | EvmSubsquidDatasourceConfig diff --git a/src/dipdup/config/evm.py b/src/dipdup/config/evm.py index 9b0e4714b..4ef3f8eaa 100644 --- a/src/dipdup/config/evm.py +++ b/src/dipdup/config/evm.py @@ -13,15 +13,23 @@ from dipdup.config import ContractConfig from dipdup.config import Hex from dipdup.config import IndexConfig +from dipdup.config.evm_blockvision import EvmBlockvisionDatasourceConfig from dipdup.config.evm_etherscan import EvmEtherscanDatasourceConfig from dipdup.config.evm_node import EvmNodeDatasourceConfig +from dipdup.config.evm_sourcify import EvmSourcifyDatasourceConfig from dipdup.config.evm_subsquid import EvmSubsquidDatasourceConfig from dipdup.exceptions import ConfigurationError EVM_ADDRESS_PREFIXES = ('0x',) EVM_ADDRESS_LENGTH = 42 -EvmDatasourceConfigU: TypeAlias = EvmSubsquidDatasourceConfig | EvmNodeDatasourceConfig | EvmEtherscanDatasourceConfig +EvmDatasourceConfigU: TypeAlias = ( + EvmSubsquidDatasourceConfig + | EvmNodeDatasourceConfig + | EvmSourcifyDatasourceConfig + | EvmBlockvisionDatasourceConfig + | EvmEtherscanDatasourceConfig +) def _validate_evm_address(v: str) -> str: diff --git a/src/dipdup/config/evm_blockvision.py b/src/dipdup/config/evm_blockvision.py new file mode 100644 index 000000000..7e3b3bfe1 --- /dev/null +++ b/src/dipdup/config/evm_blockvision.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +from typing import Literal + +from pydantic import ConfigDict +from pydantic.dataclasses import dataclass + +from dipdup.config import DatasourceConfig +from dipdup.config import HttpConfig +from dipdup.config import Url + +_logger = logging.getLogger(__name__) + + +@dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) +class EvmBlockvisionDatasourceConfig(DatasourceConfig): + """Blockvision datasource config + + :param kind: always 'evm.blockvision' + :param url: API URL + :param api_key: API key + :param http: HTTP client configuration + """ + + kind: Literal['evm.blockvision'] = 'evm.blockvision' + url: Url + api_key: str | None = None + + http: HttpConfig | None = None diff --git a/src/dipdup/config/evm_sourcify.py b/src/dipdup/config/evm_sourcify.py new file mode 100644 index 000000000..9345b6b94 --- /dev/null +++ b/src/dipdup/config/evm_sourcify.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import logging +from typing import Literal + +from pydantic import ConfigDict +from pydantic.dataclasses import dataclass + +from dipdup.config import DatasourceConfig +from dipdup.config import HttpConfig +from dipdup.config import Url + +_logger = logging.getLogger(__name__) + + +@dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) +class EvmSourcifyDatasourceConfig(DatasourceConfig): + """Sourcify datasource config + + :param kind: always 'evm.sourcify' + :param url: API URL + :param chain_id: Chain ID + :param api_key: API key + :param http: HTTP client configuration + """ + + kind: Literal['evm.sourcify'] = 'evm.sourcify' + url: Url = 'https://sourcify.dev/server' + chain_id: int + api_key: str | None = None + + http: HttpConfig | None = None diff --git a/src/dipdup/context.py b/src/dipdup/context.py index 46275f2cc..b3d76c355 100644 --- a/src/dipdup/context.py +++ b/src/dipdup/context.py @@ -45,8 +45,10 @@ from dipdup.datasources import Datasource from dipdup.datasources import IndexDatasource from dipdup.datasources.coinbase import CoinbaseDatasource +from dipdup.datasources.evm_blockvision import EvmBlockvisionDatasource from dipdup.datasources.evm_etherscan import EvmEtherscanDatasource from dipdup.datasources.evm_node import EvmNodeDatasource +from dipdup.datasources.evm_sourcify import EvmSourcifyDatasource from dipdup.datasources.evm_subsquid import EvmSubsquidDatasource from dipdup.datasources.http import HttpDatasource from dipdup.datasources.ipfs import IpfsDatasource @@ -526,9 +528,17 @@ def get_evm_etherscan_datasource(self, name: str) -> EvmEtherscanDatasource: # NOTE: Alias, remove in 9.0 get_abi_etherscan_datasource = get_evm_etherscan_datasource - def get_evm_datasource(self, name: str) -> EvmSubsquidDatasource | EvmNodeDatasource | EvmEtherscanDatasource: + def get_evm_datasource( + self, name: str + ) -> ( + EvmSubsquidDatasource + | EvmNodeDatasource + | EvmEtherscanDatasource + | EvmSourcifyDatasource + | EvmBlockvisionDatasource + ): """Get `evm` datasource by name""" - return self._get_datasource(name, EvmSubsquidDatasource, EvmNodeDatasource, EvmEtherscanDatasource) # type: ignore[return-value] + return self._get_datasource(name, EvmSubsquidDatasource, EvmNodeDatasource, EvmEtherscanDatasource, EvmSourcifyDatasource, EvmBlockvisionDatasource) # type: ignore[return-value] def get_starknet_datasource(self, name: str) -> StarknetSubsquidDatasource | StarknetNodeDatasource: """Get `starknet` datasource by name""" diff --git a/src/dipdup/datasources/__init__.py b/src/dipdup/datasources/__init__.py index f87e1aa92..fc9be659e 100644 --- a/src/dipdup/datasources/__init__.py +++ b/src/dipdup/datasources/__init__.py @@ -283,8 +283,10 @@ async def get_head_level(self) -> int: ... def create_datasource(config: DatasourceConfig) -> Datasource[Any]: from dipdup.config.coinbase import CoinbaseDatasourceConfig + from dipdup.config.evm_blockvision import EvmBlockvisionDatasourceConfig from dipdup.config.evm_etherscan import EvmEtherscanDatasourceConfig from dipdup.config.evm_node import EvmNodeDatasourceConfig + from dipdup.config.evm_sourcify import EvmSourcifyDatasourceConfig from dipdup.config.evm_subsquid import EvmSubsquidDatasourceConfig from dipdup.config.http import HttpDatasourceConfig from dipdup.config.ipfs import IpfsDatasourceConfig @@ -296,8 +298,10 @@ def create_datasource(config: DatasourceConfig) -> Datasource[Any]: from dipdup.config.tezos_tzkt import TezosTzktDatasourceConfig from dipdup.config.tzip_metadata import TzipMetadataDatasourceConfig from dipdup.datasources.coinbase import CoinbaseDatasource + from dipdup.datasources.evm_blockvision import EvmBlockvisionDatasource from dipdup.datasources.evm_etherscan import EvmEtherscanDatasource from dipdup.datasources.evm_node import EvmNodeDatasource + from dipdup.datasources.evm_sourcify import EvmSourcifyDatasource from dipdup.datasources.evm_subsquid import EvmSubsquidDatasource from dipdup.datasources.http import HttpDatasource from dipdup.datasources.ipfs import IpfsDatasource @@ -311,6 +315,8 @@ def create_datasource(config: DatasourceConfig) -> Datasource[Any]: by_config: dict[type[DatasourceConfig], type[Datasource[Any]]] = { EvmEtherscanDatasourceConfig: EvmEtherscanDatasource, + EvmSourcifyDatasourceConfig: EvmSourcifyDatasource, + EvmBlockvisionDatasourceConfig: EvmBlockvisionDatasource, CoinbaseDatasourceConfig: CoinbaseDatasource, TezosTzktDatasourceConfig: TezosTzktDatasource, TzipMetadataDatasourceConfig: TzipMetadataDatasource, diff --git a/src/dipdup/datasources/evm_blockvision.py b/src/dipdup/datasources/evm_blockvision.py new file mode 100644 index 000000000..faff07f5e --- /dev/null +++ b/src/dipdup/datasources/evm_blockvision.py @@ -0,0 +1,20 @@ +from typing import Any + +import orjson + +from dipdup.config.evm_blockvision import EvmBlockvisionDatasourceConfig +from dipdup.datasources import AbiDatasource + + +class EvmBlockvisionDatasource(AbiDatasource[EvmBlockvisionDatasourceConfig]): + + async def run(self) -> None: + pass + + async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: + response = await self.request( + 'get', + url='verifyContractV2/data', + params={'address': address}, + ) + return orjson.loads(response['result']['contractABI']) diff --git a/src/dipdup/datasources/evm_sourcify.py b/src/dipdup/datasources/evm_sourcify.py new file mode 100644 index 000000000..3b97e716c --- /dev/null +++ b/src/dipdup/datasources/evm_sourcify.py @@ -0,0 +1,21 @@ +from typing import Any + + +from dipdup.config.evm_sourcify import EvmSourcifyDatasourceConfig +from dipdup.datasources import AbiDatasource + + +class EvmSourcifyDatasource(AbiDatasource[EvmSourcifyDatasourceConfig]): + + async def run(self) -> None: + pass + + async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: + response = await self.request( + 'get', + url=f'v2/contract/{self._config.chain_id}/{address}', + params={ + 'fields': 'abi', + }, + ) + return response['abi'] diff --git a/src/dipdup/indexes/evm_node.py b/src/dipdup/indexes/evm_node.py index e55da16a6..2a24a8508 100644 --- a/src/dipdup/indexes/evm_node.py +++ b/src/dipdup/indexes/evm_node.py @@ -14,7 +14,7 @@ EVM_NODE_READAHEAD_LIMIT = 2500 MIN_BATCH_SIZE = 10 -MAX_BATCH_SIZE = 10000 +MAX_BATCH_SIZE = 500 BATCH_SIZE_UP = 1.1 BATCH_SIZE_DOWN = 0.65 From b8a453b92785617e3712105bb2a7c88e59f32d9d Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Wed, 19 Feb 2025 11:29:34 -0300 Subject: [PATCH 13/14] docs and jsonschema --- docs/1.getting-started/7.datasources.md | 22 ++++--- docs/2.indexes/_starknet.md | 2 +- docs/2.indexes/_substrate.md | 2 +- ...e_subsquid.md => 10.substrate_subsquid.md} | 0 .../{9.tezos_tzkt.md => 11.tezos_tzkt.md} | 0 ...0.tzip_metadata.md => 12.tzip_metadata.md} | 0 .../{11.coinbase.md => 13.coinbase.md} | 0 docs/3.datasources/{12.ipfs.md => 14.ipfs.md} | 0 docs/3.datasources/{13.http.md => 15.http.md} | 0 docs/3.datasources/3.evm_etherscan.md | 2 +- docs/3.datasources/4.evm_blockvision.md | 31 +++++++++ docs/3.datasources/5.evm_sourcify.md | 31 +++++++++ ...net_subsquid.md => 6.starknet_subsquid.md} | 0 ...{5.starknet_node.md => 7.starknet_node.md} | 0 ....substrate_node.md => 8.substrate_node.md} | 0 ...rate_subscan.md => 9.substrate_subscan.md} | 0 docs/7.references/2.config.md | 2 +- docs/9.release-notes/3.v8.0.md | 2 +- docs/9.release-notes/_8.0_changelog.md | 3 + schemas/dipdup-3.0.json | 64 +++++++++++++++++++ 20 files changed, 146 insertions(+), 15 deletions(-) rename docs/3.datasources/{8.substrate_subsquid.md => 10.substrate_subsquid.md} (100%) rename docs/3.datasources/{9.tezos_tzkt.md => 11.tezos_tzkt.md} (100%) rename docs/3.datasources/{10.tzip_metadata.md => 12.tzip_metadata.md} (100%) rename docs/3.datasources/{11.coinbase.md => 13.coinbase.md} (100%) rename docs/3.datasources/{12.ipfs.md => 14.ipfs.md} (100%) rename docs/3.datasources/{13.http.md => 15.http.md} (100%) create mode 100644 docs/3.datasources/4.evm_blockvision.md create mode 100644 docs/3.datasources/5.evm_sourcify.md rename docs/3.datasources/{4.starknet_subsquid.md => 6.starknet_subsquid.md} (100%) rename docs/3.datasources/{5.starknet_node.md => 7.starknet_node.md} (100%) rename docs/3.datasources/{6.substrate_node.md => 8.substrate_node.md} (100%) rename docs/3.datasources/{7.substrate_subscan.md => 9.substrate_subscan.md} (100%) diff --git a/docs/1.getting-started/7.datasources.md b/docs/1.getting-started/7.datasources.md index 4b08b3c87..78425952c 100644 --- a/docs/1.getting-started/7.datasources.md +++ b/docs/1.getting-started/7.datasources.md @@ -14,16 +14,18 @@ Index datasources, ones that can be attached to a specific index, are prefixed w | [evm.subsquid](../3.datasources/1.evm_subsquid.md) | โŸ  EVM-compatible | Subsquid Network API | | [evm.node](../3.datasources/2.evm_node.md) | โŸ  EVM-compatible | Ethereum node | | [evm.etherscan](../3.datasources/3.evm_etherscan.md) | โŸ  EVM-compatible | Provides ABIs for EVM contracts | -| [starknet.subsquid](../3.datasources/4.starknet_subsquid.md) | ๐Ÿบ Starknet | Subsquid Network API | -| [starknet.node](../3.datasources/5.starknet_node.md) | ๐Ÿบ Starknet | Starknet node | -| [substrate.node](../3.datasources/6.substrate_node.md) | ๐Ÿ”ฎ Substrate | Substrate node | -| [substrate.subscan](../3.datasources/7.substrate_subscan.md) | ๐Ÿ”ฎ Substrate | Provides pallet metadata for Substrate networks | -| [substrate.subsquid](../3.datasources/8.substrate_subsquid.md) | ๐Ÿ”ฎ Substrate | Subsquid Network API | -| [tezos.tzkt](../3.datasources/9.tezos_tzkt.md) | ๊œฉ Tezos | TzKT API | -| [tzip_metadata](../3.datasources/10.tzip_metadata.md) | ๊œฉ Tezos | TZIP-16 metadata | -| [coinbase](../3.datasources/11.coinbase.md) | any | Coinbase price feed | -| [ipfs](../3.datasources/12.ipfs.md) | any | IPFS gateway | -| [http](../3.datasources/13.http.md) | any | Generic HTTP API | +| [evm.blockvision](../3.datasources/4.evm_blockvision.md) | โŸ  EVM-compatible | Provides ABIs for EVM contracts | +| [evm.sourcify](../3.datasources/5.evm_sourcify.md) | โŸ  EVM-compatible | Provides ABIs for EVM contracts | +| [starknet.subsquid](../3.datasources/6.starknet_subsquid.md) | ๐Ÿบ Starknet | Subsquid Network API | +| [starknet.node](../3.datasources/7.starknet_node.md) | ๐Ÿบ Starknet | Starknet node | +| [substrate.node](../3.datasources/8.substrate_node.md) | ๐Ÿ”ฎ Substrate | Substrate node | +| [substrate.subscan](../3.datasources/9.substrate_subscan.md) | ๐Ÿ”ฎ Substrate | Provides pallet metadata for Substrate networks | +| [substrate.subsquid](../3.datasources/10.substrate_subsquid.md) | ๐Ÿ”ฎ Substrate | Subsquid Network API | +| [tezos.tzkt](../3.datasources/11.tezos_tzkt.md) | ๊œฉ Tezos | TzKT API | +| [tzip_metadata](../3.datasources/12.tzip_metadata.md) | ๊œฉ Tezos | TZIP-16 metadata | +| [coinbase](../3.datasources/13.coinbase.md) | any | Coinbase price feed | +| [ipfs](../3.datasources/14.ipfs.md) | any | IPFS gateway | +| [http](../3.datasources/15.http.md) | any | Generic HTTP API | ## Connection settings diff --git a/docs/2.indexes/_starknet.md b/docs/2.indexes/_starknet.md index f5f1a1f3c..98450bf68 100644 --- a/docs/2.indexes/_starknet.md +++ b/docs/2.indexes/_starknet.md @@ -1,4 +1,4 @@ ## Datasources -DipDup indexes for Starknet use [Subsquid Network](https://docs.subsquid.io/subsquid-network/overview/) as a main source of historical data. Starknet nodes are not required for DipDup to operate, but in future updates, it will be possible to use them to get the latest data (not yet in Subsquid Network) and realtime updates. See [starknet.subsquid](../3.datasources/4.starknet_subsquid.md) page for more info on how to configure the datasource. +DipDup indexes for Starknet use [Subsquid Network](https://docs.subsquid.io/subsquid-network/overview/) as a main source of historical data. Starknet nodes are not required for DipDup to operate, but in future updates, it will be possible to use them to get the latest data (not yet in Subsquid Network) and realtime updates. See [starknet.subsquid](../3.datasources/6.starknet_subsquid.md) page for more info on how to configure the datasource. diff --git a/docs/2.indexes/_substrate.md b/docs/2.indexes/_substrate.md index 3cc3f580d..ad4e2a8d1 100644 --- a/docs/2.indexes/_substrate.md +++ b/docs/2.indexes/_substrate.md @@ -1,6 +1,6 @@ ## Datasources -DipDup indexes for Substrate networks use [Subsquid Network](https://docs.subsquid.io/subsquid-network/overview/) as a main source of historical data. Substrate nodes are not required for DipDup to operate, but they can be used to get the latest data (not yet in Subsquid Network) and realtime updates. See [substrate.subsquid](../3.datasources/8.substrate_subsquid.md) and [substrate.node](../3.datasources/6.substrate_node.md) pages for more info on how to configure both datasources. +DipDup indexes for Substrate networks use [Subsquid Network](https://docs.subsquid.io/subsquid-network/overview/) as a main source of historical data. Substrate nodes are not required for DipDup to operate, but they can be used to get the latest data (not yet in Subsquid Network) and realtime updates. See [substrate.subsquid](../3.datasources/10.substrate_subsquid.md) and [substrate.node](../3.datasources/8.substrate_node.md) pages for more info on how to configure both datasources. For testing purposes, you can use EVM node as a single datasource, but indexing will be significantly slower. For production, it's recommended to use Subsquid Network as the main datasource and EVM node(s) as a secondary one. If there are multiple `substrate.node` datasources attached to index, DipDup will use random one for each request. diff --git a/docs/3.datasources/8.substrate_subsquid.md b/docs/3.datasources/10.substrate_subsquid.md similarity index 100% rename from docs/3.datasources/8.substrate_subsquid.md rename to docs/3.datasources/10.substrate_subsquid.md diff --git a/docs/3.datasources/9.tezos_tzkt.md b/docs/3.datasources/11.tezos_tzkt.md similarity index 100% rename from docs/3.datasources/9.tezos_tzkt.md rename to docs/3.datasources/11.tezos_tzkt.md diff --git a/docs/3.datasources/10.tzip_metadata.md b/docs/3.datasources/12.tzip_metadata.md similarity index 100% rename from docs/3.datasources/10.tzip_metadata.md rename to docs/3.datasources/12.tzip_metadata.md diff --git a/docs/3.datasources/11.coinbase.md b/docs/3.datasources/13.coinbase.md similarity index 100% rename from docs/3.datasources/11.coinbase.md rename to docs/3.datasources/13.coinbase.md diff --git a/docs/3.datasources/12.ipfs.md b/docs/3.datasources/14.ipfs.md similarity index 100% rename from docs/3.datasources/12.ipfs.md rename to docs/3.datasources/14.ipfs.md diff --git a/docs/3.datasources/13.http.md b/docs/3.datasources/15.http.md similarity index 100% rename from docs/3.datasources/13.http.md rename to docs/3.datasources/15.http.md diff --git a/docs/3.datasources/3.evm_etherscan.md b/docs/3.datasources/3.evm_etherscan.md index b23725564..baea4466b 100644 --- a/docs/3.datasources/3.evm_etherscan.md +++ b/docs/3.datasources/3.evm_etherscan.md @@ -22,7 +22,7 @@ datasources: api_key: ${ETHERSCAN_API_KEY:-''} ``` -During initialization, DipDup will use this datasource to fetch contract ABIs. If your config contains definitions for multiple networks, you can assign the datasource explicitly in `evm.subsquid` index definitions: +During initialization, DipDup will use this datasource to fetch contract ABIs. If your config contains definitions for multiple networks, you can assign the datasource explicitly in `evm` index definitions: ```yaml [dipdup.yaml] indexes: diff --git a/docs/3.datasources/4.evm_blockvision.md b/docs/3.datasources/4.evm_blockvision.md new file mode 100644 index 000000000..d567f58ca --- /dev/null +++ b/docs/3.datasources/4.evm_blockvision.md @@ -0,0 +1,31 @@ +--- +title: "Blockvision" +description: "Blockvision is multi-chain Node, Token, NFT, and DeFi API provider and data retrieval portal for developers. It provides a public API to fetch ABIs of verified contracts. DipDup can use its API to fetch ABIs for contracts being indexed." +network: "ethereum" +--- + +# Blockvision + +{{ #include 3.datasources/_evm_banner.md }} + +[Blockvision](https://docs.blockvision.org/reference/welcome-to-blockvision) is multi-chain Node, Token, NFT, and DeFi API provider and data retrieval portal for developers. It provides a public API to fetch ABIs of verified contracts. DipDup can use its API to fetch ABIs for contracts being indexed. + +To use this datasource, add the following section in config: + +```yaml [dipdup.yaml] +datasources: + blockvision: + kind: evm.blockvision + url: https://monad-api.blockvision.org/testnet/api +``` + +During initialization, DipDup will use this datasource to fetch contract ABIs. If your config contains definitions for multiple networks, you can assign the datasource explicitly in `evm` index definitions: + +```yaml [dipdup.yaml] +indexes: + evm_events: + kind: evm.events + datasources: + - blockvision + ... +``` diff --git a/docs/3.datasources/5.evm_sourcify.md b/docs/3.datasources/5.evm_sourcify.md new file mode 100644 index 000000000..db47a86cb --- /dev/null +++ b/docs/3.datasources/5.evm_sourcify.md @@ -0,0 +1,31 @@ +--- +title: "Sourcify" +description: "Sourcify is a source-code verification service for Ethereum smart contracts supporting Solidity and Vyper. It provides a public API to fetch ABIs of verified contracts. DipDup can use its API to fetch ABIs for contracts being indexed." +network: "ethereum" +--- + +# Sourcify + +{{ #include 3.datasources/_evm_banner.md }} + +[Sourcify](https://docs.sourcify.dev/docs/intro/index.html) is a source-code verification service for Ethereum smart contracts supporting Solidity and Vyper. It provides a public API to fetch ABIs of verified contracts. DipDup can use its API to fetch ABIs for contracts being indexed. + +To use this datasource, add the following section in config: + +```yaml [dipdup.yaml] +datasources: + sourcify: + kind: evm.sourcify + chain_id: 1 # Ethereum mainnet +``` + +During initialization, DipDup will use this datasource to fetch contract ABIs. If your config contains definitions for multiple networks, you can assign the datasource explicitly in `evm` index definitions: + +```yaml [dipdup.yaml] +indexes: + evm_events: + kind: evm.events + datasources: + - sourcify + ... +``` diff --git a/docs/3.datasources/4.starknet_subsquid.md b/docs/3.datasources/6.starknet_subsquid.md similarity index 100% rename from docs/3.datasources/4.starknet_subsquid.md rename to docs/3.datasources/6.starknet_subsquid.md diff --git a/docs/3.datasources/5.starknet_node.md b/docs/3.datasources/7.starknet_node.md similarity index 100% rename from docs/3.datasources/5.starknet_node.md rename to docs/3.datasources/7.starknet_node.md diff --git a/docs/3.datasources/6.substrate_node.md b/docs/3.datasources/8.substrate_node.md similarity index 100% rename from docs/3.datasources/6.substrate_node.md rename to docs/3.datasources/8.substrate_node.md diff --git a/docs/3.datasources/7.substrate_subscan.md b/docs/3.datasources/9.substrate_subscan.md similarity index 100% rename from docs/3.datasources/7.substrate_subscan.md rename to docs/3.datasources/9.substrate_subscan.md diff --git a/docs/7.references/2.config.md b/docs/7.references/2.config.md index e453c14e9..a3c70946c 100644 --- a/docs/7.references/2.config.md +++ b/docs/7.references/2.config.md @@ -17,7 +17,7 @@ description: "Config file reference"

    • spec_version (ToStr) โ€“ Version of config specification, currently always 3.0

    • package (str) โ€“ Name of indexerโ€™s Python package, existing or not

      • -

    • datasources (dict[str, CoinbaseDatasourceConfig | EvmEtherscanDatasourceConfig | EvmBlockvisionDatasourceConfig | HttpDatasourceConfig | IpfsDatasourceConfig | EvmSubsquidDatasourceConfig | EvmNodeDatasourceConfig | TzipMetadataDatasourceConfig | TezosTzktDatasourceConfig | StarknetSubsquidDatasourceConfig | StarknetNodeDatasourceConfig | SubstrateSubsquidDatasourceConfig | SubstrateSubscanDatasourceConfig | SubstrateNodeDatasourceConfig]) โ€“ Mapping of datasource aliases and datasource configs

      • +

    • datasources (dict[str, CoinbaseDatasourceConfig | EvmEtherscanDatasourceConfig | EvmSourcifyDatasourceConfig | EvmBlockvisionDatasourceConfig | HttpDatasourceConfig | IpfsDatasourceConfig | EvmSubsquidDatasourceConfig | EvmNodeDatasourceConfig | TzipMetadataDatasourceConfig | TezosTzktDatasourceConfig | StarknetSubsquidDatasourceConfig | StarknetNodeDatasourceConfig | SubstrateSubsquidDatasourceConfig | SubstrateSubscanDatasourceConfig | SubstrateNodeDatasourceConfig]) โ€“ Mapping of datasource aliases and datasource configs

    • database (SqliteDatabaseConfig | PostgresDatabaseConfig) โ€“ Database config

    • runtimes (dict[str, SubstrateRuntimeConfig]) โ€“ Mapping of runtime aliases and runtime configs

    • contracts (dict[str, EvmContractConfig | TezosContractConfig | StarknetContractConfig]) โ€“ Mapping of contract aliases and contract configs

      • diff --git a/docs/9.release-notes/3.v8.0.md b/docs/9.release-notes/3.v8.0.md index 487129458..b9220b50b 100644 --- a/docs/9.release-notes/3.v8.0.md +++ b/docs/9.release-notes/3.v8.0.md @@ -17,7 +17,7 @@ GM, Starknet! ๐Ÿบ [Starknet](https://docs.starknet.io/) is a permissionless zero-knowledge (ZK) rollup on Ethereum, allowing developers to scale their dapps without compromising on security and composability of the Ethereum ecosystem. -We welcome Starknet to the large family of supported networks! DipDup 8.0 introduces a new index kind [`starknet.events`](../2.indexes/3.starknet_events.md) and new datasources [`starknet.subsquid`](../3.datasources/4.starknet_subsquid.md), [`starknet.node`](../3.datasources/5.starknet_node.md) to work with Starknet events. +We welcome Starknet to the large family of supported networks! DipDup 8.0 introduces a new index kind [`starknet.events`](../2.indexes/3.starknet_events.md) and new datasources [`starknet.subsquid`](../3.datasources/6.starknet_subsquid.md), [`starknet.node`](../3.datasources/7.starknet_node.md) to work with Starknet events. Starknet smart contracts are written in [Cairo](https://github.com/starkware-libs/cairo) language. It's not EVM-compatible, but many concepts are similar. To start indexing Starknet events, you need to add a new index definition to the project config, then place the contract ABI to `abi//cairo_abi.json` file and run `dipdup init` command to generate Python types and handler stubs. You can use [Starkscan](https://starkscan.co/contract/0x068f5c6a61780768455de69077e07e89787839bf8166decfbf92b645209c0fb8#class-code-history) explorer to get the ABI and other information about the contract. We are going to add support for automatic fetching ABIs from node in the future. diff --git a/docs/9.release-notes/_8.0_changelog.md b/docs/9.release-notes/_8.0_changelog.md index afde020c6..b3e0b862e 100644 --- a/docs/9.release-notes/_8.0_changelog.md +++ b/docs/9.release-notes/_8.0_changelog.md @@ -15,6 +15,8 @@ - env: Added `DIPDUP_JSON_LOG` environment variable to enable JSON logging. - env: Added `DIPDUP_LOW_MEMORY` variable to reduce the size of caches and buffers. - env: Added `DIPDUP_PACKAGE_PATH` environment variable to override discovered package path. +- evm.blockvision: Added `evm.blockvision` datasource to fetch ABIs from Blockvision API. +- evm.sourcify: Added `evm.sourcify` datasource to fetch ABIs from Sourcify API. - package: Added built-in `batch` handler to modify higher-level indexing logic. - starknet.events: Added `starknet.events` index kind to process Starknet events. - starknet.node: Added Starknet node datasource for last mile indexing. @@ -38,6 +40,7 @@ - database: Fixed concurrency issue when using `get_or_create` method. - evm.events: Fixed matching logs when filtering by topic0. - evm.events: Improve fetching event batches from node. +- evm.node: Fixed crash when block range goes out of bounds. - evm.subsquid: Fixed typo in `iter_events` method name. - evm: Fixed crash when contract ABI contains overloaded methods. - install: Fixed reinstalling package when `--force` flag is used. diff --git a/schemas/dipdup-3.0.json b/schemas/dipdup-3.0.json index 5d48209c5..a88b398a0 100644 --- a/schemas/dipdup-3.0.json +++ b/schemas/dipdup-3.0.json @@ -390,6 +390,9 @@ { "$ref": "#/$defs/EvmNodeDatasourceConfig" }, + { + "$ref": "#/$defs/EvmSourcifyDatasourceConfig" + }, { "$ref": "#/$defs/EvmBlockvisionDatasourceConfig" }, @@ -491,6 +494,61 @@ "title": "EvmNodeDatasourceConfig", "type": "object" }, + "EvmSourcifyDatasourceConfig": { + "additionalProperties": false, + "description": "Sourcify datasource config", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "API key", + "title": "api_key" + }, + "chain_id": { + "description": "Chain ID", + "title": "chain_id", + "type": "integer" + }, + "http": { + "anyOf": [ + { + "$ref": "#/$defs/HttpConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "HTTP client configuration", + "title": "http" + }, + "kind": { + "const": "evm.sourcify", + "default": "evm.sourcify", + "description": "always 'evm.sourcify'", + "title": "kind", + "type": "string" + }, + "url": { + "$ref": "#/$defs/Url", + "default": "https://sourcify.dev/server", + "description": "API URL", + "title": "url" + } + }, + "required": [ + "chain_id" + ], + "title": "EvmSourcifyDatasourceConfig", + "type": "object" + }, "EvmSubsquidDatasourceConfig": { "additionalProperties": false, "description": "Subsquid datasource config", @@ -618,6 +676,9 @@ { "$ref": "#/$defs/EvmNodeDatasourceConfig" }, + { + "$ref": "#/$defs/EvmSourcifyDatasourceConfig" + }, { "$ref": "#/$defs/EvmBlockvisionDatasourceConfig" }, @@ -3112,6 +3173,9 @@ { "$ref": "#/$defs/EvmEtherscanDatasourceConfig" }, + { + "$ref": "#/$defs/EvmSourcifyDatasourceConfig" + }, { "$ref": "#/$defs/EvmBlockvisionDatasourceConfig" }, From fffb14d238be45be307f4a9660d814af815b1f84 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Wed, 19 Feb 2025 11:31:48 -0300 Subject: [PATCH 14/14] lint --- src/dipdup/datasources/evm_blockvision.py | 3 ++- src/dipdup/datasources/evm_sourcify.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dipdup/datasources/evm_blockvision.py b/src/dipdup/datasources/evm_blockvision.py index faff07f5e..4763d546b 100644 --- a/src/dipdup/datasources/evm_blockvision.py +++ b/src/dipdup/datasources/evm_blockvision.py @@ -1,4 +1,5 @@ from typing import Any +from typing import cast import orjson @@ -17,4 +18,4 @@ async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: url='verifyContractV2/data', params={'address': address}, ) - return orjson.loads(response['result']['contractABI']) + return cast(list[Any], orjson.loads(response['result']['contractABI'])) diff --git a/src/dipdup/datasources/evm_sourcify.py b/src/dipdup/datasources/evm_sourcify.py index 3b97e716c..99c28977f 100644 --- a/src/dipdup/datasources/evm_sourcify.py +++ b/src/dipdup/datasources/evm_sourcify.py @@ -1,5 +1,5 @@ from typing import Any - +from typing import cast from dipdup.config.evm_sourcify import EvmSourcifyDatasourceConfig from dipdup.datasources import AbiDatasource @@ -18,4 +18,4 @@ async def get_abi(self, address: str) -> dict[str, Any] | list[Any]: 'fields': 'abi', }, ) - return response['abi'] + return cast(list[Any], response['abi'])