diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4b01189..2c9973223 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic Releases prior to 7.0 has been removed from this file to declutter search results; see the [archived copy](https://github.com/dipdup-io/dipdup/blob/8.0.0b5/CHANGELOG.md) for the full list. +## [Unreleased] + +### Added + +- cli: Rewritten interactive mode for `new` command. + +### Fixed + +- coinbase: Fixed crash when using coinbase datasource. + ## [8.2.0] - 2025-02-10 ### Added diff --git a/docs/9.release-notes/_8.0_changelog.md b/docs/9.release-notes/_8.0_changelog.md index e0555feed..afde020c6 100644 --- a/docs/9.release-notes/_8.0_changelog.md +++ b/docs/9.release-notes/_8.0_changelog.md @@ -9,6 +9,7 @@ - cli: Added `package verify` command to check the package consistency. - cli: Added full project migration support for 3.0 spec. - cli: Added spec_version 3.0 support to `migrate` command. +- cli: Rewritten interactive mode for `new` command. - config: Publish JSON schemas for config validation and autocompletion. - database: Added `dipdup_status` view to the schema. - env: Added `DIPDUP_JSON_LOG` environment variable to enable JSON logging. @@ -30,6 +31,7 @@ - cli: Fixed progress estimation when there are indexes with `last_level` option set. - cli: Import some dependencies on demand to reduce memory footprint. - cli: Improved logging of indexer status. +- coinbase: Fixed crash when using coinbase datasource. - config: Allow `sentry.dsn` to be empty string. - config: Fixed (de)serialization of hex strings in config. - config: Fixed setting logging levels according to the config. diff --git a/requirements.txt b/requirements.txt index 77551c887..8dd56fada 100644 --- a/requirements.txt +++ b/requirements.txt @@ -172,9 +172,9 @@ cytoolz==1.0.1 ; implementation_name == 'cpython' \ --hash=sha256:c8231b9abbd8e368e036f4cc2e16902c9482d4cf9e02a6147ed0e9a3cd4a9ab0 \ --hash=sha256:fb988c333f05ee30ad4693fe4da55d95ec0bb05775d2b60191236493ea2e01f9 \ --hash=sha256:fcb8f7d0d65db1269022e7e0428471edee8c937bc288ebdcb72f13eaa67c2fe4 -datamodel-code-generator==0.27.3 \ - --hash=sha256:01e928c00b800aec8d2ee77b5d4b47e1bc159a3a1c32f0f405df0a442d9ab5e7 \ - --hash=sha256:ddef49e66e2b90a4c9b238f6ce42dc5a2a23f6ab1b8370eaca08576777921e43 +datamodel-code-generator==0.28.1 \ + --hash=sha256:1ff8a56f9550a82bcba3e1ad7ebdb89bc655eeabbc4bc6acfb05977cbdc6381c \ + --hash=sha256:37ef5f3b488f7d7a3f0b5b3ba0f2bc1ae01bab4dc7e0f6b99ff6c40713a6beb3 dictdiffer==0.9.0 \ --hash=sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578 \ --hash=sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595 @@ -530,9 +530,9 @@ toolz==1.0.0 ; implementation_name == 'cpython' or implementation_name == 'pypy' tortoise-orm==0.24.0 \ --hash=sha256:ae0704a93ea27931724fc899e57268c8081afce3b32b110b00037ec206553e7d \ --hash=sha256:ee3b72b226767293b24c5c4906ae5f027d7cc84496cd503352c918564b4fd687 -typeguard==4.4.1 \ - --hash=sha256:0d22a89d00b453b47c49875f42b6601b961757541a2e1e0ef517b6e24213c21b \ - --hash=sha256:9324ec07a27ec67fc54a9c063020ca4c0ae6abad5e9f0f9804ca59aee68c6e21 +typeguard==4.4.2 \ + --hash=sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c \ + --hash=sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49 types-requests==2.32.0.20241016 \ --hash=sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95 \ --hash=sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747 diff --git a/schemas/dipdup-3.0.json b/schemas/dipdup-3.0.json index 04dc08524..b2df1ad24 100644 --- a/schemas/dipdup-3.0.json +++ b/schemas/dipdup-3.0.json @@ -134,6 +134,7 @@ }, "kind": { "const": "coinbase", + "default": "coinbase", "description": "always 'coinbase'", "title": "kind", "type": "string" @@ -165,9 +166,6 @@ "title": "secret_key" } }, - "required": [ - "kind" - ], "title": "CoinbaseDatasourceConfig", "type": "object" }, @@ -212,6 +210,7 @@ }, "kind": { "const": "evm", + "default": "evm", "description": "Always `evm`", "title": "kind", "type": "string" @@ -230,9 +229,6 @@ "title": "typename" } }, - "required": [ - "kind" - ], "title": "EvmContractConfig", "type": "object" }, @@ -277,17 +273,17 @@ "type": "string" } ], + "default": "evm.etherscan", "description": "always 'evm.etherscan'", "title": "kind" }, "url": { + "$ref": "#/$defs/Url", "description": "API URL", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "url" ], "title": "EvmEtherscanDatasourceConfig", @@ -369,6 +365,7 @@ }, "kind": { "const": "evm.events", + "default": "evm.events", "description": "Always 'evm.events'", "title": "kind", "type": "string" @@ -381,7 +378,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -407,6 +403,7 @@ }, "kind": { "const": "evm.node", + "default": "evm.node", "description": "Always 'evm.node'", "title": "kind", "type": "string" @@ -437,7 +434,6 @@ } }, "required": [ - "kind", "url" ], "title": "EvmNodeDatasourceConfig", @@ -462,6 +458,7 @@ }, "kind": { "const": "evm.subsquid", + "default": "evm.subsquid", "description": "always 'evm.subsquid'", "title": "kind", "type": "string" @@ -473,7 +470,6 @@ } }, "required": [ - "kind", "url" ], "title": "EvmSubsquidDatasourceConfig", @@ -594,6 +590,7 @@ }, "kind": { "const": "evm.transactions", + "default": "evm.transactions", "description": "always 'evm.transactions'", "title": "kind", "type": "string" @@ -606,7 +603,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -957,18 +953,18 @@ }, "kind": { "const": "http", + "default": "http", "description": "always 'http'", "title": "kind", "type": "string" }, "url": { + "$ref": "#/$defs/Url", "description": "URL to fetch data from", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "url" ], "title": "HttpDatasourceConfig", @@ -984,6 +980,13 @@ "title": "first_level", "type": "integer" }, + "kind": { + "const": "template", + "default": "template", + "description": "always 'template'", + "title": "kind", + "type": "string" + }, "last_level": { "default": 0, "description": "Level to stop indexing at", @@ -1027,20 +1030,18 @@ }, "kind": { "const": "ipfs", + "default": "ipfs", "description": "always 'ipfs'", "title": "kind", "type": "string" }, "url": { + "$ref": "#/$defs/Url", "default": "https://ipfs.io/ipfs", "description": "IPFS node URL, e.g. https://ipfs.io/ipfs/", - "title": "url", - "type": "string" + "title": "url" } }, - "required": [ - "kind" - ], "title": "IpfsDatasourceConfig", "type": "object" }, @@ -1136,6 +1137,7 @@ }, "kind": { "const": "postgres", + "default": "postgres", "description": "always 'postgres'", "title": "kind", "type": "string" @@ -1166,7 +1168,6 @@ } }, "required": [ - "kind", "host" ], "title": "PostgresDatabaseConfig", @@ -1324,6 +1325,7 @@ }, "kind": { "const": "sqlite", + "default": "sqlite", "description": "always 'sqlite'", "title": "kind", "type": "string" @@ -1335,9 +1337,6 @@ "type": "string" } }, - "required": [ - "kind" - ], "title": "SqliteDatabaseConfig", "type": "object" }, @@ -1382,6 +1381,7 @@ }, "kind": { "const": "starknet", + "default": "starknet", "description": "Always `starknet`", "title": "kind", "type": "string" @@ -1400,9 +1400,6 @@ "title": "typename" } }, - "required": [ - "kind" - ], "title": "StarknetContractConfig", "type": "object" }, @@ -1479,6 +1476,7 @@ }, "kind": { "const": "starknet.events", + "default": "starknet.events", "description": "Always 'starknet.events'", "title": "kind", "type": "string" @@ -1491,7 +1489,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -1517,6 +1514,7 @@ }, "kind": { "const": "starknet.node", + "default": "starknet.node", "description": "Always 'starknet.node'", "title": "kind", "type": "string" @@ -1547,7 +1545,6 @@ } }, "required": [ - "kind", "url" ], "title": "StarknetNodeDatasourceConfig", @@ -1572,6 +1569,7 @@ }, "kind": { "const": "starknet.subsquid", + "default": "starknet.subsquid", "description": "always 'starknet.subsquid'", "title": "kind", "type": "string" @@ -1583,7 +1581,6 @@ } }, "required": [ - "kind", "url" ], "title": "StarknetSubsquidDatasourceConfig", @@ -1652,6 +1649,7 @@ }, "kind": { "const": "substrate.events", + "default": "substrate.events", "description": "Always 'substrate.events'", "title": "kind", "type": "string" @@ -1676,7 +1674,6 @@ } }, "required": [ - "kind", "datasources", "runtime", "handlers" @@ -1703,6 +1700,7 @@ }, "kind": { "const": "substrate.node", + "default": "substrate.node", "description": "Always 'substrate.node'", "title": "kind", "type": "string" @@ -1727,7 +1725,6 @@ } }, "required": [ - "kind", "url" ], "title": "SubstrateNodeDatasourceConfig", @@ -1793,18 +1790,18 @@ }, "kind": { "const": "substrate.subscan", + "default": "substrate.subscan", "description": "always 'substrate.subscan'", "title": "kind", "type": "string" }, "url": { + "$ref": "#/$defs/Url", "description": "API URL", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "url" ], "title": "SubstrateSubscanDatasourceConfig", @@ -1829,6 +1826,7 @@ }, "kind": { "const": "substrate.subsquid", + "default": "substrate.subsquid", "description": "always 'substrate.subsquid'", "title": "kind", "type": "string" @@ -1840,7 +1838,6 @@ } }, "required": [ - "kind", "url" ], "title": "SubstrateSubsquidDatasourceConfig", @@ -1919,6 +1916,7 @@ }, "kind": { "const": "tezos.big_maps", + "default": "tezos.big_maps", "description": "always 'tezos.big_maps'", "title": "kind", "type": "string" @@ -1937,7 +1935,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -1979,6 +1976,7 @@ }, "kind": { "const": "tezos", + "default": "tezos", "description": "Always `tezos`", "title": "kind", "type": "string" @@ -1997,9 +1995,6 @@ "title": "typename" } }, - "required": [ - "kind" - ], "title": "TezosContractConfig", "type": "object" }, @@ -2080,6 +2075,7 @@ }, "kind": { "const": "tezos.events", + "default": "tezos.events", "description": "always 'tezos.events'", "title": "kind", "type": "string" @@ -2092,7 +2088,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2154,13 +2149,13 @@ }, "kind": { "const": "tezos.head", + "default": "tezos.head", "description": "always 'tezos.head'", "title": "kind", "type": "string" } }, "required": [ - "kind", "datasources", "callback" ], @@ -2550,6 +2545,7 @@ }, "kind": { "const": "tezos.operations", + "default": "tezos.operations", "description": "always 'tezos.operations'", "title": "kind", "type": "string" @@ -2573,7 +2569,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2612,6 +2607,7 @@ }, "kind": { "const": "tezos.operations_unfiltered", + "default": "tezos.operations_unfiltered", "description": "always 'tezos.operations_unfiltered'", "title": "kind", "type": "string" @@ -2635,7 +2631,6 @@ } }, "required": [ - "kind", "datasources", "callback" ], @@ -2722,6 +2717,7 @@ }, "kind": { "const": "tezos.token_balances", + "default": "tezos.token_balances", "description": "always 'tezos.token_balances'", "title": "kind", "type": "string" @@ -2734,7 +2730,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2853,6 +2848,7 @@ }, "kind": { "const": "tezos.token_transfers", + "default": "tezos.token_transfers", "description": "always 'tezos.token_transfers'", "title": "kind", "type": "string" @@ -2865,7 +2861,6 @@ } }, "required": [ - "kind", "datasources", "handlers" ], @@ -2897,6 +2892,7 @@ }, "kind": { "const": "tezos.tzkt", + "default": "tezos.tzkt", "description": "always 'tezos.tzkt'", "title": "kind", "type": "string" @@ -2920,9 +2916,6 @@ "title": "url" } }, - "required": [ - "kind" - ], "title": "TezosTzktDatasourceConfig", "type": "object" }, @@ -2955,6 +2948,7 @@ }, "kind": { "const": "tzip_metadata", + "default": "tzip_metadata", "description": "always 'tzip_metadata'", "title": "kind", "type": "string" @@ -2965,14 +2959,13 @@ "title": "network" }, "url": { + "$ref": "#/$defs/Url", "default": "https://metadata.dipdup.net", "description": "GraphQL API URL, e.g. https://metadata.dipdup.net", - "title": "url", - "type": "string" + "title": "url" } }, "required": [ - "kind", "network" ], "title": "TzipMetadataDatasourceConfig", diff --git a/src/dipdup/_survey.py b/src/dipdup/_survey.py deleted file mode 100644 index 5d88c16d6..000000000 --- a/src/dipdup/_survey.py +++ /dev/null @@ -1,503 +0,0 @@ -from typing import Any -from typing import TypedDict - -import survey # type: ignore[import-untyped] - -from dipdup.cli import echo -from dipdup.install import ask - - -class Pattern(TypedDict): - destination: str - entrypoint: str - - -class Handler(TypedDict): - name: str | None - callback: str | None - contract: str | None - path: str | None - tag: str | None - pattern: Pattern | None - to: str | None - method: str | None - - -class Datasource(TypedDict): - name: str - kind: str - url: str - ws_url: str | None - api_key: str | None - - -class Contract(TypedDict): - name: str - kind: str - address: str - typename: str - - -class Index(TypedDict): - name: str - kind: str - datasources: list[str] - handlers: list[Handler] | None - skip_history: str | None - first_level: int | None - last_level: int | None - callback: str | None - types: list[str] | None - contracts: list[str] | None - - -class DipDupSurveyConfig(TypedDict): - datasources: list[Datasource] - contracts: list[Contract] | None - indexes: list[Index] - - -class DatasourceConfig(TypedDict): - kind: str - requires_api_key: bool - default_url: str - name: str - - -class IndexerConfig(TypedDict): - handler_fields: list[str] - optional_fields: dict[str, str | int] - kind: str - - -class BlockchainConfig(TypedDict): - datasources: list[DatasourceConfig] - contract_kind: str - indexes: dict[str, IndexerConfig] - - -CONFIG_STRUCTURE: dict[str, BlockchainConfig] = { - 'evm': { - 'datasources': [ - { - 'kind': 'evm.subsquid', - 'requires_api_key': False, - 'default_url': 'https://v2.archive.subsquid.io/network/ethereum-mainnet', - 'name': 'subsquid', - }, - { - 'kind': 'evm.node', - 'requires_api_key': True, - 'default_url': 'https://eth-mainnet.g.alchemy.com/v2', - 'name': 'node', - }, - { - 'kind': 'evm.etherscan', - 'requires_api_key': True, - 'default_url': 'https://api.etherscan.io/api', - 'name': 'etherscan', - }, - ], - 'contract_kind': 'evm', - 'indexes': { - 'events': { - 'handler_fields': ['callback', 'contract', 'name'], - 'optional_fields': {}, - 'kind': 'evm.events', - }, - 'transactions': { - 'handler_fields': ['callback', 'to', 'method'], - 'optional_fields': { - 'first_level': 'integer', - }, - 'kind': 'evm.transactions', - }, - }, - }, - 'tezos': { - 'datasources': [ - { - 'kind': 'tezos.tzkt', - 'requires_api_key': False, - 'default_url': 'https://api.ghostnet.tzkt.io', - 'name': 'tzkt', - } - ], - 'contract_kind': 'tezos', - 'indexes': { - 'big_maps': { - 'handler_fields': ['callback', 'contract', 'path'], - 'optional_fields': { - 'skip_history': 'select', - }, - 'kind': 'tezos.big_maps', - }, - 'events': { - 'handler_fields': ['callback', 'contract', 'tag'], - 'optional_fields': {}, - 'kind': 'tezos.events', - }, - 'head': { - 'handler_fields': [], - 'optional_fields': { - 'callback': 'string', - }, - 'kind': 'tezos.head', - }, - 'operations': { - 'handler_fields': ['callback', 'pattern'], - 'optional_fields': {}, - 'kind': 'tezos.operations', - }, - 'operations_unfiltered': { - 'handler_fields': [], - 'optional_fields': { - 'types': 'select', - 'callback': 'string', - 'first_level': 'integer', - 'last_level': 'integer', - }, - 'kind': 'tezos.operations_unfiltered', - }, - 'token_balances': { - 'handler_fields': ['callback', 'contract'], - 'optional_fields': {}, - 'kind': 'tezos.token_balances', - }, - 'token_transfers': { - 'handler_fields': ['callback', 'contract'], - 'optional_fields': {}, - 'kind': 'tezos.token_transfers', - }, - }, - }, - 'starknet': { - 'datasources': [ - { - 'kind': 'starknet.subsquid', - 'requires_api_key': False, - 'default_url': 'https://v2.archive.subsquid.io/network/starknet-mainnet', - 'name': 'subsquid', - }, - { - 'kind': 'starknet.node', - 'requires_api_key': True, - 'default_url': 'https://starknet-mainnet.g.alchemy.com/v2', - 'name': 'node', - }, - ], - 'contract_kind': 'starknet', - 'indexes': { - 'events': { - 'handler_fields': ['callback', 'contract', 'name'], - 'optional_fields': {}, - 'kind': 'starknet.events', - } - }, - }, -} - - -def prompt_anyof( - question: str, - options: tuple[str, ...], - comments: tuple[str, ...], - default: int, -) -> tuple[int, str]: - """Ask user to choose one of the options; returns index and value""" - from tabulate import tabulate - - table = tabulate( - zip(options, comments, strict=False), - tablefmt='plain', - ) - index = survey.routines.select( - question + '\n', - options=table.split('\n'), - index=default, - ) - return index, options[index] - - -# Helper function to generate comments for datasource options -def get_datasource_comments(datasources: list[str]) -> tuple[str, ...]: - default_comments = { - 'evm.subsquid': 'Use Subsquid as your datasource for EVM.', - 'evm.node': 'Connect to an EVM node.', - 'evm.etherscan': 'Fetch ABI from Etherscan.', - 'tezos.tzkt': 'Use TzKT API for Tezos.', - 'starknet.subsquid': 'Use Subsquid for Starknet.', - 'starknet.node': 'Connect to a Starknet node.', - } - return tuple(default_comments.get(option, 'No description available') for option in datasources) - - -# Helper function to generate comments for indexes options -def get_index_comments(indexes: dict[str, IndexerConfig]) -> tuple[str, ...]: - default_comments: dict[str, str] = { - 'evm.events': 'Listen to EVM blockchain events.', - 'evm.transactions': 'Track EVM blockchain transactions.', - 'tezos.big_maps': 'Monitor changes in Tezos big maps.', - 'tezos.events': 'Track specific events in Tezos.', - 'tezos.head': 'Monitor Tezos chain head updates.', - 'tezos.operations': 'Track operations on Tezos blockchain.', - 'tezos.operations_unfiltered': 'Monitor unfiltered Tezos operations.', - 'tezos.token_balances': 'Track Tezos token balances.', - 'tezos.token_transfers': 'Monitor token transfers on Tezos.', - 'starknet.events': 'Listen to Starknet blockchain events.', - } - comments = [] - for _, config in indexes.items(): - kind = config.get('kind') - comment = default_comments.get(kind, 'No description available') # type: ignore[arg-type] - comments.append(comment) - return tuple(comments) - - -def query_handlers( - contract_names: list[str], - additional_fields: list[str] | None, -) -> list[Handler] | None: - - handlers: list[Handler] = [] - - if not additional_fields: - return None - - first = True - while first or ask('Add another handler?', False): - first = False - - handler: Handler = { - 'callback': '', - 'path': None, - 'tag': None, - 'pattern': None, - 'name': None, - 'contract': None, - 'to': None, - 'method': None, - } - - # Prompt for additional fields - for field in additional_fields: - if field in ['contract', 'to']: - handler[field] = prompt_anyof( # type: ignore[literal-required] - 'Choose contract for the handler', - tuple(contract_names), - ('Contract to listen to',) * len(contract_names), - 0, - )[1] - elif field == 'pattern': - handler['pattern'] = { - 'destination': prompt_anyof( - 'Choose contract for the handler', - tuple(contract_names), - ('Contract to listen to',) * len(contract_names), - 0, - )[1], - 'entrypoint': validate_non_empty_input( - survey.routines.input('Enter pattern entrypoint: ', value=''), 'entrypoint' - ), - } - else: - handler[field] = validate_non_empty_input( # type: ignore[literal-required] - survey.routines.input(f'Enter handler {field}: ', value=''), - field, - ) - - handlers.append(handler) - - return handlers - - -def query_optional_fields(optional_fields: dict[str, str | int]) -> dict[str, Any]: - field_values: dict[str, Any] = {} - for field, field_type in optional_fields.items(): - if field_type == 'select': - if field == 'types': - types: list[str] = [] - while True: - _, type = prompt_anyof( - 'Select operation type', - ('origination', 'transaction', 'migration'), - ('origination', 'transaction', 'migration'), - 0, - ) - types.append(type) - if not ask('Add another type?', False): - break - field_values[field] = types - else: - _, field_values[field] = prompt_anyof( - f'Select {field}', - ('never', 'always', 'auto'), - ('Never', 'Always', 'Auto'), - 0, - ) - else: - field_values[field] = survey.routines.input(f'Enter {field}: ') - - return field_values - - -def validate_non_empty_input(input_value: str, field_name: str) -> str: - while not input_value.strip(): - echo(f'{field_name} cannot be empty. Please enter a valid value.', fg='red') - input_value = survey.routines.input(f'Enter {field_name}: ') - return input_value - - -def filter_index_options( - indexes: tuple[str, ...], - index_comments: tuple[str, ...], - contracts: list[Contract], - blockchain: str, -) -> tuple[tuple[str, ...], tuple[str, ...]]: - blockchain_config = CONFIG_STRUCTURE[blockchain] - index_dict = blockchain_config['indexes'] - required_contract_fields = {'contract', 'to', 'pattern'} - if not contracts: - valid_indexes = [ - idx - for idx, index in enumerate(indexes) - if not required_contract_fields.intersection(set(index_dict[index].get('handler_fields', []))) or contracts - ] - - indexes = tuple(indexes[idx] for idx in valid_indexes) - index_comments = tuple(index_comments[idx] for idx in valid_indexes) - - return indexes, index_comments - - -def query_survey_config(blockchain: str) -> DipDupSurveyConfig: - blockchain_config = CONFIG_STRUCTURE[blockchain] - datasources: list[Datasource] = [] - contracts: list[Contract] = [] - indexes: list[Index] = [] - survey_config = DipDupSurveyConfig( - datasources=datasources, - contracts=contracts, - indexes=indexes, - ) - - first = True - while ask(f'Add {'' if first else 'another '}datasource?', first): - first = False - - datasource_options = blockchain_config['datasources'] - datasource_comments = get_datasource_comments([ds['kind'] for ds in datasource_options]) - - selected_index, _ = prompt_anyof( - f'Select the datasource kind for {blockchain}:', - tuple(ds['name'] for ds in datasource_options), - datasource_comments, - 0, - ) - selected_datasource = datasource_options[selected_index] - datasource_kind = selected_datasource['kind'] - - final_url = validate_non_empty_input( - survey.routines.input('Enter Datasource URL: ', value=selected_datasource.get('default_url', '')), - 'Datasource URL', - ) - api_key = None - ws_url = None - - if selected_datasource['requires_api_key']: - api_key = validate_non_empty_input( - survey.routines.input('Enter API key: ', value=''), - 'API key', - ) - if 'node' in datasource_kind: - final_url = '${NODE_URL:-' + final_url + '}/${NODE_API_KEY:-' + api_key + '}' - if datasource_kind == 'evm.node': - ws_url = survey.routines.input( - 'Enter WebSocket (wss) URL: ', value='wss://eth-mainnet.g.alchemy.com/v2' - ) - ws_url = '${NODE_WS_URL:-' + ws_url + '}/${NODE_API_KEY:-' + api_key + '}' - api_key = None - else: - api_key = '${ETHERSCAN_API_KEY:-' + api_key + '}' - - if datasource_kind != 'evm.etherscan': - api_key = None - - if 'subsquid' in datasource_kind: - final_url = '${SUBSQUID_URL:-' + final_url + '}' - - datasource: Datasource = { - 'kind': datasource_kind, - 'url': final_url, - 'ws_url': ws_url, - 'api_key': api_key, - 'name': selected_datasource['name'], - } - - datasources.append(datasource) - - first = True - while ask(f'Add {'' if first else 'another '}contract?', first): - first = False - - contract_name = validate_non_empty_input( - survey.routines.input('Enter contract name: '), - 'Contract name', - ) - contract_address = validate_non_empty_input( - survey.routines.input('Enter contract address: '), - 'Contract address', - ) - - contract: Contract = { - 'name': contract_name, - 'kind': blockchain_config['contract_kind'], - 'address': contract_address, - 'typename': contract_name, - } - contracts.append(contract) - - if not datasources: - return survey_config - - index_options = tuple(blockchain_config['indexes'].keys()) - index_comments = get_index_comments(blockchain_config['indexes']) - index_options, index_comments = filter_index_options(index_options, index_comments, contracts, blockchain) - - if not index_options: - return survey_config - - first = True - while ask(f'Add {'' if first else 'another '}index?', first): - first = False - - _, kind = prompt_anyof(f'Select the type of index for {blockchain}:', index_options, index_comments, 0) - - index_config = blockchain_config['indexes'][kind] - index_name = validate_non_empty_input( - survey.routines.input('Enter the index name: '), - 'Indexer name', - ) - - index: Index = { - 'name': index_name, - 'kind': index_config['kind'], - 'datasources': [ds['name'] for ds in datasources], - 'handlers': None, - 'skip_history': None, - 'first_level': None, - 'last_level': None, - 'callback': None, - 'types': None, - 'contracts': None, - } - - handlers = query_handlers([c['name'] for c in contracts], index_config.get('handler_fields')) - index['handlers'] = handlers - - if 'optional_fields' in index_config: - index.update(query_optional_fields(index_config['optional_fields'])) # type: ignore[typeddict-item] - - indexes.append(index) - - return survey_config diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 6eccc226a..9e210da44 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -27,6 +27,7 @@ from dipdup.install import EPILOG from dipdup.install import WELCOME_ASCII from dipdup.sys import set_up_process +from dipdup.yaml import DipDupYAMLConfig if TYPE_CHECKING: from dipdup.config import DipDupConfig @@ -849,6 +850,9 @@ async def new( from dipdup.project import answers_from_terminal from dipdup.project import get_default_answers from dipdup.project import render_project + from dipdup.project import template_from_terminal + + config_dict: dict[str, Any] | None = None if quiet: answers = get_default_answers() @@ -860,13 +864,30 @@ async def new( answers['template'] = template else: try: - answers = answers_from_terminal(template) + answers = answers_from_terminal() + answers['template'] = template or 'demo_blank' + + if template: + echo(f'Using template `{template}`\n') + else: + template, config_dict = template_from_terminal(answers['package']) + except Escape: return _logger.info('Rendering project') render_project(answers, force) + if config_dict: + # NOTE: Preserve the header at the top of the file + config_dict = { + 'package': answers['package'], + 'spec_version': '3.0', + **config_dict, + } + path = env.get_package_path(config_dict['package']) / ROOT_CONFIG + path.write_text(DipDupYAMLConfig(**config_dict).dump()) + _logger.info('Initializing project') config = DipDupConfig.load([Path(answers['package'])]) config.initialize() diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index a0cb103c9..48a5a34b7 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -30,8 +30,10 @@ from typing import Annotated from typing import Any from typing import Literal +from typing import Self from typing import TypeVar from typing import cast +from typing import get_args from urllib.parse import quote_plus import orjson @@ -47,8 +49,10 @@ from dipdup import __spec_version__ from dipdup import env from dipdup.config._mixin import CallbackMixin +from dipdup.config._mixin import InteractiveMixin from dipdup.config._mixin import NameMixin from dipdup.config._mixin import ParentMixin +from dipdup.config._mixin import TerminalOptions from dipdup.exceptions import ConfigInitializationException from dipdup.exceptions import ConfigurationError from dipdup.exceptions import IndexAlreadyExistsError @@ -71,6 +75,8 @@ def _valid_url(v: str, ws: bool) -> str: + if not v: + raise ConfigurationError('URL is required') if not ws and not v.startswith(('http://', 'https://')): raise ConfigurationError(f'`{v}` is not a valid HTTP URL') if ws and not v.startswith(('ws://', 'wss://')): @@ -100,7 +106,7 @@ class SqliteDatabaseConfig: :param immune_tables: List of tables to preserve during reindexing """ - kind: Literal['sqlite'] + kind: Literal['sqlite'] = 'sqlite' path: str = DEFAULT_SQLITE_PATH immune_tables: set[str] = Field(default_factory=set) @@ -139,7 +145,7 @@ class PostgresDatabaseConfig: :param connection_timeout: Connection timeout """ - kind: Literal['postgres'] + kind: Literal['postgres'] = 'postgres' host: str user: str = DEFAULT_POSTGRES_USER database: str = DEFAULT_POSTGRES_DATABASE @@ -291,6 +297,10 @@ class DatasourceConfig(ABC, NameMixin): ws_url: WsUrl | None = None http: HttpConfig | None = None + # @classmethod + # def from_terminal(cls, opts): + # return super().from_terminal(opts) + @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) class HandlerConfig(CallbackMixin, ParentMixin['IndexConfig']): @@ -316,7 +326,7 @@ class IndexTemplateConfig(NameMixin): """ - kind = 'template' + kind: Literal['template'] = 'template' template: str values: dict[str, Any] first_level: int = 0 @@ -561,7 +571,7 @@ class AdvancedConfig: @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) -class DipDupConfig: +class DipDupConfig(InteractiveMixin): """DipDup project configuration file :param spec_version: Version of config specification, currently always `3.0` @@ -586,9 +596,7 @@ class DipDupConfig: spec_version: ToStr package: str datasources: dict[str, DatasourceConfigU] = Field(default_factory=dict) - database: SqliteDatabaseConfig | PostgresDatabaseConfig = Field( - default_factory=lambda *a, **kw: SqliteDatabaseConfig(kind='sqlite') - ) + database: DatabaseConfigU = Field(default_factory=lambda *a, **kw: SqliteDatabaseConfig(kind='sqlite')) runtimes: dict[str, RuntimeConfigU] = Field(default_factory=dict) contracts: dict[str, ContractConfigU] = Field(default_factory=dict) indexes: dict[str, IndexConfigU] = Field(default_factory=dict) @@ -619,6 +627,82 @@ def schema_name(self) -> str: def package_path(self) -> Path: return env.get_package_path(self.package) + @classmethod + def from_terminal(cls, opts: TerminalOptions) -> Self: + import survey # type: ignore[import-untyped] + + from dipdup.project import SINGULAR_FORMS + from dipdup.project import fill_type_from_input + from dipdup.project import prompt_bool + from dipdup.project import prompt_kind + + config_dict: defaultdict[str, dict[str, Any]] = defaultdict(dict) + + sections = { + 'datasources': get_args(DatasourceConfigU), + 'runtimes': get_args(RuntimeConfigU), + 'contracts': get_args(ContractConfigU), + # NOTE: Skip the `template` kind + 'indexes': get_args(ResolvedIndexConfigU), + } + # NOTE: Substrate or multichain + if opts.namespace in {'substrate', None}: + sections['runtimes'] = get_args(RuntimeConfigU) + + for section, types in sections.items(): + another = False + + while True: + section_singular = SINGULAR_FORMS[section] + + if not prompt_bool( + f'Do you want to add {'another' if another else 'the first'} {section_singular}?', + default=not another, + ): + break + + # NOTE: All sections are mappings alias to dict + name = None + while True: + name = survey.routines.input( + f'Enter {section_singular} name: ', + ) + if not name: + print('Name is required') + continue + if name in config_dict[section]: + print(f'{section_singular.capitalize()} with name `{name}` already exists') + continue + break + + type_ = prompt_kind( + section_singular, + types, + opts.namespace, + ) + + if issubclass(type_, InteractiveMixin): + + res = type_.from_terminal(opts) + else: + _logger.debug('Not an `InteractiveMixin`; falling back to field inspection', type_.__name__) + res = fill_type_from_input(type_) + + if res is not None: + config_dict[section][name] = res + another = True + + # NOTE: Make sure that header is above other sections + config_dict = { # type: ignore[assignment] + 'package': opts.package, + 'spec_version': '3.0', + **config_dict, + } + + self = cls(**config_dict) # type: ignore[arg-type] + self._json = config_dict # type: ignore[assignment] + return self + @classmethod def load( cls, @@ -635,9 +719,6 @@ def load( ) try: - # from pydantic.dataclasses import rebuild_dataclass - # rebuild_dataclass(cls, force=True) - config = TypeAdapter(cls).validate_python(config_json) except ConfigurationError: raise @@ -1162,6 +1243,7 @@ def _set_names(self) -> None: from dipdup.config.tzip_metadata import TzipMetadataDatasourceConfig # NOTE: Unions for Pydantic config deserialization +DatabaseConfigU = SqliteDatabaseConfig | PostgresDatabaseConfig RuntimeConfigU = SubstrateRuntimeConfig ContractConfigU = EvmContractConfig | TezosContractConfig | StarknetContractConfig DatasourceConfigU = ( diff --git a/src/dipdup/config/_mixin.py b/src/dipdup/config/_mixin.py index 270e88ca4..98890924d 100644 --- a/src/dipdup/config/_mixin.py +++ b/src/dipdup/config/_mixin.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from typing import Any from typing import Generic +from typing import Self from typing import TypeVar from typing import cast @@ -121,3 +122,17 @@ def subgroup_index(self) -> int: @subgroup_index.setter def subgroup_index(self, value: int) -> None: self._subgroup_index = value + + +@dataclass +class TerminalOptions: + package: str + namespace: str | None = None + + +@dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) +class InteractiveMixin: + + @classmethod + @abstractmethod + def from_terminal(cls, opts: TerminalOptions) -> Self: ... diff --git a/src/dipdup/config/coinbase.py b/src/dipdup/config/coinbase.py index fe194af3f..a1dc34918 100644 --- a/src/dipdup/config/coinbase.py +++ b/src/dipdup/config/coinbase.py @@ -21,9 +21,13 @@ class CoinbaseDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['coinbase'] + kind: Literal['coinbase'] = 'coinbase' api_key: str | None = None secret_key: str | None = Field(default=None, repr=False) passphrase: str | None = Field(default=None, repr=False) http: HttpConfig | None = None + + @property + def url(self) -> str: # type: ignore[override] + return 'https://api.pro.coinbase.com' diff --git a/src/dipdup/config/evm.py b/src/dipdup/config/evm.py index d903937a1..9b0e4714b 100644 --- a/src/dipdup/config/evm.py +++ b/src/dipdup/config/evm.py @@ -55,7 +55,7 @@ class EvmContractConfig(ContractConfig): :param typename: Alias for the contract script """ - kind: Literal['evm'] + kind: Literal['evm'] = 'evm' address: EvmAddress | None = None abi: EvmAddress | None = None typename: str | None = None diff --git a/src/dipdup/config/evm_etherscan.py b/src/dipdup/config/evm_etherscan.py index 2ff8ba722..9d68377a2 100644 --- a/src/dipdup/config/evm_etherscan.py +++ b/src/dipdup/config/evm_etherscan.py @@ -6,8 +6,11 @@ from pydantic import ConfigDict from pydantic.dataclasses import dataclass +from dipdup import env from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url +from dipdup.exceptions import ConfigurationError _logger = logging.getLogger(__name__) @@ -23,14 +26,15 @@ class EvmEtherscanDatasourceConfig(DatasourceConfig): """ # NOTE: Alias, remove in 9.0 - kind: Literal['evm.etherscan'] | Literal['abi.etherscan'] - url: str + kind: Literal['evm.etherscan'] | Literal['abi.etherscan'] = 'evm.etherscan' + url: Url api_key: str | None = None http: HttpConfig | None = None def __post_init__(self) -> None: if self.kind == 'abi.etherscan': - _logger.warning( - '`abi.etherscan` datasource has been renamed to `evm.etherscan`. Please, update your config.' - ) + msg = '`abi.etherscan` datasource has been renamed to `evm.etherscan`. Please, update your config.' + if env.NEXT: + raise ConfigurationError(msg) + _logger.warning(msg) diff --git a/src/dipdup/config/evm_events.py b/src/dipdup/config/evm_events.py index aedb92fe3..f277227c1 100644 --- a/src/dipdup/config/evm_events.py +++ b/src/dipdup/config/evm_events.py @@ -61,7 +61,7 @@ class EvmEventsIndexConfig(EvmIndexConfig): :param last_level: Level to stop indexing and disable this index """ - kind: Literal['evm.events'] + kind: Literal['evm.events'] = 'evm.events' datasources: tuple[Alias[EvmDatasourceConfigU], ...] handlers: tuple[EvmEventsHandlerConfig, ...] diff --git a/src/dipdup/config/evm_node.py b/src/dipdup/config/evm_node.py index 9ee1eb170..fe7a0ec95 100644 --- a/src/dipdup/config/evm_node.py +++ b/src/dipdup/config/evm_node.py @@ -22,7 +22,7 @@ class EvmNodeDatasourceConfig(DatasourceConfig): :param rollback_depth: A number of blocks to store in database for rollback """ - kind: Literal['evm.node'] + kind: Literal['evm.node'] = 'evm.node' url: Url ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/evm_subsquid.py b/src/dipdup/config/evm_subsquid.py index 9ce1ac9a4..c8848480e 100644 --- a/src/dipdup/config/evm_subsquid.py +++ b/src/dipdup/config/evm_subsquid.py @@ -19,7 +19,7 @@ class EvmSubsquidDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['evm.subsquid'] + kind: Literal['evm.subsquid'] = 'evm.subsquid' url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/evm_transactions.py b/src/dipdup/config/evm_transactions.py index 956685206..3fdeb06c7 100644 --- a/src/dipdup/config/evm_transactions.py +++ b/src/dipdup/config/evm_transactions.py @@ -93,7 +93,7 @@ class EvmTransactionsIndexConfig(EvmIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['evm.transactions'] + kind: Literal['evm.transactions'] = 'evm.transactions' datasources: tuple[Alias[EvmDatasourceConfigU], ...] handlers: tuple[EvmTransactionsHandlerConfig, ...] diff --git a/src/dipdup/config/http.py b/src/dipdup/config/http.py index 6b3cfcf0c..772e7c918 100644 --- a/src/dipdup/config/http.py +++ b/src/dipdup/config/http.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) @@ -18,6 +19,6 @@ class HttpDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['http'] - url: str + kind: Literal['http'] = 'http' + url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/ipfs.py b/src/dipdup/config/ipfs.py index 148ce6df1..df1bc962d 100644 --- a/src/dipdup/config/ipfs.py +++ b/src/dipdup/config/ipfs.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url DEFAULT_IPFS_URL = 'https://ipfs.io/ipfs' @@ -20,6 +21,6 @@ class IpfsDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['ipfs'] - url: str = DEFAULT_IPFS_URL + kind: Literal['ipfs'] = 'ipfs' + url: Url = DEFAULT_IPFS_URL http: HttpConfig | None = None diff --git a/src/dipdup/config/starknet.py b/src/dipdup/config/starknet.py index 52469d5a9..8e026c70d 100644 --- a/src/dipdup/config/starknet.py +++ b/src/dipdup/config/starknet.py @@ -65,7 +65,7 @@ class StarknetContractConfig(ContractConfig): :param typename: Alias for the contract script """ - kind: Literal['starknet'] + kind: Literal['starknet'] = 'starknet' address: StarknetAddress | None = None abi: StarknetAddress | None = None typename: str | None = None diff --git a/src/dipdup/config/starknet_events.py b/src/dipdup/config/starknet_events.py index e09b69392..14e7cf901 100644 --- a/src/dipdup/config/starknet_events.py +++ b/src/dipdup/config/starknet_events.py @@ -61,7 +61,7 @@ class StarknetEventsIndexConfig(StarknetIndexConfig): """ - kind: Literal['starknet.events'] + kind: Literal['starknet.events'] = 'starknet.events' datasources: tuple[Alias[StarknetDatasourceConfigU], ...] handlers: tuple[StarknetEventsHandlerConfig, ...] diff --git a/src/dipdup/config/starknet_node.py b/src/dipdup/config/starknet_node.py index 5e2b72251..54318a36a 100644 --- a/src/dipdup/config/starknet_node.py +++ b/src/dipdup/config/starknet_node.py @@ -22,7 +22,7 @@ class StarknetNodeDatasourceConfig(DatasourceConfig): :param rollback_depth: A number of blocks to store in database for rollback """ - kind: Literal['starknet.node'] + kind: Literal['starknet.node'] = 'starknet.node' url: Url ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/starknet_subsquid.py b/src/dipdup/config/starknet_subsquid.py index 581f8400d..6b270ec7a 100644 --- a/src/dipdup/config/starknet_subsquid.py +++ b/src/dipdup/config/starknet_subsquid.py @@ -19,7 +19,7 @@ class StarknetSubsquidDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['starknet.subsquid'] + kind: Literal['starknet.subsquid'] = 'starknet.subsquid' url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/substrate_events.py b/src/dipdup/config/substrate_events.py index c814fe712..5e008b656 100644 --- a/src/dipdup/config/substrate_events.py +++ b/src/dipdup/config/substrate_events.py @@ -62,7 +62,7 @@ class SubstrateEventsIndexConfig(SubstrateIndexConfig): :param runtime: Substrate runtime """ - kind: Literal['substrate.events'] + kind: Literal['substrate.events'] = 'substrate.events' datasources: tuple[Alias[SubstrateDatasourceConfigU], ...] handlers: tuple[SubstrateEventsHandlerConfig, ...] runtime: Alias[SubstrateRuntimeConfig] diff --git a/src/dipdup/config/substrate_node.py b/src/dipdup/config/substrate_node.py index a7eeb402c..a0bd09f4f 100644 --- a/src/dipdup/config/substrate_node.py +++ b/src/dipdup/config/substrate_node.py @@ -21,7 +21,7 @@ class SubstrateNodeDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['substrate.node'] + kind: Literal['substrate.node'] = 'substrate.node' url: Url ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/substrate_subscan.py b/src/dipdup/config/substrate_subscan.py index 30dc2d178..296ab629e 100644 --- a/src/dipdup/config/substrate_subscan.py +++ b/src/dipdup/config/substrate_subscan.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) @@ -19,8 +20,8 @@ class SubstrateSubscanDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['substrate.subscan'] - url: str + kind: Literal['substrate.subscan'] = 'substrate.subscan' + url: Url api_key: str | None = None http: HttpConfig | None = None diff --git a/src/dipdup/config/substrate_subsquid.py b/src/dipdup/config/substrate_subsquid.py index b2de5eb3e..777ae1880 100644 --- a/src/dipdup/config/substrate_subsquid.py +++ b/src/dipdup/config/substrate_subsquid.py @@ -19,7 +19,7 @@ class SubstrateSubsquidDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['substrate.subsquid'] + kind: Literal['substrate.subsquid'] = 'substrate.subsquid' url: Url http: HttpConfig | None = None diff --git a/src/dipdup/config/tezos.py b/src/dipdup/config/tezos.py index 707cfe5a5..83e3490cf 100644 --- a/src/dipdup/config/tezos.py +++ b/src/dipdup/config/tezos.py @@ -59,7 +59,7 @@ class TezosContractConfig(ContractConfig): :param typename: Alias for the contract script """ - kind: Literal['tezos'] + kind: Literal['tezos'] = 'tezos' address: TezosAddress | None = None code_hash: int | TezosAddress | None = None typename: str | None = None diff --git a/src/dipdup/config/tezos_big_maps.py b/src/dipdup/config/tezos_big_maps.py index 0777a1ea3..d09e664d6 100644 --- a/src/dipdup/config/tezos_big_maps.py +++ b/src/dipdup/config/tezos_big_maps.py @@ -79,7 +79,7 @@ class TezosBigMapsIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.big_maps'] + kind: Literal['tezos.big_maps'] = 'tezos.big_maps' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosBigMapsHandlerConfig, ...] diff --git a/src/dipdup/config/tezos_events.py b/src/dipdup/config/tezos_events.py index 1c39d4d3d..a38bb394c 100644 --- a/src/dipdup/config/tezos_events.py +++ b/src/dipdup/config/tezos_events.py @@ -83,7 +83,7 @@ class TezosEventsIndexConfig(TezosIndexConfig): :param last_level: Last block level to index """ - kind: Literal['tezos.events'] + kind: Literal['tezos.events'] = 'tezos.events' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosEventsHandlerConfigU, ...] diff --git a/src/dipdup/config/tezos_head.py b/src/dipdup/config/tezos_head.py index f727a2d2d..c76b57ecc 100644 --- a/src/dipdup/config/tezos_head.py +++ b/src/dipdup/config/tezos_head.py @@ -41,7 +41,7 @@ class TezosHeadIndexConfig(TezosIndexConfig): :param datasources: `tezos` datasources to use """ - kind: Literal['tezos.head'] + kind: Literal['tezos.head'] = 'tezos.head' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] callback: str diff --git a/src/dipdup/config/tezos_operations.py b/src/dipdup/config/tezos_operations.py index 702282d28..e6fa68bfe 100644 --- a/src/dipdup/config/tezos_operations.py +++ b/src/dipdup/config/tezos_operations.py @@ -306,7 +306,7 @@ class TezosOperationsIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.operations'] + kind: Literal['tezos.operations'] = 'tezos.operations' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosOperationsHandlerConfig, ...] contracts: list[Alias[TezosContractConfig]] = Field(default_factory=list) @@ -431,7 +431,7 @@ class TezosOperationsUnfilteredIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.operations_unfiltered'] + kind: Literal['tezos.operations_unfiltered'] = 'tezos.operations_unfiltered' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] callback: str types: tuple[TezosOperationType, ...] = (TezosOperationType.transaction,) diff --git a/src/dipdup/config/tezos_token_balances.py b/src/dipdup/config/tezos_token_balances.py index 3dbdbe0f4..ecd7667ed 100644 --- a/src/dipdup/config/tezos_token_balances.py +++ b/src/dipdup/config/tezos_token_balances.py @@ -56,7 +56,7 @@ class TezosTokenBalancesIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.token_balances'] + kind: Literal['tezos.token_balances'] = 'tezos.token_balances' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosTokenBalancesHandlerConfig, ...] diff --git a/src/dipdup/config/tezos_token_transfers.py b/src/dipdup/config/tezos_token_transfers.py index bd76a29b7..caf00e227 100644 --- a/src/dipdup/config/tezos_token_transfers.py +++ b/src/dipdup/config/tezos_token_transfers.py @@ -59,7 +59,7 @@ class TezosTokenTransfersIndexConfig(TezosIndexConfig): :param last_level: Level to stop indexing at """ - kind: Literal['tezos.token_transfers'] + kind: Literal['tezos.token_transfers'] = 'tezos.token_transfers' datasources: tuple[Alias[TezosTzktDatasourceConfig], ...] handlers: tuple[TezosTokenTransfersHandlerConfig, ...] diff --git a/src/dipdup/config/tezos_tzkt.py b/src/dipdup/config/tezos_tzkt.py index 9eba47d18..564fe414a 100644 --- a/src/dipdup/config/tezos_tzkt.py +++ b/src/dipdup/config/tezos_tzkt.py @@ -34,7 +34,7 @@ class TezosTzktDatasourceConfig(DatasourceConfig): :param rollback_depth: Number of blocks to keep in the database to handle reorgs """ - kind: Literal['tezos.tzkt'] + kind: Literal['tezos.tzkt'] = 'tezos.tzkt' url: Url = DEFAULT_TZKT_URL http: HttpConfig | None = None buffer_size: int = 0 diff --git a/src/dipdup/config/tzip_metadata.py b/src/dipdup/config/tzip_metadata.py index 1a26ab5e7..52665c4db 100644 --- a/src/dipdup/config/tzip_metadata.py +++ b/src/dipdup/config/tzip_metadata.py @@ -7,6 +7,7 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +from dipdup.config import Url from dipdup.models.tzip_metadata import TzipMetadataNetwork DEFAULT_TZIP_METADATA_URL = 'https://metadata.dipdup.net' @@ -22,7 +23,7 @@ class TzipMetadataDatasourceConfig(DatasourceConfig): :param http: HTTP client configuration """ - kind: Literal['tzip_metadata'] + kind: Literal['tzip_metadata'] = 'tzip_metadata' network: TzipMetadataNetwork - url: str = DEFAULT_TZIP_METADATA_URL + url: Url = DEFAULT_TZIP_METADATA_URL http: HttpConfig | None = None diff --git a/src/dipdup/datasources/coinbase.py b/src/dipdup/datasources/coinbase.py index 2871fd23e..346fe1087 100644 --- a/src/dipdup/datasources/coinbase.py +++ b/src/dipdup/datasources/coinbase.py @@ -11,7 +11,6 @@ from dipdup.models.coinbase import CoinbaseCandleInterval CANDLES_REQUEST_LIMIT = 300 -API_URL = 'https://api.pro.coinbase.com' class CoinbaseDatasource(Datasource[CoinbaseDatasourceConfig]): diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 5b983f668..5fa5eba4c 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -3,22 +3,25 @@ Ask user some question with Click; render Jinja2 templates with answers. """ +import dataclasses import logging import re +from itertools import chain from pathlib import Path +from typing import Any from pydantic import ConfigDict from pydantic import TypeAdapter from pydantic.dataclasses import dataclass +from pydantic.fields import FieldInfo from typing_extensions import TypedDict from dipdup import __version__ -from dipdup._survey import DipDupSurveyConfig -from dipdup._survey import prompt_anyof -from dipdup._survey import query_survey_config from dipdup.cli import big_yellow_echo from dipdup.cli import echo +from dipdup.config import DipDupConfig from dipdup.config import ToStr +from dipdup.config._mixin import TerminalOptions from dipdup.env import get_package_path from dipdup.env import get_pyproject_name from dipdup.utils import load_template @@ -29,6 +32,14 @@ CODEGEN_HEADER = f'# generated by DipDup {__version__.split("+")[0]}' +SINGULAR_FORMS = { + 'datasources': 'datasource', + 'contracts': 'contract', + 'indexes': 'index', + 'hooks': 'hook', + 'jobs': 'job', + 'runtimes': 'runtime', +} # NOTE: All templates are stored in src/dipdup/projects TEMPLATES: dict[str, tuple[str, ...]] = { @@ -80,7 +91,6 @@ class Answers(TypedDict): hasura_image: str line_length: ToStr package_manager: str - _survey_config: DipDupSurveyConfig | None def get_default_answers() -> Answers: @@ -98,7 +108,6 @@ def get_default_answers() -> Answers: hasura_image='hasura/graphql-engine:latest', line_length='120', package_manager='uv', - _survey_config=None, ) @@ -125,7 +134,32 @@ def get_replay_path(name: str) -> Path: return Path(__file__).parent / 'projects' / name / 'replay.yaml' -def template_from_terminal() -> tuple[str | None, DipDupSurveyConfig | None]: +def namespace_from_terminal() -> str | None: + _, res = prompt_anyof( + question='What blockchain are you going to index?', + options=( + 'EVM', + 'Starknet', + 'Substrate', + 'Tezos', + '[multiple]', + ), + comments=( + 'EVM-compatible', + 'Starknet', + 'Substrate', + 'Tezos', + '(disable filtering)', + ), + default=0, + ) + print(res) + if res == '[multiple]': + return None + return res.lower() + + +def template_from_terminal(package: str) -> tuple[str | None, dict[str, Any] | None]: _, mode = prompt_anyof( question='How would you like to set up your new DipDup project?', options=( @@ -144,33 +178,22 @@ def template_from_terminal() -> tuple[str | None, DipDupSurveyConfig | None]: if mode == 'Blank': return ('demo_blank', None) - res = prompt_anyof( - question='What blockchain are you going to index?', - options=( - 'EVM', - 'Starknet', - 'Substrate', - 'Tezos', - ), - comments=( - 'EVM-compatible blockchains', - 'Starknet', - 'Substrate', - 'Tezos', - ), - default=0, - ) - blockchain = res[1].lower() + namespace = namespace_from_terminal() if mode == 'Interactively': replay_path = get_replay_path('demo_blank') - survey_config = query_survey_config(blockchain) - return ('demo_blank', survey_config) + opts = TerminalOptions( + package=package, + namespace=namespace, + ) + config = DipDupConfig.from_terminal(opts) + return ('demo_blank', config._json) if mode == 'From template': options, comments = [], [] - for name in TEMPLATES[blockchain]: - replay_path = get_replay_path(name) + templates = TEMPLATES[namespace] if namespace else chain(v for v in TEMPLATES.values()) + for name in templates: + replay_path = get_replay_path(name) # type: ignore[arg-type] _answers = answers_from_replay(replay_path) options.append(_answers['template']) comments.append(_answers['description']) @@ -186,7 +209,7 @@ def template_from_terminal() -> tuple[str | None, DipDupSurveyConfig | None]: raise NotImplementedError -def answers_from_terminal(template: str | None) -> Answers: +def answers_from_terminal() -> Answers: """Script running on dipdup new command and will create a new project base from interactive survey""" import survey # type: ignore[import-untyped] @@ -197,14 +220,6 @@ def answers_from_terminal(template: str | None) -> Answers: answers = get_default_answers() - if template: - echo(f'Using template `{template}`\n') - else: - template, survey_config = template_from_terminal() - answers['_survey_config'] = survey_config - - answers['template'] = template or 'demo_blank' - big_yellow_echo('Set up project') while True: @@ -221,10 +236,10 @@ def answers_from_terminal(template: str | None) -> Answers: ) answers['package'] = package - answers['version'] = survey.routines.input( - 'Enter project version: ', - value=answers['version'], - ) + # answers['version'] = survey.routines.input( + # 'Enter project version: ', + # value=answers['version'], + # ) # NOTE: Used in pyproject.toml, README.md and some other places answers['description'] = survey.routines.input( @@ -269,29 +284,30 @@ def answers_from_terminal(template: str | None) -> Answers: fg='yellow', ) - big_yellow_echo('Miscellaneous tunables; leave default values if unsure') + # big_yellow_echo('Miscellaneous tunables; leave default values if unsure') + + # _, answers['package_manager'] = prompt_anyof( + # question='Choose package manager', + # options=( + # 'uv', + # 'poetry', + # 'pdm', + # 'none', + # ), + # comments=( + # 'uv (recommended)', + # 'Poetry', + # 'PDM', + # '[none]', + # ), + # default=0, + # ) + + # answers['line_length'] = survey.routines.input( + # 'Enter maximum line length for linters: ', + # value=answers['line_length'], + # ) - _, answers['package_manager'] = prompt_anyof( - question='Choose package manager', - options=( - 'uv', - 'poetry', - 'pdm', - 'none', - ), - comments=( - 'uv (recommended)', - 'Poetry', - 'PDM', - '[none]', - ), - default=0, - ) - - answers['line_length'] = survey.routines.input( - 'Enter maximum line length for linters: ', - value=answers['line_length'], - ) return answers @@ -409,3 +425,128 @@ def _render(answers: Answers, template_path: Path, output_path: Path, force: boo ) write(output_path, content, overwrite=force) + + +def prompt_bool( + question: str, + default: bool = False, +) -> bool: + choices = ('yes', 'no') if default else ('no', 'yes') + _, value = prompt_anyof( + question, + choices, + ('', ''), + default=0, + ) + return value == 'yes' + + +def prompt_kind( + entity: str, + types: tuple[type, ...], + filter: str | None, +) -> type: + + matched = {} + for entity_type in types: + try: + kind = entity_type.__dataclass_fields__['kind'].default # type: ignore[attr-defined] + except KeyError: + kind = entity_type.__name__ + + assert kind + + if filter and '.' in kind and not kind.startswith(filter): + continue + + matched[kind] = entity_type + + if len(matched) == 1: + return next(iter(matched)) # type: ignore[no-any-return] + + kinds = tuple(sorted(matched.keys())) + comments = tuple('' for _ in kinds) + _, kind = prompt_anyof( + f'Choose {entity} kind: ', + kinds, + comments, + default=0, + ) + return matched[kind] + + +def fill_type_from_input( + type_: type, +) -> Any: + import survey + + # Gather input for the fields of the chosen type + entity_data = {} + for field_name, field in type_.__dataclass_fields__.items(): # type: ignore[attr-defined] + # print(field) + + if field_name in {'kind', 'http'}: + continue + if field_name.startswith('_'): + continue + if field_name == 'handlers': + entity_data[field_name] = () + continue + + default = field.default + + if default == dataclasses.MISSING: + default = None + elif isinstance(default, FieldInfo): + default = default.default_factory() if default.default_factory else default.default # type: ignore[call-arg] + + field_value = survey.routines.input( + f'Enter value for `{field_name}` [{field.type}]: ', + value=str(default) if default is not None else '', + ) + + if field.default_factory == dataclasses.MISSING: + default = None + + # NOTE: Basic transformations: string lists + if 'tuple' in field.type or 'list' in field.type: + if not field_value: + field_value = default + elif ',' in field_value: + field_value = field_value.split(',') + else: + field_value = [field_value] + + entity_data[field_name] = field_value or default + + # print(default, 'default') + # print(entity_data[field_name], 'field_value') + + # Validate and add the entity + try: + obj = type_(**entity_data) + return {k: v for k, v in obj.__dict__.items() if not k.startswith('_')} + except Exception as e: + print(f'Error: {e}. Please try again.') + + +def prompt_anyof( + question: str, + options: tuple[str, ...], + comments: tuple[str, ...], + default: int, +) -> tuple[int, str]: + """Ask user to choose one of the options; returns index and value""" + import survey + from tabulate import tabulate + + table = tabulate( + zip(options, comments, strict=False), + tablefmt='plain', + ) + index = survey.routines.select( + question + '\n', + options=table.split('\n'), + index=default, + ) + return index, options[index] diff --git a/src/dipdup/yaml.py b/src/dipdup/yaml.py index 771c20747..3f30fb710 100644 --- a/src/dipdup/yaml.py +++ b/src/dipdup/yaml.py @@ -45,7 +45,7 @@ def exclude_none(config_json: Any) -> Any: if isinstance(config_json, list | tuple): return [exclude_none(i) for i in config_json if i is not None] if isinstance(config_json, dict): - return {k: exclude_none(v) for k, v in config_json.items() if v is not None} + return {k: exclude_none(v) for k, v in config_json.items() if v is not None and not k.startswith('_')} return config_json diff --git a/uv.lock b/uv.lock index 91d95efd6..4372a6db0 100644 --- a/uv.lock +++ b/uv.lock @@ -446,7 +446,7 @@ wheels = [ [[package]] name = "datamodel-code-generator" -version = "0.27.3" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "argcomplete" }, @@ -459,9 +459,9 @@ dependencies = [ { name = "pydantic" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/17/53cec24a4edb2021c3b89a3c50c83a9a8dd0e506495d5f7ce486be4a24fa/datamodel_code_generator-0.27.3.tar.gz", hash = "sha256:01e928c00b800aec8d2ee77b5d4b47e1bc159a3a1c32f0f405df0a442d9ab5e7", size = 440612 } +sdist = { url = "https://files.pythonhosted.org/packages/cb/d3/80f6a2394bbf3b46b150fc75afa5b0050f91baa5771e9be87df148013d83/datamodel_code_generator-0.28.1.tar.gz", hash = "sha256:37ef5f3b488f7d7a3f0b5b3ba0f2bc1ae01bab4dc7e0f6b99ff6c40713a6beb3", size = 434901 } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/28/a72154c09d09b4c331fee613bb43451a8c43860b52f349777db7935dde04/datamodel_code_generator-0.27.3-py3-none-any.whl", hash = "sha256:ddef49e66e2b90a4c9b238f6ce42dc5a2a23f6ab1b8370eaca08576777921e43", size = 116023 }, + { url = "https://files.pythonhosted.org/packages/c8/17/2876ca0a4ac7dd7cb5f56a2f0f6d9ac910969f467e8142c847c45a76b897/datamodel_code_generator-0.28.1-py3-none-any.whl", hash = "sha256:1ff8a56f9550a82bcba3e1ad7ebdb89bc655eeabbc4bc6acfb05977cbdc6381c", size = 115601 }, ] [[package]] @@ -1869,14 +1869,14 @@ wheels = [ [[package]] name = "typeguard" -version = "4.4.1" +version = "4.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/c3/400917dd37d7b8c07e9723f3046818530423e1e759a56a22133362adab00/typeguard-4.4.1.tar.gz", hash = "sha256:0d22a89d00b453b47c49875f42b6601b961757541a2e1e0ef517b6e24213c21b", size = 74959 } +sdist = { url = "https://files.pythonhosted.org/packages/70/60/8cd6a3d78d00ceeb2193c02b7ed08f063d5341ccdfb24df88e61f383048e/typeguard-4.4.2.tar.gz", hash = "sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49", size = 75746 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/53/9465dedf2d69fe26008e7732cf6e0a385e387c240869e7d54eed49782a3c/typeguard-4.4.1-py3-none-any.whl", hash = "sha256:9324ec07a27ec67fc54a9c063020ca4c0ae6abad5e9f0f9804ca59aee68c6e21", size = 35635 }, + { url = "https://files.pythonhosted.org/packages/cf/4b/9a77dc721aa0b7f74440a42e4ef6f9a4fae7324e17f64f88b96f4c25cc05/typeguard-4.4.2-py3-none-any.whl", hash = "sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c", size = 35801 }, ] [[package]]