Watchdog service

CHANGELOG.md

@@ -8,6 +8,10 @@ Releases prior to 7.0 has been removed from this file to declutter search result
 
 ## [Unreleased]
 
+### Added
+
+- context: Added configurable watchdog service to notify about long-running callbacks and transactions.
+
 ### Fixed
 
 - codegen: Fixed loading ABIs from the project with no ABI datasources configured.

docs/7.references/3.context.md

@@ -329,7 +329,7 @@ description: "Context reference"
 <dl class="field-list simple">
 <dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>reason</strong> (<em>str</em><em> | </em><a class="reference internal" href="models#dipdupmodelsreindexingreason" title="dipdup.models.ReindexingReason" target="_self"><em>ReindexingReason</em></a><em> | </em><em>None</em>) – Reason for reindexing in free-form string</p></li>
+<li><p><strong>reason</strong> (<em>str</em><em> | </em><a class="reference internal" href="config#dipdupconfigreindexingreason" title="dipdup.config.ReindexingReason" target="_self"><em>ReindexingReason</em></a><em> | </em><em>None</em>) – Reason for reindexing in free-form string</p></li>
 <li><p><strong>context</strong> (<em>Any</em>) – Additional information to include in exception message</p></li>
 </ul>
 </dd>

docs/7.references/4.models.md

@@ -137,59 +137,6 @@ description: "Models reference"
 
 <dl class="py class">
 
-## dipdup.models.ReindexingAction
-
-<em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.</span></span><span class="sig-name descname"><span class="pre">ReindexingAction</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">values</span></span></em><span class="sig-paren">)</span></dt>
-<dd><p>Action that should be performed on reindexing</p>
-<dl class="field-list simple">
-<dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>exception</strong> – Raise <cite>ReindexingRequiredError</cite> exception.</p></li>
-<li><p><strong>wipe</strong> – Wipe the database and reindex from scratch. (WARNING: This action is irreversible! All indexed data will be lost!)</p></li>
-<li><p><strong>ignore</strong> – Ignore the reindexing cause and continue.</p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-<dl class="py class">
-
-## dipdup.models.ReindexingReason
-
-<em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.</span></span><span class="sig-name descname"><span class="pre">ReindexingReason</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">values</span></span></em><span class="sig-paren">)</span></dt>
-<dd><p>Reason that caused reindexing</p>
-<dl class="field-list simple">
-<dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>manual</strong> – Manual reindexing.</p></li>
-<li><p><strong>migration</strong> – Migration of the database schema.</p></li>
-<li><p><strong>rollback</strong> – Rollback that couldn’t be handled automatically.</p></li>
-<li><p><strong>config_modified</strong> – Index config was modified.</p></li>
-<li><p><strong>schema_modified</strong> – Project models or database schema were modified.</p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-<dl class="py class">
-
-## dipdup.models.SkipHistory
-
-<em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.</span></span><span class="sig-name descname"><span class="pre">SkipHistory</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">values</span></span></em><span class="sig-paren">)</span></dt>
-<dd><p>Whether to skip indexing big map history and use only current state</p>
-<dl class="field-list simple">
-<dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>never</strong> – Always index big map historical updates.</p></li>
-<li><p><strong>once</strong> – Skip history once after reindexing; process updates as usual on the next resync.</p></li>
-<li><p><strong>always</strong> – Always skip big map history.</p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-<dl class="py class">
-
 ## dipdup.models.Meta
 
 <em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.</span></span><span class="sig-name descname"><span class="pre">Meta</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span></dt>

docs/8.examples/_demos_table.md

@@ -1,22 +1,22 @@
 <!-- markdownlint-disable first-line-h1 -->
 | name | network | description | source |
 |-|-|-|-|
-| demo_blank |  | Empty config for a fresh start | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_blank) |
-| demo_evm_events | EVM | ERC-20 token transfers (from event logs) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_evm_events) |
-| demo_evm_transactions | EVM | ERC-20 token transfers (from transactions) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_evm_transactions) |
-| demo_evm_uniswap | EVM | Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_evm_uniswap) |
-| demo_starknet_events | Starknet | ERC-20 token transfers (from events) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_starknet_events) |
-| demo_substrate_events | Substrate | Substrate balance transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_substrate_events) |
-| demo_tezos_auction | Tezos | NFT marketplace (TzColors) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_auction) |
-| demo_tezos_dao | Tezos | DAO registry (Homebase DAO) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_dao) |
-| demo_tezos_dex | Tezos | DEX balances and liquidity (Quipuswap) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_dex) |
-| demo_tezos_domains | Tezos | Domain name service (Tezos Domains) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_domains) |
-| demo_tezos_etherlink | Tezos | Etherlink smart rollup transactions | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_etherlink) |
-| demo_tezos_events | Tezos | Processing contract events | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_events) |
-| demo_tezos_factories | Tezos | Example of spawning indexes in runtime | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_factories) |
-| demo_tezos_head | Tezos | Processing head block metadata (realtime only) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_head) |
-| demo_tezos_nft_marketplace | Tezos | NFT marketplace (hic at nunc) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_nft_marketplace) |
-| demo_tezos_raw | Tezos | Process raw operations without filtering and typed payloads | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_raw) |
-| demo_tezos_token | Tezos | FA1.2 token contract operations | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_token) |
-| demo_tezos_token_balances | Tezos | FA1.2 token balances | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_token_balances) |
-| demo_tezos_token_transfers | Tezos | FA1.2 token transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.3.3/src/demo_tezos_token_transfers) |
+| demo_blank |  | Empty config for a fresh start | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_blank) |
+| demo_evm_events | EVM | ERC-20 token transfers (from event logs) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_evm_events) |
+| demo_evm_transactions | EVM | ERC-20 token transfers (from transactions) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_evm_transactions) |
+| demo_evm_uniswap | EVM | Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_evm_uniswap) |
+| demo_starknet_events | Starknet | ERC-20 token transfers (from events) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_starknet_events) |
+| demo_substrate_events | Substrate | Substrate balance transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_substrate_events) |
+| demo_tezos_auction | Tezos | NFT marketplace (TzColors) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_auction) |
+| demo_tezos_dao | Tezos | DAO registry (Homebase DAO) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_dao) |
+| demo_tezos_dex | Tezos | DEX balances and liquidity (Quipuswap) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_dex) |
+| demo_tezos_domains | Tezos | Domain name service (Tezos Domains) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_domains) |
+| demo_tezos_etherlink | Tezos | Etherlink smart rollup transactions | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_etherlink) |
+| demo_tezos_events | Tezos | Processing contract events | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_events) |
+| demo_tezos_factories | Tezos | Example of spawning indexes in runtime | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_factories) |
+| demo_tezos_head | Tezos | Processing head block metadata (realtime only) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_head) |
+| demo_tezos_nft_marketplace | Tezos | NFT marketplace (hic at nunc) | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_nft_marketplace) |
+| demo_tezos_raw | Tezos | Process raw operations without filtering and typed payloads | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_raw) |
+| demo_tezos_token | Tezos | FA1.2 token contract operations | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_token) |
+| demo_tezos_token_balances | Tezos | FA1.2 token balances | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_token_balances) |
+| demo_tezos_token_transfers | Tezos | FA1.2 token transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.3.4/src/demo_tezos_token_transfers) |

docs/9.release-notes/_8.0_changelog.md

@@ -10,6 +10,7 @@
 - cli: Added full project migration support for 3.0 spec.
 - cli: Added spec_version 3.0 support to `migrate` command.
 - config: Publish JSON schemas for config validation and autocompletion.
+- context: Added configurable watchdog service to notify about long-running callbacks and transactions.
 - database: Added `dipdup_status` view to the schema.
 - env: Added `DIPDUP_JSON_LOG` environment variable to enable JSON logging.
 - env: Added `DIPDUP_LOW_MEMORY` variable to reduce the size of caches and buffers.
@@ -48,6 +49,7 @@
 - project: Fixed `make image` command and default workdir.
 - starknet.events: Fixed filtering events by key.
 - subsquid: Fixed missing entry in `dipdup_head` internal table.
+- substrate.node: Fixed Substrate Node Event Index.
 - tezos.big_maps: Fixed logging status message in `skip_history` mode.
 - tezos.big_maps: Respect order of handlers in `skip_history` mode.
 - tezos.operations: Fixed `sr_cement` operation index subscription.

docs/config.rst

@@ -2,8 +2,6 @@
 
 .. autoclass:: dipdup.config.DipDupConfig
 
-.. autoclass:: dipdup.config.abi_etherscan.AbiEtherscanDatasourceConfig
-.. autoclass:: dipdup.config.ContractConfig
 .. autoclass:: dipdup.config.AdvancedConfig
 .. autoclass:: dipdup.config.ApiConfig
 .. autoclass:: dipdup.config.coinbase.CoinbaseDatasourceConfig
@@ -20,8 +18,6 @@
 .. autoclass:: dipdup.config.HookConfig
 .. autoclass:: dipdup.config.HttpConfig
 .. autoclass:: dipdup.config.http.HttpDatasourceConfig
-.. autoclass:: dipdup.config.IndexConfig
-.. autoclass:: dipdup.config.DatasourceConfig
 .. autoclass:: dipdup.config.IndexTemplateConfig
 .. autoclass:: dipdup.config.ipfs.IpfsDatasourceConfig
 .. autoclass:: dipdup.config.JobConfig
@@ -66,3 +62,12 @@
 .. autoclass:: dipdup.config.substrate_subsquid.SubstrateSubsquidDatasourceConfig
 .. autoclass:: dipdup.config.substrate_subscan.SubstrateSubscanDatasourceConfig
 .. autoclass:: dipdup.config.tzip_metadata.TzipMetadataDatasourceConfig
+.. autoclass:: dipdup.config.evm_sourcify.EvmSourcifyDatasourceConfig
+.. autoclass:: dipdup.config.SkipHistory
+.. autoclass:: dipdup.config.substrate_node.SubstrateNodeDatasourceConfig
+.. autoclass:: dipdup.config.McpConfig
+.. autoclass:: dipdup.config.ReindexingAction
+.. autoclass:: dipdup.config.WatchdogTrigger
+.. autoclass:: dipdup.config.WatchdogAction
+.. autoclass:: dipdup.config.evm_blockvision.EvmBlockvisionDatasourceConfig
+.. autoclass:: dipdup.config.ReindexingReason

docs/models.rst

@@ -20,9 +20,6 @@ Internal models
 .. autoclass:: dipdup.models.IndexStatus
 .. autoclass:: dipdup.models.IndexType
 .. autoclass:: dipdup.models.Schema
-.. autoclass:: dipdup.models.ReindexingAction
-.. autoclass:: dipdup.models.ReindexingReason
-.. autoclass:: dipdup.models.SkipHistory
 .. autoclass:: dipdup.models.Meta
 .. autoclass:: dipdup.models.ModelUpdate
 .. autoclass:: dipdup.models.ModelUpdateAction

schemas/dipdup-3.0.json

@@ -56,7 +56,7 @@
             }
           ],
           "default": null,
-          "description": "A number of levels to keep for rollback.",
+          "description": "A number of blocks to keep for rollback (affects all datasources)",
           "title": "rollback_depth"
         },
         "scheduler": {
@@ -77,6 +77,17 @@
           "description": "Disable journaling and data integrity checks. Use only for testing.",
           "title": "unsafe_sqlite",
           "type": "boolean"
+        },
+        "watchdog": {
+          "additionalProperties": {
+            "$ref": "#/$defs/WatchdogAction"
+          },
+          "description": "Mapping of watchdog triggers and actions DipDup performs.",
+          "propertyNames": {
+            "$ref": "#/$defs/WatchdogTrigger"
+          },
+          "title": "watchdog",
+          "type": "object"
         }
       },
       "title": "AdvancedConfig",
@@ -3135,6 +3146,24 @@
     "Url": {
       "type": "string"
     },
+    "WatchdogAction": {
+      "enum": [
+        "exception",
+        "warning",
+        "ignore"
+      ],
+      "title": "WatchdogAction",
+      "type": "string"
+    },
+    "WatchdogTrigger": {
+      "enum": [
+        "callback",
+        "transaction",
+        "websocket"
+      ],
+      "title": "WatchdogTrigger",
+      "type": "string"
+    },
     "WsUrl": {
       "type": "string"
     }

scripts/docs.py

@@ -531,6 +531,7 @@ def _compare(ref: str, ignore: set[str]) -> None:
             exit(1)
 
     _compare('models', IGNORED_MODEL_CLASSES)
+    _compare('config', set())
 
     green_echo('=> Building Sphinx docs')
     rmtree('docs/_build', ignore_errors=True)

src/dipdup/config/__init__.py

@@ -23,6 +23,7 @@
 from abc import abstractmethod
 from collections import defaultdict
 from contextlib import suppress
+from enum import StrEnum
 from itertools import chain
 from pathlib import Path
 from types import NoneType
@@ -56,9 +57,6 @@
 from dipdup.exceptions import ConfigInitializationException
 from dipdup.exceptions import ConfigurationError
 from dipdup.exceptions import IndexAlreadyExistsError
-from dipdup.models import ReindexingAction
-from dipdup.models import ReindexingReason
-from dipdup.models import SkipHistory
 from dipdup.utils import pascal_to_snake
 from dipdup.yaml import DipDupYAMLConfig
 
@@ -569,22 +567,80 @@ def default_api_url(self) -> Url:
         return self.api_url or f'http://{self.host}:{DEFAULT_API_PORT}'
 
 
+# NOTE: Used as a key in config, must inherit from str
+class ReindexingReason(StrEnum):
+    """Reason that caused reindexing
+
+    :param manual: Manual reindexing.
+    :param migration: Migration of the database schema.
+    :param rollback: Rollback that couldn't be handled automatically.
+    :param config_modified: Index config was modified.
+    :param schema_modified: Project models or database schema were modified.
+    """
+
+    manual = 'manual'
+    migration = 'migration'
+    rollback = 'rollback'
+    config_modified = 'config_modified'
+    schema_modified = 'schema_modified'
+
+
+class ReindexingAction(StrEnum):
+    """Action that should be performed on reindexing
+
+    :param exception: Raise `ReindexingRequiredError` exception.
+    :param wipe: Wipe the database and reindex from scratch. (WARNING: This action is irreversible! All indexed data will be lost!)
+    :param ignore: Ignore the reindexing cause and continue.
+    """
+
+    exception = 'exception'
+    wipe = 'wipe'
+    ignore = 'ignore'
+
+
+class WatchdogTrigger(StrEnum):
+    callback = 'callback'
+    transaction = 'transaction'
+    websocket = 'websocket'
+
+
+class WatchdogAction(StrEnum):
+    exception = 'exception'
+    warning = 'warning'
+    ignore = 'ignore'
+
+
+class SkipHistory(StrEnum):
+    """Whether to skip indexing big map history and use only current state
+
+    :param never: Always index big map historical updates.
+    :param once: Skip history once after reindexing; process updates as usual on the next resync.
+    :param always: Always skip big map history.
+    """
+
+    never = 'never'
+    once = 'once'
+    always = 'always'
+
+
 # NOTE: Should be the only place where extras are allowed
 @dataclass(config=ConfigDict(extra='allow', defer_build=True), kw_only=True)
 class AdvancedConfig:
     """This section allows users to tune some system-wide options, either experimental or unsuitable for generic configurations.
 
     :param reindex: Mapping of reindexing reasons and actions DipDup performs.
+    :param watchdog: Mapping of watchdog triggers and actions DipDup performs.
     :param scheduler: `apscheduler` scheduler config.
     :param postpone_jobs: Do not start job scheduler until all indexes reach the realtime state.
     :param early_realtime: Establish realtime connection and start collecting messages while sync is in progress (faster, but consumes more RAM).
-    :param rollback_depth: A number of levels to keep for rollback.
+    :param rollback_depth: A number of blocks to keep for rollback (affects all datasources)
     :param decimal_precision: Overwrite precision if it's not guessed correctly based on project models.
     :param unsafe_sqlite: Disable journaling and data integrity checks. Use only for testing.
     :param alt_operation_matcher: Use different algorithm to match Tezos operations (dev only)
     """
 
     reindex: dict[ReindexingReason, ReindexingAction] = Field(default_factory=dict)
+    watchdog: dict[WatchdogTrigger, WatchdogAction] = Field(default_factory=dict)
     scheduler: dict[str, Any] | None = None
     postpone_jobs: bool = False
     early_realtime: bool = False

src/dipdup/config/tezos_big_maps.py

@@ -10,10 +10,10 @@
 from dipdup.config import Alias
 from dipdup.config import ContractConfig
 from dipdup.config import HandlerConfig
+from dipdup.config import SkipHistory
 from dipdup.config.tezos import TezosContractConfig
 from dipdup.config.tezos import TezosIndexConfig
 from dipdup.config.tezos_tzkt import TezosTzktDatasourceConfig
-from dipdup.models import SkipHistory
 from dipdup.models.tezos_tzkt import BigMapSubscription
 from dipdup.utils import pascal_to_snake
 from dipdup.utils import snake_to_pascal