Add Prometheus metrics tracking for ClickHouse and RPC operations (#257)

* Add Prometheus metrics tracking for ClickHouse and RPC operations

* adjust buckets

* Update README.md

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fix Prometheus metric naming, security, and documentation issues (#259)

* Initial plan

* Address PR review comments: fix metric naming, JSDoc, security, and add tests

Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>

* Improve JSDoc clarity and simplify fallback pattern

Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>

* Use sanitized node_host instead of boolean node_url_configured

Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>

* Fix Prometheus server state leak on startup failure (#261)

* Initial plan

* Fix Prometheus test failures by cleaning up server state on error

Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>

* use bun sleep

* lint

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>
Co-authored-by: YaroShkvorets <shkvorets@gmail.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: YaroShkvorets <29608734+YaroShkvorets@users.noreply.github.com>

Author: 3 people

README.md

@@ -7,7 +7,7 @@ A specialized tool for scraping and indexing ERC-20 token data on the TRON block
 - **ERC-20 Focus**: Designed specifically for TRON ERC-20 tokens
 - **Continuous Query Mechanism**: Tracks block numbers to enable incremental balance updates
 - **Efficient Processing**: Only queries new or updated transfers, avoiding redundant RPC calls
-- **Prometheus Metrics**: Real-time monitoring with Prometheus metrics support
+- **Prometheus Metrics**: Real-time monitoring with comprehensive metrics (see [Monitoring](#monitoring))
 - **Concurrent Processing**: Configurable concurrency for optimal RPC throughput
 - **RPC Batch Requests**: Optional batching of multiple RPC calls for improved performance
 
@@ -107,6 +107,44 @@ docker run --env-file .env -p 9090:9090 token-api-scraper \
 
 See [Docker Guide](docs/DOCKER.md) for Docker Compose examples and production deployment.
 
+## Monitoring
+
+The scraper exposes Prometheus metrics on port `9090` (configurable via `PROMETHEUS_PORT`).
+
+### Available Metrics
+
+**ClickHouse Operations**
+- `scraper_clickhouse_operations_seconds` (histogram) - Duration of ClickHouse operations in seconds
+  - Labels: `operation_type` (`read`, `write`), `status` (`success`, `error`)
+  - Automatically tracked in `lib/clickhouse.ts` and `lib/batch-insert.ts`
+
+**RPC Requests**
+- `scraper_rpc_requests_seconds` (histogram) - Duration of RPC requests in seconds
+  - Labels: `method` (e.g., `eth_call`, `eth_getBalance`), `status` (`success`, `error`)
+  - Automatically tracked in `lib/rpc.ts`
+
+**Task Completion**
+- `scraper_completed_tasks_total` (counter) - Total completed tasks
+  - Labels: `service`, `status` (`success`, `error`)
+- `scraper_error_tasks_total` (counter) - Total failed tasks
+  - Labels: `service`
+
+**Configuration Info**
+- `scraper_config_info` (gauge) - Configuration metadata
+  - Labels: `clickhouse_host` (sanitized hostname only), `clickhouse_database`, `node_host` (sanitized hostname only)
+
+### Accessing Metrics
+
+```bash
+# View metrics endpoint
+curl http://localhost:9090/metrics
+
+# Example Prometheus queries
+scraper_clickhouse_operations_seconds_bucket{operation_type="write"}
+rate(scraper_rpc_requests_seconds_count{status="success"}[5m])
+histogram_quantile(0.95, sum(rate(scraper_clickhouse_operations_seconds_bucket{operation_type="read",status="success"}[5m])) by (le))
+```
+
 ## Testing
 
 ```bash

biome.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "https://biomejs.dev/schemas/2.3.11/schema.json",
+    "$schema": "./node_modules/@biomejs/biome/configuration_schema.json",
     "vcs": {
         "enabled": true,
         "clientKind": "git",

lib/batch-insert.ts

@@ -1,6 +1,7 @@
 import { insertClient } from './clickhouse';
 import { NODE_URL } from './config';
 import { createLogger } from './logger';
+import { trackClickHouseOperation } from './prometheus';
 
 const log = createLogger('batch-insert');
 
@@ -60,11 +61,14 @@ export class BatchInsertQueue {
 
         // Get all items and clear the queue
         const items = queue.splice(0);
+        const startTime = performance.now();
 
         try {
             await this.insertImmediate(table, items);
+            trackClickHouseOperation('write', 'success', startTime);
         } catch (error) {
             this.handleInsertError(error, table, items.length);
+            trackClickHouseOperation('write', 'error', startTime);
         }
     }
 

lib/clickhouse.ts

@@ -1,5 +1,6 @@
 import { type ClickHouseClient, createClient } from '@clickhouse/client';
 import { createLogger } from './logger';
+import { trackClickHouseOperation } from './prometheus';
 
 const log = createLogger('clickhouse');
 
@@ -156,6 +157,7 @@ export async function query<T = any>(
             query_params,
             format: 'JSONEachRow',
         });
+        trackClickHouseOperation('read', 'success', startTime);
         const queryEndTime = performance.now();
 
         // Track data parsing time
@@ -189,6 +191,7 @@ export async function query<T = any>(
             },
         };
     } catch (error: unknown) {
+        trackClickHouseOperation('read', 'error', startTime);
         const url = process.env.CLICKHOUSE_URL || 'http://localhost:8123';
         const urlObj = new URL(url);
         const host = urlObj.hostname;

lib/prometheus.test.ts

@@ -1,9 +1,12 @@
 import { describe, expect, test } from 'bun:test';
+import { sleep } from 'bun';
 import {
     incrementError,
     incrementSuccess,
     startPrometheusServer,
     stopPrometheusServer,
+    trackClickHouseOperation,
+    trackRpcRequest,
 } from './prometheus';
 
 describe('Prometheus Server', () => {
@@ -12,8 +15,7 @@ describe('Prometheus Server', () => {
 
         await startPrometheusServer(port);
 
-        // Wait for server to start
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await sleep(100);
 
         // Verify the Prometheus metrics endpoint is accessible
         const response = await fetch(`http://localhost:${port}/metrics`);
@@ -26,7 +28,7 @@ describe('Prometheus Server', () => {
         await stopPrometheusServer();
 
         // Wait for server to close
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await sleep(100);
 
         // Verify server is closed
         try {
@@ -50,8 +52,7 @@ describe('Prometheus Server', () => {
 
         await startPrometheusServer(port);
 
-        // Wait for server to start
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await sleep(100);
 
         // Fetch metrics
         const response = await fetch(`http://localhost:${port}/metrics`);
@@ -65,9 +66,9 @@ describe('Prometheus Server', () => {
         expect(metricsText).toContain('scraper_config_info');
 
         // Verify config info has labels
-        expect(metricsText).toContain('clickhouse_url');
+        expect(metricsText).toContain('clickhouse_host');
         expect(metricsText).toContain('clickhouse_database');
-        expect(metricsText).toContain('node_url');
+        expect(metricsText).toContain('node_host');
 
         await stopPrometheusServer();
     });
@@ -78,8 +79,7 @@ describe('Prometheus Server', () => {
 
         await startPrometheusServer(port);
 
-        // Wait for server to start
-        await new Promise((resolve) => setTimeout(resolve, 100));
+        await sleep(100);
 
         // Set metrics
         incrementSuccess(serviceName);
@@ -93,6 +93,9 @@ describe('Prometheus Server', () => {
         expect(metricsText).toContain(serviceName);
 
         await stopPrometheusServer();
+
+        // Wait for server to fully close
+        await sleep(100);
     });
 
     test('should handle starting server on already used port', async () => {
@@ -108,6 +111,9 @@ describe('Prometheus Server', () => {
         expect(response.ok).toBe(true);
 
         await stopPrometheusServer();
+
+        // Wait for server to fully close
+        await sleep(100);
     });
 
     test('should reject when port is already used by external process', async () => {
@@ -130,3 +136,69 @@ describe('Prometheus Server', () => {
         }
     });
 });
+
+describe('Prometheus Histogram Helpers', () => {
+    test('should track ClickHouse operations with correct labels', async () => {
+        const port = 19006;
+
+        await startPrometheusServer(port);
+
+        await sleep(100);
+
+        // Track some ClickHouse operations
+        const startTime = performance.now();
+        trackClickHouseOperation('read', 'success', startTime);
+        trackClickHouseOperation('write', 'success', startTime);
+        trackClickHouseOperation('read', 'error', startTime);
+
+        // Fetch metrics
+        const response = await fetch(`http://localhost:${port}/metrics`);
+        const metricsText = await response.text();
+
+        // Verify histogram metric is present with correct name
+        expect(metricsText).toContain('scraper_clickhouse_operations_seconds');
+
+        // Verify labels are present
+        expect(metricsText).toContain('operation_type="read"');
+        expect(metricsText).toContain('operation_type="write"');
+        expect(metricsText).toContain('status="success"');
+        expect(metricsText).toContain('status="error"');
+
+        await stopPrometheusServer();
+
+        // Wait for server to fully close
+        await sleep(100);
+    });
+
+    test('should track RPC requests with correct labels', async () => {
+        const port = 19007;
+
+        await startPrometheusServer(port);
+
+        await sleep(100);
+
+        // Track some RPC requests
+        const startTime = performance.now();
+        trackRpcRequest('eth_call', 'success', startTime);
+        trackRpcRequest('eth_getBalance', 'success', startTime);
+        trackRpcRequest('eth_call', 'error', startTime);
+
+        // Fetch metrics
+        const response = await fetch(`http://localhost:${port}/metrics`);
+        const metricsText = await response.text();
+
+        // Verify histogram metric is present with correct name
+        expect(metricsText).toContain('scraper_rpc_requests_seconds');
+
+        // Verify labels are present
+        expect(metricsText).toContain('method="eth_call"');
+        expect(metricsText).toContain('method="eth_getBalance"');
+        expect(metricsText).toContain('status="success"');
+        expect(metricsText).toContain('status="error"');
+
+        await stopPrometheusServer();
+
+        // Wait for server to fully close
+        await sleep(100);
+    });
+});

lib/prometheus.ts

@@ -31,18 +31,53 @@ const errorTasksCounter = new promClient.Counter({
     registers: [register],
 });
 
+// ClickHouse operation metrics
+const clickhouseOperations = new promClient.Histogram({
+    name: 'scraper_clickhouse_operations_seconds',
+    help: 'Duration of ClickHouse operations in seconds',
+    labelNames: ['operation_type', 'status'],
+    buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
+    registers: [register],
+});
+
+// RPC request metrics
+const rpcRequests = new promClient.Histogram({
+    name: 'scraper_rpc_requests_seconds',
+    help: 'Duration of RPC requests in seconds',
+    labelNames: ['method', 'status'],
+    buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
+    registers: [register],
+});
+
 // Configuration info metrics
 const configInfoGauge = new promClient.Gauge({
     name: 'scraper_config_info',
     help: 'Configuration information for the scraper',
-    labelNames: ['clickhouse_url', 'clickhouse_database', 'node_url'],
+    labelNames: ['clickhouse_host', 'clickhouse_database', 'node_host'],
     registers: [register],
 });
 
 // Track whether config metrics have been initialized
 let configMetricsInitialized = false;
 let prometheusServer: http.Server | undefined;
 
+/**
+ * Sanitize a URL to extract only the hostname, removing credentials, port, and path
+ * @param url - The URL to sanitize
+ * @returns The sanitized hostname or 'not_set' if invalid
+ */
+function sanitizeUrl(url: string | undefined): string {
+    if (!url || url === 'not_set') {
+        return 'not_set';
+    }
+    try {
+        const parsed = new URL(url);
+        return parsed.hostname;
+    } catch {
+        return 'redacted';
+    }
+}
+
 /**
  * Initialize Prometheus server
  * @param port - Port to listen on
@@ -56,8 +91,18 @@ export function startPrometheusServer(
     return new Promise((resolve, reject) => {
         // Set configuration info metrics once (only on first initialization)
         if (!configMetricsInitialized) {
+            // Read from process.env at runtime to get CLI overrides
+            const clickhouseUrl = process.env.CLICKHOUSE_URL || CLICKHOUSE_URL;
+            const clickhouseDatabase =
+                process.env.CLICKHOUSE_DATABASE || CLICKHOUSE_DATABASE;
+            const nodeUrl = process.env.NODE_URL || NODE_URL;
+
             configInfoGauge
-                .labels(CLICKHOUSE_URL, CLICKHOUSE_DATABASE, NODE_URL)
+                .labels(
+                    sanitizeUrl(clickhouseUrl),
+                    clickhouseDatabase || 'not_set',
+                    sanitizeUrl(nodeUrl),
+                )
                 .set(1);
             configMetricsInitialized = true;
         }
@@ -89,6 +134,7 @@ export function startPrometheusServer(
 
         prometheusServer.on('error', (err) => {
             log.error('Prometheus server error', { error: err.message });
+            prometheusServer = undefined;
             reject(err);
         });
     });
@@ -133,3 +179,33 @@ export function incrementError(serviceName: string): void {
     completedTasksCounter.labels(serviceName, 'error').inc();
     errorTasksCounter.labels(serviceName).inc();
 }
+
+/**
+ * Track a ClickHouse operation
+ * @param operationType - Type of operation ('read' or 'write')
+ * @param status - Operation status ('success' or 'error')
+ * @param startTime - Start time in milliseconds (from performance.now())
+ */
+export function trackClickHouseOperation(
+    operationType: 'read' | 'write',
+    status: 'success' | 'error',
+    startTime: number,
+): void {
+    const durationSeconds = (performance.now() - startTime) / 1000;
+    clickhouseOperations.labels(operationType, status).observe(durationSeconds);
+}
+
+/**
+ * Track an RPC request
+ * @param method - RPC method name (e.g., 'eth_getBlockByNumber')
+ * @param status - Request status ('success' or 'error')
+ * @param startTime - Start time in milliseconds (from performance.now())
+ */
+export function trackRpcRequest(
+    method: string,
+    status: 'success' | 'error',
+    startTime: number,
+): void {
+    const durationSeconds = (performance.now() - startTime) / 1000;
+    rpcRequests.labels(method, status).observe(durationSeconds);
+}