Fix Prometheus metric naming, security, and documentation issues

README.md

@@ -114,12 +114,12 @@ The scraper exposes Prometheus metrics on port `9090` (configurable via `PROMETH
 ### Available Metrics
 
 **ClickHouse Operations**
-- `scraper_clickhouse_operations` (histogram) - Duration of ClickHouse operations in seconds
+- `scraper_clickhouse_operations_seconds` (histogram) - Duration of ClickHouse operations in seconds
   - Labels: `operation_type` (`read`, `write`), `status` (`success`, `error`)
   - Automatically tracked in `lib/clickhouse.ts` and `lib/batch-insert.ts`
 
 **RPC Requests**
-- `scraper_rpc_requests` (histogram) - Duration of RPC requests in seconds
+- `scraper_rpc_requests_seconds` (histogram) - Duration of RPC requests in seconds
   - Labels: `method` (e.g., `eth_call`, `eth_getBalance`), `status` (`success`, `error`)
   - Automatically tracked in `lib/rpc.ts`
 
@@ -131,7 +131,7 @@ The scraper exposes Prometheus metrics on port `9090` (configurable via `PROMETH
 
 **Configuration Info**
 - `scraper_config_info` (gauge) - Configuration metadata
-  - Labels: `clickhouse_url`, `clickhouse_database`, `node_url`
+  - Labels: `clickhouse_host` (sanitized hostname only), `clickhouse_database`, `node_host` (sanitized hostname only)
 
 ### Accessing Metrics
 
@@ -140,9 +140,9 @@ The scraper exposes Prometheus metrics on port `9090` (configurable via `PROMETH
 curl http://localhost:9090/metrics
 
 # Example Prometheus queries
-scraper_clickhouse_operations_bucket{operation_type="write"}
-rate(scraper_rpc_requests_count{status="success"}[5m])
-histogram_quantile(0.95, sum(rate(scraper_clickhouse_operations_bucket{operation_type="read",status="success"}[5m])) by (le))
+scraper_clickhouse_operations_seconds_bucket{operation_type="write"}
+rate(scraper_rpc_requests_seconds_count{status="success"}[5m])
+histogram_quantile(0.95, sum(rate(scraper_clickhouse_operations_seconds_bucket{operation_type="read",status="success"}[5m])) by (le))
 ```
 
 ## Testing

lib/prometheus.test.ts

@@ -4,6 +4,8 @@ import {
     incrementSuccess,
     startPrometheusServer,
     stopPrometheusServer,
+    trackClickHouseOperation,
+    trackRpcRequest,
 } from './prometheus';
 
 describe('Prometheus Server', () => {
@@ -65,9 +67,9 @@ describe('Prometheus Server', () => {
         expect(metricsText).toContain('scraper_config_info');
 
         // Verify config info has labels
-        expect(metricsText).toContain('clickhouse_url');
+        expect(metricsText).toContain('clickhouse_host');
         expect(metricsText).toContain('clickhouse_database');
-        expect(metricsText).toContain('node_url');
+        expect(metricsText).toContain('node_host');
 
         await stopPrometheusServer();
     });
@@ -130,3 +132,65 @@ describe('Prometheus Server', () => {
         }
     });
 });
+
+describe('Prometheus Histogram Helpers', () => {
+    test('should track ClickHouse operations with correct labels', async () => {
+        const port = 19006;
+
+        await startPrometheusServer(port);
+
+        // Wait for server to start
+        await new Promise((resolve) => setTimeout(resolve, 100));
+
+        // Track some ClickHouse operations
+        const startTime = performance.now();
+        trackClickHouseOperation('read', 'success', startTime);
+        trackClickHouseOperation('write', 'success', startTime);
+        trackClickHouseOperation('read', 'error', startTime);
+
+        // Fetch metrics
+        const response = await fetch(`http://localhost:${port}/metrics`);
+        const metricsText = await response.text();
+
+        // Verify histogram metric is present with correct name
+        expect(metricsText).toContain('scraper_clickhouse_operations_seconds');
+
+        // Verify labels are present
+        expect(metricsText).toContain('operation_type="read"');
+        expect(metricsText).toContain('operation_type="write"');
+        expect(metricsText).toContain('status="success"');
+        expect(metricsText).toContain('status="error"');
+
+        await stopPrometheusServer();
+    });
+
+    test('should track RPC requests with correct labels', async () => {
+        const port = 19007;
+
+        await startPrometheusServer(port);
+
+        // Wait for server to start
+        await new Promise((resolve) => setTimeout(resolve, 100));
+
+        // Track some RPC requests
+        const startTime = performance.now();
+        trackRpcRequest('eth_call', 'success', startTime);
+        trackRpcRequest('eth_getBalance', 'success', startTime);
+        trackRpcRequest('eth_call', 'error', startTime);
+
+        // Fetch metrics
+        const response = await fetch(`http://localhost:${port}/metrics`);
+        const metricsText = await response.text();
+
+        // Verify histogram metric is present with correct name
+        expect(metricsText).toContain('scraper_rpc_requests_seconds');
+
+        // Verify labels are present
+        expect(metricsText).toContain('method="eth_call"');
+        expect(metricsText).toContain('method="eth_getBalance"');
+        expect(metricsText).toContain('status="success"');
+        expect(metricsText).toContain('status="error"');
+
+        await stopPrometheusServer();
+    });
+});

lib/prometheus.ts

@@ -33,7 +33,7 @@ const errorTasksCounter = new promClient.Counter({
 
 // ClickHouse operation metrics
 const clickhouseOperations = new promClient.Histogram({
-    name: 'scraper_clickhouse_operations',
+    name: 'scraper_clickhouse_operations_seconds',
     help: 'Duration of ClickHouse operations in seconds',
     labelNames: ['operation_type', 'status'],
     buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
@@ -42,7 +42,7 @@ const clickhouseOperations = new promClient.Histogram({
 
 // RPC request metrics
 const rpcRequests = new promClient.Histogram({
-    name: 'scraper_rpc_requests',
+    name: 'scraper_rpc_requests_seconds',
     help: 'Duration of RPC requests in seconds',
     labelNames: ['method', 'status'],
     buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
@@ -53,14 +53,31 @@ const rpcRequests = new promClient.Histogram({
 const configInfoGauge = new promClient.Gauge({
     name: 'scraper_config_info',
     help: 'Configuration information for the scraper',
-    labelNames: ['clickhouse_url', 'clickhouse_database', 'node_url'],
+    labelNames: ['clickhouse_host', 'clickhouse_database', 'node_host'],
     registers: [register],
 });
 
 // Track whether config metrics have been initialized
 let configMetricsInitialized = false;
 let prometheusServer: http.Server | undefined;
 
+/**
+ * Sanitize a URL to extract only the hostname, removing credentials, port, and path
+ * @param url - The URL to sanitize
+ * @returns The sanitized hostname or 'not_set' if invalid
+ */
+function sanitizeUrl(url: string | undefined): string {
+    if (!url || url === 'not_set') {
+        return 'not_set';
+    }
+    try {
+        const parsed = new URL(url);
+        return parsed.hostname;
+    } catch {
+        return 'redacted';
+    }
+}
+
 /**
  * Initialize Prometheus server
  * @param port - Port to listen on
@@ -74,11 +91,16 @@ export function startPrometheusServer(
     return new Promise((resolve, reject) => {
         // Set configuration info metrics once (only on first initialization)
         if (!configMetricsInitialized) {
+            // Read from process.env at runtime to get CLI overrides
+            const clickhouseUrl = process.env.CLICKHOUSE_URL || CLICKHOUSE_URL;
+            const clickhouseDatabase = process.env.CLICKHOUSE_DATABASE || CLICKHOUSE_DATABASE;
+            const nodeUrl = process.env.NODE_URL || NODE_URL;
+
             configInfoGauge
                 .labels(
-                    CLICKHOUSE_URL,
-                    CLICKHOUSE_DATABASE || 'not_set',
-                    NODE_URL || 'not_set',
+                    sanitizeUrl(clickhouseUrl),
+                    clickhouseDatabase || 'not_set',
+                    sanitizeUrl(nodeUrl),
                 )
                 .set(1);
             configMetricsInitialized = true;
@@ -159,7 +181,8 @@ export function incrementError(serviceName: string): void {
 /**
  * Track a ClickHouse operation
  * @param operationType - Type of operation ('read' or 'write')
- * @param durationSeconds - Duration of the operation in seconds
+ * @param status - Operation status ('success' or 'error')
+ * @param startTime - Start time in milliseconds (from performance.now())
  */
 export function trackClickHouseOperation(
     operationType: 'read' | 'write',
@@ -174,7 +197,7 @@ export function trackClickHouseOperation(
  * Track an RPC request
  * @param method - RPC method name (e.g., 'eth_getBlockByNumber')
  * @param status - Request status ('success' or 'error')
- * @param durationSeconds - Duration of the request in seconds
+ * @param startTime - Start time in milliseconds (from performance.now())
  */
 export function trackRpcRequest(
     method: string,