feat(ocap-kernel): add permanent failure detection for reconnection (#789)

Closes #688

## Summary

- Add error pattern tracking to `ReconnectionManager` to detect
permanently unreachable peers
- When the same error code (ECONNREFUSED, EHOSTUNREACH, ENOTFOUND, or
ENETUNREACH) occurs consecutively N times (default 5), the peer is
marked as permanently failed
- Permanent failures stop reconnection attempts to avoid wasted retries
for unreachable peers

## Changes

**kernel-errors:**
- Add `getNetworkErrorCode()` helper to extract error codes from errors

**ocap-kernel:**
- Extend `ReconnectionState` with `errorHistory` and `permanentlyFailed`
fields
- Add `recordError()`, `isPermanentlyFailed()`,
`clearPermanentFailure()` methods to `ReconnectionManager`
- Update `startReconnection()` to return `false` for permanently failed
peers and reset error history
- Integrate error recording into reconnection lifecycle
- Check permanent failure status before attempting reconnection

## Test plan

- [x] Unit tests for `getNetworkErrorCode` helper
- [x] Unit tests for error tracking in `ReconnectionManager`
- [x] Unit tests for permanent failure detection (consecutive identical
errors)
- [x] Unit tests for clearing permanent failure state
- [x] Integration tests for reconnection lifecycle with permanent
failure
- [x] All existing tests pass

🤖 Generated with [Claude Code](https://claude.com/claude-code)

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> **Medium Risk**
> Changes reconnection control-flow to stop retry loops based on tracked
error patterns and modifies `startReconnection` semantics (now returns a
boolean), which could alter peer recovery behavior if misclassified or
mis-integrated.
> 
> **Overview**
> Adds permanent-failure detection to the remote reconnection flow: the
`ReconnectionManager` now tracks recent retryable network error codes
per peer and marks a peer as permanently failed after a configurable
threshold of consecutive identical “unreachable” codes.
> 
> The reconnection lifecycle records retryable failures (via new
`kernel-errors` export `getNetworkErrorCode`) and short-circuits future
attempts when a peer is permanently failed, calling `onRemoteGiveUp`;
manual `reconnectPeer` now clears permanent-failure state before
restarting. Also treats `ENOTFOUND` as retryable to support this
detection, with expanded unit/integration test coverage.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
e08af9e. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: sirtimid

packages/kernel-errors/src/index.test.ts

@@ -20,6 +20,7 @@ describe('index', () => {
       'VatAlreadyExistsError',
       'VatDeletedError',
       'VatNotFoundError',
+      'getNetworkErrorCode',
       'isMarshaledError',
       'isMarshaledOcapError',
       'isOcapError',

packages/kernel-errors/src/index.ts

@@ -27,4 +27,5 @@ export { unmarshalError } from './marshal/unmarshalError.ts';
 export { isMarshaledError } from './marshal/isMarshaledError.ts';
 export { isMarshaledOcapError } from './marshal/isMarshaledOcapError.ts';
 export { isRetryableNetworkError } from './utils/isRetryableNetworkError.ts';
+export { getNetworkErrorCode } from './utils/getNetworkErrorCode.ts';
 export { isResourceLimitError } from './utils/isResourceLimitError.ts';

packages/kernel-errors/src/utils/getNetworkErrorCode.test.ts

@@ -0,0 +1,88 @@
+import { describe, it, expect } from 'vitest';
+
+import { getNetworkErrorCode } from './getNetworkErrorCode.ts';
+
+describe('getNetworkErrorCode', () => {
+  describe('Node.js network error codes', () => {
+    it.each([
+      { code: 'ECONNRESET' },
+      { code: 'ETIMEDOUT' },
+      { code: 'EPIPE' },
+      { code: 'ECONNREFUSED' },
+      { code: 'EHOSTUNREACH' },
+      { code: 'ENETUNREACH' },
+      { code: 'ENOTFOUND' },
+    ])('returns $code from error with code property', ({ code }) => {
+      const error = new Error('Network error') as Error & { code: string };
+      error.code = code;
+      expect(getNetworkErrorCode(error)).toBe(code);
+    });
+  });
+
+  describe('libp2p and other named errors', () => {
+    it.each([
+      { name: 'DialError' },
+      { name: 'TransportError' },
+      { name: 'MuxerClosedError' },
+      { name: 'WebRTCDialError' },
+    ])('returns $name from error with name property', ({ name }) => {
+      const error = new Error('Connection failed');
+      error.name = name;
+      expect(getNetworkErrorCode(error)).toBe(name);
+    });
+
+    it('prefers code over name when both are present', () => {
+      const error = Object.assign(new Error('Network error'), {
+        code: 'ECONNREFUSED',
+        name: 'DialError',
+      });
+      expect(getNetworkErrorCode(error)).toBe('ECONNREFUSED');
+    });
+  });
+
+  describe('relay reservation errors', () => {
+    it('returns name for Error with NO_RESERVATION in message (name takes precedence)', () => {
+      const error = new Error(
+        'failed to connect via relay with status NO_RESERVATION',
+      );
+      // name ('Error') takes precedence over message parsing
+      expect(getNetworkErrorCode(error)).toBe('Error');
+    });
+
+    it('returns NO_RESERVATION when error has empty name', () => {
+      const error = Object.assign(
+        new Error('failed to connect via relay with status NO_RESERVATION'),
+        { name: '' },
+      );
+      expect(getNetworkErrorCode(error)).toBe('NO_RESERVATION');
+    });
+
+    it('returns name when both name and NO_RESERVATION message are present', () => {
+      const error = Object.assign(
+        new Error('failed to connect via relay with status NO_RESERVATION'),
+        { name: 'InvalidMessageError' },
+      );
+      // name takes precedence over message parsing
+      expect(getNetworkErrorCode(error)).toBe('InvalidMessageError');
+    });
+  });
+
+  describe('unknown errors', () => {
+    it.each([
+      { name: 'null', value: null },
+      { name: 'undefined', value: undefined },
+      { name: 'string', value: 'error string' },
+      { name: 'number', value: 42 },
+      { name: 'plain object', value: { message: 'error' } },
+      { name: 'empty error name and code', value: { name: '', code: '' } },
+    ])('returns UNKNOWN for $name', ({ value }) => {
+      expect(getNetworkErrorCode(value)).toBe('UNKNOWN');
+    });
+
+    it('returns UNKNOWN for generic Error with no code', () => {
+      const error = new Error('Generic error');
+      // Generic Error has name 'Error', so it returns 'Error'
+      expect(getNetworkErrorCode(error)).toBe('Error');
+    });
+  });
+});

packages/kernel-errors/src/utils/getNetworkErrorCode.ts

@@ -0,0 +1,36 @@
+/**
+ * Extract a network error code from an error object.
+ *
+ * Returns error codes like 'ECONNREFUSED', 'ETIMEDOUT', etc., or
+ * the error name for libp2p errors, or 'UNKNOWN' for unrecognized errors.
+ *
+ * @param error - The error to extract the code from.
+ * @returns The error code string.
+ */
+export function getNetworkErrorCode(error: unknown): string {
+  const anyError = error as {
+    code?: string;
+    name?: string;
+    message?: string;
+  };
+
+  // Node.js network error codes (ECONNREFUSED, ETIMEDOUT, etc.)
+  if (typeof anyError?.code === 'string' && anyError.code.length > 0) {
+    return anyError.code;
+  }
+
+  // libp2p errors and other named errors
+  if (typeof anyError?.name === 'string' && anyError.name.length > 0) {
+    return anyError.name;
+  }
+
+  // Check message for relay reservation errors
+  if (
+    typeof anyError?.message === 'string' &&
+    anyError.message.includes('NO_RESERVATION')
+  ) {
+    return 'NO_RESERVATION';
+  }
+
+  return 'UNKNOWN';
+}

packages/kernel-errors/src/utils/isRetryableNetworkError.test.ts

@@ -34,6 +34,7 @@ describe('isRetryableNetworkError', () => {
       { code: 'ECONNREFUSED', description: 'connection refused' },
       { code: 'EHOSTUNREACH', description: 'no route to host' },
       { code: 'ENETUNREACH', description: 'network unreachable' },
+      { code: 'ENOTFOUND', description: 'DNS lookup failed' },
     ])('returns true for $code ($description)', ({ code }) => {
       const error = new Error('Network error') as Error & { code: string };
       error.code = code;

packages/kernel-errors/src/utils/isRetryableNetworkError.ts

@@ -21,14 +21,18 @@ export function isRetryableNetworkError(error: unknown): boolean {
   }
 
   // Node.js network error codes
+  // Note: ENOTFOUND (DNS lookup failed) is included to allow permanent failure
+  // detection to work - after multiple consecutive ENOTFOUND errors, the peer
+  // will be marked as permanently failed rather than giving up immediately.
   const code = anyError?.code;
   if (
     code === 'ECONNRESET' ||
     code === 'ETIMEDOUT' ||
     code === 'EPIPE' ||
     code === 'ECONNREFUSED' ||
     code === 'EHOSTUNREACH' ||
-    code === 'ENETUNREACH'
+    code === 'ENETUNREACH' ||
+    code === 'ENOTFOUND'
   ) {
     return true;
   }

packages/ocap-kernel/src/remotes/platform/reconnection-lifecycle.test.ts

@@ -17,14 +17,15 @@ vi.mock('@metamask/kernel-utils', async () => {
   };
 });
 
-// Mock kernel-errors for isRetryableNetworkError
+// Mock kernel-errors for isRetryableNetworkError and getNetworkErrorCode
 vi.mock('@metamask/kernel-errors', async () => {
   const actual = await vi.importActual<typeof kernelErrors>(
     '@metamask/kernel-errors',
   );
   return {
     ...actual,
     isRetryableNetworkError: vi.fn(),
+    getNetworkErrorCode: vi.fn().mockReturnValue('ECONNREFUSED'),
   };
 });
 
@@ -75,9 +76,11 @@ describe('reconnection-lifecycle', () => {
         incrementAttempt: vi.fn().mockReturnValue(1),
         decrementAttempt: vi.fn(),
         calculateBackoff: vi.fn().mockReturnValue(100),
-        startReconnection: vi.fn(),
+        startReconnection: vi.fn().mockReturnValue(true),
         stopReconnection: vi.fn(),
         resetBackoff: vi.fn(),
+        isPermanentlyFailed: vi.fn().mockReturnValue(false),
+        recordError: vi.fn(),
       },
       maxRetryAttempts: 3,
       onRemoteGiveUp: vi.fn(),
@@ -616,5 +619,168 @@ describe('reconnection-lifecycle', () => {
       // stopReconnection should be called on success
       expect(deps.reconnectionManager.stopReconnection).toHaveBeenCalled();
     });
+
+    describe('permanent failure detection', () => {
+      it('gives up when peer is permanently failed at start of loop', async () => {
+        (
+          deps.reconnectionManager.isPermanentlyFailed as ReturnType<
+            typeof vi.fn
+          >
+        ).mockReturnValue(true);
+
+        const lifecycle = makeReconnectionLifecycle(deps);
+
+        await lifecycle.attemptReconnection('peer1');
+
+        expect(deps.reconnectionManager.stopReconnection).toHaveBeenCalledWith(
+          'peer1',
+        );
+        expect(deps.onRemoteGiveUp).toHaveBeenCalledWith('peer1');
+        expect(deps.logger.log).toHaveBeenCalledWith(
+          expect.stringContaining('permanently failed'),
+        );
+      });
+
+      it('records error after failed dial attempt', async () => {
+        const error = new Error('Connection refused');
+        (error as Error & { code: string }).code = 'ECONNREFUSED';
+        (deps.dialPeer as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
+          error,
+        );
+        (
+          kernelErrors.isRetryableNetworkError as ReturnType<typeof vi.fn>
+        ).mockReturnValue(true);
+        (deps.reconnectionManager.isReconnecting as ReturnType<typeof vi.fn>)
+          .mockReturnValueOnce(true)
+          .mockReturnValueOnce(false);
+
+        const lifecycle = makeReconnectionLifecycle(deps);
+
+        await lifecycle.attemptReconnection('peer1');
+
+        expect(deps.reconnectionManager.recordError).toHaveBeenCalledWith(
+          'peer1',
+          'ECONNREFUSED',
+        );
+      });
+
+      it('gives up when error triggers permanent failure', async () => {
+        const error = new Error('Connection refused');
+        (deps.dialPeer as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
+          error,
+        );
+        (
+          kernelErrors.isRetryableNetworkError as ReturnType<typeof vi.fn>
+        ).mockReturnValue(true);
+        (
+          deps.reconnectionManager.isPermanentlyFailed as ReturnType<
+            typeof vi.fn
+          >
+        )
+          .mockReturnValueOnce(false) // At start of loop
+          .mockReturnValueOnce(true); // After recording error
+
+        const lifecycle = makeReconnectionLifecycle(deps);
+
+        await lifecycle.attemptReconnection('peer1');
+
+        expect(deps.reconnectionManager.stopReconnection).toHaveBeenCalledWith(
+          'peer1',
+        );
+        expect(deps.onRemoteGiveUp).toHaveBeenCalledWith('peer1');
+        expect(deps.outputError).toHaveBeenCalledWith(
+          'peer1',
+          expect.stringContaining('permanent failure detected'),
+          expect.any(Error),
+        );
+      });
+
+      it('continues retrying when error does not trigger permanent failure', async () => {
+        (deps.dialPeer as ReturnType<typeof vi.fn>)
+          .mockRejectedValueOnce(new Error('Temporary failure'))
+          .mockResolvedValueOnce(mockChannel);
+        (
+          kernelErrors.isRetryableNetworkError as ReturnType<typeof vi.fn>
+        ).mockReturnValue(true);
+        (
+          deps.reconnectionManager.isPermanentlyFailed as ReturnType<
+            typeof vi.fn
+          >
+        ).mockReturnValue(false);
+        (deps.reconnectionManager.isReconnecting as ReturnType<typeof vi.fn>)
+          .mockReturnValueOnce(true)
+          .mockReturnValueOnce(true)
+          .mockReturnValueOnce(false);
+
+        const lifecycle = makeReconnectionLifecycle(deps);
+
+        await lifecycle.attemptReconnection('peer1');
+
+        expect(deps.dialPeer).toHaveBeenCalledTimes(2);
+        expect(deps.reconnectionManager.recordError).toHaveBeenCalled();
+      });
+
+      it('does not record non-retryable errors in history', async () => {
+        const error = new Error('Auth failed');
+        (deps.dialPeer as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
+          error,
+        );
+        // Non-retryable error
+        (
+          kernelErrors.isRetryableNetworkError as ReturnType<typeof vi.fn>
+        ).mockReturnValue(false);
+
+        const lifecycle = makeReconnectionLifecycle(deps);
+
+        await lifecycle.attemptReconnection('peer1');
+
+        // Non-retryable errors should NOT be recorded - they don't contribute
+        // to permanent failure detection
+        expect(deps.reconnectionManager.recordError).not.toHaveBeenCalled();
+        // But should still give up
+        expect(deps.onRemoteGiveUp).toHaveBeenCalledWith('peer1');
+      });
+    });
+  });
+
+  describe('handleConnectionLoss with permanent failure', () => {
+    it('skips reconnection and calls onRemoteGiveUp for permanently failed peer', () => {
+      (
+        deps.reconnectionManager.isReconnecting as ReturnType<typeof vi.fn>
+      ).mockReturnValue(false);
+      (
+        deps.reconnectionManager.startReconnection as ReturnType<typeof vi.fn>
+      ).mockReturnValue(false);
+
+      const lifecycle = makeReconnectionLifecycle(deps);
+
+      lifecycle.handleConnectionLoss('peer1');
+
+      expect(deps.reconnectionManager.startReconnection).toHaveBeenCalledWith(
+        'peer1',
+      );
+      expect(deps.onRemoteGiveUp).toHaveBeenCalledWith('peer1');
+      expect(deps.logger.log).toHaveBeenCalledWith(
+        expect.stringContaining('permanently failed'),
+      );
+    });
+
+    it('proceeds with reconnection when startReconnection returns true', () => {
+      (
+        deps.reconnectionManager.isReconnecting as ReturnType<typeof vi.fn>
+      ).mockReturnValue(false);
+      (
+        deps.reconnectionManager.startReconnection as ReturnType<typeof vi.fn>
+      ).mockReturnValue(true);
+
+      const lifecycle = makeReconnectionLifecycle(deps);
+
+      lifecycle.handleConnectionLoss('peer1');
+
+      expect(deps.reconnectionManager.startReconnection).toHaveBeenCalledWith(
+        'peer1',
+      );
+      expect(deps.onRemoteGiveUp).not.toHaveBeenCalled();
+    });
   });
 });