feat(new-compiler): migrate metadata storage from lockfile to LMDB

.changeset/common-teeth-reply.md

@@ -0,0 +1,9 @@
+---
+"@lingo.dev/compiler": patch
+---
+
+- Migrate metadata storage from JSON files to LMDB
+- New storage locations: .lingo/metadata-dev/ and .lingo/metadata-build/
+- Use pure functions with short-lived connections for multi-worker safety
+- Update compiler docs
+- Remove proper-lockfile dependency

packages/new-compiler/README.md

@@ -395,9 +395,10 @@ The compiler is organized into several key modules:
 
 #### `src/metadata/` - Translation metadata management
 
-- **`manager.ts`** - CRUD operations for `.lingo/metadata.json`
-- Thread-safe metadata file operations with file locking
+- **`manager.ts`** - CRUD operations for LMDB metadata database
+- Uses LMDB for high-performance key-value storage with built-in concurrency
 - Manages translation entries with hash-based identifiers
+- Stores metadata in `.lingo/metadata-dev/` (development) or `.lingo/metadata-build/` (production)
 
 #### `src/translators/` - Translation provider abstraction
 

packages/new-compiler/docs/TRANSLATION_ARCHITECTURE.md

@@ -7,8 +7,8 @@ metadata management, translation execution, and caching.
 
 ## Architectural Principles
 
-1. **Metadata file structure** is only known by:
-   - Metadata Manager (reads/writes metadata.json)
+1. **Metadata storage** is only known by:
+   - Metadata functions (reads/writes LMDB database)
    - Translation Service (orchestrator that coordinates everything)
 
 2. **Translators are stateless** and work with abstract `TranslatableEntry` types
@@ -36,9 +36,9 @@ metadata management, translation execution, and caching.
                  │ writes
                  ↓
 ┌──────────────────────────────────────────────────┐
-│ MetadataManager                                  │
-│  - ONLY component that reads/writes metadata.json│
-│  - Provides metadata loading/saving              │
+│ Metadata Functions (saveMetadata/loadMetadata)   │
+│  - Pure functions for LMDB database access       │
+│  - Short-lived connections (multi-worker safe)   │
 │  - Returns TranslationEntry[]                    │
 └────────────────┬─────────────────────────────────┘
                  │ reads from

packages/new-compiler/package.json

@@ -143,7 +143,6 @@
     "@types/babel__traverse": "7.28.0",
     "@types/ini": "4.1.1",
     "@types/node": "25.0.3",
-    "@types/proper-lockfile": "4.1.4",
     "@types/react": "19.2.7",
     "@types/react-dom": "19.2.3",
     "@types/ws": "8.18.1",
@@ -178,7 +177,7 @@
     "lodash": "4.17.21",
     "node-machine-id": "1.1.12",
     "posthog-node": "5.14.0",
-    "proper-lockfile": "4.1.2",
+    "lmdb": "3.2.6",
     "ws": "8.18.3"
   },
   "peerDependencies": {

packages/new-compiler/src/metadata/manager.test.ts

@@ -0,0 +1,219 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import fs from "fs";
+import path from "path";
+import os from "os";
+import {
+  createEmptyMetadata,
+  loadMetadata,
+  saveMetadata,
+  cleanupExistingMetadata,
+  getMetadataPath,
+} from "./manager";
+import type { TranslationEntry } from "../types";
+
+function createTestEntry(
+  overrides: Partial<TranslationEntry> & {
+    hash?: string;
+    sourceText?: string;
+  } = {},
+): TranslationEntry {
+  const hash = overrides.hash ?? `hash_${Math.random().toString(36).slice(2)}`;
+  return {
+    type: "content",
+    hash,
+    sourceText: overrides.sourceText ?? `Source text for ${hash}`,
+    context: { filePath: "test.tsx", componentName: "TestComponent" },
+    location: { filePath: "test.tsx", line: 1, column: 1 },
+    ...overrides,
+  } as TranslationEntry;
+}
+
+function createUniqueDbPath(): string {
+  return path.join(
+    os.tmpdir(),
+    `lmdb-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+}
+
+describe("metadata", () => {
+  let testDbPath: string;
+
+  beforeEach(() => {
+    testDbPath = createUniqueDbPath();
+  });
+
+  afterEach(() => {
+    cleanupExistingMetadata(testDbPath);
+  });
+
+  describe("createEmptyMetadata", () => {
+    it("should return valid empty metadata structure", () => {
+      const metadata = createEmptyMetadata();
+
+      expect(metadata.entries).toEqual({});
+      expect(metadata.stats!.totalEntries).toBe(0);
+      // Verify valid ISO date
+      const date = new Date(metadata.stats!.lastUpdated);
+      expect(date.getTime()).not.toBeNaN();
+    });
+  });
+
+  describe("loadMetadata", () => {
+    it("should return empty metadata for new database", async () => {
+      const metadata = await loadMetadata(testDbPath);
+      expect(metadata.entries).toEqual({});
+      expect(metadata.stats!.totalEntries).toBe(0);
+    });
+
+    it("should load and preserve all entry fields", async () => {
+      const entry: TranslationEntry = {
+        type: "content",
+        hash: "full-entry",
+        sourceText: "Hello world",
+        context: { filePath: "app.tsx", componentName: "AppComponent" },
+        location: { filePath: "app.tsx", line: 42, column: 10 },
+      };
+
+      await saveMetadata(testDbPath, [entry]);
+      const metadata = await loadMetadata(testDbPath);
+
+      expect(metadata.entries["full-entry"]).toEqual(entry);
+      expect(metadata.stats!.totalEntries).toBe(1);
+    });
+
+    it("should handle entries with very long sourceText", async () => {
+      const longText = "A".repeat(100000);
+      await saveMetadata(testDbPath, [
+        createTestEntry({ hash: "long-text", sourceText: longText }),
+      ]);
+
+      const metadata = await loadMetadata(testDbPath);
+      expect(metadata.entries["long-text"].sourceText).toBe(longText);
+    });
+  });
+
+  describe("saveMetadata", () => {
+    it("should save, accumulate, and update entries correctly", async () => {
+      // Save single entry
+      await saveMetadata(testDbPath, [
+        createTestEntry({ hash: "entry-1", sourceText: "v1" }),
+      ]);
+      expect((await loadMetadata(testDbPath)).stats!.totalEntries).toBe(1);
+
+      // Accumulate multiple entries
+      await saveMetadata(testDbPath, [
+        createTestEntry({ hash: "entry-2" }),
+        createTestEntry({ hash: "entry-3" }),
+      ]);
+      expect((await loadMetadata(testDbPath)).stats!.totalEntries).toBe(3);
+
+      // Update existing entry (count should not increase)
+      await saveMetadata(testDbPath, [
+        createTestEntry({ hash: "entry-1", sourceText: "v2" }),
+      ]);
+      const updated = await loadMetadata(testDbPath);
+      expect(updated.stats!.totalEntries).toBe(3);
+      expect(updated.entries["entry-1"].sourceText).toBe("v2");
+
+      // Empty array should not change anything
+      await saveMetadata(testDbPath, []);
+      expect((await loadMetadata(testDbPath)).stats!.totalEntries).toBe(3);
+    });
+
+    it("should handle large batch of entries", async () => {
+      const entries = Array.from({ length: 100 }, (_, i) =>
+        createTestEntry({ hash: `batch-${i}` }),
+      );
+
+      await saveMetadata(testDbPath, entries);
+      expect((await loadMetadata(testDbPath)).stats!.totalEntries).toBe(100);
+    });
+
+    it("should maintain data integrity after many operations", async () => {
+      // Many saves with overlapping keys
+      for (let i = 0; i < 10; i++) {
+        await saveMetadata(testDbPath, [
+          createTestEntry({
+            hash: `persistent-${i % 5}`,
+            sourceText: `v${i}`,
+          }),
+          createTestEntry({ hash: `unique-${i}` }),
+        ]);
+      }
+
+      const final = await loadMetadata(testDbPath);
+      // 5 persistent + 10 unique = 15
+      expect(final.stats!.totalEntries).toBe(15);
+    });
+  });
+
+  describe("concurrent access (single process)", () => {
+    it("should handle concurrent operations from multiple calls", async () => {
+      // LMDB handles concurrent writes via OS-level locking
+      const promises = Array.from({ length: 10 }, async (_, i) => {
+        await saveMetadata(testDbPath, [
+          createTestEntry({ hash: `concurrent-${i}` }),
+        ]);
+      });
+      await Promise.all(promises);
+
+      // Verify all entries are present
+      expect((await loadMetadata(testDbPath)).stats!.totalEntries).toBe(10);
+    });
+  });
+
+  describe("cleanupExistingMetadata", () => {
+    it("should remove database and allow reopening with fresh state", async () => {
+      await saveMetadata(testDbPath, [createTestEntry({ hash: "before" })]);
+      expect(fs.existsSync(testDbPath)).toBe(true);
+
+      // Cleanup should succeed because saveMetadata closes the DB
+      cleanupExistingMetadata(testDbPath);
+      expect(fs.existsSync(testDbPath)).toBe(false);
+
+      // Should work with fresh state after cleanup
+      const metadata = await loadMetadata(testDbPath);
+      expect(metadata.entries["before"]).toBeUndefined();
+      expect(metadata.stats!.totalEntries).toBe(0);
+    });
+
+    it("should handle non-existent path and multiple calls gracefully", () => {
+      const nonExistent = path.join(os.tmpdir(), "does-not-exist-db");
+      expect(() => cleanupExistingMetadata(nonExistent)).not.toThrow();
+      expect(() => cleanupExistingMetadata(nonExistent)).not.toThrow();
+    });
+  });
+
+  describe("getMetadataPath", () => {
+    it("should return correct path based on environment and config", () => {
+      const devResult = getMetadataPath({
+        sourceRoot: "/app",
+        lingoDir: ".lingo",
+        environment: "development",
+      });
+      expect(devResult).toContain("metadata-dev");
+      expect(devResult).not.toContain("metadata-build");
+
+      const prodResult = getMetadataPath({
+        sourceRoot: "/app",
+        lingoDir: ".lingo",
+        environment: "production",
+      });
+      expect(prodResult).toContain("metadata-build");
+
+      const customResult = getMetadataPath({
+        sourceRoot: "/app",
+        lingoDir: ".custom-lingo",
+        environment: "development",
+      });
+      expect(customResult).toContain(".custom-lingo");
+    });
+  });
+
+  describe("error handling", () => {
+    it("should throw descriptive error for invalid path", async () => {
+      const invalidPath = "/root/definitely/cannot/create/this/path";
+      await expect(loadMetadata(invalidPath)).rejects.toThrow();
+    });
+  });
+});