diff --git a/package.json b/package.json index 8eaceaf61e..27a97fd3b1 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,7 @@ "backend:updateDendronDeps": "node bootstrap/backend/updateDendronhqDeps.js", "stash:unstaged": "git stash save -k 'pre-linting-stash'", "stash:pop": "git stash && git stash pop stash@{1} && git read-tree stash && git stash drop", - "test": "cross-env LOG_LEVEL=error yarn jest", + "test": "yarn jest", "test:cli": "cross-env LOG_LEVEL=error npx jest --selectProjects non-plugin-tests --forceExit", "test:cli:update-snapshots": "yarn test:cli -u" }, diff --git a/packages/common-server/src/logger.ts b/packages/common-server/src/logger.ts index b7d65a79f0..2fd9c58eb3 100644 --- a/packages/common-server/src/logger.ts +++ b/packages/common-server/src/logger.ts @@ -1,7 +1,6 @@ // import pino from "pino"; import { Disposable, env } from "@dendronhq/common-all"; -import _ from "lodash"; import pino from "pino"; export type LogLvl = "debug" | "info" | "error"; diff --git a/packages/engine-server/src/DendronEngineV3.ts b/packages/engine-server/src/DendronEngineV3.ts index 0bf5c7424c..6d7fd9824b 100644 --- a/packages/engine-server/src/DendronEngineV3.ts +++ b/packages/engine-server/src/DendronEngineV3.ts @@ -66,6 +66,7 @@ import { WriteSchemaResp, BacklinkUtils, DLinkUtils, + TimeUtils, } from "@dendronhq/common-all"; import { createLogger, @@ -85,6 +86,7 @@ import { LinkUtils } from "@dendronhq/unified"; import { NodeJSFileStore } from "./store"; import { HookUtils, RequireHookResp } from "./topics/hooks"; import { EngineUtils } from "./utils/engineUtils"; +import { SQLiteMetadataStore } from "./drivers"; type DendronEngineOptsV3 = { wsRoot: string; @@ -184,6 +186,7 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine { * Does not throw error but returns it */ async init(): Promise { + const ctx = "DendronEngineV3:init"; const defaultResp = { notes: {}, schemas: {}, @@ -262,7 +265,7 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine { error = new DendronCompositeError(allErrors); } this.logger.info({ - ctx: "init:ext", + ctx, error, storeError: allErrors, hookErrors, @@ -942,11 +945,47 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine { private async initNotes( schemas: SchemaModuleDict ): Promise> { - const ctx = "DEngine:initNotes"; - this.logger.info({ ctx, msg: "enter" }); + const ctx = "DEngineV3:initNotes"; let errors: IDendronError[] = []; let notesFname: NotePropsByFnameDict = {}; const start = process.hrtime(); + const enableSQLITE = + DConfig.readConfigSync(this.wsRoot).workspace.metadataStore === "sqlite"; + this.logger.info({ ctx, msg: "enter", enableSQLITE }); + if (enableSQLITE) { + // eslint-disable-next-line no-new + const store = new SQLiteMetadataStore({ + wsRoot: this.wsRoot, + force: true, + }); + + // sleep until store is done + const output = await TimeUtils.awaitWithLimit( + { limitMs: 6e4 }, + async () => { + while (store.status === "loading") { + this.logger.info({ ctx, msg: "downloading sql dependencies..." }); + // eslint-disable-next-line no-await-in-loop + await TimeUtils.sleep(1000); + } + return; + } + ); + + this.logger.info({ + ctx, + msg: "checking if sql is initialized...", + output, + }); + if (!(await SQLiteMetadataStore.isDBInitialized())) { + this.logger.info({ + ctx, + msg: "db not initialized", + }); + await SQLiteMetadataStore.createAllTables(); + await SQLiteMetadataStore.createWorkspace(this.wsRoot); + } + } const allNotesList = await Promise.all( this.vaults.map(async (vault) => { @@ -996,6 +1035,13 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine { numEntries: _.size(notesById), numCacheUpdates: notesCache.numCacheMisses, }); + if (enableSQLITE) { + if (!(await SQLiteMetadataStore.isVaultInitialized(vault))) { + await SQLiteMetadataStore.prisma().dVault.create({ + data: { fsPath: vault.fsPath, wsRoot: this.wsRoot }, + }); + } + } return notesById; } return {}; @@ -1010,12 +1056,27 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine { }, notesWithLinks ); + + if (enableSQLITE) { + this.logger.info({ ctx, msg: "updating notes in sql" }); + // TODO: OPTIMIZE + await SQLiteMetadataStore.deleteAllNotes(); + await SQLiteMetadataStore.bulkInsertAllNotes({ + notesIdDict: allNotes, + }); + + // const allNotes = await SQLiteetadataStore.prisma().note.findMany(); + this.logger.info({ + ctx, + msg: "post:update notes in sql", + }); + } const duration = getDurationMilliseconds(start); this.logger.info({ ctx, msg: `time to init notes: "${duration}" ms` }); return { data: allNotes, - error: new DendronCompositeError(errors), + error: errors.length > 0 ? new DendronCompositeError(errors) : undefined, }; } diff --git a/packages/engine-server/src/drivers/SQLiteMetadataStore.ts b/packages/engine-server/src/drivers/SQLiteMetadataStore.ts index fb0b2b813f..d2fa930979 100644 --- a/packages/engine-server/src/drivers/SQLiteMetadataStore.ts +++ b/packages/engine-server/src/drivers/SQLiteMetadataStore.ts @@ -227,7 +227,8 @@ export class SQLiteMetadataStore implements IDataStore { const fullQuery = sqlBegin + sqlEnd; // eslint-disable-next-line no-useless-catch try { - await prisma.$queryRawUnsafe(fullQuery); + const resp = await prisma.$executeRawUnsafe(fullQuery); + return { query: fullQuery, resp }; } catch (error) { // uncomment to log // console.log("---> ERROR START"); @@ -238,7 +239,11 @@ export class SQLiteMetadataStore implements IDataStore { // console.log("---> ERROR END"); throw error; } - return { query: fullQuery }; + } + + static deleteAllNotes() { + const raw = `DELETE FROM Note`; + return getPrismaClient().$queryRawUnsafe(raw); } static async search( diff --git a/packages/engine-server/src/drivers/file/storev2.ts b/packages/engine-server/src/drivers/file/storev2.ts index ba1947c4cd..b1172845c7 100644 --- a/packages/engine-server/src/drivers/file/storev2.ts +++ b/packages/engine-server/src/drivers/file/storev2.ts @@ -110,7 +110,7 @@ export class FileStorage implements DStore { this.anchors = []; this.logger = logger; this.config = config; - const ctx = "FileStorageV2"; + const ctx = "FileStorage:constructor"; this.logger.info({ ctx, wsRoot, vaults, level: this.logger.level }); this.engine = props.engine; } @@ -119,6 +119,7 @@ export class FileStorage implements DStore { const ctx = "FileStorage:init"; let errors: IDendronError[] = []; try { + this.logger.info({ ctx, msg: "pre:initSchema" }); const resp = await this.initSchema(); if (resp.error) { errors.push(FileStorage.createMalformedSchemaError(resp)); @@ -126,7 +127,9 @@ export class FileStorage implements DStore { resp.data.map((ent) => { this.schemas[ent.root.id] = ent; }); + this.logger.info({ ctx, msg: "pre:initNotes" }); const { errors: initErrors } = await this.initNotes(); + this.logger.info({ ctx, msg: "post:initNotes", initErrors }); errors = errors.concat(initErrors); this.logger.info({ ctx, msg: "post:initNotes", errors }); @@ -504,7 +507,7 @@ export class FileStorage implements DStore { async initNotes(): Promise<{ errors: IDendronError[]; }> { - const ctx = "initNotes"; + const ctx = "FileStorage:initNotes"; this.logger.info({ ctx, msg: "enter" }); let notesWithLinks: NoteProps[] = []; @@ -578,7 +581,10 @@ export class FileStorage implements DStore { this._addBacklinks({ notesWithLinks, allNotes }); const duration = getDurationMilliseconds(start); - this.logger.info({ ctx, msg: `time to init notes: "${duration}" ms` }); + this.logger.info({ + ctx, + msg: `time to init notes: "${duration}" ms`, + }); return { errors }; } diff --git a/packages/engine-test-utils/src/__tests__/engine-server/drivers/sqlite.spec.ts b/packages/engine-test-utils/src/__tests__/engine-server/drivers/sqlite.spec.ts index c00da107df..85f99f725d 100644 --- a/packages/engine-test-utils/src/__tests__/engine-server/drivers/sqlite.spec.ts +++ b/packages/engine-test-utils/src/__tests__/engine-server/drivers/sqlite.spec.ts @@ -1,9 +1,13 @@ -import { NoteUtils } from "@dendronhq/common-all"; +import { DEngineClient, NotePropsMeta, NoteUtils } from "@dendronhq/common-all"; import { SQLiteMetadataStore } from "@dendronhq/engine-server"; import fs from "fs-extra"; import _ from "lodash"; import sinon from "sinon"; -import { ENGINE_HOOKS, runEngineTestV5 } from "../../.."; +import { + createEngineV3FromEngine, + ENGINE_HOOKS, + runEngineTestV5, +} from "../../.."; // import os from "os"; // current issue with windows test @@ -12,6 +16,19 @@ import { ENGINE_HOOKS, runEngineTestV5 } from "../../.."; // const describeSkipWindows = // os.platform() === "win32" ? describe.skip : describe; +const createEngine = createEngineV3FromEngine; + +async function expectNotesAreEqual(opts: { + engine: DEngineClient; + notes: NotePropsMeta[]; +}) { + const resp = await opts.engine.bulkGetNotes( + opts.notes.map((note: NotePropsMeta) => note.id) + ); + const sortedResp = _.sortBy(_.pick(resp.data, "fname"), "fname"); + expect(sortedResp).toEqual(_.sortBy(_.pick(opts.notes, "fname"), "fname")); +} + describe.skip("GIVEN sqlite store", () => { afterEach(async () => { await SQLiteMetadataStore.prisma().$disconnect(); @@ -25,11 +42,12 @@ describe.skip("GIVEN sqlite store", () => { expect(dirList).toMatchSnapshot(); expect(dirList.includes("metadata.db")).toBeTruthy(); const notes = await SQLiteMetadataStore.prisma().note.findMany(); - const engineNotes = await engine.findNotesMeta({ excludeStub: false }); - expect(engineNotes.length).toEqual(notes.length); + expect(notes.length).toEqual(6); + await expectNotesAreEqual({ engine, notes }); }, { expect, + createEngine, preSetupHook: ENGINE_HOOKS.setupBasic, modConfigCb: (config) => { config.workspace.metadataStore = "sqlite"; @@ -46,8 +64,7 @@ describe.skip("GIVEN sqlite store", () => { expect(dirList).toMatchSnapshot(); expect(dirList.includes("metadata.db")).toBeTruthy(); const notes = await SQLiteMetadataStore.prisma().note.findMany(); - const engineNotes = await engine.findNotesMeta({ excludeStub: false }); - expect(engineNotes.length).toEqual(notes.length); + await expectNotesAreEqual({ engine, notes }); const { error } = await engine.init(); expect(error).toBeFalsy(); @@ -56,6 +73,7 @@ describe.skip("GIVEN sqlite store", () => { }, { expect, + createEngine, preSetupHook: ENGINE_HOOKS.setupBasic, modConfigCb: (config) => { config.workspace.metadataStore = "sqlite"; @@ -72,8 +90,7 @@ describe.skip("GIVEN sqlite store", () => { expect(dirList).toMatchSnapshot(); expect(dirList.includes("metadata.db")).toBeTruthy(); const notes = await SQLiteMetadataStore.prisma().note.findMany(); - const engineNotes = await engine.findNotesMeta({ excludeStub: false }); - expect(engineNotes.length).toEqual(notes.length); + await expectNotesAreEqual({ engine, notes }); const newNote = NoteUtils.create({ id: "new-note", fname: "new-note", @@ -90,6 +107,7 @@ describe.skip("GIVEN sqlite store", () => { }, { expect, + createEngine, preSetupHook: ENGINE_HOOKS.setupBasic, modConfigCb: (config) => { config.workspace.metadataStore = "sqlite";