Yield rows individually

Author: ebardelli

packages/datasources/duckdb/index.cjs

@@ -125,6 +125,9 @@ const runQuery = async (queryString, database, batchSize = 100000) => {
 		} else if (database.directory) {
 			// Local database stored in source directory
 			filename = path.join(database.directory, database.filename);
+		} else {
+			// filename provided without a directory; use as given (may be absolute or relative)
+			filename = database.filename;
 		}
 	}
 
@@ -185,7 +188,10 @@ const runQuery = async (queryString, database, batchSize = 100000) => {
 				await reader.readUntil(target);
 				const allRows = reader.getRowObjectsJS();
 				const newRows = allRows.slice(prev).map(standardizeRow);
-				if (newRows.length > 0) yield newRows;
+				// yield rows one-by-one (asyncIterableToBatchedAsyncGenerator expects single-row yields)
+				for (const r of newRows) {
+					yield r;
+				}
 				prev = reader.currentRowCount;
 			}
 		} finally {

packages/datasources/duckdb/test/test.js

@@ -1,9 +1,26 @@
 import { test } from 'uvu';
 import * as assert from 'uvu/assert';
+import path from 'path';
+import { fileURLToPath } from 'url';
 import runQuery from '../index.cjs';
 import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons';
 import 'dotenv/config';
 
+test('basic select from needful_things.duckdb', async () => {
+	// Resolve the database file path relative to the repository root (from this test file)
+	const __filename = fileURLToPath(import.meta.url);
+	const __dirname = path.dirname(__filename);
+	const dbPath = path.join(__dirname, '..', '..', '..', '..', 'needful_things.duckdb');
+	// Select the rows from sqlite_master; this test database contains 7 entries
+	const query = 'SELECT name FROM sqlite_master';
+	const { rows: rowGen, expectedRowCount } = await runQuery(query, { filename: dbPath });
+	const rows = await batchedAsyncGeneratorToArray(rowGen);
+	assert.instance(rows, Array);
+	assert.type(rows[0], 'object');
+	// Expect exactly 7 rows in this test database
+	assert.equal(rows.length, 7);
+});
+
 // Types to test
 // BOOLEAN
 // TINYINT