Added other languages

.parcelrc

@@ -5,6 +5,9 @@
 	"transformers": {
 		"*.txt": [
 			"@parcel/transformer-raw"
+		],
+		"*.x-tmpl-mustache": [
+			"@parcel/transformer-inline-string"
 		]
 	}
 }

README.md

@@ -14,9 +14,9 @@ Development makes use of NodeJS.
 
 Various NPM scripts are defined:
 
--   clean -- remove generated content
--   validate -- validate input files
--   build -- build the result
--   test -- test the results
+- clean -- remove generated content
+- validate -- validate input files
+- build -- build the result
+- test -- test the results
 
 The start command will execute: validate, build, test

bin/extract-texts.mjs

@@ -0,0 +1,212 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import HQRLib from "@lbalab/hqr";
+import iconv from "iconv-lite";
+import yaml from "yaml";
+
+const gameInfo = {
+	lba1: {
+		lang: ["en", "fr", "de", "es", "it"],
+		// no idea what \x01 does
+		// '@' are newlines
+		postProcess: (text) =>
+			text.replaceAll("\x01", "").replaceAll(/ ?@ ?/g, "\n"),
+	},
+	lba2: {
+		lang: ["en", "fr", "de", "es", "it", "pt"],
+		// first byte is the dialog type
+		// '@' are newlines
+		postProcess: (text) => text.substr(1).replaceAll(/ ?@ ?/g, "\n"),
+	},
+};
+
+const textIdLookupTable = {
+	lba1: {},
+	lba2: {},
+};
+
+function getLbtInfo(mode, bundleCount, hqrIndex) {
+	// First text bundle ID is 2
+	const bundleId = ("00" + ((hqrIndex % bundleCount) + 2)).slice(-2);
+	const lang = gameInfo[mode].lang[Math.floor(hqrIndex / bundleCount)];
+	return { mode, hqrIndex, bundleId, lang };
+}
+
+async function readTextIdTable(byteArray) {
+	const table = [];
+	const wordArray = new Uint16Array(byteArray);
+	for (let i = 0; i < wordArray.length; ++i) {
+		table.push(wordArray[i]);
+	}
+	return table;
+}
+
+function seedTextIdLookupTable(lbtInfo, textIdTable) {
+	// first LTI file per bundle is leading, which should be "en"
+	// Quote IDs are based on the index of the "en" text ID table
+	if (textIdLookupTable[lbtInfo.mode][lbtInfo.bundleId]) {
+		return;
+	}
+	let table = {};
+	textIdLookupTable[lbtInfo.mode][lbtInfo.bundleId] = table;
+	for (let i = 0; i < textIdTable.length; ++i) {
+		table[textIdTable[i]] = ("000" + (i + 1)).slice(-3);
+	}
+}
+
+async function exportLookupTable(lbtInfo, textIdTable) {
+	let table = textIdTable.map((textId, index) => {
+		return { textId, index, quoteId: resolveTextId(lbtInfo, textId) };
+	});
+	let filename = `${lbtInfo.mode}/tables/${lbtInfo.bundleId}-${lbtInfo.lang}.json`;
+	await fs.mkdir(path.dirname(filename), { recursive: true });
+	await fs.writeFile(filename, JSON.stringify(table, null, 4));
+}
+
+function resolveTextId(lbtInfo, textId) {
+	const table = textIdLookupTable[lbtInfo.mode][lbtInfo.bundleId];
+	return table[textId];
+}
+
+function readShort(byteArray, index) {
+	return new Uint16Array(byteArray, index, 2)[0];
+}
+
+function isEmpty(text, lang) {
+	const trimmed = text.trim().toLowerCase();
+	if (trimmed === "") {
+		return true;
+	}
+	// Looks like empty is sometimes translated
+	switch (lang) {
+		case "en":
+			return trimmed === "empty";
+		case "fr":
+			return trimmed === "vide";
+		case "de":
+			return trimmed === "leer";
+		case "it":
+			return trimmed === "vuoto";
+		case "pt":
+			return trimmed === "vazio";
+	}
+	return false;
+}
+
+async function procLbtEntry(lbtInfo, textId, text) {
+	if (isEmpty(text, lbtInfo.lang)) {
+		return;
+	}
+
+	const id = resolveTextId(lbtInfo, textId);
+	const filename = `${lbtInfo.mode}/${lbtInfo.bundleId}/${id}.yaml`;
+
+	let data = { textId, location: null, speaker: null, message: null };
+	try {
+		const filedata = await fs.readFile(filename, "utf8");
+		const ymlData = yaml.parse(filedata);
+		data = { ...data, ...ymlData };
+	} catch (e) {}
+
+	if (lbtInfo.lang === "en") {
+		data.message = text;
+	} else {
+		if (!data.message) {
+			// missing en message is probably not good either
+			return;
+		}
+		if (!data[lbtInfo.lang]) {
+			data[lbtInfo.lang] = {};
+		}
+		data[lbtInfo.lang].message = text;
+	}
+
+	await fs.mkdir(path.dirname(filename), { recursive: true });
+	await fs.writeFile(
+		filename,
+		yaml.stringify(data, {
+			nullStr: "",
+			defaultKeyType: "PLAIN",
+		}),
+	);
+}
+
+async function procLbt(lbtInfo, textIdTable, byteArray) {
+	console.log("Processing LBT:", lbtInfo);
+	const entries = [];
+	const end = byteArray.byteLength;
+	for (let i = 0; i < end; i += 2) {
+		const val = readShort(byteArray, i);
+		if (val === end) {
+			break;
+		}
+		entries.push({ start: val, end: val });
+		if (entries.length > 1) {
+			entries[entries.length - 2].end = val;
+		}
+	}
+	if (entries.length > 0) {
+		entries[entries.length - 1].end = end;
+	}
+	console.log("Bundle entries:", entries.length);
+	for (let i = 0; i < entries.length; ++i) {
+		let text = iconv.decode(
+			new Uint8Array(
+				byteArray.slice(entries[i].start, entries[i].end - 1),
+			),
+			"cp437",
+		);
+		text = gameInfo[lbtInfo.mode].postProcess(text);
+		await procLbtEntry(lbtInfo, textIdTable[i], text.trim());
+	}
+}
+
+async function procFile(mode, file) {
+	console.log("Processing file:", file);
+	const data = await fs.readFile(file);
+	const hqr = HQRLib.HQR.fromArrayBuffer(
+		data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength),
+	);
+	console.log("Number of entries in file:", hqr.entries.length);
+	if (hqr.entries.length % gameInfo[mode].lang.length !== 0) {
+		throw new Error(
+			"Bundle count not a round number. Invalid language config?",
+		);
+	}
+	const bundles = Math.floor(hqr.entries.length / gameInfo[mode].lang.length);
+	let textIdTable;
+	let lbtInfo;
+	for (let i = 0; i < hqr.entries.length; ++i) {
+		let entry = hqr.entries[i];
+		if (!entry) {
+			continue;
+		}
+		if (i % 2 == 0) {
+			// Even entries are LTI (Text Index)
+			// Build the index -> textId table
+			lbtInfo = getLbtInfo(mode, bundles, i);
+			textIdTable = await readTextIdTable(entry.content);
+			// seed the textId -> quoteId table
+			seedTextIdLookupTable(lbtInfo, textIdTable);
+			await exportLookupTable(lbtInfo, textIdTable);
+		} else {
+			// Odd entries are LBTs
+			await procLbt(lbtInfo, textIdTable, entry.content);
+		}
+	}
+}
+
+if (process.argv.length !== 4) {
+	throw new Error(
+		`Usage: ${process.argv[0]} ${process.argv[1]} <lba1|lba2> <text.hqr>`,
+	);
+}
+
+switch (process.argv[2]) {
+	case "lba1":
+	case "lba2":
+		await procFile(process.argv[2], process.argv[3]);
+		break;
+	default:
+		throw new Error(`Unknown mode: ${process.argv[2]}`);
+}

bin/generate-json-bundle.mjs

@@ -24,9 +24,30 @@ async function load_yaml(file) {
 	}
 }
 
-async function resolveAudioFile(file) {
+function explodeLanguage(data) {
+	let langMap = Object.keys(data).filter(
+		(k) => /^[a-z]{2}(-[a-z]*)?$/i.test(k) && typeof data[k] === "object",
+	);
+	let base = { ...data };
+	for (let lang of langMap) {
+		delete base[lang];
+	}
+	let result = { en: base };
+	for (let lang of langMap) {
+		let entry = { ...base, ...data[lang] };
+		result[lang] = entry;
+	}
+	return result;
+}
+
+async function resolveAudioFile(file, lang = "en") {
 	let fp = path.parse(file);
-	let audioFile = `${fp.dir}/${fp.name}.webm`;
+	if (lang === "en") {
+		lang = "";
+	} else {
+		lang = `-${lang}`;
+	}
+	let audioFile = `${fp.dir}/${fp.name}${lang}.webm`;
 	try {
 		await fs.stat(path.join("dist", audioFile));
 		return audioFile;
@@ -37,44 +58,65 @@ async function resolveAudioFile(file) {
 
 async function load_quotes(game, bundlefile) {
 	console.log("Processing bundle:", bundlefile);
-	let result = [];
+	let result = { en: [] };
 	const bundledir = path.dirname(bundlefile);
-	const bundle = {
+	const bundle = explodeLanguage({
 		...(await load_yaml(bundlefile)),
-	};
+	});
 	let quotes = await glob(`${bundledir}/*.yaml`);
 	for (let quotefile of quotes) {
 		if (!path.basename(quotefile).match(/^[0-9]{3}\.yaml$/)) {
 			// Not a quote file
 			continue;
 		}
-		let quote = {
-			id: `${path.basename(bundledir)}:${path.parse(quotefile).name}`,
-			...bundle,
-			audio: await resolveAudioFile(quotefile),
-			...(await load_yaml(quotefile)),
-		};
-		result.push(quote);
+		let quoteData = await load_yaml(quotefile);
+		delete quoteData["textId"];
+		quoteData = explodeLanguage(quoteData);
+		for (let lang of Object.keys(quoteData)) {
+			let quote = {
+				id: `${path.basename(bundledir)}:${path.parse(quotefile).name}`,
+				...bundle[lang],
+				audio: await resolveAudioFile(quotefile, lang),
+				...quoteData[lang],
+			};
+			result[lang] = result[lang] || [];
+			result[lang].push(quote);
+		}
 	}
 	return result;
 }
 
 async function generate_game_bundle(dirname, destination) {
-	let quotes = [];
+	let quotes = {};
 	let bundles = await glob(`${dirname}/*/_bundle.yaml`);
 	for (let bundle of bundles) {
-		quotes.push(...(await load_quotes(dirname, bundle)));
+		let bundleQuotes = await load_quotes(dirname, bundle);
+		for (let lang of Object.keys(bundleQuotes)) {
+			quotes[lang] = quotes[lang] || [];
+			quotes[lang].push(...bundleQuotes[lang]);
+		}
 	}
 	await fs.mkdir(path.dirname(destination), { recursive: true });
-	await fs.writeFile(destination, JSON.stringify(quotes));
-	console.log(`Merged ${dirname} into ${destination}`);
+	const destDir = path.dirname(destination);
+	const destBasename = path.basename(destination);
+	const manifest = {};
+	for (let lang of Object.keys(quotes)) {
+		let dest = `${destBasename}-${lang}.json`;
+		manifest[lang] = { src: dest };
+		await fs.writeFile(`${destDir}/${dest}`, JSON.stringify(quotes[lang]));
+		console.log(`Merged ${dirname} [${lang}] into ${dest}`);
+	}
+	await fs.writeFile(
+		`${destDir}/${destBasename}.manifest.json`,
+		JSON.stringify(manifest),
+	);
 }
 
 let procs = [];
 for (let i = 2; i < process.argv.length; i++) {
 	let dirname = process.argv[i];
 	if (await is_dir(dirname)) {
-		procs.push(generate_game_bundle(dirname, `dist/${dirname}.json`));
+		procs.push(generate_game_bundle(dirname, `dist/${dirname}`));
 	} else {
 		console.warn("Not a directory:", dirname);
 	}