From 367cc7902c8d97bd6a487c0021d5ee3172b3bb78 Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 27 Jan 2025 14:11:58 -0500 Subject: [PATCH 1/2] fix: `XAccountController` should use streaming for large archives --- src/account_x/x_account_controller.ts | 60 ++++++++++++++++++++------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/src/account_x/x_account_controller.ts b/src/account_x/x_account_controller.ts index d8006e38..23b82161 100644 --- a/src/account_x/x_account_controller.ts +++ b/src/account_x/x_account_controller.ts @@ -1222,26 +1222,47 @@ export class XAccountController { log.info(`XAccountController.archiveBuild: archive has ${tweets.length} tweets, ${retweets.length} retweets, ${likes.length} likes, ${bookmarks.length} bookmarks, ${users.length} users, ${conversations.length} conversations, and ${messages.length} messages`); - const archive: XArchiveTypes.XArchive = { - appVersion: app.getVersion(), - username: this.account.username, - createdAt: new Date().toLocaleString(), - tweets: formattedTweets, - retweets: formattedRetweets, - likes: formattedLikes, - bookmarks: formattedBookmarks, - users: formattedUsers, - conversations: formattedConversations, - messages: formattedMessages, - } - - // Save the archive object to a file + // Save the archive object to a file using streaming const assetsPath = path.join(getAccountDataPath("X", this.account.username), "assets"); if (!fs.existsSync(assetsPath)) { fs.mkdirSync(assetsPath); } const archivePath = path.join(assetsPath, "archive.js"); - fs.writeFileSync(archivePath, `window.archiveData=${JSON.stringify(archive, null, 2)};`); + + const streamWriter = fs.createWriteStream(archivePath); + try { + // Write the window.archiveData prefix + streamWriter.write('window.archiveData='); + + // Write the archive metadata + streamWriter.write('{\n'); + streamWriter.write(` "appVersion": ${JSON.stringify(app.getVersion())},\n`); + streamWriter.write(` "username": ${JSON.stringify(this.account.username)},\n`); + streamWriter.write(` "createdAt": ${JSON.stringify(new Date().toLocaleString())},\n`); + + // Write each array separately using a streaming approach in case the arrays are large + await this.writeJSONArray(streamWriter, formattedTweets, "tweets"); + streamWriter.write(',\n'); + await this.writeJSONArray(streamWriter, formattedRetweets, "retweets"); + streamWriter.write(',\n'); + await this.writeJSONArray(streamWriter, formattedLikes, "likes"); + streamWriter.write(',\n'); + await this.writeJSONArray(streamWriter, formattedBookmarks, "bookmarks"); + streamWriter.write(',\n'); + await this.writeJSONArray(streamWriter, Object.values(formattedUsers), "users"); + streamWriter.write(',\n'); + await this.writeJSONArray(streamWriter, formattedConversations, "conversations"); + streamWriter.write(',\n'); + await this.writeJSONArray(streamWriter, formattedMessages, "messages"); + + // Close the object + streamWriter.write('};'); + + await new Promise((resolve) => streamWriter.end(resolve)); + } catch (error) { + streamWriter.end(); + throw error; + } log.info(`XAccountController.archiveBuild: archive saved to ${archivePath}`); @@ -1251,6 +1272,15 @@ export class XAccountController { await archiveZip.extract({ path: getAccountDataPath("X", this.account.username) }); } + async writeJSONArray(streamWriter: fs.WriteStream, items: any[], propertyName: string) { + streamWriter.write(` "${propertyName}": [\n`); + for (let i = 0; i < items.length; i++) { + const suffix = i < items.length - 1 ? ',\n' : '\n'; + streamWriter.write(' ' + JSON.stringify(items[i]) + suffix); + } + streamWriter.write(' ]'); + } + // When you start deleting tweets, return a list of tweets to delete async deleteTweetsStart(): Promise { if (!this.db) { From 6df4a971c38596c6068e0f818ab60ac938395174 Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 27 Jan 2025 15:26:52 -0500 Subject: [PATCH 2/2] fix: make `writeJSONArray` generic --- src/account_x/x_account_controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/account_x/x_account_controller.ts b/src/account_x/x_account_controller.ts index 23b82161..420ab736 100644 --- a/src/account_x/x_account_controller.ts +++ b/src/account_x/x_account_controller.ts @@ -1272,7 +1272,7 @@ export class XAccountController { await archiveZip.extract({ path: getAccountDataPath("X", this.account.username) }); } - async writeJSONArray(streamWriter: fs.WriteStream, items: any[], propertyName: string) { + async writeJSONArray(streamWriter: fs.WriteStream, items: T[], propertyName: string) { streamWriter.write(` "${propertyName}": [\n`); for (let i = 0; i < items.length; i++) { const suffix = i < items.length - 1 ? ',\n' : '\n';