From fa6ac17f51a2bb848d5fa232601e8413613af3a7 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 28 Feb 2025 14:23:40 -0800 Subject: [PATCH 1/8] Only save URLs if all of the fields are available --- src/account_x/x_account_controller.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/account_x/x_account_controller.ts b/src/account_x/x_account_controller.ts index 0f666ed3..b25f6eb0 100644 --- a/src/account_x/x_account_controller.ts +++ b/src/account_x/x_account_controller.ts @@ -887,6 +887,11 @@ export class XAccountController { // Loop over all URL items tweetLegacy.entities?.urls.forEach((url: XAPILegacyURL) => { + // Make sure we have all of the URL information before importing + if (!url["url"] || !url["display_url"] || !url["expanded_url"] || !url["indices"]) { + return; + } + // Have we seen this URL before? const existing: XTweetURLRow[] = exec(this.db, 'SELECT * FROM tweet_url WHERE url = ? AND tweetID = ?', [url["url"], tweetLegacy["id_str"]], "all") as XTweetURLRow[]; if (existing.length > 0) { @@ -2378,7 +2383,7 @@ export class XAccountController { const filename = `${media["id_str"]}.${mediaExtension}`; let archiveMediaFilename = null; - if ((media.type === "video" || media.type === "animated_gif" ) && media.video_info?.variants) { + if ((media.type === "video" || media.type === "animated_gif") && media.video_info?.variants) { // For videos, find the highest quality MP4 variant const mp4Variants = media.video_info.variants .filter(v => v.content_type === "video/mp4") From 4bda9282e6fa931c98e10230bfe2363e9901bef0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 28 Feb 2025 15:04:27 -0800 Subject: [PATCH 2/8] Add support for X API errors, and write tests for isXAPIBookmarksData, isXAPIError, and isXAPIData --- src/account_x.test.ts | 32 +++++++++++++++-- src/account_x/types.ts | 25 ++++++++++++-- src/account_x/x_account_controller.ts | 5 +++ src/renderer/src/view_models/XViewModel.ts | 1 + testdata/XAPIUserTweetsAndRepliesError.json | 38 +++++++++++++++++++++ 5 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 testdata/XAPIUserTweetsAndRepliesError.json diff --git a/src/account_x.test.ts b/src/account_x.test.ts index f16fde6e..fab15276 100644 --- a/src/account_x.test.ts +++ b/src/account_x.test.ts @@ -8,9 +8,7 @@ import { XAPILegacyUser, XAPILegacyTweet, XAPIConversation, - XAPIUser -} from './account_x'; -import { + XAPIUser, XTweetRow, XTweetMediaRow, XTweetURLRow, @@ -18,6 +16,9 @@ import { XConversationRow, XConversationParticipantRow, XMessageRow, + isXAPIError, + isXAPIBookmarksData, + isXAPIData, } from './account_x'; // Mock the util module @@ -802,6 +803,31 @@ test("XAccountController.indexParsedTweets() should index and parse links", asyn expect(linkRows[2].expandedURL).toBe('https://x.com/nexamind91326/status/1890513848811090236'); }) +test("types.isXAPIBookmarksData() should recognize bookmarks data", async () => { + const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIBookmarks.json'), 'utf8'); + const data = JSON.parse(body); + expect(isXAPIBookmarksData(data)).toBe(true); + expect(isXAPIError(data)).toBe(false); + expect(isXAPIData(data)).toBe(false); +}) + +test("types.isXAPIError() should recognize errors", async () => { + const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesError.json'), 'utf8'); + const data = JSON.parse(body); + expect(isXAPIError(data)).toBe(true); + expect(isXAPIBookmarksData(data)).toBe(false); + expect(isXAPIData(data)).toBe(false); +}) + +test("types.isXAPIData() should recognize data", async () => { + const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndReplies1.json'), 'utf8'); + const data = JSON.parse(body); + expect(isXAPIData(data)).toBe(true); + expect(isXAPIBookmarksData(data)).toBe(false); + expect(isXAPIError(data)).toBe(false); +}) + + // Testing the X migrations test("test migration: 20241016_add_config", async () => { diff --git a/src/account_x/types.ts b/src/account_x/types.ts index 4f54d461..c1817d01 100644 --- a/src/account_x/types.ts +++ b/src/account_x/types.ts @@ -312,6 +312,23 @@ export interface XAPITimeline { } export interface XAPIData { + errors?: [ + { + message: string; + locations: { + line: number; + column: number; + }[]; + path: string[]; + extensions: any; + code: number; + kind: string; + name: string; + source: string; + retry_after: number; + tracing: any; + } + ], data: { user: { result: { @@ -329,11 +346,15 @@ export interface XAPIBookmarksData { } export function isXAPIBookmarksData(body: any): body is XAPIBookmarksData { - return body.data && body.data.bookmark_timeline_v2; + return !!(body.data && body.data.bookmark_timeline_v2); +} + +export function isXAPIError(body: any): body is XAPIData { + return !!(body.errors && body.errors.length > 0); } export function isXAPIData(body: any): body is XAPIData { - return body.data && body.data.user && body.data.user.result && body.data.user.result.timeline_v2; + return !!(body.data && body.data.user && body.data.user.result && body.data.user.result.timeline_v2); } // Index direct messages diff --git a/src/account_x/x_account_controller.ts b/src/account_x/x_account_controller.ts index b25f6eb0..b3fcc192 100644 --- a/src/account_x/x_account_controller.ts +++ b/src/account_x/x_account_controller.ts @@ -87,6 +87,7 @@ import { XArchiveLikeContainer, isXArchiveLikeContainer, isXAPIBookmarksData, + isXAPIError, isXAPIData, } from './types' import * as XArchiveTypes from '../../archive-static-sites/x-archive/src/types'; @@ -703,6 +704,10 @@ export class XAccountController { timeline = (body as XAPIBookmarksData).data.bookmark_timeline_v2; } else if (isXAPIData(body)) { timeline = (body as XAPIData).data.user.result.timeline_v2; + } else if (isXAPIError(body)) { + log.error('XAccountController.indexParseTweetsResponseData: XAPIError', body); + this.mitmController.responseData[responseIndex].processed = true; + return false; } else { throw new Error('Invalid response data'); } diff --git a/src/renderer/src/view_models/XViewModel.ts b/src/renderer/src/view_models/XViewModel.ts index 9240f702..ad1936b4 100644 --- a/src/renderer/src/view_models/XViewModel.ts +++ b/src/renderer/src/view_models/XViewModel.ts @@ -441,6 +441,7 @@ export class XViewModel extends BaseViewModel { async indexTweetsHandleRateLimit(): Promise { this.log("indexTweetsHandleRateLimit", this.progress); + this.pause(); await this.waitForPause(); if (await this.doesSelectorExist('section [data-testid="cellInnerDiv"]')) { diff --git a/testdata/XAPIUserTweetsAndRepliesError.json b/testdata/XAPIUserTweetsAndRepliesError.json new file mode 100644 index 00000000..a7449c96 --- /dev/null +++ b/testdata/XAPIUserTweetsAndRepliesError.json @@ -0,0 +1,38 @@ +{ + "errors": [ + { + "message": "Timeout: Unspecified", + "locations": [ + { + "line": 3, + "column": 5 + } + ], + "path": [ + "user", + "result" + ], + "extensions": { + "name": "TimeoutError", + "source": "Server", + "retry_after": 0, + "code": 29, + "kind": "ServiceLevel", + "tracing": { + "trace_id": "c685ec3405bf1b67" + } + }, + "code": 29, + "kind": "ServiceLevel", + "name": "TimeoutError", + "source": "Server", + "retry_after": 0, + "tracing": { + "trace_id": "c685ec3405bf1b67" + } + } + ], + "data": { + "user": {} + } +} \ No newline at end of file From d4516a3702fd0b69571582c80054a283730fa93c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 28 Feb 2025 15:05:30 -0800 Subject: [PATCH 3/8] Oops, do not pause here --- src/renderer/src/view_models/XViewModel.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/renderer/src/view_models/XViewModel.ts b/src/renderer/src/view_models/XViewModel.ts index ad1936b4..9240f702 100644 --- a/src/renderer/src/view_models/XViewModel.ts +++ b/src/renderer/src/view_models/XViewModel.ts @@ -441,7 +441,6 @@ export class XViewModel extends BaseViewModel { async indexTweetsHandleRateLimit(): Promise { this.log("indexTweetsHandleRateLimit", this.progress); - this.pause(); await this.waitForPause(); if (await this.doesSelectorExist('section [data-testid="cellInnerDiv"]')) { From 283675fa861cf964f4d1de4564856230c0e86979 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 28 Feb 2025 15:25:17 -0800 Subject: [PATCH 4/8] During indexTweets, X sometimes responds with 429 even when it is not really rated limited. This leads the indexing to stop early because it cannot find the retry button. Check to make sure this is not the case before looking for the retry button. --- src/renderer/src/view_models/XViewModel.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/view_models/XViewModel.ts b/src/renderer/src/view_models/XViewModel.ts index 9240f702..73cc477b 100644 --- a/src/renderer/src/view_models/XViewModel.ts +++ b/src/renderer/src/view_models/XViewModel.ts @@ -459,6 +459,19 @@ export class XViewModel extends BaseViewModel { // // + // Check if we get more tweets by scrolling down, even without clicking any buttons + let numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + await this.sleep(2000); + await this.scrollUp(2000); + await this.sleep(2000); + await this.scrollToBottom(); + await this.sleep(2000); + let numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + if (numberOfDivsAfter > numberOfDivsBefore) { + // More tweets loaded + return true; + } + // If the retry button does not exist, try scrolling up and down again to trigger it // The retry button should be in the last cellInnerDiv, and it should have only 1 button in it if (await this.countSelectorsWithinElementLastFound('main[role="main"] nav[role="navigation"] + section div[data-testid=cellInnerDiv]', 'button') != 1) { @@ -473,7 +486,7 @@ export class XViewModel extends BaseViewModel { } // Count divs before clicking retry button - let numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); if (numberOfDivsBefore > 0) { // The last one is the one with the button numberOfDivsBefore--; @@ -484,7 +497,7 @@ export class XViewModel extends BaseViewModel { await this.sleep(2000); // Count divs after clicking retry button - const numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); // If there are more divs after, it means more tweets loaded return numberOfDivsAfter > numberOfDivsBefore; From fdb2ed572a5ef5760d3f48b711cc368a083ab9c0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 28 Feb 2025 16:21:18 -0800 Subject: [PATCH 5/8] Add a "Enable Clicking in Browser" link to the X job status, which turns clicking on and off --- src/renderer/src/App.vue | 5 +++++ src/renderer/src/views/x/XJobStatusComponent.vue | 11 +++++++++-- src/renderer/src/views/x/XView.vue | 13 +++++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/renderer/src/App.vue b/src/renderer/src/App.vue index 54a262eb..54efa8b4 100644 --- a/src/renderer/src/App.vue +++ b/src/renderer/src/App.vue @@ -350,6 +350,11 @@ body { opacity: 0.7; } +.webview-automation-border.webview-clickable { + opacity: 1; + pointer-events: auto; +} + .webview-input-border { border: 5px solid #c1fac4; } diff --git a/src/renderer/src/views/x/XJobStatusComponent.vue b/src/renderer/src/views/x/XJobStatusComponent.vue index a9d16c83..3040d19b 100644 --- a/src/renderer/src/views/x/XJobStatusComponent.vue +++ b/src/renderer/src/views/x/XJobStatusComponent.vue @@ -4,10 +4,11 @@ import RunningIcon from '../shared_components/RunningIcon.vue' defineProps<{ jobs: XJob[], - isPaused: boolean + isPaused: boolean, + clickingEnabled: boolean, }>(); -const emit = defineEmits(['onPause', 'onResume', 'onCancel', 'onReportBug']); +const emit = defineEmits(['onPause', 'onResume', 'onCancel', 'onReportBug', 'onClickingDisabled', 'onClickingEnabled']); const getStatusIcon = (status: string) => { const statusIcons: { [key: string]: string } = { @@ -69,6 +70,12 @@ const getJobTypeText = (jobType: string) => { Report a Bug +
+ +
diff --git a/src/renderer/src/views/x/XView.vue b/src/renderer/src/views/x/XView.vue index 4e9d18d1..0fbbfcd8 100644 --- a/src/renderer/src/views/x/XView.vue +++ b/src/renderer/src/views/x/XView.vue @@ -214,6 +214,9 @@ const reloadMediaPath = async () => { console.log('mediaPath', mediaPath.value); }; +// Enable/disable clicking in the webview +const clickingEnabled = ref(false); + // User variables const userAuthenticated = ref(false); const userPremium = ref(false); @@ -458,9 +461,10 @@ onUnmounted(async () => {
+ :jobs="currentJobs" :is-paused="isPaused" :clicking-enabled="clickingEnabled" + class="job-status-component" @on-pause="model.pause()" @on-resume="model.resume()" + @on-cancel="emit('onRefreshClicked')" @on-report-bug="onReportBug" + @on-clicking-enabled="clickingEnabled = true" @on-clicking-disabled="clickingEnabled = false" />
@@ -479,7 +483,8 @@ onUnmounted(async () => { :class="{ 'hidden': !model.showBrowser, 'webview-automation-border': model.showAutomationNotice, - 'webview-input-border': !model.showAutomationNotice + 'webview-input-border': !model.showAutomationNotice, + 'webview-clickable': clickingEnabled }" />