Various X bug fixes

src/account_x.test.ts

@@ -8,16 +8,17 @@ import {
     XAPILegacyUser,
     XAPILegacyTweet,
     XAPIConversation,
-    XAPIUser
-} from './account_x';
-import {
+    XAPIUser,
     XTweetRow,
     XTweetMediaRow,
     XTweetURLRow,
     XUserRow,
     XConversationRow,
     XConversationParticipantRow,
     XMessageRow,
+    isXAPIError,
+    isXAPIBookmarksData,
+    isXAPIData,
 } from './account_x';
 
 // Mock the util module
@@ -802,6 +803,31 @@ test("XAccountController.indexParsedTweets() should index and parse links", asyn
     expect(linkRows[2].expandedURL).toBe('https://x.com/nexamind91326/status/1890513848811090236');
 })
 
+test("types.isXAPIBookmarksData() should recognize bookmarks data", async () => {
+    const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIBookmarks.json'), 'utf8');
+    const data = JSON.parse(body);
+    expect(isXAPIBookmarksData(data)).toBe(true);
+    expect(isXAPIError(data)).toBe(false);
+    expect(isXAPIData(data)).toBe(false);
+})
+
+test("types.isXAPIError() should recognize errors", async () => {
+    const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesError.json'), 'utf8');
+    const data = JSON.parse(body);
+    expect(isXAPIError(data)).toBe(true);
+    expect(isXAPIBookmarksData(data)).toBe(false);
+    expect(isXAPIData(data)).toBe(false);
+})
+
+test("types.isXAPIData() should recognize data", async () => {
+    const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndReplies1.json'), 'utf8');
+    const data = JSON.parse(body);
+    expect(isXAPIData(data)).toBe(true);
+    expect(isXAPIBookmarksData(data)).toBe(false);
+    expect(isXAPIError(data)).toBe(false);
+})
+
+
 // Testing the X migrations
 
 test("test migration: 20241016_add_config", async () => {

src/account_x/types.ts

@@ -312,6 +312,23 @@ export interface XAPITimeline {
 }
 
 export interface XAPIData {
+    errors?: [
+        {
+            message: string;
+            locations: {
+                line: number;
+                column: number;
+            }[];
+            path: string[];
+            extensions: any;
+            code: number;
+            kind: string;
+            name: string;
+            source: string;
+            retry_after: number;
+            tracing: any;
+        }
+    ],
     data: {
         user: {
             result: {
@@ -329,11 +346,15 @@ export interface XAPIBookmarksData {
 }
 
 export function isXAPIBookmarksData(body: any): body is XAPIBookmarksData {
-    return body.data && body.data.bookmark_timeline_v2;
+    return !!(body.data && body.data.bookmark_timeline_v2);
+}
+
+export function isXAPIError(body: any): body is XAPIData {
+    return !!(body.errors && body.errors.length > 0);
 }
 
 export function isXAPIData(body: any): body is XAPIData {
-    return body.data && body.data.user && body.data.user.result && body.data.user.result.timeline_v2;
+    return !!(body.data && body.data.user && body.data.user.result && body.data.user.result.timeline_v2);
 }
 
 // Index direct messages

src/account_x/x_account_controller.ts

@@ -87,6 +87,7 @@ import {
     XArchiveLikeContainer,
     isXArchiveLikeContainer,
     isXAPIBookmarksData,
+    isXAPIError,
     isXAPIData,
 } from './types'
 import * as XArchiveTypes from '../../archive-static-sites/x-archive/src/types';
@@ -703,6 +704,10 @@ export class XAccountController {
                 timeline = (body as XAPIBookmarksData).data.bookmark_timeline_v2;
             } else if (isXAPIData(body)) {
                 timeline = (body as XAPIData).data.user.result.timeline_v2;
+            } else if (isXAPIError(body)) {
+                log.error('XAccountController.indexParseTweetsResponseData: XAPIError', body);
+                this.mitmController.responseData[responseIndex].processed = true;
+                return false;
             } else {
                 throw new Error('Invalid response data');
             }
@@ -887,6 +892,11 @@ export class XAccountController {
 
         // Loop over all URL items
         tweetLegacy.entities?.urls.forEach((url: XAPILegacyURL) => {
+            // Make sure we have all of the URL information before importing
+            if (!url["url"] || !url["display_url"] || !url["expanded_url"] || !url["indices"]) {
+                return;
+            }
+
             // Have we seen this URL before?
             const existing: XTweetURLRow[] = exec(this.db, 'SELECT * FROM tweet_url WHERE url = ? AND tweetID = ?', [url["url"], tweetLegacy["id_str"]], "all") as XTweetURLRow[];
             if (existing.length > 0) {
@@ -1550,7 +1560,7 @@ export class XAccountController {
             streamWriter.write(',\n');
             await this.writeJSONArray(streamWriter, formattedBookmarks, "bookmarks");
             streamWriter.write(',\n');
-            await this.writeJSONArray(streamWriter, Object.values(formattedUsers), "users");
+            await this.writeJSONObject(streamWriter, formattedUsers, "users");
             streamWriter.write(',\n');
             await this.writeJSONArray(streamWriter, formattedConversations, "conversations");
             streamWriter.write(',\n');
@@ -1582,6 +1592,18 @@ export class XAccountController {
         streamWriter.write('  ]');
     }
 
+    async writeJSONObject(streamWriter: fs.WriteStream, item: object, propertyName: string) {
+        streamWriter.write(`  "${propertyName}": {\n`);
+        const keys = Object.keys(item);
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            const suffix = i < keys.length - 1 ? ',\n' : '\n';
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            streamWriter.write(`    "${key}": ${JSON.stringify((item as any)[key])}${suffix}`);
+        }
+        streamWriter.write('  }');
+    }
+
     // When you start deleting tweets, return a list of tweets to delete
     async deleteTweetsStart(): Promise<XDeleteTweetsStartResponse> {
         if (!this.db) {
@@ -2378,7 +2400,7 @@ export class XAccountController {
         const filename = `${media["id_str"]}.${mediaExtension}`;
 
         let archiveMediaFilename = null;
-        if ((media.type === "video" || media.type === "animated_gif" ) && media.video_info?.variants) {
+        if ((media.type === "video" || media.type === "animated_gif") && media.video_info?.variants) {
             // For videos, find the highest quality MP4 variant
             const mp4Variants = media.video_info.variants
                 .filter(v => v.content_type === "video/mp4")

src/renderer/src/App.vue

@@ -350,6 +350,11 @@ body {
   opacity: 0.7;
 }
 
+.webview-automation-border.webview-clickable {
+  opacity: 1;
+  pointer-events: auto;
+}
+
 .webview-input-border {
   border: 5px solid #c1fac4;
 }

src/renderer/src/view_models/XViewModel.ts

@@ -459,6 +459,19 @@ export class XViewModel extends BaseViewModel {
             //     </div>
             // </section>
 
+            // Check if we get more tweets by scrolling down, even without clicking any buttons
+            let numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]');
+            await this.sleep(2000);
+            await this.scrollUp(2000);
+            await this.sleep(2000);
+            await this.scrollToBottom();
+            await this.sleep(2000);
+            let numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]');
+            if (numberOfDivsAfter > numberOfDivsBefore) {
+                // More tweets loaded
+                return true;
+            }
+
             // If the retry button does not exist, try scrolling up and down again to trigger it
             // The retry button should be in the last cellInnerDiv, and it should have only 1 button in it
             if (await this.countSelectorsWithinElementLastFound('main[role="main"] nav[role="navigation"] + section div[data-testid=cellInnerDiv]', 'button') != 1) {
@@ -473,7 +486,7 @@ export class XViewModel extends BaseViewModel {
             }
 
             // Count divs before clicking retry button
-            let numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]');
+            numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]');
             if (numberOfDivsBefore > 0) {
                 // The last one is the one with the button
                 numberOfDivsBefore--;
@@ -484,7 +497,7 @@ export class XViewModel extends BaseViewModel {
             await this.sleep(2000);
 
             // Count divs after clicking retry button
-            const numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]');
+            numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]');
 
             // If there are more divs after, it means more tweets loaded
             return numberOfDivsAfter > numberOfDivsBefore;
@@ -1190,13 +1203,22 @@ Hang on while I scroll down to your earliest direct message conversations...`;
 
             // Load the messages page
             await this.loadURLWithRateLimit("https://x.com/messages");
+            this.log("runJobIndexConversations", "loaded messages page, waiting for conversations to load");
+
+            // Just for good measure, scroll down, wait, and scroll back up, to encourage the web page to load conversations
+            await this.sleep(1000);
+            await this.scrollToBottom();
+            await this.sleep(1000);
+            await this.scrollUp(2000);
+            await this.sleep(1000);
 
             // If the conversations list is empty, there is no search text field
             try {
-                // Wait for the search text field to appear with a 2 second timeout
-                await this.waitForSelector('section input[type="text"]', "https://x.com/messages", 2000);
+                // Wait for the search text field to appear with a 10 second timeout
+                await this.waitForSelector('section input[type="text"]', "https://x.com/messages", 10000);
             } catch (e) {
                 // There are no conversations
+                this.log("runJobIndexConversations", ["no conversations found", e]);
                 await this.waitForLoadingToFinish();
                 this.progress.isIndexConversationsFinished = true;
                 this.progress.conversationsIndexed = 0;
@@ -1279,6 +1301,7 @@ Hang on while I scroll down to your earliest direct message conversations...`;
                 break;
             } else {
                 if (!moreToScroll) {
+                    this.log("runJobIndexConversations", "we scrolled to the bottom but we're not finished, so scroll up a bit to trigger infinite scroll next time");
                     // We scrolled to the bottom but we're not finished, so scroll up a bit to trigger infinite scroll next time
                     await this.sleep(500);
                     await this.scrollUp(1000);

src/renderer/src/views/x/XJobStatusComponent.vue

@@ -4,10 +4,11 @@ import RunningIcon from '../shared_components/RunningIcon.vue'
 
 defineProps<{
     jobs: XJob[],
-    isPaused: boolean
+    isPaused: boolean,
+    clickingEnabled: boolean,
 }>();
 
-const emit = defineEmits(['onPause', 'onResume', 'onCancel', 'onReportBug']);
+const emit = defineEmits(['onPause', 'onResume', 'onCancel', 'onReportBug', 'onClickingDisabled', 'onClickingEnabled']);
 
 const getStatusIcon = (status: string) => {
     const statusIcons: { [key: string]: string } = {
@@ -69,6 +70,12 @@ const getJobTypeText = (jobType: string) => {
                 Report a Bug
             </button>
         </div>
+        <div class="d-flex justify-content-center">
+            <button class="btn btn-link btn-sm"
+                @click="clickingEnabled ? emit('onClickingDisabled') : emit('onClickingEnabled')">
+                {{ clickingEnabled ? 'Disable' : 'Enable' }} Clicking in Browser
+            </button>
+        </div>
     </div>
 </template>
 

src/renderer/src/views/x/XView.vue

@@ -214,6 +214,9 @@ const reloadMediaPath = async () => {
     console.log('mediaPath', mediaPath.value);
 };
 
+// Enable/disable clicking in the webview
+const clickingEnabled = ref(false);
+
 // User variables
 const userAuthenticated = ref(false);
 const userPremium = ref(false);
@@ -458,9 +461,10 @@ onUnmounted(async () => {
                 <div class="d-flex align-items-center">
                     <!-- Job status -->
                     <XJobStatusComponent v-if="currentJobs.length > 0 && model.state == State.RunJobs"
-                        :jobs="currentJobs" :is-paused="isPaused" class="job-status-component" @on-pause="model.pause()"
-                        @on-resume="model.resume()" @on-cancel="emit('onRefreshClicked')"
-                        @on-report-bug="onReportBug" />
+                        :jobs="currentJobs" :is-paused="isPaused" :clicking-enabled="clickingEnabled"
+                        class="job-status-component" @on-pause="model.pause()" @on-resume="model.resume()"
+                        @on-cancel="emit('onRefreshClicked')" @on-report-bug="onReportBug"
+                        @on-clicking-enabled="clickingEnabled = true" @on-clicking-disabled="clickingEnabled = false" />
                 </div>
             </div>
 
@@ -479,7 +483,8 @@ onUnmounted(async () => {
             :class="{
                 'hidden': !model.showBrowser,
                 'webview-automation-border': model.showAutomationNotice,
-                'webview-input-border': !model.showAutomationNotice
+                'webview-input-border': !model.showAutomationNotice,
+                'webview-clickable': clickingEnabled
             }" />
 
         <template v-if="model.state != State.WizardStart">

testdata/XAPIUserTweetsAndRepliesError.json

@@ -0,0 +1,38 @@
+{
+    "errors": [
+        {
+            "message": "Timeout: Unspecified",
+            "locations": [
+                {
+                    "line": 3,
+                    "column": 5
+                }
+            ],
+            "path": [
+                "user",
+                "result"
+            ],
+            "extensions": {
+                "name": "TimeoutError",
+                "source": "Server",
+                "retry_after": 0,
+                "code": 29,
+                "kind": "ServiceLevel",
+                "tracing": {
+                    "trace_id": "c685ec3405bf1b67"
+                }
+            },
+            "code": 29,
+            "kind": "ServiceLevel",
+            "name": "TimeoutError",
+            "source": "Server",
+            "retry_after": 0,
+            "tracing": {
+                "trace_id": "c685ec3405bf1b67"
+            }
+        }
+    ],
+    "data": {
+        "user": {}
+    }
+}