diff --git a/src/account_x.test.ts b/src/account_x.test.ts index f16fde6e..fab15276 100644 --- a/src/account_x.test.ts +++ b/src/account_x.test.ts @@ -8,9 +8,7 @@ import { XAPILegacyUser, XAPILegacyTweet, XAPIConversation, - XAPIUser -} from './account_x'; -import { + XAPIUser, XTweetRow, XTweetMediaRow, XTweetURLRow, @@ -18,6 +16,9 @@ import { XConversationRow, XConversationParticipantRow, XMessageRow, + isXAPIError, + isXAPIBookmarksData, + isXAPIData, } from './account_x'; // Mock the util module @@ -802,6 +803,31 @@ test("XAccountController.indexParsedTweets() should index and parse links", asyn expect(linkRows[2].expandedURL).toBe('https://x.com/nexamind91326/status/1890513848811090236'); }) +test("types.isXAPIBookmarksData() should recognize bookmarks data", async () => { + const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIBookmarks.json'), 'utf8'); + const data = JSON.parse(body); + expect(isXAPIBookmarksData(data)).toBe(true); + expect(isXAPIError(data)).toBe(false); + expect(isXAPIData(data)).toBe(false); +}) + +test("types.isXAPIError() should recognize errors", async () => { + const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesError.json'), 'utf8'); + const data = JSON.parse(body); + expect(isXAPIError(data)).toBe(true); + expect(isXAPIBookmarksData(data)).toBe(false); + expect(isXAPIData(data)).toBe(false); +}) + +test("types.isXAPIData() should recognize data", async () => { + const body = fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndReplies1.json'), 'utf8'); + const data = JSON.parse(body); + expect(isXAPIData(data)).toBe(true); + expect(isXAPIBookmarksData(data)).toBe(false); + expect(isXAPIError(data)).toBe(false); +}) + + // Testing the X migrations test("test migration: 20241016_add_config", async () => { diff --git a/src/account_x/types.ts b/src/account_x/types.ts index 4f54d461..c1817d01 100644 --- a/src/account_x/types.ts +++ b/src/account_x/types.ts @@ -312,6 +312,23 @@ export interface XAPITimeline { } export interface XAPIData { + errors?: [ + { + message: string; + locations: { + line: number; + column: number; + }[]; + path: string[]; + extensions: any; + code: number; + kind: string; + name: string; + source: string; + retry_after: number; + tracing: any; + } + ], data: { user: { result: { @@ -329,11 +346,15 @@ export interface XAPIBookmarksData { } export function isXAPIBookmarksData(body: any): body is XAPIBookmarksData { - return body.data && body.data.bookmark_timeline_v2; + return !!(body.data && body.data.bookmark_timeline_v2); +} + +export function isXAPIError(body: any): body is XAPIData { + return !!(body.errors && body.errors.length > 0); } export function isXAPIData(body: any): body is XAPIData { - return body.data && body.data.user && body.data.user.result && body.data.user.result.timeline_v2; + return !!(body.data && body.data.user && body.data.user.result && body.data.user.result.timeline_v2); } // Index direct messages diff --git a/src/account_x/x_account_controller.ts b/src/account_x/x_account_controller.ts index 0f666ed3..5af0ff5f 100644 --- a/src/account_x/x_account_controller.ts +++ b/src/account_x/x_account_controller.ts @@ -87,6 +87,7 @@ import { XArchiveLikeContainer, isXArchiveLikeContainer, isXAPIBookmarksData, + isXAPIError, isXAPIData, } from './types' import * as XArchiveTypes from '../../archive-static-sites/x-archive/src/types'; @@ -703,6 +704,10 @@ export class XAccountController { timeline = (body as XAPIBookmarksData).data.bookmark_timeline_v2; } else if (isXAPIData(body)) { timeline = (body as XAPIData).data.user.result.timeline_v2; + } else if (isXAPIError(body)) { + log.error('XAccountController.indexParseTweetsResponseData: XAPIError', body); + this.mitmController.responseData[responseIndex].processed = true; + return false; } else { throw new Error('Invalid response data'); } @@ -887,6 +892,11 @@ export class XAccountController { // Loop over all URL items tweetLegacy.entities?.urls.forEach((url: XAPILegacyURL) => { + // Make sure we have all of the URL information before importing + if (!url["url"] || !url["display_url"] || !url["expanded_url"] || !url["indices"]) { + return; + } + // Have we seen this URL before? const existing: XTweetURLRow[] = exec(this.db, 'SELECT * FROM tweet_url WHERE url = ? AND tweetID = ?', [url["url"], tweetLegacy["id_str"]], "all") as XTweetURLRow[]; if (existing.length > 0) { @@ -1550,7 +1560,7 @@ export class XAccountController { streamWriter.write(',\n'); await this.writeJSONArray(streamWriter, formattedBookmarks, "bookmarks"); streamWriter.write(',\n'); - await this.writeJSONArray(streamWriter, Object.values(formattedUsers), "users"); + await this.writeJSONObject(streamWriter, formattedUsers, "users"); streamWriter.write(',\n'); await this.writeJSONArray(streamWriter, formattedConversations, "conversations"); streamWriter.write(',\n'); @@ -1582,6 +1592,18 @@ export class XAccountController { streamWriter.write(' ]'); } + async writeJSONObject(streamWriter: fs.WriteStream, item: object, propertyName: string) { + streamWriter.write(` "${propertyName}": {\n`); + const keys = Object.keys(item); + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const suffix = i < keys.length - 1 ? ',\n' : '\n'; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + streamWriter.write(` "${key}": ${JSON.stringify((item as any)[key])}${suffix}`); + } + streamWriter.write(' }'); + } + // When you start deleting tweets, return a list of tweets to delete async deleteTweetsStart(): Promise { if (!this.db) { @@ -2378,7 +2400,7 @@ export class XAccountController { const filename = `${media["id_str"]}.${mediaExtension}`; let archiveMediaFilename = null; - if ((media.type === "video" || media.type === "animated_gif" ) && media.video_info?.variants) { + if ((media.type === "video" || media.type === "animated_gif") && media.video_info?.variants) { // For videos, find the highest quality MP4 variant const mp4Variants = media.video_info.variants .filter(v => v.content_type === "video/mp4") diff --git a/src/renderer/src/App.vue b/src/renderer/src/App.vue index 54a262eb..54efa8b4 100644 --- a/src/renderer/src/App.vue +++ b/src/renderer/src/App.vue @@ -350,6 +350,11 @@ body { opacity: 0.7; } +.webview-automation-border.webview-clickable { + opacity: 1; + pointer-events: auto; +} + .webview-input-border { border: 5px solid #c1fac4; } diff --git a/src/renderer/src/view_models/XViewModel.ts b/src/renderer/src/view_models/XViewModel.ts index 9240f702..759a89ef 100644 --- a/src/renderer/src/view_models/XViewModel.ts +++ b/src/renderer/src/view_models/XViewModel.ts @@ -459,6 +459,19 @@ export class XViewModel extends BaseViewModel { // // + // Check if we get more tweets by scrolling down, even without clicking any buttons + let numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + await this.sleep(2000); + await this.scrollUp(2000); + await this.sleep(2000); + await this.scrollToBottom(); + await this.sleep(2000); + let numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + if (numberOfDivsAfter > numberOfDivsBefore) { + // More tweets loaded + return true; + } + // If the retry button does not exist, try scrolling up and down again to trigger it // The retry button should be in the last cellInnerDiv, and it should have only 1 button in it if (await this.countSelectorsWithinElementLastFound('main[role="main"] nav[role="navigation"] + section div[data-testid=cellInnerDiv]', 'button') != 1) { @@ -473,7 +486,7 @@ export class XViewModel extends BaseViewModel { } // Count divs before clicking retry button - let numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + numberOfDivsBefore = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); if (numberOfDivsBefore > 0) { // The last one is the one with the button numberOfDivsBefore--; @@ -484,7 +497,7 @@ export class XViewModel extends BaseViewModel { await this.sleep(2000); // Count divs after clicking retry button - const numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); + numberOfDivsAfter = await this.countSelectorsFound('section div[data-testid=cellInnerDiv]'); // If there are more divs after, it means more tweets loaded return numberOfDivsAfter > numberOfDivsBefore; @@ -1190,13 +1203,22 @@ Hang on while I scroll down to your earliest direct message conversations...`; // Load the messages page await this.loadURLWithRateLimit("https://x.com/messages"); + this.log("runJobIndexConversations", "loaded messages page, waiting for conversations to load"); + + // Just for good measure, scroll down, wait, and scroll back up, to encourage the web page to load conversations + await this.sleep(1000); + await this.scrollToBottom(); + await this.sleep(1000); + await this.scrollUp(2000); + await this.sleep(1000); // If the conversations list is empty, there is no search text field try { - // Wait for the search text field to appear with a 2 second timeout - await this.waitForSelector('section input[type="text"]', "https://x.com/messages", 2000); + // Wait for the search text field to appear with a 10 second timeout + await this.waitForSelector('section input[type="text"]', "https://x.com/messages", 10000); } catch (e) { // There are no conversations + this.log("runJobIndexConversations", ["no conversations found", e]); await this.waitForLoadingToFinish(); this.progress.isIndexConversationsFinished = true; this.progress.conversationsIndexed = 0; @@ -1279,6 +1301,7 @@ Hang on while I scroll down to your earliest direct message conversations...`; break; } else { if (!moreToScroll) { + this.log("runJobIndexConversations", "we scrolled to the bottom but we're not finished, so scroll up a bit to trigger infinite scroll next time"); // We scrolled to the bottom but we're not finished, so scroll up a bit to trigger infinite scroll next time await this.sleep(500); await this.scrollUp(1000); diff --git a/src/renderer/src/views/x/XJobStatusComponent.vue b/src/renderer/src/views/x/XJobStatusComponent.vue index a9d16c83..3040d19b 100644 --- a/src/renderer/src/views/x/XJobStatusComponent.vue +++ b/src/renderer/src/views/x/XJobStatusComponent.vue @@ -4,10 +4,11 @@ import RunningIcon from '../shared_components/RunningIcon.vue' defineProps<{ jobs: XJob[], - isPaused: boolean + isPaused: boolean, + clickingEnabled: boolean, }>(); -const emit = defineEmits(['onPause', 'onResume', 'onCancel', 'onReportBug']); +const emit = defineEmits(['onPause', 'onResume', 'onCancel', 'onReportBug', 'onClickingDisabled', 'onClickingEnabled']); const getStatusIcon = (status: string) => { const statusIcons: { [key: string]: string } = { @@ -69,6 +70,12 @@ const getJobTypeText = (jobType: string) => { Report a Bug +
+ +
diff --git a/src/renderer/src/views/x/XView.vue b/src/renderer/src/views/x/XView.vue index 4e9d18d1..0fbbfcd8 100644 --- a/src/renderer/src/views/x/XView.vue +++ b/src/renderer/src/views/x/XView.vue @@ -214,6 +214,9 @@ const reloadMediaPath = async () => { console.log('mediaPath', mediaPath.value); }; +// Enable/disable clicking in the webview +const clickingEnabled = ref(false); + // User variables const userAuthenticated = ref(false); const userPremium = ref(false); @@ -458,9 +461,10 @@ onUnmounted(async () => {
+ :jobs="currentJobs" :is-paused="isPaused" :clicking-enabled="clickingEnabled" + class="job-status-component" @on-pause="model.pause()" @on-resume="model.resume()" + @on-cancel="emit('onRefreshClicked')" @on-report-bug="onReportBug" + @on-clicking-enabled="clickingEnabled = true" @on-clicking-disabled="clickingEnabled = false" />
@@ -479,7 +483,8 @@ onUnmounted(async () => { :class="{ 'hidden': !model.showBrowser, 'webview-automation-border': model.showAutomationNotice, - 'webview-input-border': !model.showAutomationNotice + 'webview-input-border': !model.showAutomationNotice, + 'webview-clickable': clickingEnabled }" />