Support for indexing video (and fix indexing URLs)

src/account_x.test.ts

@@ -12,6 +12,8 @@ import {
 } from './account_x';
 import {
     XTweetRow,
+    XTweetMediaRow,
+    XTweetURLRow,
     XUserRow,
     XConversationRow,
     XConversationParticipantRow,
@@ -70,6 +72,10 @@ vi.mock('electron', () => ({
     }
 }));
 
+// Mock fetch
+const fetchMock = vi.fn();
+global.fetch = fetchMock;
+
 // Import the local modules after stuff has been mocked
 import { Account, ResponseData, XProgress } from './shared_types'
 import { XAccountController } from './account_x'
@@ -174,6 +180,31 @@ class MockMITMController implements IMITMController {
                 }
             ];
         }
+        if (testdata == "indexTweetsMedia") {
+            this.responseData = [
+                {
+                    host: 'x.com',
+                    url: '/i/api/graphql/pZXwh96YGRqmBbbxu7Vk2Q/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221769426369526771712%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D',
+                    status: 200,
+                    headers: {},
+                    body: fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesMedia.json'), 'utf8'),
+                    processed: false
+                }
+            ];
+        }
+        if (testdata == "indexTweetsLinks") {
+            this.responseData = [
+                {
+                    host: 'x.com',
+                    url: '/i/api/graphql/pZXwh96YGRqmBbbxu7Vk2Q/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221769426369526771712%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D',
+                    status: 200,
+                    headers: {},
+                    body: fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesLinks.json'), 'utf8'),
+                    processed: false
+                }
+            ];
+        }
+
     }
     setAutomationErrorReportTestdata(filename: string) {
         const testData = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'testdata', 'automation-errors', filename), 'utf8'));
@@ -210,13 +241,17 @@ afterEach(() => {
         controller.cleanup();
     }
 
-    // Delete databases from disk
-    fs.readdirSync(getSettingsPath()).forEach(file => {
-        fs.unlinkSync(path.join(getSettingsPath(), file));
-    });
-    fs.readdirSync(getAccountDataPath("X", "test")).forEach(file => {
-        fs.unlinkSync(path.join(getAccountDataPath("X", "test"), file));
-    });
+    // Delete data from disk
+    const settingsPath = getSettingsPath();
+    const accountDataPath = getAccountDataPath("X", "test");
+
+    if (fs.existsSync(settingsPath)) {
+        fs.rmSync(settingsPath, { recursive: true, force: true });
+    }
+
+    if (fs.existsSync(accountDataPath)) {
+        fs.rmSync(accountDataPath, { recursive: true, force: true });
+    }
 });
 
 // Fixtures
@@ -728,6 +763,57 @@ test("XAccountController.indexParsedTweets() should index bookmarks", async () =
     expect(rows.length).toBe(14);
 })
 
+test("XAccountController.indexParsedTweets() should index and download media", async () => {
+    mitmController.setTestdata("indexTweetsMedia");
+    if (controller.account) {
+        controller.account.username = 'nexamind91326';
+    }
+
+    await controller.indexParseTweets()
+
+    // Verify the video tweet
+    let tweetRows: XTweetRow[] = database.exec(controller.db, "SELECT * FROM tweet WHERE tweetID=?", ['1890513848811090236'], "all") as XTweetRow[];
+    expect(tweetRows.length).toBe(1);
+    expect(tweetRows[0].tweetID).toBe('1890513848811090236');
+    expect(tweetRows[0].text).toBe('check out this video i found https://t.co/MMfXeoZEdi');
+
+    let mediaRows: XTweetMediaRow[] = database.exec(controller.db, "SELECT * FROM tweet_media WHERE tweetID=?", ['1890513848811090236'], "all") as XTweetMediaRow[];
+    expect(mediaRows.length).toBe(1);
+    expect(mediaRows[0].mediaType).toBe('video');
+    expect(mediaRows[0].filename).toBe('7_1890513743144185859.mp4');
+    expect(mediaRows[0].startIndex).toBe(29);
+    expect(mediaRows[0].endIndex).toBe(52);
+
+    // Verify the image tweet
+    tweetRows = database.exec(controller.db, "SELECT * FROM tweet WHERE tweetID=?", ['1890512076189114426'], "all") as XTweetRow[];
+    expect(tweetRows.length).toBe(1);
+    expect(tweetRows[0].tweetID).toBe('1890512076189114426');
+    expect(tweetRows[0].text).toBe('what a pretty photo https://t.co/eBFfDPOPz6');
+
+    mediaRows = database.exec(controller.db, "SELECT * FROM tweet_media WHERE tweetID=?", ['1890512076189114426'], "all") as XTweetMediaRow[];
+    expect(mediaRows.length).toBe(1);
+    expect(mediaRows[0].mediaType).toBe('photo');
+    expect(mediaRows[0].filename).toBe('3_1890512052424466432.jpg');
+    expect(mediaRows[0].startIndex).toBe(20);
+    expect(mediaRows[0].endIndex).toBe(43);
+})
+
+test("XAccountController.indexParsedTweets() should index and parse links", async () => {
+    mitmController.setTestdata("indexTweetsLinks");
+    if (controller.account) {
+        controller.account.username = 'nexamind91326';
+    }
+
+    await controller.indexParseTweets()
+
+    // Verify the link tweet
+    const linkRows: XTweetURLRow[] = database.exec(controller.db, "SELECT * FROM tweet_url WHERE tweetID=?", ['1891186072995946783'], "all") as XTweetURLRow[];
+    expect(linkRows.length).toBe(3);
+    expect(linkRows[0].expandedURL).toBe('https://en.wikipedia.org/wiki/Moon');
+    expect(linkRows[1].expandedURL).toBe('https://en.wikipedia.org/wiki/Sun');
+    expect(linkRows[2].expandedURL).toBe('https://x.com/nexamind91326/status/1890513848811090236');
+})
+
 // Testing the X migrations
 
 test("test migration: 20241016_add_config", async () => {

src/account_x/types.ts

@@ -18,8 +18,22 @@ export interface XJobRow {
 export interface XTweetMediaRow {
     id: number;
     mediaID: string;
+    mediaType: string;
+    tweetID: string;
+    url: string;
+    filename: string;
+    startIndex: number;
+    endIndex: number;
+}
+
+export interface XTweetURLRow {
+    id: number;
+    url: string;
+    displayURL: string;
+    expandedURL: string;
+    startIndex: number;
+    endIndex: number;
     tweetID: string;
-    type: string;
 }
 
 export interface XTweetRow {
@@ -135,6 +149,50 @@ export function convertTweetRowToXTweetItemArchive(row: XTweetRow): XTweetItemAr
 
 // Index tweets
 
+export interface XAPILegacyTweetMediaVideoVariant {
+    bitrate?: number;
+    content_type: string;
+    url: string;
+}
+
+export interface XAPILegacyTweetMedia {
+    display_url: string;
+    expanded_url: string;
+    id_str: string;
+    indices: number[];
+    media_key: string;
+    media_url_https: string;
+    type: string;
+    url: string;
+    additional_media_info: any;
+    ext_media_availability: any;
+    features?: any;
+    sizes: any;
+    original_info: any;
+    allow_download_status?: any;
+    video_info?: {
+        aspect_ratio: number[];
+        duration_millis: number;
+        variants: XAPILegacyTweetMediaVideoVariant[];
+    };
+    media_results?: any;
+}
+
+export interface XAPILegacyURL {
+    display_url: string;
+    expanded_url: string;
+    url: string;
+    indices: number[];
+}
+
+export interface XAPILegacyEntities {
+    hashtags: any[];
+    symbols: any[];
+    timestamps: any[];
+    urls: XAPILegacyURL[];
+    user_mentions: any[];
+}
+
 export interface XAPILegacyTweet {
     bookmark_count: number;
     bookmarked: boolean;
@@ -156,7 +214,8 @@ export interface XAPILegacyTweet {
     user_id_str: string;
     id_str: string;
     entities: any;
-    quoted_status_permalink: any;
+    quoted_status_permalink?: any;
+    media?: XAPILegacyTweetMedia[];
 }
 
 export interface XAPILegacyUser {

src/account_x/x_account_controller.ts

@@ -55,6 +55,9 @@ import {
     // X API types
     XAPILegacyUser,
     XAPILegacyTweet,
+    XAPILegacyTweetMedia,
+    XAPILegacyTweetMediaVideoVariant,
+    XAPILegacyURL,
     XAPIData,
     XAPIBookmarksData,
     XAPITimeline,
@@ -354,15 +357,15 @@ export class XAccountController {
                         url TEXT NOT NULL,
                         displayURL TEXT NOT NULL,
                         expandedURL TEXT NOT NULL,
-                        start_index INTEGER NOT NULL,
-                        end_index INTEGER NOT NULL,
+                        startIndex INTEGER NOT NULL,
+                        endIndex INTEGER NOT NULL,
                         tweetID TEXT NOT NULL,
                         UNIQUE(url, tweetID)
                     );`,
                     `ALTER TABLE tweet_media ADD COLUMN url TEXT;`,
                     `ALTER TABLE tweet_media ADD COLUMN filename TEXT;`,
-                    `ALTER TABLE tweet_media ADD COLUMN start_index INTEGER;`,
-                    `ALTER TABLE tweet_media ADD COLUMN end_index INTEGER;`
+                    `ALTER TABLE tweet_media ADD COLUMN startIndex INTEGER;`,
+                    `ALTER TABLE tweet_media ADD COLUMN endIndex INTEGER;`
                 ]
             },
         ])
@@ -511,13 +514,13 @@ export class XAccountController {
 
         // Check if tweet has media and call indexTweetMedia
         let hasMedia: boolean = false;
-        if (tweetLegacy["entities"]["media"] && tweetLegacy["entities"]["media"].length){
+        if (tweetLegacy["entities"]["media"] && tweetLegacy["entities"]["media"].length) {
             hasMedia = true;
             this.indexTweetMedia(tweetLegacy)
         }
 
         // Check if tweet has URLs and index it
-        if (tweetLegacy["entities"]["url"] && tweetLegacy["entities"]["url"].length) {
+        if (tweetLegacy["entities"]["urls"] && tweetLegacy["entities"]["urls"].length) {
             this.indexTweetURLs(tweetLegacy)
         }
 
@@ -545,9 +548,6 @@ export class XAccountController {
             new Date(),
         ]);
 
-        // Add media information to tweet_media table
-
-
         // Update progress
         if (tweetLegacy["favorited"]) {
             // console.log("DEBUG-### LIKE: ", tweetLegacy["id_str"], userLegacy["screen_name"], tweetLegacy["full_text"]);
@@ -757,10 +757,36 @@ export class XAccountController {
         log.debug("XAccountController.indexMedia");
 
         // Loop over all media items
-        tweetLegacy["entities"]["media"].forEach((media: any) => {
+        tweetLegacy["entities"]["media"].forEach((media: XAPILegacyTweetMedia) => {
+            // Get the HTTPS URL of the media -- this works for photos
+            let mediaURL = media["media_url_https"];
+
+            // If it's a video, set mediaURL to the video variant with the highest bitrate
+            if (media["type"] == "video") {
+                let highestBitrate = 0;
+                if (media["video_info"] && media["video_info"]["variants"]) {
+                    media["video_info"]["variants"].forEach((variant: XAPILegacyTweetMediaVideoVariant) => {
+                        if (variant["bitrate"] && variant["bitrate"] > highestBitrate) {
+                            highestBitrate = variant["bitrate"];
+                            mediaURL = variant["url"];
+
+                            // Stripe query parameters from the URL.
+                            // For some reason video variants end with `?tag=12`, and when we try downloading with that
+                            // it responds with 404.
+                            const queryIndex = mediaURL.indexOf("?");
+                            if (queryIndex > -1) {
+                                mediaURL = mediaURL.substring(0, queryIndex);
+                            }
+                        }
+                    });
+                };
+            }
+
+            const mediaExtension = mediaURL.substring(mediaURL.lastIndexOf(".") + 1);
+
             // Download media locally
-            const filename = `${media["media_key"]}.${media["media_url_https"].substring(media["media_url_https"].lastIndexOf(".") + 1)}`;
-            this.saveTweetMedia(media["media_url_https"], filename);
+            const filename = `${media["media_key"]}.${mediaExtension}`;
+            this.saveTweetMedia(mediaURL, filename);
 
             // Have we seen this media before?
             const existing: XTweetMediaRow[] = exec(this.db, 'SELECT * FROM tweet_media WHERE mediaID = ?', [media["media_key"]], "all") as XTweetMediaRow[];
@@ -770,7 +796,7 @@ export class XAccountController {
             }
 
             // Index media information in tweet_media table
-            exec(this.db, 'INSERT INTO tweet_media (mediaID, mediaType, url, filename, start_index, end_index, tweetID) VALUES (?, ?, ?, ?, ?, ?, ?)', [
+            exec(this.db, 'INSERT INTO tweet_media (mediaID, mediaType, url, filename, startIndex, endIndex, tweetID) VALUES (?, ?, ?, ?, ?, ?, ?)', [
                 media["media_key"],
                 media["type"],
                 media["url"],
@@ -786,9 +812,9 @@ export class XAccountController {
         log.debug("XAccountController.indexTweetURL");
 
         // Loop over all URL items
-        tweetLegacy["entities"]["urls"].forEach((url: any) => {
+        tweetLegacy["entities"]["urls"].forEach((url: XAPILegacyURL) => {
             // Index url information in tweet_url table
-            exec(this.db, 'INSERT INTO tweet_url (url, displayURL, expandedURL, start_index, end_index, tweetID) VALUES (?, ?, ?, ?, ?, ?)', [
+            exec(this.db, 'INSERT INTO tweet_url (url, displayURL, expandedURL, startIndex, endIndex, tweetID) VALUES (?, ?, ?, ?, ?, ?)', [
                 url["url"],
                 url["display_url"],
                 url["expanded_url"],