Index media, reply status, quote status and extended URL

src/account_x/types.ts

@@ -15,6 +15,13 @@ export interface XJobRow {
     error: string | null;
 }
 
+export interface XTweetMediaRow {
+    id: number;
+    mediaID: string;
+    tweetID: string;
+    type: string;
+}
+
 export interface XTweetRow {
     id: number;
     username: string;

src/account_x/x_account_controller.ts

@@ -44,6 +44,7 @@
 import {
     XJobRow,
     XTweetRow,
+    XTweetMediaRow,
     XUserRow,
     XConversationRow,
     XMessageRow,
@@ -317,6 +318,20 @@
                     `UPDATE tweet SET deletedLikeAt = deletedAt WHERE deletedAt IS NOT NULL AND isLiked = 1;`
                 ]
             },
+            // Add hasMediato the tweet table, and update isBookarked for all tweets
+            {
+                name: "20250206_add_hasMedia_and_tweet_media",
+                sql: [
+                    `CREATE TABLE tweet_media (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        mediaID TEXT NOT NULL UNIQUE,
+                        mediaType TEXT NOT NULL,
+                        tweetID TEXT NOT NULL
+                    );`,
+                    `ALTER TABLE tweet ADD COLUMN hasMedia BOOLEAN;`,
+                    `UPDATE tweet SET hasMedia = 0;`
+                ]
+            },
         ])
         log.info("XAccountController.initDB: database initialized");
     }
@@ -461,8 +476,15 @@
             exec(this.db, 'DELETE FROM tweet WHERE tweetID = ?', [tweetLegacy["id_str"]]);
         }
 
+        // Check if tweet has media and call indexTweetMedia
+        let hasMedia: boolean = false;
+        if (tweetLegacy["entities"]["media"] && tweetLegacy["entities"]["media"].length){
+            hasMedia = true;
+            this.indexTweetMedia(tweetLegacy)
+        }
+
         // Add the tweet
-        exec(this.db, 'INSERT INTO tweet (username, tweetID, conversationID, createdAt, likeCount, quoteCount, replyCount, retweetCount, isLiked, isRetweeted, isBookmarked, text, path, addedToDatabaseAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
+        exec(this.db, 'INSERT INTO tweet (username, tweetID, conversationID, createdAt, likeCount, quoteCount, replyCount, retweetCount, isLiked, isRetweeted, isBookmarked, text, path, hasMedia, addedToDatabaseAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
             userLegacy["screen_name"],
             tweetLegacy["id_str"],
             tweetLegacy["conversation_id_str"],
@@ -476,9 +498,13 @@
             tweetLegacy["bookmarked"] ? 1 : 0,
             tweetLegacy["full_text"],
             `${userLegacy['screen_name']}/status/${tweetLegacy['id_str']}`,
+            hasMedia ? 1 : 0,
             new Date(),
         ]);
 
+        // Add media information to tweet_media table
+
+
         // Update progress
         if (tweetLegacy["favorited"]) {
             // console.log("DEBUG-### LIKE: ", tweetLegacy["id_str"], userLegacy["screen_name"], tweetLegacy["full_text"]);
@@ -655,6 +681,59 @@
         return this.progress;
     }
 
+    async saveTweetMedia(mediaPath: string, filename: string) {
+        if (this.account) {
+            // Create path to store tweet media if it doesn't exist already
+            const accountDataPath = getAccountDataPath("X", this.account.username);
+            const outputPath = path.join(accountDataPath, "Tweet Media");
+            if (!fs.existsSync(outputPath)) {
+                fs.mkdirSync(outputPath);
+            }
+
+            // Download and save media from the mediaPath
+            try {
+                const response = await fetch(mediaPath, {});
+                if (!response.ok) {
+                    return "";
+                }
+
+                const arrayBuffer = await response.arrayBuffer();
+                const buffer = Buffer.from(arrayBuffer);
+                const fileType = mediaPath.substring(mediaPath.lastIndexOf(".") + 1);
+                const outputFileName = path.join(outputPath, `${filename}.${fileType}`);
+                fs.createWriteStream(outputFileName).write(buffer);
+                return outputFileName;
+            } catch {
+                return "";
+            }
+        }
+        throw new Error("Account not found");
+    }
+
+    async indexTweetMedia(tweetLegacy: XAPILegacyTweet) {
+        log.debug("XAccountController.indexMedia");
+
+        // Loop over all media items
+        tweetLegacy["entities"]["media"].forEach((media: any) => {
+            // Download media locally
+            this.saveTweetMedia(media["media_url_https"], media["media_key"]);
+
+            // Have we seen this media before?
+            const existing: XTweetMediaRow[] = exec(this.db, 'SELECT * FROM tweet_media WHERE mediaID = ?', [media["media_key"]], "all") as XTweetMediaRow[];
+            if (existing.length > 0) {
+                // Delete it, so we can re-add it
+                exec(this.db, 'DELETE FROM tweet_media WHERE mediaID = ?', [media["media_key"]]);
+            }
+
+            // Index media information in tweet_media table
+            exec(this.db, 'INSERT INTO tweet_media (mediaID, mediaType, tweetID) VALUES (?, ?, ?)', [
+                media["media_key"],
+                media["type"],
+                tweetLegacy["id_str"],
+            ]);
+        })
+    }
+
     async getProfileImageDataURI(user: XAPIUser): Promise<string> {
         if (!user.profile_image_url_https) {
             return "";