Merge pull request #437 from lockdown-systems/facebook-media

[facebook 4] import media from archive

Author: redshiftzero

archive-static-sites/facebook-archive/src/components/PostComponent.vue

@@ -14,6 +14,14 @@ const formattedText = computed(() => {
     text = text.replace(/(?:\r\n|\r|\n)/g, '<br>');
     return text.trim();
 });
+
+const shouldShowText = computed(() => {
+    if (!props.post.text) return false;
+    if (!props.post.media || props.post.media.length === 0) return true;
+
+    // Show text if it's different from all media descriptions
+    return !props.post.media.some(media => media.description === props.post.text);
+});
 </script>
 
 <template>
@@ -28,20 +36,41 @@ const formattedText = computed(() => {
             <small v-else class="text-muted">unknown date</small>
         </div>
         <div class="mt-2">
-            <!-- Text -->
-            <p v-html="formattedText"></p>
+            <!-- Text (only show if it's different from media description) -->
+            <p v-if="shouldShowText" v-html="formattedText"></p>
+
+            <!-- Media -->
+            <div v-if="post.media" class="media-container mt-2">
+                <div v-for="media in post.media" :key="media.mediaId" class="media-item mb-2">
+                    <img v-if="media.type === 'photo'"
+                         :src="`./media/${media.uri}`"
+                         :alt="media.description || ''"
+                         class="img-fluid rounded">
+                    <video v-else-if="media.type === 'video'"
+                           controls
+                           class="w-100 rounded">
+                        <source :src="`./media/${media.uri}`" type="video/mp4">
+                        Your browser does not support the video tag.
+                    </video>
+                    <p v-if="media.description" class="text-muted small mt-1">{{ media.description }}</p>
+                </div>
+            </div>
         </div>
         <div class="meta d-flex gap-2">
             <span v-if="post.archivedAt" class="date text-muted ms-2">
                 archived {{ formattedDate(post.archivedAt) }}
             </span>
         </div>
-
     </div>
 </template>
 
 <style scoped>
-.meta {
-    font-size: 0.8rem;
+.media-container {
+    max-width: 100%;
+}
+.media-item img,
+.media-item video {
+    max-height: 400px;
+    object-fit: contain;
 }
 </style>

archive-static-sites/facebook-archive/src/types.ts

@@ -1,11 +1,18 @@
-export type Post = {
+export interface Post {
     postID: string;
     createdAt: string;
     text: string;
     title: string;
-    archivedAt: string | null;
     isReposted: boolean;
-};
+    archivedAt: string | null;
+    media?: {
+        mediaId: string;
+        type: string;
+        uri: string;
+        description: string | null;
+        createdAt: string | null;
+    }[];
+}
 
 export type FacebookArchive = {
     posts: Post[];

src/account_facebook/facebook_account_controller.ts

@@ -32,6 +32,8 @@ import {
     FacebookJobRow,
     convertFacebookJobRowToFacebookJob,
     FacebookArchivePost,
+    FacebookArchiveMedia,
+    FacebookPostWithMedia,
     FacebookPostRow
 } from './types'
 import * as FacebookArchiveTypes from '../../archive-static-sites/facebook-archive/src/types';
@@ -180,6 +182,22 @@ export class FacebookAccountController {
                     `ALTER TABLE post_new RENAME TO post;`
                 ]
             },
+            {
+                name: "20250302_add_media_table",
+                sql: [
+                    `CREATE TABLE post_media (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        mediaId TEXT NOT NULL UNIQUE,
+                        postId TEXT NOT NULL,
+                        type TEXT NOT NULL,
+                        uri TEXT NOT NULL,
+                        description TEXT,
+                        createdAt DATETIME,
+                        addedToDatabaseAt DATETIME NOT NULL,
+                        FOREIGN KEY(postId) REFERENCES post(postID)
+                    );`
+                ]
+            },
         ])
         log.info("FacebookAccountController.initDB: database initialized");
     }
@@ -234,29 +252,60 @@ export class FacebookAccountController {
         }
 
         log.info("FacebookAccountController.archiveBuild: building archive");
-
-        // Posts
-        const posts: FacebookPostRow[] = exec(
+        // Posts with optional media
+        const postsFromDb = exec(
             this.db,
-            "SELECT * FROM post ORDER BY createdAt DESC",
+            `SELECT
+                p.*,
+                CASE
+                    WHEN pm.mediaId IS NOT NULL
+                    THEN GROUP_CONCAT(
+                        json_object(
+                            'mediaId', pm.mediaId,
+                            'postId', pm.postId,
+                            'type', pm.type,
+                            'uri', pm.uri,
+                            'description', pm.description,
+                            'createdAt', pm.createdAt,
+                            'addedToDatabaseAt', pm.addedToDatabaseAt
+                        )
+                    )
+                    ELSE NULL
+                END as media
+            FROM post p
+            LEFT JOIN post_media pm ON p.postID = pm.postId
+            GROUP BY p.postID
+            ORDER BY p.createdAt DESC`,
             [],
             "all"
-        ) as FacebookPostRow[];
+        );
+        // Transform into FacebookPostWithMedia
+        const posts: FacebookPostWithMedia[] = (postsFromDb as Array<FacebookPostRow & { media?: string }>).map((post) => ({
+            ...post,
+            media: post.media ? JSON.parse(`[${post.media}]`) : undefined
+        }));
 
         // Get the current account's userID
         // const accountUser = users.find((user) => user.screenName == this.account?.username);
         // const accountUserID = accountUser?.userID;
 
-        const postRowToArchivePost = (post: FacebookPostRow): FacebookArchiveTypes.Post => {
+        const postRowToArchivePost = (post: FacebookPostWithMedia): FacebookArchiveTypes.Post => {
             const archivePost: FacebookArchiveTypes.Post = {
                 postID: post.postID,
                 createdAt: post.createdAt,
                 text: post.text,
                 title: post.title,
                 isReposted: post.isReposted,
                 archivedAt: post.archivedAt,
+                media: post.media?.map(m => ({
+                    mediaId: m.mediaId,
+                    type: m.type,
+                    uri: m.uri,
+                    description: m.description,
+                    createdAt: m.createdAt
+                }))
             };
-            return archivePost
+            return archivePost;
         }
 
         // Build the archive object
@@ -524,30 +573,46 @@ export class FacebookAccountController {
                     const posts = JSON.parse(postsFile);
 
                     for (const post of posts) {
-                        // Skip if no post text
                         const postText = post.data?.find((d: { post?: string }) => 'post' in d && typeof d.post === 'string')?.post;
-                        if (!postText) {
-                            log.info("FacebookAccountController.importFacebookArchive: skipping post with no text");
-                            continue;
-                        }
 
                         // Check if it's a shared post by looking for external_context in attachments
                         const isSharedPost = post.attachments?.[0]?.data?.[0]?.external_context !== undefined;
                         log.info("FacebookAccountController.importFacebookArchive: isSharedPost", isSharedPost);
 
-                        // Skip if it's a group post, shares a group, etc. We will extend the import logic
-                        // to include other data types in the future.
-                        if (post.attachments && !isSharedPost) {
-                            log.info("FacebookAccountController.importFacebookArchive: skipping unknown post type");
-                            continue;
+                        // Check if it's a share of a group post
+                        const isGroupPost = post.attachments?.[0]?.data?.[0]?.name !== undefined;
+                        const groupName = isGroupPost ? post.attachments[0].data[0].name : undefined;
+
+                        // For group posts, if there's no explicit post text, use the group name
+                        const finalText = isGroupPost
+                            ? (postText || `Shared the group: ${groupName}`)
+                            : postText;
+
+                        // Process media attachments
+                        const media: FacebookArchiveMedia[] = [];
+                        if (post.attachments) {
+                            for (const attachment of post.attachments) {
+                                for (const data of attachment.data) {
+                                    if (data.media) {
+                                        media.push({
+                                            uri: data.media.uri,
+                                            type: data.media.uri.endsWith('.mp4') ? 'video' : 'photo',
+                                            description: data.media.description,
+                                            creationTimestamp: data.media.creation_timestamp
+                                        });
+                                    }
+                                }
+                            }
                         }
+                        log.info("FacebookAccountController.importFacebookArchive: media", media);
 
                         postsData.push({
                             id_str: post.timestamp.toString(),
                             title: post.title || '',
-                            full_text: postText,
+                            full_text: finalText,
                             created_at: new Date(post.timestamp * 1000).toISOString(),
-                            isReposted: isSharedPost,
+                            isReposted: isSharedPost || isGroupPost, // Group shares are reposts too
+                            media: media.length > 0 ? media : undefined,
                         });
                     }
                 } catch (e) {
@@ -561,15 +626,14 @@ export class FacebookAccountController {
 
                 // Loop through the posts and add them to the database
                 try {
-                    postsData.forEach((post) => {
+                    postsData.forEach(async (post) => {
                         // Is this post already there?
                         const existingPost = exec(this.db, 'SELECT * FROM post WHERE postID = ?', [post.id_str], "get") as FacebookPostRow;
                         if (existingPost) {
                             // Delete the existing post to re-import
                             exec(this.db, 'DELETE FROM post WHERE postID = ?', [post.id_str]);
                         }
 
-                        // TODO: implement media import for facebook
                         // TODO: implement urls import for facebook
 
                         // Import it
@@ -581,9 +645,15 @@ export class FacebookAccountController {
                             post.isReposted ? 1 : 0,
                             new Date(),
                         ]);
+
+                        if (post.media && post.media.length > 0) {
+                            log.info("FacebookAccountController.importFacebookArchive: importing media for post", post.id_str);
+                            await this.importFacebookArchiveMedia(post.id_str, post.media, archivePath);
+                        }
                         importCount++;
                     });
                 } catch (e) {
+                    log.error("FacebookAccountController.importFacebookArchive: error importing posts", e);
                     return {
                         status: "error",
                         errorMessage: "Error importing posts: " + e,
@@ -608,4 +678,37 @@ export class FacebookAccountController {
             skipCount: skipCount,
         };
     }
+
+    async importFacebookArchiveMedia(postId: string, media: FacebookArchiveMedia[], archivePath: string): Promise<void> {
+        for (const mediaItem of media) {
+            const sourcePath = path.join(archivePath, mediaItem.uri);
+            const mediaId = `${postId}_${path.basename(mediaItem.uri)}`;
+
+            // Create destination directory if it doesn't exist
+            const mediaDir = path.join(this.accountDataPath, 'media');
+            if (!fs.existsSync(mediaDir)) {
+                fs.mkdirSync(mediaDir, { recursive: true });
+            }
+
+            const destPath = path.join(mediaDir, path.basename(mediaItem.uri));
+            try {
+                await fs.promises.copyFile(sourcePath, destPath);
+
+                exec(this.db,
+                    'INSERT INTO post_media (mediaId, postId, type, uri, description, createdAt, addedToDatabaseAt) VALUES (?, ?, ?, ?, ?, ?, ?)',
+                    [
+                        mediaId,
+                        postId,
+                        mediaItem.type,
+                        path.basename(mediaItem.uri),
+                        mediaItem.description || null,
+                        mediaItem.creationTimestamp ? new Date(mediaItem.creationTimestamp * 1000) : null,
+                        new Date()
+                    ]
+                );
+            } catch (error) {
+                log.error(`FacebookAccountController.importFacebookArchiveMedia: Error importing media: ${error}`);
+            }
+        }
+    }
 }

src/account_facebook/types.ts

@@ -38,9 +38,17 @@ export interface FacebookArchivePost {
     full_text: string;
     title: string;
     isReposted: boolean;
+    media?: FacebookArchiveMedia[];  // Media attachments
     // lang: string;
 }
 
+export interface FacebookArchiveMedia {
+    uri: string;
+    type: 'photo' | 'video';
+    description?: string;  // Some media items have descriptions
+    creationTimestamp?: number;  // From media.creation_timestamp
+}
+
 export interface FacebookPostRow {
     id: number;
     username: string;
@@ -55,3 +63,17 @@ export interface FacebookPostRow {
     hasMedia: boolean;
     isReposted: boolean;
 }
+
+export interface FacebookPostMediaRow {
+    mediaId: string;
+    postId: string;  // Foreign key to post.postID
+    type: string;
+    uri: string;
+    description: string | null;
+    createdAt: string | null;
+    addedToDatabaseAt: string;
+}
+
+export interface FacebookPostWithMedia extends FacebookPostRow {
+    media?: FacebookPostMediaRow[];
+}