From 4186a608f31b37fdb9763c7d5812c28a99803822 Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 3 Mar 2025 15:35:08 -0500 Subject: [PATCH 1/5] facebook: support media imports --- .../facebook_account_controller.ts | 77 ++++++++++++++++++- src/account_facebook/types.ts | 8 ++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/account_facebook/facebook_account_controller.ts b/src/account_facebook/facebook_account_controller.ts index 990bbbdc..18037e60 100644 --- a/src/account_facebook/facebook_account_controller.ts +++ b/src/account_facebook/facebook_account_controller.ts @@ -32,6 +32,7 @@ import { FacebookJobRow, convertFacebookJobRowToFacebookJob, FacebookArchivePost, + FacebookArchiveMedia, FacebookPostRow } from './types' import * as FacebookArchiveTypes from '../../archive-static-sites/facebook-archive/src/types'; @@ -180,6 +181,22 @@ export class FacebookAccountController { `ALTER TABLE post_new RENAME TO post;` ] }, + { + name: "20250302_add_media_table", + sql: [ + `CREATE TABLE media ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + mediaId TEXT NOT NULL UNIQUE, + postId TEXT NOT NULL, + type TEXT NOT NULL, + uri TEXT NOT NULL, + description TEXT, + createdAt DATETIME, + addedToDatabaseAt DATETIME NOT NULL, + FOREIGN KEY(postId) REFERENCES post(postID) + );` + ] + }, ]) log.info("FacebookAccountController.initDB: database initialized"); } @@ -535,6 +552,23 @@ export class FacebookAccountController { const isSharedPost = post.attachments?.[0]?.data?.[0]?.external_context !== undefined; log.info("FacebookAccountController.importFacebookArchive: isSharedPost", isSharedPost); + // Process media attachments + const media: FacebookArchiveMedia[] = []; + if (post.attachments) { + for (const attachment of post.attachments) { + for (const data of attachment.data) { + if (data.media) { + media.push({ + uri: data.media.uri, + type: data.media.uri.endsWith('.mp4') ? 'video' : 'photo', + description: data.media.description, + creationTimestamp: data.media.creation_timestamp + }); + } + } + } + } + // Skip if it's a group post, shares a group, etc. We will extend the import logic // to include other data types in the future. if (post.attachments && !isSharedPost) { @@ -548,6 +582,7 @@ export class FacebookAccountController { full_text: postText, created_at: new Date(post.timestamp * 1000).toISOString(), isReposted: isSharedPost, + media: media.length > 0 ? media : undefined, }); } } catch (e) { @@ -561,7 +596,7 @@ export class FacebookAccountController { // Loop through the posts and add them to the database try { - postsData.forEach((post) => { + postsData.forEach(async (post) => { // Is this post already there? const existingPost = exec(this.db, 'SELECT * FROM post WHERE postID = ?', [post.id_str], "get") as FacebookPostRow; if (existingPost) { @@ -569,7 +604,10 @@ export class FacebookAccountController { exec(this.db, 'DELETE FROM post WHERE postID = ?', [post.id_str]); } - // TODO: implement media import for facebook + if (post.media && post.media.length > 0) { + await this.importFacebookArchiveMedia(post.id_str, post.media, archivePath); + } + // TODO: implement urls import for facebook // Import it @@ -608,4 +646,39 @@ export class FacebookAccountController { skipCount: skipCount, }; } + + async importFacebookArchiveMedia(postId: string, media: FacebookArchiveMedia[], archivePath: string): Promise { + for (const mediaItem of media) { + const sourcePath = path.join(archivePath, mediaItem.uri); + const mediaId = `${postId}_${path.basename(mediaItem.uri)}`; + + // Create destination directory if it doesn't exist + const mediaDir = path.join(this.accountDataPath, 'media'); + if (!fs.existsSync(mediaDir)) { + fs.mkdirSync(mediaDir, { recursive: true }); + } + + const destPath = path.join(mediaDir, path.basename(mediaItem.uri)); + + try { + await fs.promises.copyFile(sourcePath, destPath); + + // Store media info in database + exec(this.db, + 'INSERT INTO media (mediaId, postId, type, uri, description, createdAt, addedToDatabaseAt) VALUES (?, ?, ?, ?, ?, ?, ?)', + [ + mediaId, + postId, + mediaItem.type, + path.basename(mediaItem.uri), + mediaItem.description || null, + mediaItem.creationTimestamp ? new Date(mediaItem.creationTimestamp * 1000) : null, + new Date() + ] + ); + } catch (error) { + log.error(`FacebookAccountController.importFacebookArchiveMedia: Error importing media: ${error}`); + } + } + } } diff --git a/src/account_facebook/types.ts b/src/account_facebook/types.ts index 99bc22e2..e38d1c1a 100644 --- a/src/account_facebook/types.ts +++ b/src/account_facebook/types.ts @@ -38,9 +38,17 @@ export interface FacebookArchivePost { full_text: string; title: string; isReposted: boolean; + media?: FacebookArchiveMedia[]; // Media attachments // lang: string; } +export interface FacebookArchiveMedia { + uri: string; + type: 'photo' | 'video'; + description?: string; // Some media items have descriptions + creationTimestamp?: number; // From media.creation_timestamp +} + export interface FacebookPostRow { id: number; username: string; From 9e3c181c505db36222c45675eeefd704f9097c63 Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 3 Mar 2025 16:14:16 -0500 Subject: [PATCH 2/5] fix: media gets saved to post_media, and media/ dir --- .../facebook_account_controller.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/account_facebook/facebook_account_controller.ts b/src/account_facebook/facebook_account_controller.ts index 18037e60..3e305eb4 100644 --- a/src/account_facebook/facebook_account_controller.ts +++ b/src/account_facebook/facebook_account_controller.ts @@ -184,7 +184,7 @@ export class FacebookAccountController { { name: "20250302_add_media_table", sql: [ - `CREATE TABLE media ( + `CREATE TABLE post_media ( id INTEGER PRIMARY KEY AUTOINCREMENT, mediaId TEXT NOT NULL UNIQUE, postId TEXT NOT NULL, @@ -568,10 +568,11 @@ export class FacebookAccountController { } } } + log.info("FacebookAccountController.importFacebookArchive: media", media); // Skip if it's a group post, shares a group, etc. We will extend the import logic // to include other data types in the future. - if (post.attachments && !isSharedPost) { + if (post.attachments && !isSharedPost && media.length === 0) { log.info("FacebookAccountController.importFacebookArchive: skipping unknown post type"); continue; } @@ -604,10 +605,6 @@ export class FacebookAccountController { exec(this.db, 'DELETE FROM post WHERE postID = ?', [post.id_str]); } - if (post.media && post.media.length > 0) { - await this.importFacebookArchiveMedia(post.id_str, post.media, archivePath); - } - // TODO: implement urls import for facebook // Import it @@ -619,9 +616,15 @@ export class FacebookAccountController { post.isReposted ? 1 : 0, new Date(), ]); + + if (post.media && post.media.length > 0) { + log.info("FacebookAccountController.importFacebookArchive: importing media for post", post.id_str); + await this.importFacebookArchiveMedia(post.id_str, post.media, archivePath); + } importCount++; }); } catch (e) { + log.error("FacebookAccountController.importFacebookArchive: error importing posts", e); return { status: "error", errorMessage: "Error importing posts: " + e, @@ -659,13 +662,12 @@ export class FacebookAccountController { } const destPath = path.join(mediaDir, path.basename(mediaItem.uri)); - try { await fs.promises.copyFile(sourcePath, destPath); // Store media info in database exec(this.db, - 'INSERT INTO media (mediaId, postId, type, uri, description, createdAt, addedToDatabaseAt) VALUES (?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO post_media (mediaId, postId, type, uri, description, createdAt, addedToDatabaseAt) VALUES (?, ?, ?, ?, ?, ?, ?)', [ mediaId, postId, From 3a727340520a8738a41d7f2dd859a5bf23f277eb Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 3 Mar 2025 20:55:45 -0500 Subject: [PATCH 3/5] facebook: add video/image support to static site --- .../src/components/PostComponent.vue | 27 +++++++++-- .../facebook-archive/src/types.ts | 13 ++++-- .../facebook_account_controller.ts | 46 ++++++++++++++++--- src/account_facebook/types.ts | 14 ++++++ 4 files changed, 87 insertions(+), 13 deletions(-) diff --git a/archive-static-sites/facebook-archive/src/components/PostComponent.vue b/archive-static-sites/facebook-archive/src/components/PostComponent.vue index c1c2172d..c00b9237 100644 --- a/archive-static-sites/facebook-archive/src/components/PostComponent.vue +++ b/archive-static-sites/facebook-archive/src/components/PostComponent.vue @@ -30,18 +30,39 @@ const formattedText = computed(() => {

+ + +
+
+ + +

{{ media.description }}

+
+
archived {{ formattedDate(post.archivedAt) }}
- \ No newline at end of file diff --git a/archive-static-sites/facebook-archive/src/types.ts b/archive-static-sites/facebook-archive/src/types.ts index 8ee738b8..9359f247 100644 --- a/archive-static-sites/facebook-archive/src/types.ts +++ b/archive-static-sites/facebook-archive/src/types.ts @@ -1,11 +1,18 @@ -export type Post = { +export interface Post { postID: string; createdAt: string; text: string; title: string; - archivedAt: string | null; isReposted: boolean; -}; + archivedAt: string | null; + media?: { + mediaId: string; + type: string; + uri: string; + description: string | null; + createdAt: string | null; + }[]; +} export type FacebookArchive = { posts: Post[]; diff --git a/src/account_facebook/facebook_account_controller.ts b/src/account_facebook/facebook_account_controller.ts index 3e305eb4..faabc01d 100644 --- a/src/account_facebook/facebook_account_controller.ts +++ b/src/account_facebook/facebook_account_controller.ts @@ -33,6 +33,7 @@ import { convertFacebookJobRowToFacebookJob, FacebookArchivePost, FacebookArchiveMedia, + FacebookPostWithMedia, FacebookPostRow } from './types' import * as FacebookArchiveTypes from '../../archive-static-sites/facebook-archive/src/types'; @@ -251,20 +252,44 @@ export class FacebookAccountController { } log.info("FacebookAccountController.archiveBuild: building archive"); - - // Posts - const posts: FacebookPostRow[] = exec( + // Posts with optional media + const postsFromDb = exec( this.db, - "SELECT * FROM post ORDER BY createdAt DESC", + `SELECT + p.*, + CASE + WHEN pm.mediaId IS NOT NULL + THEN GROUP_CONCAT( + json_object( + 'mediaId', pm.mediaId, + 'postId', pm.postId, + 'type', pm.type, + 'uri', pm.uri, + 'description', pm.description, + 'createdAt', pm.createdAt, + 'addedToDatabaseAt', pm.addedToDatabaseAt + ) + ) + ELSE NULL + END as media + FROM post p + LEFT JOIN post_media pm ON p.postID = pm.postId + GROUP BY p.postID + ORDER BY p.createdAt DESC`, [], "all" - ) as FacebookPostRow[]; + ); + // Transform into FacebookPostWithMedia + const posts: FacebookPostWithMedia[] = (postsFromDb as Array).map((post) => ({ + ...post, + media: post.media ? JSON.parse(`[${post.media}]`) : undefined + })); // Get the current account's userID // const accountUser = users.find((user) => user.screenName == this.account?.username); // const accountUserID = accountUser?.userID; - const postRowToArchivePost = (post: FacebookPostRow): FacebookArchiveTypes.Post => { + const postRowToArchivePost = (post: FacebookPostWithMedia): FacebookArchiveTypes.Post => { const archivePost: FacebookArchiveTypes.Post = { postID: post.postID, createdAt: post.createdAt, @@ -272,8 +297,15 @@ export class FacebookAccountController { title: post.title, isReposted: post.isReposted, archivedAt: post.archivedAt, + media: post.media?.map(m => ({ + mediaId: m.mediaId, + type: m.type, + uri: m.uri, + description: m.description, + createdAt: m.createdAt + })) }; - return archivePost + return archivePost; } // Build the archive object diff --git a/src/account_facebook/types.ts b/src/account_facebook/types.ts index e38d1c1a..866fbf42 100644 --- a/src/account_facebook/types.ts +++ b/src/account_facebook/types.ts @@ -63,3 +63,17 @@ export interface FacebookPostRow { hasMedia: boolean; isReposted: boolean; } + +export interface FacebookPostMediaRow { + mediaId: string; + postId: string; // Foreign key to post.postID + type: string; + uri: string; + description: string | null; + createdAt: string | null; + addedToDatabaseAt: string; +} + +export interface FacebookPostWithMedia extends FacebookPostRow { + media?: FacebookPostMediaRow[]; +} From 83fe944231368bd92f9f1e2dbabc0a13f55da0c3 Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 3 Mar 2025 21:03:38 -0500 Subject: [PATCH 4/5] fix: don't show post text twice if it's the same text as the image --- .../src/components/PostComponent.vue | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/archive-static-sites/facebook-archive/src/components/PostComponent.vue b/archive-static-sites/facebook-archive/src/components/PostComponent.vue index c00b9237..2d33aa69 100644 --- a/archive-static-sites/facebook-archive/src/components/PostComponent.vue +++ b/archive-static-sites/facebook-archive/src/components/PostComponent.vue @@ -14,6 +14,14 @@ const formattedText = computed(() => { text = text.replace(/(?:\r\n|\r|\n)/g, '
'); return text.trim(); }); + +const shouldShowText = computed(() => { + if (!props.post.text) return false; + if (!props.post.media || props.post.media.length === 0) return true; + + // Show text if it's different from all media descriptions + return !props.post.media.some(media => media.description === props.post.text); +});