New history method

packages/scripts/migrate-to-supabase/notes.md

@@ -1,61 +1,27 @@
-## Better data sync
-- send out notice
-- handle entries load errors and saving status
+## TODO
+- New entry history method
+  - remove content-import.interface.ts
+  - show entry history from pop-up entry modal
+  - remove Entry History i18n
+  - drop entry_updates and content_updates tables
 - show green recent update in list and table view based on all an entries' updated_at fields
+- use line-clamp instead of truncateString in SelectedDict.svelte and also look at inline-children-elements purpose
+- handle entries load errors and saving status
 - solve circular dependencies issues
 - make plan for cleaning up connected senses and join tables for deleted entries, and for clearing out deleted items over 2 months old
 - listen to can_edit store changes and re-init if it changes
-
-## Final Migration cleanup
-- remove content-import.interface.ts code after getting new history method working
-- use line-clamp instead of truncateString in SelectedDict.svelte and also look at inline-children-elements purpose
-- show entry history from pop-up entry modal
-- test admin rls, alternative is auth.jwt() read https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt to see if better
-- Diego: AuthModal.svelte translations
-- review created_by forcers to see which tables need set_created_by
-  - audio_speakers
-  - dictionary_info
-  - dictionary_partners
-  - entry_tags
-  - invites
-  - sense_photos
-- remove Entry History i18n
-- remove import_meta from content update endpoint
-- remove unneeded urls from https://console.cloud.google.com/auth/clients/215143435444-fugm4gpav71r3l89n6i0iath4m436qnv.apps.googleusercontent.com?inv=1&invt=AboyXQ&project=talking-dictionaries-alpha
+- don't show my dictionaries that are private in the public dictionaries listing to avoid confusion
 - move featured images to photos table and make a connection to the dictionary
 - delete dev Firebase project and create new gcs dev bucket
-- adjust user migration to set these fields to empty strings and not null to avoid db errors: `confirmation_token`, `recovery_token`, `email_change_token_new`, `email_change`
-
-# Migrate Entries and Speakers from Firestore to Supabase
-- don't show my dictionaries that are private in the public dictionaries listing to avoid confusion
-- unpack content-update to be handle client-sides after adding RLS policies for photos, audio, video
-- build new Orama indexes every hour offset after materialized view is updated
+- test admin rls, alternative is auth.jwt() read https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt to see if better
 - Remove extra row in dictionary downloads csv and entries download csv
-- deal with content-update and content-import interface differences
 - If an audio file does not have a speaker still let it play even though speaker needs chosen
-- ensure all auth users are brought over
 - Orama: replaceState in createQueryParamStore? look into improving the history to change for view and page changes but not for the others
-- test new db triggers, especially when deleting relationships
-- get failed tests working again
-- bring back in variants and tests that relied on old forms of test data
-- save backup files to cloud bucket instead of github repo (Cloudflare R2 has 10GB free/mo)
-- optimize entries loading error handling
-
-- clean up old history data in content_updates
-- look at deletedEntries to see if they should be saved somewhere
-- cleaner format for content-updates and refactor current ones
 - prefetching dictionary will cause entry store value to download twice at the same time
-- drop content_updates' table column
-- drop entry_updates
 - make alternate writing systems of the sentence translations as different bcp keys (same as for glosses)
-- change old senses created_by/updated_by from firebase ids to user_ids and then connect relationships and change type to uuid
-- add 331 megabytes of content_updates to db, saved sql queries to avoid upgrading to the $25/month
 - think about find-replacing the "pn/v": "prenoun / preverb", and one other pos with dash when filtering by pos
 
 ## Notes
-- 1st manual backup was before any action
-- 11:37 at 50, 13:54 at 100 = 2 min 17 seconds for 50 entries, 387000/50 = 7740 chunks, 7740 * 2:17 = 17492 minutes = 12 days (1440 minutes in a day), 18:30 at 200
-- `pnpm -F scripts run-migration`
 - relocate tip: Google's Magic Image serving url reference: https://medium.com/google-cloud/uploading-resizing-and-serving-images-with-google-cloud-platform-ca9631a2c556
 
 ### No lexeme

packages/site/src/routes/api/email/announcement/+server.ts

@@ -8,24 +8,24 @@ import { ResponseCodes } from '$lib/constants'
 import { dev } from '$app/environment'
 import { getAdminSupabaseClient } from '$lib/supabase/admin'
 
-const batchSize = 50
-
 export const GET: RequestHandler = async () => {
   if (!dev)
     error(ResponseCodes.INTERNAL_SERVER_ERROR, { message: 'Not allowed' })
 
   try {
     const admin_supabase = getAdminSupabaseClient()
 
-    let user_emails: { email: string }[] = []
+    let user_emails: { email: string, last_sign_in_at: string }[] = []
     let from = 0
     const pageSize = 1000
 
     while (true) {
       const { data, error } = await admin_supabase
         .from('user_emails')
-        .select('email')
-        .order('email', { ascending: true })
+        .select('email, last_sign_in_at')
+        .order('last_sign_in_at', { ascending: true })
+        .gt('last_sign_in_at', '2025-02-01T00:00:00Z')
+        // .order('email', { ascending: true })
         .range(from, from + pageSize - 1)
 
       if (error) {
@@ -41,29 +41,19 @@ export const GET: RequestHandler = async () => {
       }
     }
 
-    const email_batches: { email: string }[][] = []
-    for (let i = 0; i < user_emails.length; i += batchSize) {
-      email_batches.push(user_emails.slice(i, i + batchSize))
-    }
-
-    for (let index = 0; index < email_batches.length; index++) {
-      const email_batch = email_batches[index]
-      console.info({ index, emails: email_batch.map(({ email }) => email) })
-
+    const emails = user_emails.map(({ email }) => email)
+    for (const email of emails) {
       await send_email({
         from: no_reply_address, // must use a livingdictionaries.app email for domain verification and not livingdictionaries.org
         reply_to: jacobAddress,
-        to: [jacobAddress],
-        bcc: email_batch,
+        to: [{ email }],
         subject: '🔧 Recent Entry Loading Issues Resolved',
         type: 'text/html',
         body: render_component_to_html({ component: Announcement }),
       })
-      console.info('sent batch')
-      await new Promise(resolve => setTimeout(resolve, 5000))
     }
 
-    return json({ result: 'success', email_count: user_emails.length, email_batches })
+    return json({ result: 'success', email_count: emails.length, user_emails })
   } catch (err) {
     console.error(`Error with email send request: ${err.message}`)
     error(ResponseCodes.INTERNAL_SERVER_ERROR, `Error with email send request: ${err.message}`)

packages/site/src/routes/api/email/announcement/Announcement.svelte

@@ -25,7 +25,7 @@
     Good news: we recently resolved some bugs on our platform! Over the past 6 weeks, there were entry loading issues, especially for the large Living Dictionaries. Some of you may have seen a "timeout" error message occur on your dictionary homepage. All loading problems have now been resolved.
   </Paragraph>
   <Paragraph row={{ top: 20 }}>
-    If you haven't been on the platform in the last week and notice that loading problems persist for you when you visit next time, please refresh the page once after letting everything load.
+    If you haven't been on the platform in the last two weeks and notice that loading problems persist for you when you visit next time, please refresh the page once after waiting a minute for everything to load.
   </Paragraph>
   <Paragraph row={{ top: 20 }}>
     If you still have trouble accessing your dictionary or experience loading issues, please use the "Contact Us" form to send us a descriptive message of your issue and we will take a look. Keep in mind that our platform is run by a small, dedicated team here at Living Tongues Institute.

packages/site/src/routes/api/email/send-email.ts

@@ -1,4 +1,4 @@
-// import { DKIM_PRIVATE_KEY, MAILCHANNELS_API_KEY } from '$env/static/private'
+// AWS Limit caps sends at 14 emails per second and 50,000 per day
 import { SESClient, SendEmailCommand, type SendEmailCommandOutput } from '@aws-sdk/client-ses'
 import { dictionary_address, no_reply_address } from './addresses'
 import { AWS_SES_ACCESS_KEY_ID, AWS_SES_REGION, AWS_SES_SECRET_ACCESS_KEY } from '$env/static/private'

supabase/migrations/20250525151430_history-to-bigquery.sql

@@ -0,0 +1,113 @@
+CREATE extension IF NOT EXISTS wrappers WITH SCHEMA extensions;
+
+create foreign data wrapper bigquery_wrapper
+  handler big_query_fdw_handler
+  validator big_query_fdw_validator;
+
+-- Save BigQuery service account json in Vault and retrieve the created `key_id`
+select vault.create_secret(
+  '
+    {
+      "type": "service_account",
+      "project_id": "your_gcp_project_id",
+      "private_key_id": "your_private_key_id",
+      "private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n",
+      ...
+    }
+  ',
+  'bigquery',
+  'BigQuery service account json for Wrappers'
+);
+
+create server bigquery_server
+  foreign data wrapper bigquery_wrapper
+  options (
+    sa_key_id '<key_ID>', -- The Key ID from above.
+    project_id 'your_gcp_project_id',
+    dataset_id 'your_gcp_dataset_id'
+  );
+
+create schema if not exists bigquery;
+
+create foreign table bigquery.my_bigquery_table (
+  id bigint,
+  name text,
+  ts timestamp
+)
+  server bigquery_server
+  options (
+    table 'people',
+    location 'EU'
+  );
+
+
+------------------------------------------------------------
+
+CREATE TRIGGER set_created_by_trigger_senses_in_sentences
+BEFORE UPDATE ON senses_in_sentences
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_audio_speakers
+BEFORE UPDATE ON audio_speakers
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_video_speakers
+BEFORE UPDATE ON video_speakers
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sense_videos
+BEFORE UPDATE ON sense_videos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sentence_videos
+BEFORE UPDATE ON sentence_videos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sense_photos
+BEFORE UPDATE ON sense_photos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sentence_photos
+BEFORE UPDATE ON sentence_photos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_entry_dialects
+BEFORE UPDATE ON entry_dialects
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_entry_tags
+BEFORE UPDATE ON entry_tags
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_invites
+BEFORE UPDATE ON invites
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_dictionary_info
+BEFORE UPDATE ON dictionary_info
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_dictionary_partners
+BEFORE UPDATE ON dictionary_partners
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_api_keys
+BEFORE UPDATE ON api_keys
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+
+
+

supabase/summarized-migrations.sql

@@ -1685,3 +1685,67 @@ CREATE INDEX IF NOT EXISTS idx_senses_in_sentences_dictionary_id_where_not_delet
 CREATE INDEX IF NOT EXISTS idx_sentence_photos_dictionary_id_where_not_deleted ON sentence_photos (dictionary_id, sentence_id) WHERE deleted IS NULL;
 CREATE INDEX IF NOT EXISTS idx_sentence_videos_dictionary_id_where_not_deleted ON sentence_videos (dictionary_id, sentence_id) WHERE deleted IS NULL;
 
+CREATE TRIGGER set_created_by_trigger_senses_in_sentences
+BEFORE UPDATE ON senses_in_sentences
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_audio_speakers
+BEFORE UPDATE ON audio_speakers
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_video_speakers
+BEFORE UPDATE ON video_speakers
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sense_videos
+BEFORE UPDATE ON sense_videos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sentence_videos
+BEFORE UPDATE ON sentence_videos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sense_photos
+BEFORE UPDATE ON sense_photos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_sentence_photos
+BEFORE UPDATE ON sentence_photos
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_entry_dialects
+BEFORE UPDATE ON entry_dialects
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_entry_tags
+BEFORE UPDATE ON entry_tags
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_invites
+BEFORE UPDATE ON invites
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_dictionary_info
+BEFORE UPDATE ON dictionary_info
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_dictionary_partners
+BEFORE UPDATE ON dictionary_partners
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();
+
+CREATE TRIGGER set_created_by_trigger_api_keys
+BEFORE UPDATE ON api_keys
+FOR EACH ROW
+EXECUTE FUNCTION set_created_by();