Merge pull request #38 from opencrvs/multiple-registrations-and-skip

Fix registration number and add skip functionality

Author: pankaj-pant

.github/workflows/migrate-prod.yml

@@ -20,6 +20,11 @@ on:
         required: true
         type: string
         default: 'farajaland.opencrvs.org'
+      skip:
+        description: 'Number of records to skip'
+        required: false
+        type: number
+        default: 0
 
 jobs:
   migrate:
@@ -53,6 +58,7 @@ jobs:
         env:
           OPENCRVS_EVENT: ${{ github.event.inputs.event }}
           OPENCRVS_DOMAIN: ${{ github.event.inputs.domain }}
+          RECORD_SKIP: ${{ github.event.inputs.skip }}
           OPENCRVS_CLIENT_ID: ${{ secrets.OPENCRVS_CLIENT_ID }}
           OPENCRVS_CLIENT_SECRET: ${{ secrets.OPENCRVS_CLIENT_SECRET }}
         run: |

v1-to-v2-data-migration/helpers/transform.ts

@@ -525,9 +525,33 @@ function postProcess(
 
   const approvedCorrections = []
 
-  const rev = document.actions.slice().reverse()
+  let rev = document.actions.slice().reverse()
+  let firstRegisterAction = null
+  const actionsToRemove: string[] = []
 
   for (const action of rev) {
+    if (action.type === 'REGISTER') {
+      if (action.registrationNumber) {
+        if (firstRegisterAction) {
+          console.warn(
+            `Multiple REGISTER actions found for document ${document.id}`,
+          )
+          action.registrationNumber = undefined
+          action.status = 'Requested'
+
+          firstRegisterAction.originalActionId = action.id
+        } else {
+          firstRegisterAction = action
+        }
+      } else {
+        if (!firstRegisterAction || firstRegisterAction.originalActionId) {
+          actionsToRemove.push(action.id)
+        } else {
+          firstRegisterAction.originalActionId = action.id
+        }
+      }
+    }
+
     if (action.type === 'APPROVE_CORRECTION') {
       approvedCorrections.push(action.requestId)
     }
@@ -555,6 +579,10 @@ function postProcess(
     }
   }
 
+  if (actionsToRemove.length > 0) {
+    rev = rev.filter((action) => !actionsToRemove.includes(action.id))
+  }
+
   document.actions = rev.reverse()
 
   return document

v1-to-v2-data-migration/helpers/types.ts

@@ -265,6 +265,7 @@ export interface Action {
   registrationNumber?: string
   assignedTo?: string
   requestId?: string
+  originalActionId?: string
 }
 
 // Document output types

v1-to-v2-data-migration/helpers/vars.ts

@@ -1,6 +1,7 @@
 // @ts-nocheck - Using Deno-specific environment variables
 export const DOMAIN = Deno?.env?.get('OPENCRVS_DOMAIN') || 'localhost'
 export const EVENT = Deno?.env?.get('OPENCRVS_EVENT') || 'birth'
+export const RECORD_SKIP = Number(Deno?.env?.get('RECORD_SKIP')) || 0
 export const CLIENT_ID = Deno?.env?.get('OPENCRVS_CLIENT_ID')
 export const CLIENT_SECRET = Deno?.env?.get('OPENCRVS_CLIENT_SECRET')
 export const ADMIN_USERNAME =

v1-to-v2-data-migration/migrate.ipynb

@@ -192,8 +192,7 @@
     "} from './helpers/gqlHandlers.ts'\n",
     "import { transform } from './helpers/transform.ts'\n",
     "import { batch, getIndexErrors } from './helpers/utils.ts'\n",
-    "\n",
-    "const saved = []\n",
+    "import { RECORD_SKIP } from './helpers/vars.ts'\n",
     "\n",
     "const migrateBirth = async (entryIds) => {\n",
     "  const transformed = []\n",
@@ -231,12 +230,16 @@
     "}\n",
     "\n",
     "if (EVENT === 'birth') {\n",
+    "  const skippedPages = RECORD_SKIP ? Math.floor(RECORD_SKIP / 1000) : 0\n",
     "  const pageSize = 1000\n",
     "  const batchSize = 100\n",
     "  let itemsRemaining = 0\n",
-    "  let page = 1\n",
-    "  let totalProcessed = 0\n",
+    "  let page = 1 + skippedPages\n",
+    "  let totalProcessed = skippedPages * pageSize\n",
     "\n",
+    "  if (RECORD_SKIP) {\n",
+    "    console.log(`Skipping first ${skippedPages * pageSize} records`)\n",
+    "  }\n",
     "  do {\n",
     "    const birthRegistrations = await fetchAllBirthRegistrations(\n",
     "      sysToken,\n",
@@ -250,6 +253,7 @@
     "\n",
     "    const { results, totalItems } = birthRegistrations.data.searchEvents\n",
     "    const birthIds = results.map((x) => x.id)\n",
+    "\n",
     "    console.log(\n",
     "      `Processing next page of ${birthIds.length} of ${totalItems} total records`\n",
     "    )\n",
@@ -340,11 +344,16 @@
     "}\n",
     "\n",
     "if (EVENT === 'death') {\n",
+    "  const skippedPages = RECORD_SKIP ? Math.floor(RECORD_SKIP / 1000) : 0\n",
     "  const pageSize = 1000\n",
     "  const batchSize = 100\n",
     "  let itemsRemaining = 0\n",
-    "  let page = 1\n",
-    "  let totalProcessed = 0\n",
+    "  let page = 1 + skippedPages\n",
+    "  let totalProcessed = skippedPages * pageSize\n",
+    "\n",
+    "  if (RECORD_SKIP) {\n",
+    "    console.log(`Skipping first ${skippedPages * pageSize} records`)\n",
+    "  }\n",
     "\n",
     "  do {\n",
     "    const deathRegistrations = await fetchAllDeathRegistrations(\n",