Add Data Type to Variables

apps/platform/db/migrations/20250630020727_add_rule_path_data_type.js

@@ -0,0 +1,13 @@
+exports.up = async function(knex) {
+    await knex.schema.table('project_rule_paths', function(table) {
+        table.string('data_type').defaultTo('string')
+        table.enum('visibility', ['public', 'hidden', 'classified']).defaultTo('public').notNullable()
+    })
+}
+
+exports.down = async function(knex) {
+    await knex.schema.table('project_rule_paths', function(table) {
+        table.dropColumn('data_type')
+        table.dropColumn('visibility')
+    })
+}

apps/platform/src/projects/ProjectController.ts

@@ -1,20 +1,21 @@
 import Router from '@koa/router'
-import { ProjectParams } from './Project'
-import { JSONSchemaType, validate } from '../core/validate'
-import { extractQueryParams } from '../utilities'
-import { searchParamsSchema } from '../core/searchParams'
 import { ParameterizedContext } from 'koa'
-import { allProjects, createProject, getProject, pagedProjects, requireProjectRole, updateProject } from './ProjectService'
+import App from '../app'
+import { getAdmin } from '../auth/AdminRepository'
 import { AuthState, ProjectState } from '../auth/AuthMiddleware'
-import { getProjectAdmin } from './ProjectAdminRepository'
 import { RequestError } from '../core/errors'
-import { ProjectError } from './ProjectError'
-import { ProjectRulePath } from '../rules/ProjectRulePath'
-import { getAdmin } from '../auth/AdminRepository'
-import UserSchemaSyncJob from '../schema/UserSchemaSyncJob'
-import App from '../app'
-import { hasProvider } from '../providers/ProviderService'
+import { searchParamsSchema } from '../core/searchParams'
+import { JSONSchemaType, validate } from '../core/validate'
 import { requireOrganizationRole } from '../organizations/OrganizationService'
+import { hasProvider } from '../providers/ProviderService'
+import { RulePathVisibility } from '../rules/ProjectRulePath'
+import UserSchemaSyncJob from '../schema/UserSchemaSyncJob'
+import { extractQueryParams } from '../utilities'
+import { ProjectParams } from './Project'
+import { getProjectAdmin } from './ProjectAdminRepository'
+import { ProjectError } from './ProjectError'
+import { getRulePaths, pagedUserRulePaths, updateRulePath } from './ProjectRulePathRepository'
+import { allProjects, createProject, getProject, pagedProjects, requireProjectRole, updateProject } from './ProjectService'
 
 export async function projectMiddleware(ctx: ParameterizedContext<ProjectState>, next: () => void) {
 
@@ -177,22 +178,29 @@ subrouter.patch('/', async ctx => {
 })
 
 subrouter.get('/data/paths', async ctx => {
-    ctx.body = await ProjectRulePath
-        .all(q => q.where('project_id', ctx.state.project.id))
-        .then(list => list.reduce((a, { type, name, path }) => {
-            if (type === 'event') {
-                (a.eventPaths[name!] ?? (a.eventPaths[name!] = [])).push(path)
-            } else {
-                a.userPaths.push(path)
-            }
-            return a
-        }, {
-            userPaths: [],
-            eventPaths: {},
-        } as {
-            userPaths: string[]
-            eventPaths: { [name: string]: string[] }
-        }))
+    const visibilities: RulePathVisibility[] = ctx.state.projectRole === 'admin'
+        ? ['public', 'classified']
+        : ['public']
+    ctx.body = await getRulePaths(ctx.state.project.id, visibilities)
+})
+
+subrouter.get('/data/paths/users', async ctx => {
+    requireProjectRole(ctx, 'admin')
+    const search = extractQueryParams(ctx.query, searchParamsSchema)
+    const visibilities: RulePathVisibility[] = ctx.state.projectRole === 'admin'
+        ? ['public', 'classified', 'hidden']
+        : ['public']
+    ctx.body = await pagedUserRulePaths({
+        search,
+        projectId: ctx.state.project.id,
+        visibilities,
+    })
+})
+
+subrouter.put('/data/paths/users/:pathId', async ctx => {
+    requireProjectRole(ctx, 'admin')
+
+    ctx.body = await updateRulePath(parseInt(ctx.params.pathId), ctx.request.body.visibility)
 })
 
 subrouter.post('/data/paths/sync', async ctx => {

apps/platform/src/projects/ProjectRulePathRepository.ts

@@ -0,0 +1,66 @@
+import { PageParams } from '../core/searchParams'
+import { GetProjectRulePath, ProjectRulePath, RulePathVisibility } from '../rules/ProjectRulePath'
+import { KeyedSet } from '../utilities'
+
+type PagedRulePathParams = {
+    search: PageParams,
+    projectId: number,
+    visibilities: RulePathVisibility[]
+}
+export const pagedUserRulePaths = async ({ search, projectId, visibilities }: PagedRulePathParams) => {
+    return await ProjectRulePath.search(
+        search,
+        q => q.where('project_id', projectId)
+            .whereIn('visibility', visibilities)
+            .where('type', 'user'),
+    )
+}
+
+export const pagedEventRulePaths = async ({ search, projectId, visibilities }: PagedRulePathParams) => {
+    return await ProjectRulePath.search(
+        search,
+        q => q.where('project_id', projectId)
+            .whereIn('visibility', visibilities)
+            .where('type', 'event')
+            .groupBy('name')
+            .select('name'),
+    )
+}
+
+export const updateRulePath = async (id: number, visibility: RulePathVisibility) => {
+    return await ProjectRulePath.update(qb => qb.where('id', id), { visibility })
+}
+
+type RulePaths = {
+    userPaths: KeyedSet<GetProjectRulePath>,
+    eventPaths: { [name: string]: KeyedSet<GetProjectRulePath> }
+}
+export const getRulePaths = async (
+    projectId: number,
+    visibilities: RulePathVisibility[] = ['public'],
+): Promise<RulePaths> => {
+    const rulePaths = await ProjectRulePath.all(q => q
+        .where('project_id', projectId)
+        .whereIn('visibility', visibilities)
+        .select('path', 'type', 'name', 'data_type', 'visibility'),
+    )
+
+    return rulePaths.reduce((a, rulePath) => {
+        const { path, type, name, data_type } = rulePath
+        if (type === 'event' && name) {
+            if (!a.eventPaths[name]) {
+                const set = new KeyedSet<GetProjectRulePath>(item => item.path)
+                set.add({ path, type, name, data_type })
+                a.eventPaths[name] = set
+            } else {
+                a.eventPaths[name].add({ path, type, name, data_type })
+            }
+        } else {
+            a.userPaths.add({ path, type, name, data_type })
+        }
+        return a
+    }, {
+        userPaths: new KeyedSet<GetProjectRulePath>(item => item.path),
+        eventPaths: {},
+    } as RulePaths)
+}

apps/platform/src/rules/ProjectRulePath.ts

@@ -1,10 +1,16 @@
 import Model from '../core/Model'
 
+export type RulePathDataType = 'string' | 'number' | 'boolean' | 'date' | 'array'
+export type RulePathVisibility = 'public' | 'hidden' | 'classified'
+export type RulePathEventName = string
 export class ProjectRulePath extends Model {
 
     project_id!: number
     path!: string
     type!: 'user' | 'event'
-    name?: string // event name
-
+    name?: RulePathEventName // event name
+    data_type?: RulePathDataType
+    visibility!: RulePathVisibility
 }
+
+export type GetProjectRulePath = Pick<ProjectRulePath, 'path' | 'type' | 'name' | 'data_type'>

apps/platform/src/rules/RuleHelpers.ts

@@ -3,6 +3,7 @@ import { AnyJson, EventRulePeriod, EventRuleTree, Operator, RuleGroup, RuleTree
 import { compileTemplate } from '../render'
 import { visit } from '../utilities'
 import { subSeconds } from 'date-fns'
+import { RulePathDataType } from './ProjectRulePath'
 
 export const queryValue = <T>(
     value: Record<string, unknown>,
@@ -78,6 +79,10 @@ export const reservedPaths: Record<RuleGroup, string[]> = {
     parent: [],
 }
 
+export const reservedPathDataType = (path: string): RulePathDataType => {
+    return path === 'created_at' ? 'date' : 'string'
+}
+
 export const isEventWrapper = (rule: RuleTree): rule is EventRuleTree => {
     return rule.group === 'event'
         && (rule.path === '$.name' || rule.path === 'name')

apps/platform/src/schema/UserSchemaService.ts

@@ -1,8 +1,9 @@
 import { User } from '../users/User'
 import App from '../app'
-import { ProjectRulePath } from '../rules/ProjectRulePath'
+import { ProjectRulePath, RulePathDataType } from '../rules/ProjectRulePath'
 import { UserEvent } from '../users/UserEvent'
-import { reservedPaths } from '../rules/RuleHelpers'
+import { reservedPathDataType, reservedPaths } from '../rules/RuleHelpers'
+import { KeyedSet } from '../utilities'
 
 export async function listUserPaths(project_id: number) {
     const paths: Array<{ path: string }> = await ProjectRulePath.query()
@@ -31,7 +32,7 @@ export async function listEventPaths(project_id: number, name: string) {
     return paths.map(p => p.path)
 }
 
-export function addLeafPaths(set: Set<string>, value: any, path = '$') {
+export function addLeafPaths(set: KeyedSet<[string, RulePathDataType]>, value: any, path = '$') {
     if (typeof value === 'undefined') return
     if (Array.isArray(value)) {
         for (const item of value) {
@@ -43,7 +44,8 @@ export function addLeafPaths(set: Set<string>, value: any, path = '$') {
         }
     } else {
         if (path !== '$') {
-            set.add(path)
+            const type = inferDataType(value)
+            set.add([path, type])
         }
     }
 }
@@ -54,6 +56,25 @@ const joinPath = (path: string, key: string) => {
     return `${path}['${key}']`
 }
 
+const inferDataType = (value: any): RulePathDataType => {
+    if (Array.isArray(value)) {
+        return 'array'
+    }
+    if (typeof value === 'string') {
+        return 'string'
+    }
+    if (value instanceof Date) {
+        return 'date'
+    }
+    if (typeof value === 'number') {
+        return 'number'
+    }
+    if (typeof value === 'boolean') {
+        return 'boolean'
+    }
+    return 'string'
+}
+
 interface SyncProjectRulePathsParams {
     project_id: number
     updatedAfter?: Date
@@ -65,8 +86,8 @@ export async function syncUserDataPaths({
 }: SyncProjectRulePathsParams) {
     await App.main.db.transaction(async trx => {
 
-        const userPaths = new Set<string>()
-        const eventPaths = new Map<string, Set<string>>()
+        const userPaths = new KeyedSet<[string, RulePathDataType]>(item => item[0])
+        const eventPaths = new Map<string, KeyedSet<[string, RulePathDataType]>>()
 
         const userQuery = User.query(trx)
             .where('project_id', project_id)
@@ -80,7 +101,7 @@ export async function syncUserDataPaths({
             }
         })
         for (const path of reservedPaths.user) {
-            userPaths.add(joinPath('$', path))
+            userPaths.add([joinPath('$', path), reservedPathDataType(path)])
         }
 
         const eventQuery = await UserEvent.clickhouse().query(`SELECT name, data FROM user_events WHERE project_id = {projectId: UInt32} ${updatedAfter ? 'AND created_at >= {updatedAfter: DateTime64(3, \'UTC\')}' : ''}`, {
@@ -93,11 +114,12 @@ export async function syncUserDataPaths({
                 const { name, data } = result.json()
                 let set = eventPaths.get(name)
                 if (!set) {
-                    eventPaths.set(name, set = new Set())
+                    set = new KeyedSet<[string, RulePathDataType]>(item => item[0])
+                    eventPaths.set(name, set)
                 }
                 addLeafPaths(set, data)
                 for (const path of reservedPaths.event) {
-                    set.add(joinPath('$', path))
+                    set.add([joinPath('$', path), reservedPathDataType(path)])
                 }
             }
         }
@@ -106,52 +128,49 @@ export async function syncUserDataPaths({
 
         if (!updatedAfter && existing.length) {
             const removeIds: number[] = []
-            let i = 0
-            let remove = false
-            while (i < existing.length) {
-                const e = existing[i]
-                if (e.type === 'user') {
-                    remove = !userPaths.has(e.path)
-                } else if (e.type === 'event') {
-                    remove = !eventPaths.get(e.name ?? '')?.has(e.path)
+            for (const { id, name, type, path } of existing) {
+                let remove = false
+                if (type === 'user') {
+                    remove = !userPaths.has(path)
+                } else if (type === 'event') {
+                    remove = !eventPaths.get(name ?? '')?.has(path)
                 } else {
                     remove = true
                 }
                 if (remove) {
-                    removeIds.push(e.id)
-                    existing.splice(i, 1)
-                } else {
-                    i++
+                    removeIds.push(id)
                 }
             }
+
             if (removeIds.length) {
                 await ProjectRulePath.delete(q => q.whereIn('id', removeIds), trx)
             }
         }
 
         // add all new paths
-        for (const path of userPaths) {
+        for (const [path, data_type] of userPaths) {
             if (!existing.find(e => e.type === 'user' && e.path === path)) {
                 await ProjectRulePath.insert({
                     project_id,
                     path,
+                    data_type,
                     type: 'user',
                 }, trx)
             }
         }
 
         for (const [name, paths] of eventPaths.entries()) {
-            for (const path of paths) {
+            for (const [path, data_type] of paths) {
                 if (!existing.find(e => e.type === 'event' && e.path === path && e.name === name)) {
                     await ProjectRulePath.insert({
                         project_id,
                         path,
+                        data_type,
                         name,
                         type: 'event',
                     }, trx)
                 }
             }
         }
-
     })
 }

apps/platform/src/schema/__tests__/UserSchemaService.spec.ts

@@ -1,9 +1,9 @@
 import Project from '../../projects/Project'
-import { ProjectRulePath } from '../../rules/ProjectRulePath'
+import { ProjectRulePath, RulePathDataType } from '../../rules/ProjectRulePath'
 import { User } from '../../users/User'
 import { UserEvent } from '../../users/UserEvent'
 import { addLeafPaths, syncUserDataPaths } from '../UserSchemaService'
-import { sleep } from '../../utilities'
+import { KeyedSet, sleep } from '../../utilities'
 import { startOfSecond } from 'date-fns'
 import { reservedPaths } from '../../rules/RuleHelpers'
 
@@ -28,11 +28,11 @@ describe('UserSchemaService', () => {
                 },
             }
 
-            const set = new Set<string>()
+            const set = new KeyedSet<[string, RulePathDataType]>(item => item[0])
 
             addLeafPaths(set, data)
 
-            const arr = Array.from(set.values())
+            const arr = Array.from(set.keys())
 
             expect(arr).not.toContain('$')
             expect(arr).toContain('$.one')