Add granular connector caching

workspaces/mi/mi-extension/.env.example

@@ -11,5 +11,7 @@ MI_CONNECTOR_STORE_BACKEND=https://apis.wso2.com/qgpf/connector-store-backend/en
 MI_CONNECTOR_STORE_BACKEND_SEARCH=https://apis.wso2.com/qgpf/connector-store-backend/endpoint-9090-803/v1.0/connectors/details?limit=10&offset=0&searchQuery=${searchValue}&type=Connector&product=MI&runtimeVersion=${version}
 MI_CONNECTOR_STORE_BACKEND_INBOUND_ENDPOINTS=https://apis.wso2.com/qgpf/connector-store-backend/endpoint-9090-803/v1.0/connectors/details?offset=0&product=MI&type=inbound&runtimeVersion=${version}
 MI_CONNECTOR_STORE_BACKEND_GETBYVERSION=https://apis.wso2.com/qgpf/connector-store-backend/endpoint-9090-803/v1.0/connectors/${repoName}/versions/${versionId}?runtimeVersion=4.3.0&product=MI
+MI_CONNECTOR_STORE_BACKEND_SUMMARIES=https://apis.wso2.com/qgpf/connector-store-backend/endpoint-9090-803/v1.0/connectors/summaries?type=${type}&limit=100&offset=0&product=MI
+MI_CONNECTOR_STORE_BACKEND_DETAILS_FILTER=https://apis.wso2.com/qgpf/connector-store-backend/endpoint-9090-803/v1.0/connectors/details/filter
 ADOPTIUM_API_BASE_URL=https://api.adoptium.net/v3/assets/feature_releases
 COPILOT_ROOT_URL=https://7eff1239-64bb-4663-b256-30a00d187a5c-prod.e1-us-east-azure.choreoapis.dev/copilot

workspaces/mi/mi-extension/scripts/update-connector-context-db.js

@@ -0,0 +1,372 @@
+#!/usr/bin/env node
+
+/* eslint-disable no-console */
+
+const fs = require('fs/promises');
+const path = require('path');
+
+const API_BASE = process.env.CONNECTOR_STORE_BASE_URL
+    || 'https://apis.wso2.com/qgpf/connector-store-backend/endpoint-9090-803/v1.0';
+const SUMMARY_URL_TEMPLATE = `${API_BASE}/connectors/summaries?type=\${type}&limit=100&offset=0&product=MI`;
+const DETAILS_URL = `${API_BASE}/connectors/details/filter`;
+const PRODUCT = 'MI';
+const RUNTIME_VERSION = process.env.MI_RUNTIME_VERSION || '4.5.0';
+const MAX_NAMES_PER_REQUEST = 3;
+const REQUEST_TIMEOUT_MS = 120000;
+const RETRY_COUNT = 3;
+const RETRY_DELAY_MS = 1250;
+const BATCH_DELAY_MS = 250;
+
+const CONTEXT_DIR = path.resolve(__dirname, '../src/ai-features/agent-mode/context');
+const TARGETS = [
+    { type: 'Connector', fileName: 'connector_db.ts', exportName: 'CONNECTOR_DB' },
+    { type: 'Inbound', fileName: 'inbound_db.ts', exportName: 'INBOUND_DB' },
+];
+
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function chunkArray(items, size) {
+    const chunks = [];
+    for (let i = 0; i < items.length; i += size) {
+        chunks.push(items.slice(i, i + size));
+    }
+    return chunks;
+}
+
+function getConnectorName(item) {
+    if (!item || typeof item !== 'object') {
+        return '';
+    }
+
+    const rawName = item.connectorName || item.connector_name || item.name;
+    if (typeof rawName !== 'string') {
+        return '';
+    }
+
+    return rawName.trim();
+}
+
+function normalizeArrayPayload(payload, label) {
+    if (Array.isArray(payload)) {
+        return payload;
+    }
+
+    if (payload && typeof payload === 'object') {
+        if (Array.isArray(payload.data)) {
+            return payload.data;
+        }
+        if (Array.isArray(payload.items)) {
+            return payload.items;
+        }
+        if (Array.isArray(payload.connectors)) {
+            return payload.connectors;
+        }
+    }
+
+    throw new Error(`${label} payload is not an array.`);
+}
+
+async function fetchWithTimeout(url, init) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+    try {
+        return await fetch(url, {
+            ...init,
+            signal: controller.signal,
+        });
+    } finally {
+        clearTimeout(timeout);
+    }
+}
+
+async function parseResponse(response, label) {
+    const text = await response.text();
+
+    if (!response.ok) {
+        const bodySnippet = text ? ` - ${text.slice(0, 300)}` : '';
+        throw new Error(`${label} failed: HTTP ${response.status} ${response.statusText}${bodySnippet}`);
+    }
+
+    if (text.trim().length === 0) {
+        return [];
+    }
+
+    try {
+        return JSON.parse(text);
+    } catch {
+        throw new Error(`${label} returned non-JSON content.`);
+    }
+}
+
+async function requestJson(url, init, label) {
+    let lastError;
+
+    for (let attempt = 1; attempt <= RETRY_COUNT; attempt++) {
+        try {
+            const response = await fetchWithTimeout(url, init);
+            return await parseResponse(response, label);
+        } catch (error) {
+            lastError = error;
+            if (attempt < RETRY_COUNT) {
+                console.warn(`${label} attempt ${attempt}/${RETRY_COUNT} failed. Retrying...`);
+                await sleep(RETRY_DELAY_MS * attempt);
+            }
+        }
+    }
+
+    throw lastError;
+}
+
+async function fetchSummaries(type) {
+    const summaryUrl = SUMMARY_URL_TEMPLATE.replace('${type}', encodeURIComponent(type));
+    const payload = await requestJson(
+        summaryUrl,
+        {
+            method: 'GET',
+            headers: {
+                Accept: 'application/json',
+            },
+        },
+        `${type} summaries`
+    );
+
+    return normalizeArrayPayload(payload, `${type} summaries`);
+}
+
+function extractUniqueNames(summaries, type) {
+    const names = [];
+    const seen = new Set();
+
+    for (const summary of summaries) {
+        const name = getConnectorName(summary);
+        if (!name || seen.has(name)) {
+            continue;
+        }
+        seen.add(name);
+        names.push(name);
+    }
+
+    if (names.length === 0) {
+        throw new Error(`No ${type} names found from summaries.`);
+    }
+
+    return names;
+}
+
+function getConnectorDescription(item) {
+    if (!item || typeof item !== 'object') {
+        return '';
+    }
+
+    const rawDescription = item.description;
+    return typeof rawDescription === 'string' ? rawDescription : '';
+}
+
+function getConnectorTypeValue(item, fallbackType) {
+    if (!item || typeof item !== 'object') {
+        return fallbackType;
+    }
+
+    const rawType = item.connectorType || item.connector_type;
+    if (typeof rawType === 'string' && rawType.trim().length > 0) {
+        return rawType.trim();
+    }
+
+    return fallbackType;
+}
+
+function createSummaryFallbackRecord(name, summary, type) {
+    const baseRecord = {
+        connectorName: name,
+        repoName: '',
+        description: getConnectorDescription(summary),
+        connectorType: getConnectorTypeValue(summary, type),
+        mavenGroupId: '',
+        mavenArtifactId: '',
+        version: {
+            tagName: '',
+            releaseId: '',
+            isLatest: true,
+            isDeprecated: false,
+            operations: [],
+            connections: [],
+        },
+        otherVersions: {},
+        connectorRank: 0,
+        iconUrl: '',
+    };
+
+    if (type === 'Inbound') {
+        return {
+            ...baseRecord,
+            id: '',
+        };
+    }
+
+    return baseRecord;
+}
+
+async function fetchDetailsBatch(type, connectorNames) {
+    const payload = await requestJson(
+        DETAILS_URL,
+        {
+            method: 'POST',
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                connectorNames,
+                runtimeVersion: RUNTIME_VERSION,
+                product: PRODUCT,
+                latest: true,
+            }),
+        },
+        `${type} details (${connectorNames.join(', ')})`
+    );
+
+    return normalizeArrayPayload(payload, `${type} details`);
+}
+
+async function fetchAllDetails(type, names) {
+    const detailsByName = new Map();
+    let missing = names.slice();
+    const maxPasses = 3;
+
+    for (let pass = 1; pass <= maxPasses && missing.length > 0; pass++) {
+        if (pass > 1) {
+            console.warn(`[${type}] retry pass ${pass} for ${missing.length} missing item(s).`);
+        }
+
+        const batches = chunkArray(missing, MAX_NAMES_PER_REQUEST);
+        for (let i = 0; i < batches.length; i++) {
+            const batch = batches[i];
+            console.log(`[${type}] details batch ${i + 1}/${batches.length} with ${batch.length} item(s).`);
+
+            try {
+                const batchDetails = await fetchDetailsBatch(type, batch);
+                for (const detail of batchDetails) {
+                    const name = getConnectorName(detail);
+                    if (name) {
+                        detailsByName.set(name, detail);
+                    }
+                }
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.warn(`[${type}] batch failed and will be retried in next pass: ${message}`);
+            }
+
+            await sleep(BATCH_DELAY_MS);
+        }
+
+        missing = names.filter((name) => !detailsByName.has(name));
+    }
+
+    if (missing.length > 0) {
+        console.warn(`[${type}] missing API details for ${missing.length} item(s): ${missing.join(', ')}`);
+    }
+
+    return { detailsByName, missing };
+}
+
+async function readExistingRecordsByName(filePath, exportName) {
+    const existing = await fs.readFile(filePath, 'utf8');
+    const exportIndex = existing.indexOf(`export const ${exportName} =`);
+    if (exportIndex < 0) {
+        throw new Error(`Could not find export declaration for ${exportName} in ${filePath}.`);
+    }
+
+    const arrayStart = existing.indexOf('[', exportIndex);
+    const arrayEnd = existing.lastIndexOf(']');
+    if (arrayStart < 0 || arrayEnd < 0 || arrayEnd < arrayStart) {
+        throw new Error(`Could not parse array contents from ${filePath}.`);
+    }
+
+    const parsed = JSON.parse(existing.slice(arrayStart, arrayEnd + 1));
+    if (!Array.isArray(parsed)) {
+        throw new Error(`Parsed existing data from ${filePath} is not an array.`);
+    }
+
+    const recordsByName = new Map();
+    for (const record of parsed) {
+        const name = getConnectorName(record);
+        if (name) {
+            recordsByName.set(name, record);
+        }
+    }
+
+    return recordsByName;
+}
+
+async function writeTsArrayFile(filePath, exportName, records) {
+    const existing = await fs.readFile(filePath, 'utf8');
+    const exportRegex = new RegExp(`^[\\s\\S]*?export const\\s+${exportName}\\s*=\\s*`);
+    const match = existing.match(exportRegex);
+
+    if (!match) {
+        throw new Error(`Could not find export declaration for ${exportName} in ${filePath}.`);
+    }
+
+    const content = `${match[0]}${JSON.stringify(records, null, 4)}\n`;
+    await fs.writeFile(filePath, content, 'utf8');
+}
+
+async function updateTarget(target) {
+    const { type, fileName, exportName } = target;
+    const filePath = path.join(CONTEXT_DIR, fileName);
+
+    console.log(`\n=== Updating ${type} definitions ===`);
+    const summaries = await fetchSummaries(type);
+    const names = extractUniqueNames(summaries, type);
+    const summariesByName = new Map(summaries.map((summary) => [getConnectorName(summary), summary]));
+    console.log(`[${type}] fetched ${summaries.length} summaries, ${names.length} unique names.`);
+
+    const { detailsByName, missing } = await fetchAllDetails(type, names);
+
+    if (missing.length > 0) {
+        const existingRecordsByName = await readExistingRecordsByName(filePath, exportName);
+        let fallbackCount = 0;
+        let summaryFallbackCount = 0;
+
+        for (const name of missing) {
+            const fallbackRecord = existingRecordsByName.get(name);
+            if (fallbackRecord) {
+                detailsByName.set(name, fallbackRecord);
+                fallbackCount++;
+            } else {
+                const summary = summariesByName.get(name);
+                detailsByName.set(name, createSummaryFallbackRecord(name, summary, type));
+                summaryFallbackCount++;
+            }
+        }
+
+        console.warn(
+            `[${type}] used fallback records for ${fallbackCount} item(s) and summary-only placeholders for ${summaryFallbackCount} item(s).`
+        );
+    }
+
+    const details = names.map((name) => detailsByName.get(name)).filter(Boolean);
+    console.log(`[${type}] fetched ${details.length} detailed records.`);
+
+    await writeTsArrayFile(filePath, exportName, details);
+    console.log(`[${type}] wrote ${filePath}`);
+}
+
+async function main() {
+    if (MAX_NAMES_PER_REQUEST > 3) {
+        throw new Error('MAX_NAMES_PER_REQUEST must be 3 or less to avoid backend overload.');
+    }
+
+    for (const target of TARGETS) {
+        await updateTarget(target);
+    }
+}
+
+if (require.main === module) {
+    main().catch((error) => {
+        console.error(`Failed to update connector context DB files: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    });
+}

workspaces/mi/mi-extension/src/ai-features/agent-mode/agents/main/agent.ts

@@ -481,8 +481,10 @@ export async function executeAgent(
                         displayInput = { file_path: toolInput?.file_path };
                     } else if (part.toolName === CONNECTOR_TOOL_NAME) {
                         displayInput = {
-                            connector_names: toolInput?.connector_names,
-                            inbound_endpoint_names: toolInput?.inbound_endpoint_names,
+                            name: toolInput?.name,
+                            include_full_descriptions: toolInput?.include_full_descriptions,
+                            operation_names: toolInput?.operation_names,
+                            connection_names: toolInput?.connection_names,
                         };
                     } else if (part.toolName === SKILL_TOOL_NAME) {
                         displayInput = {