From 568199bcd566db5856a8723c3c45e81d063342d3 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Thu, 29 Jan 2026 15:11:55 +0000 Subject: [PATCH 1/5] feat: CodeExecutionTool types for allowing model to run code --- packages/ai/lib/types/content.ts | 89 ++++++++++++++++++++++++++++++- packages/ai/lib/types/enums.ts | 36 +++++++++++++ packages/ai/lib/types/requests.ts | 16 +++++- 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/packages/ai/lib/types/content.ts b/packages/ai/lib/types/content.ts index bd8ccf1ef1..62bdef6231 100644 --- a/packages/ai/lib/types/content.ts +++ b/packages/ai/lib/types/content.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { Role } from './enums'; +import { Language, Outcome, Role } from './enums'; /** * Content type for both prompts and response candidates. @@ -36,7 +36,9 @@ export type Part = | InlineDataPart | FunctionCallPart | FunctionResponsePart - | FileDataPart; + | FileDataPart + | ExecutableCodePart + | CodeExecutionResultPart; /** * Content part interface if the part represents a text string. @@ -52,6 +54,8 @@ export interface TextPart { * @internal */ thoughtSignature?: string; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -72,6 +76,8 @@ export interface InlineDataPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -105,6 +111,8 @@ export interface FunctionCallPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -121,6 +129,8 @@ export interface FunctionResponsePart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -138,6 +148,8 @@ export interface FileDataPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -202,3 +214,76 @@ export interface FileData { mimeType: string; fileUri: string; } + +/** + * Represents the code that is executed by the model. + * + * @beta + */ +export interface ExecutableCodePart { + text?: never; + inlineData?: never; + functionCall?: never; + functionResponse?: never; + fileData: never; + thought?: never; + /** + * @internal + */ + thoughtSignature?: never; + executableCode?: ExecutableCode; + codeExecutionResult?: never; +} + +/** + * Represents the code execution result from the model. + * + * @beta + */ +export interface CodeExecutionResultPart { + text?: never; + inlineData?: never; + functionCall?: never; + functionResponse?: never; + fileData: never; + thought?: never; + /** + * @internal + */ + thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: CodeExecutionResult; +} + +/** + * An interface for executable code returned by the model. + * + * @beta + */ +export interface ExecutableCode { + /** + * The programming language of the code. + */ + language?: Language; + /** + * The source code to be executed. + */ + code?: string; +} + +/** + * The results of code execution run by the model. + * + * @beta + */ +export interface CodeExecutionResult { + /** + * The result of the code execution. + */ + outcome?: Outcome; + /** + * The output from the code execution, or an error message + * if it failed. + */ + output?: string; +} diff --git a/packages/ai/lib/types/enums.ts b/packages/ai/lib/types/enums.ts index 73aaef699c..92d41925d1 100644 --- a/packages/ai/lib/types/enums.ts +++ b/packages/ai/lib/types/enums.ts @@ -304,3 +304,39 @@ export const InferenceMode = { * @public */ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; + +/** + * Represents the result of the code execution. + * + * @beta + */ +export const Outcome = { + UNSPECIFIED: 'OUTCOME_UNSPECIFIED', + OK: 'OUTCOME_OK', + FAILED: 'OUTCOME_FAILED', + DEADLINE_EXCEEDED: 'OUTCOME_DEADLINE_EXCEEDED', +}; + +/** + * Represents the result of the code execution. + * + * @beta + */ +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; + +/** + * The programming language of the code. + * + * @beta + */ +export const Language = { + UNSPECIFIED: 'LANGUAGE_UNSPECIFIED', + PYTHON: 'PYTHON', +}; + +/** + * The programming language of the code. + * + * @beta + */ +export type Language = (typeof Language)[keyof typeof Language]; diff --git a/packages/ai/lib/types/requests.ts b/packages/ai/lib/types/requests.ts index a3e0432ad3..839c2a31bb 100644 --- a/packages/ai/lib/types/requests.ts +++ b/packages/ai/lib/types/requests.ts @@ -250,7 +250,7 @@ export interface RequestOptions { * Defines a tool that model can call to access external knowledge. * @public */ -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; /** * Structured representation of a function declaration as defined by the @@ -316,6 +316,20 @@ export interface GoogleSearchTool { // eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface GoogleSearch {} +/** + * A tool that enables the model to use code execution. + * + * @beta + */ +export interface CodeExecutionTool { + /** + * Specifies the code execution configuration. + * Currently, this is an empty object, but it's reserved for future configuration options. + */ + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + codeExecution: {}; +} + /** * A `FunctionDeclarationsTool` is a piece of code that enables the system to * interact with external systems to perform an action, or set of actions, From 7ce4d956d8dcb9826ebb774910d13f5bdbcdea25 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Fri, 30 Jan 2026 14:15:44 +0000 Subject: [PATCH 2/5] test: update AI package tests --- .../ai/__tests__/generate-content.test.ts | 11 +++ .../ai/__tests__/generative-model.test.ts | 82 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/packages/ai/__tests__/generate-content.test.ts b/packages/ai/__tests__/generate-content.test.ts index 3f38b6ecf7..30c9428b2d 100644 --- a/packages/ai/__tests__/generate-content.test.ts +++ b/packages/ai/__tests__/generate-content.test.ts @@ -23,6 +23,8 @@ import { HarmBlockMethod, HarmBlockThreshold, HarmCategory, + Language, + Outcome, // RequestOptions, } from '../lib/types'; import { ApiSettings } from '../lib/types/internal'; @@ -172,6 +174,15 @@ describe('generateContent()', () => { ); }); + it('codeExecution', async function () { + const mockResponse = getMockResponse(BackendName.VertexAI, 'unary-success-code-execution.json'); + jest.spyOn(request, 'makeRequest').mockResolvedValue(mockResponse as Response); + const result = await generateContent(fakeApiSettings, 'model', fakeRequestParams); + const parts = result.response.candidates?.[0]?.content?.parts; + expect(parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK)).toBe(true); + expect(parts?.some(part => part.executableCode?.language === Language.PYTHON)).toBe(true); + }); + it('blocked prompt', async () => { const mockResponse = getMockResponse( BackendName.VertexAI, diff --git a/packages/ai/__tests__/generative-model.test.ts b/packages/ai/__tests__/generative-model.test.ts index 7835469e37..5f3f58d7c1 100644 --- a/packages/ai/__tests__/generative-model.test.ts +++ b/packages/ai/__tests__/generative-model.test.ts @@ -37,6 +37,50 @@ const fakeAI: AI = { }; describe('GenerativeModel', () => { + it('passes CodeExecutionTool with other tools through to generateContent', async function () { + const genModel = new GenerativeModel(fakeAI, { + model: 'my-model', + tools: [ + { + functionDeclarations: [ + { + name: 'myfunc', + description: 'mydesc', + }, + ], + }, + { googleSearch: {} }, + { codeExecution: {} }, + ], + toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, + }); + expect(genModel.tools?.length).toBe(3); + expect(genModel.toolConfig?.functionCallingConfig?.mode).toBe(FunctionCallingMode.NONE); + expect(genModel.systemInstruction?.parts[0]!.text).toBe('be friendly'); + const mockResponse = getMockResponse( + BackendName.VertexAI, + 'unary-success-basic-reply-short.json', + ); + const makeRequestStub = jest + .spyOn(request, 'makeRequest') + .mockResolvedValue(mockResponse as Response); + await genModel.generateContent('hello'); + expect(makeRequestStub).toHaveBeenCalledWith( + expect.objectContaining({ + model: 'publishers/google/models/my-model', + task: request.Task.GENERATE_CONTENT, + apiSettings: expect.anything(), + stream: false, + requestOptions: {}, + }), + expect.stringMatching( + new RegExp(`myfunc|googleSearch|codeExecution|${FunctionCallingMode.NONE}|be friendly`), + ), + ); + makeRequestStub.mockRestore(); + }); + it('passes params through to generateContent', async () => { const genModel = new GenerativeModel(fakeAI, { model: 'my-model', @@ -214,6 +258,44 @@ describe('GenerativeModel', () => { makeRequestStub.mockRestore(); }); + it('passes CodeExecutionTool through to chat.sendMessage', async function () { + const genModel = new GenerativeModel(fakeAI, { + model: 'my-model', + tools: [ + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] }, + { googleSearch: {} }, + { codeExecution: {} }, + ], + toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, + generationConfig: { topK: 1 }, + }); + expect(genModel.tools?.length).toBe(3); + const mockResponse = getMockResponse( + BackendName.VertexAI, + 'unary-success-basic-reply-short.json', + ); + const makeRequestStub = jest + .spyOn(request, 'makeRequest') + .mockResolvedValue(mockResponse as Response); + await genModel.startChat().sendMessage('hello'); + expect(makeRequestStub).toHaveBeenCalledWith( + expect.objectContaining({ + model: 'publishers/google/models/my-model', + task: request.Task.GENERATE_CONTENT, + apiSettings: expect.anything(), + stream: false, + requestOptions: {}, + }), + expect.stringMatching( + new RegExp( + `myfunc|googleSearch|codeExecution|${FunctionCallingMode.NONE}|be friendly|topK`, + ), + ), + ); + makeRequestStub.mockRestore(); + }); + it('passes text-only systemInstruction through to chat.sendMessage', async () => { const genModel = new GenerativeModel(fakeAI, { model: 'my-model', From 5938ce292ef01b33fe87c6529782105c6a9bb329 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Fri, 30 Jan 2026 14:24:35 +0000 Subject: [PATCH 3/5] fix: add two new properties --- packages/ai/lib/methods/chat-session-helpers.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ai/lib/methods/chat-session-helpers.ts b/packages/ai/lib/methods/chat-session-helpers.ts index 0bf988ae63..fecf1658d5 100644 --- a/packages/ai/lib/methods/chat-session-helpers.ts +++ b/packages/ai/lib/methods/chat-session-helpers.ts @@ -82,6 +82,8 @@ export function validateChatHistory(history: Content[]): void { functionResponse: 0, thought: 0, thoughtSignature: 0, + executableCode: 0, + codeExecutionResult: 0, }; for (const part of parts) { From c233f263c8e363436aa133937b938ad740f6b4b4 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Fri, 30 Jan 2026 14:30:39 +0000 Subject: [PATCH 4/5] chore: record diff between firebase-js-sdk and react native firebase package for audio browser api --- .cursor/rules/ai/ai-package.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.cursor/rules/ai/ai-package.md b/.cursor/rules/ai/ai-package.md index 4358aa3361..8b60cae630 100644 --- a/.cursor/rules/ai/ai-package.md +++ b/.cursor/rules/ai/ai-package.md @@ -294,6 +294,16 @@ registerVersion(name, version); **Reason**: Chrome's on-device AI is browser-specific and not available in React Native. This is a web-only feature. +### Audio Conversation (startAudioConversation, etc.) +**Firebase JS SDK has:** +- `src/methods/live-session-helpers.ts` – `startAudioConversation`, `AudioConversationController`, `StartAudioConversationOptions` +- Uses Web Audio API (`AudioContext`, `AudioWorkletNode`, `AudioWorkletProcessor`), `navigator.mediaDevices.getUserMedia()`, and `MediaStream` for mic capture and playback + +**React Native Firebase does NOT have:** +- `live-session-helpers.ts` or any audio-conversation exports + +**Reason**: Audio conversation depends on browser-only APIs (Web Audio API, getUserMedia). React Native has no equivalent; mic and playback use different native modules or libraries. Do not port. + ### Browser-Specific APIs Not Used - No `window` object references (except in comments/docs) - No `document` object usage (except in JSDoc examples) @@ -707,6 +717,7 @@ Any difference NOT documented in these cursor rules is a potential feature gap. | **Chrome Adapter** | ✅ Has | ❌ Doesn't have | Browser-only feature | | **Hybrid Mode** | ✅ Has | ❌ Doesn't have | Browser-only feature | | **HybridParams** | ✅ Has | ❌ Doesn't have | Browser-only feature | +| **Audio Conversation** (startAudioConversation, etc.) | ✅ Has | ❌ Doesn't have | Web Audio API / getUserMedia not in RN | | **Polyfills** | ❌ Doesn't need | ✅ Requires | RN environment | | **Dependencies** | `@firebase/*` | `@react-native-firebase/*` | Different ecosystem | | **AIService** | Complex (with _FirebaseService) | Simple | Different architecture | From f186fa539373809568446ced4dcc567ace11a284 Mon Sep 17 00:00:00 2001 From: russellwheatley Date: Mon, 2 Feb 2026 17:12:26 +0000 Subject: [PATCH 5/5] ci: workaround for ios workflows --- .github/workflows/tests_e2e_ios.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_e2e_ios.yml b/.github/workflows/tests_e2e_ios.yml index 71e5088ad7..4fe59c8e8f 100644 --- a/.github/workflows/tests_e2e_ios.yml +++ b/.github/workflows/tests_e2e_ios.yml @@ -128,7 +128,7 @@ jobs: - uses: maxim-lobanov/setup-xcode@v1 with: - xcode-version: '26.0.1' # temporarily pinning version until simulators are defined 'latest-stable' + xcode-version: '26.2.0' # temporarily pinning version until simulators are defined 'latest-stable' - uses: actions/checkout@v4 with: