Skip to content

Unify test functions about LLM schemas #220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
samchon merged 5 commits into from
Dec 20, 2025
Merged

Unify test functions about LLM schemas #220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8a36d98
Unify test functions about LLM schemas
samchon Dec 20, 2025
101e1e9
Update test/src/utils/LlmApplicationFactory.ts
samchon Dec 20, 2025
b15c3cf
fixed most of test functions
samchon Dec 20, 2025
fce05d4
fixed most of test functions
samchon Dec 20, 2025
75924c8
completed everything
samchon Dec 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 62 additions & 40 deletions src/composers/LlmSchemaComposer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ export namespace LlmSchemaComposer {
? JsonDescriptionUtil.cascade({
prefix: "#/components/schemas/",
components: props.components,
schema: props.schema,
schema: {
...props.schema,
description: result.value.description,
},
escape: true,
})
: result.value.description,
Expand All @@ -59,7 +62,7 @@ export namespace LlmSchemaComposer {
};

export const schema = (props: {
config: ILlmSchema.IConfig;
config?: Partial<ILlmSchema.IConfig>;
components: OpenApi.IComponents;
$defs: Record<string, ILlmSchema>;
schema: OpenApi.IJsonSchema;
Expand Down Expand Up @@ -140,6 +143,30 @@ export namespace LlmSchemaComposer {
},
};

const visitConstant = (input: OpenApi.IJsonSchema): void => {
const insert = (value: any): void => {
const matched:
| ILlmSchema.IString
| ILlmSchema.INumber
| ILlmSchema.IBoolean
| undefined = union.find(
(u) =>
(u as (IJsonSchemaAttribute & { type: string }) | undefined)
?.type === typeof value,
) as ILlmSchema.IString | undefined;
if (matched !== undefined) {
matched.enum ??= [];
matched.enum.push(value);
} else
union.push({
type: typeof value as "number",
enum: [value],
});
};
if (OpenApiTypeChecker.isConstant(input)) insert(input.const);
else if (OpenApiTypeChecker.isOneOf(input))
input.oneOf.forEach(visitConstant);
};
const visit = (input: OpenApi.IJsonSchema, accessor: string): void => {
if (OpenApiTypeChecker.isOneOf(input)) {
// UNION TYPE
Expand Down Expand Up @@ -184,7 +211,8 @@ export namespace LlmSchemaComposer {
// DISCARD THE REFERENCE TYPE
const length: number = union.length;
visit(target, accessor);
if (length === union.length - 1 && union[union.length - 1] !== null)
visitConstant(target);
if (length === union.length - 1)
union[union.length - 1] = {
...union[union.length - 1]!,
description: JsonDescriptionUtil.cascade({
Expand Down Expand Up @@ -257,6 +285,10 @@ export namespace LlmSchemaComposer {
properties,
additionalProperties,
required: input.required ?? [],
description:
props.config.strict === true
? JsonDescriptionUtil.take(input)
: input.description,
});
} else if (OpenApiTypeChecker.isArray(input)) {
// ARRAY TYPE
Expand All @@ -278,7 +310,10 @@ export namespace LlmSchemaComposer {
...input,
items: items.value,
})
: items.value,
: {
...input,
items: items.value,
},
);
} else if (OpenApiTypeChecker.isString(input))
union.push(
Expand All @@ -297,36 +332,12 @@ export namespace LlmSchemaComposer {
);
else if (OpenApiTypeChecker.isTuple(input))
return; // UNREACHABLE
else union.push({ ...input });
else if (OpenApiTypeChecker.isConstant(input) === false)
union.push({ ...input });
};

const visitConstant = (input: OpenApi.IJsonSchema): void => {
const insert = (value: any): void => {
const matched:
| ILlmSchema.IString
| ILlmSchema.INumber
| ILlmSchema.IBoolean
| undefined = union.find(
(u) =>
(u as (IJsonSchemaAttribute & { type: string }) | undefined)
?.type === typeof value,
) as ILlmSchema.IString | undefined;
if (matched !== undefined) {
matched.enum ??= [];
matched.enum.push(value);
} else
union.push({
type: typeof value as "number",
enum: [value],
});
};
if (OpenApiTypeChecker.isConstant(input)) insert(input.const);
else if (OpenApiTypeChecker.isOneOf(input))
input.oneOf.forEach(visitConstant);
};

visit(props.schema, props.accessor ?? "$input.schema");
visitConstant(props.schema);
visit(props.schema, props.accessor ?? "$input.schema");

if (reasons.length > 0)
return {
Expand All @@ -339,6 +350,7 @@ export namespace LlmSchemaComposer {
};
else if (union.length === 0)
return {
// unknown type
success: true,
value: {
...attribute,
Expand All @@ -347,18 +359,28 @@ export namespace LlmSchemaComposer {
};
else if (union.length === 1)
return {
// single type
success: true,
value: {
...attribute,
...union[0],
description: union[0].description ?? attribute.description,
description:
props.config.strict === true && LlmTypeChecker.isReference(union[0])
? undefined
: (union[0].description ?? attribute.description),
},
};
return {
success: true,
value: {
...attribute,
anyOf: union,
anyOf: union.map((u) => ({
...u,
description:
props.config.strict === true && LlmTypeChecker.isReference(u)
? undefined
: u.description,
})),
"x-discriminator":
OpenApiTypeChecker.isOneOf(props.schema) &&
props.schema.discriminator !== undefined &&
Expand Down Expand Up @@ -782,14 +804,14 @@ export namespace LlmSchemaComposer {
}),
} satisfies OpenApi.IJsonSchema;
};
}

const getConfig = (
config?: Partial<ILlmSchema.IConfig> | undefined,
): ILlmSchema.IConfig => ({
reference: config?.reference ?? true,
strict: config?.strict ?? false,
});
export const getConfig = (
config?: Partial<ILlmSchema.IConfig> | undefined,
): ILlmSchema.IConfig => ({
reference: config?.reference ?? true,
strict: config?.strict ?? false,
});
}

const validateStrict = (
schema: OpenApi.IJsonSchema,
Expand Down
12 changes: 5 additions & 7 deletions test/src/examples/chatgpt-function-call-to-sale-create.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,15 @@ const main = async (): Promise<void> => {
// convert to emended OpenAPI document,
// and compose LLM function calling application
const document: OpenApi.IDocument = OpenApi.convert(swagger);
const application: IHttpLlmApplication<"chatgpt"> = HttpLlm.application({
model: "chatgpt",
const application: IHttpLlmApplication = HttpLlm.application({
document,
});

// Let's imagine that LLM has selected a function to call
const func: IHttpLlmFunction<"chatgpt"> | undefined =
application.functions.find(
// (f) => f.name === "llm_selected_function_name"
(f) => f.path === "/shoppings/sellers/sale" && f.method === "post",
);
const func: IHttpLlmFunction | undefined = application.functions.find(
// (f) => f.name === "llm_selected_function_name"
(f) => f.path === "/shoppings/sellers/sale" && f.method === "post",
);
if (func === undefined) throw new Error("No matched function exists.");

// Get arguments by ChatGPT function calling
Expand Down
19 changes: 8 additions & 11 deletions test/src/examples/claude-function-call-separate-to-sale-create.ts.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import Anthropic from "@anthropic-ai/sdk";
import {
ClaudeTypeChecker,
HttpLlm,
IHttpLlmApplication,
IHttpLlmFunction,
LlmTypeChecker,
OpenApi,
OpenApiV3,
OpenApiV3_1,
Expand All @@ -26,23 +26,20 @@ const main = async (): Promise<void> => {
// convert to emended OpenAPI document,
// and compose LLM function calling application
const document: OpenApi.IDocument = OpenApi.convert(swagger);
const application: IHttpLlmApplication<"claude"> = HttpLlm.application({
model: "claude",
const application: IHttpLlmApplication = HttpLlm.application({
document,
options: {
reference: true,
config: {
separate: (schema) =>
ClaudeTypeChecker.isString(schema) &&
LlmTypeChecker.isString(schema) &&
!!schema.contentMediaType?.startsWith("image"),
},
});

// Let's imagine that LLM has selected a function to call
const func: IHttpLlmFunction<"claude"> | undefined =
application.functions.find(
// (f) => f.name === "llm_selected_fuction_name"
(f) => f.path === "/shoppings/sellers/sale" && f.method === "post",
);
const func: IHttpLlmFunction | undefined = application.functions.find(
// (f) => f.name === "llm_selected_fuction_name"
(f) => f.path === "/shoppings/sellers/sale" && f.method === "post",
);
if (func === undefined) throw new Error("No matched function exists.");

// Get arguments by ChatGPT function calling
Expand Down
7 changes: 3 additions & 4 deletions test/src/executable/execute.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,15 @@ const main = async (): Promise<void> => {
// convert to emended OpenAPI document,
// and compose LLM function calling application
const document: OpenApi.IDocument = OpenApi.convert(swagger);
const application: IHttpLlmApplication<"3.0"> = HttpLlm.application({
model: "3.0",
const application: IHttpLlmApplication = HttpLlm.application({
document,
});

// Let's imagine that LLM has selected a function to call
const func: IHttpLlmFunction<"3.0"> | undefined = application.functions.find(
const func: IHttpLlmFunction | undefined = application.functions.find(
(f) => f.path === "/bbs/articles" && f.method === "post",
);
typia.assertGuard<IHttpLlmFunction<"3.0">>(func);
typia.assertGuard<IHttpLlmFunction>(func);

// actual execution is by yourself
const article = await HttpLlm.execute({
Expand Down
25 changes: 11 additions & 14 deletions test/src/executable/sale.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { ILlmSchema } from "@samchon/openapi";
import fs from "fs";
import typia from "typia";

Expand All @@ -8,32 +7,30 @@ import { LlmFunctionCaller } from "../utils/LlmFunctionCaller";
import { ShoppingSalePrompt } from "../utils/ShoppingSalePrompt";
import { StopWatch } from "../utils/StopWatch";

const VENDORS: Array<[string, ILlmSchema.Model]> = [
["openai/gpt-4.1", "chatgpt"],
["anthropic/claude-sonnet-4.5", "claude"],
["deepseek/deepseek-v3.1-terminus:exacto", "claude"],
["google/gemini-2.5-pro", "gemini"],
["meta-llama/llama-3.3-70b-instruct", "claude"],
["qwen/qwen3-next-80b-a3b-instruct", "claude"],
const VENDORS: string[] = [
"openai/gpt-4.1",
"anthropic/claude-sonnet-4.5",
"deepseek/deepseek-v3.1-terminus:exacto",
"google/gemini-2.5-pro",
"meta-llama/llama-3.3-70b-instruct",
"qwen/qwen3-next-80b-a3b-instruct",
];

const main = async (): Promise<void> => {
for (const title of await ShoppingSalePrompt.documents())
for (const [vendor, model] of VENDORS)
for (const vendor of VENDORS)
try {
const application = LlmApplicationFactory.convert({
model,
application: typia.json.application<ShoppingSalePrompt.IApplication>(),
});
await StopWatch.trace(`${title} - ${model}`)(async () =>
await StopWatch.trace(`${title} - ${vendor}`)(async () =>
LlmFunctionCaller.test({
vendor,
model,
function: application.functions[0] as any,
texts: await ShoppingSalePrompt.texts(title),
handleCompletion: async (input) => {
await fs.promises.writeFile(
`${TestGlobal.ROOT}/examples/function-calling/arguments/${model}.${title}.input.json`,
`${TestGlobal.ROOT}/examples/function-calling/arguments/llm.${title}.input.json`,
JSON.stringify(input, null, 2),
"utf8",
);
Expand All @@ -43,7 +40,7 @@ const main = async (): Promise<void> => {
} catch (error) {
console.log(title, " -> Error");
await fs.promises.writeFile(
`${TestGlobal.ROOT}/examples/function-calling/arguments/${model}.${title}.error.json`,
`${TestGlobal.ROOT}/examples/function-calling/arguments/llm.${title}.error.json`,
JSON.stringify(
typia.is<object>(error) ? { ...error } : error,
null,
Expand Down
3 changes: 1 addition & 2 deletions test/src/features/issues/test_issue_104_upgrade_v20_allOf.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ export const test_issue_104_upgrade_v20_allOf = async (): Promise<void> => {
);
}

const app: IHttpLlmApplication<"claude"> = HttpLlm.application({
model: "claude",
const app: IHttpLlmApplication = HttpLlm.application({
document,
});
TestValidator.equals("errors")(app.errors.length)(0);
Expand Down
8 changes: 4 additions & 4 deletions test/src/features/issues/test_issue_127_enum_description.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import typia, { IJsonSchemaCollection } from "typia";

export const test_issue_127_enum_description = (): void => {
const collection: IJsonSchemaCollection = typia.json.schemas<[ISomething]>();
const chatgpt = LlmSchemaComposer.parameters("chatgpt")({
config: LlmSchemaComposer.defaultConfig("chatgpt"),
const chatgpt = LlmSchemaComposer.parameters({
config: LlmSchemaComposer.getConfig(),
components: collection.components,
schema: collection.schemas[0] as OpenApi.IJsonSchema.IReference,
});
Expand All @@ -17,8 +17,8 @@ export const test_issue_127_enum_description = (): void => {
: "",
)("The description.");

const gemini = LlmSchemaComposer.parameters("gemini")({
config: LlmSchemaComposer.defaultConfig("gemini"),
const gemini = LlmSchemaComposer.parameters({
config: LlmSchemaComposer.getConfig(),
components: collection.components,
schema: collection.schemas[0] as OpenApi.IJsonSchema.IReference,
});
Expand Down
Loading