Skip to content

Commit f9b959b

Browse files
SasinduDilsharaSasinduDilshara
authored
Merge pull request #37 from SasinduDilshara/handle-parse-json-error
Handle error message for parse errors
2 parents ad112f5 + d2c8f3b commit f9b959b

File tree

8 files changed

+212
-113
lines changed

8 files changed

+212
-113
lines changed

‎ballerina/Ballerina.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
org = "ballerinax"
33
name = "np"
4-
version = "0.2.0"
4+
version = "0.2.1"
55
authors = ["Ballerina"]
66
keywords = ["natural programming", "ai"]
77
repository = "https://github.com/ballerina-platform/module-ballerinax-np"
@@ -17,5 +17,5 @@ observabilityIncluded = true
1717
[[platform.java21.dependency]]
1818
groupId = "io.ballerina.lib"
1919
artifactId = "np-native"
20-
version = "0.2.0"
21-
path = "../native/build/libs/np-native-0.2.0.jar"
20+
version = "0.2.1"
21+
path = "../native/build/libs/np-native-0.2.1-SNAPSHOT.jar"

‎ballerina/CompilerPlugin.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ id = "np-compiler-plugin"
33
class = "io.ballerina.lib.np.compilerplugin.CompilerPlugin"
44

55
[[dependency]]
6-
path = "../compiler-plugin/build/libs/np-compiler-plugin-0.2.0.jar"
6+
path = "../compiler-plugin/build/libs/np-compiler-plugin-0.2.1-SNAPSHOT.jar"

‎ballerina/Dependencies.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ modules = [
335335
[[package]]
336336
org = "ballerinax"
337337
name = "np"
338-
version = "0.2.0"
338+
version = "0.2.1"
339339
dependencies = [
340340
{org = "ballerina", name = "http"},
341341
{org = "ballerina", name = "jballerina.java"},

‎ballerina/main.bal‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
// specific language governing permissions and limitations
1515
// under the License.
1616

17+
const JSON_CONVERSION_ERROR = "FromJsonStringError";
18+
const CONVERSION_ERROR = "ConversionError";
19+
const ERROR_MESSAGE = "Error occurred while converting the LLM response to the given type. Please refined your prompt to get a better result.";
20+
1721
type DefaultModelConfig DefaultAzureOpenAIModelConfig|DefaultOpenAIModelConfig|DefaultBallerinaModelConfig;
1822

1923
type DefaultAzureOpenAIModelConfig record {|
@@ -96,8 +100,25 @@ isolated function callLlmGeneric(Prompt prompt, Context context, typedesc<json>
96100

97101
isolated function parseResponseAsJson(string resp) returns json|error {
98102
string processedResponse = re `${"```json|```"}`.replaceAll(resp, "");
99-
return processedResponse.fromJsonString();
103+
json|error result = trap processedResponse.fromJsonString();
104+
if result is error {
105+
return handlepParseResponseError(result);
106+
}
107+
return result;
100108
}
101109

102-
isolated function parseResponseAsType(json resp, typedesc<json> targetType) returns json|error =>
103-
resp.fromJsonWithType(targetType);
110+
isolated function parseResponseAsType(json resp, typedesc<json> targetType) returns json|error {
111+
json|error result = trap resp.fromJsonWithType(targetType);
112+
if result is error {
113+
return handlepParseResponseError(result);
114+
}
115+
return result;
116+
}
117+
118+
isolated function handlepParseResponseError(error chatResponseError) returns error {
119+
if chatResponseError.message().includes(JSON_CONVERSION_ERROR)
120+
|| chatResponseError.message().includes(CONVERSION_ERROR) {
121+
return error(string `${ERROR_MESSAGE}`);
122+
}
123+
return chatResponseError;
124+
}

‎ballerina/tests/test_services.bal‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import ballerina/http;
2+
import ballerinax/azure.openai.chat as azureOpenAIChat;
3+
import ballerinax/openai.chat as openAIChat;
4+
import ballerina/test;
5+
6+
service /llm on new http:Listener(8080) {
7+
resource function post azureopenai/deployments/gpt4onew/chat/completions(
8+
string api\-version, azureOpenAIChat:CreateChatCompletionRequest payload)
9+
returns json|error {
10+
test:assertEquals(api\-version, "2023-08-01-preview");
11+
azureOpenAIChat:ChatCompletionRequestMessage[]? messages = payload.messages;
12+
if messages is () {
13+
test:assertFail("Expected messages in the payload");
14+
}
15+
azureOpenAIChat:ChatCompletionRequestMessage message = messages[0];
16+
anydata content = message["content"];
17+
string contentStr = content.toString();
18+
test:assertEquals(message.role, "user");
19+
test:assertEquals(content, getExpectedPrompt(contentStr));
20+
return {
21+
'object: "chat.completion",
22+
created: 0,
23+
model: "",
24+
id: "",
25+
choices: [
26+
{
27+
message: {
28+
content: getTheMockLLMResult(contentStr)
29+
}
30+
}
31+
]
32+
};
33+
}
34+
35+
resource function post openai/chat/completions(openAIChat:CreateChatCompletionRequest payload)
36+
returns json|error {
37+
38+
azureOpenAIChat:ChatCompletionRequestMessage message = payload.messages[0];
39+
anydata content = message["content"];
40+
string contentStr = content.toString();
41+
test:assertEquals(message.role, "user");
42+
test:assertEquals(content, getExpectedPrompt(content.toString()));
43+
44+
test:assertEquals(payload.model, "gpt4o");
45+
return {
46+
'object: "chat.completion",
47+
created: 0,
48+
model: "",
49+
id: "",
50+
choices: [
51+
{
52+
finish_reason: "stop",
53+
index: 0,
54+
logprobs: (),
55+
message: {
56+
role: "assistant",
57+
content: getTheMockLLMResult(contentStr),
58+
refusal: ()
59+
}
60+
}
61+
]
62+
};
63+
}
64+
}

‎ballerina/tests/test_utils.bal‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
isolated function getExpectedPrompt(string message) returns string {
2+
if message.startsWith("Rate this blog") {
3+
return expectedPromptStringForRateBlog;
4+
}
5+
6+
if message.startsWith("Please rate this blog") {
7+
return expectedPromptStringForRateBlog2;
8+
}
9+
10+
if message.startsWith("What is 1 + 1?") {
11+
return expectedPromptStringForRateBlog3;
12+
}
13+
14+
if message.startsWith("Tell me") {
15+
return expectedPromptStringForRateBlog4;
16+
}
17+
18+
return "INVALID";
19+
}
20+
21+
isolated function getTheMockLLMResult(string message) returns string {
22+
if message.startsWith("Rate this blog") {
23+
return "4";
24+
}
25+
26+
if message.startsWith("Please rate this blog") {
27+
return review2.toJsonString();
28+
}
29+
30+
if message.startsWith("What is 1 + 1?") {
31+
return "2";
32+
}
33+
34+
if message.startsWith("Tell me") {
35+
return "[{\"name\":\"Virat Kohli\",\"age\":33},{\"name\":\"Kane Williamson\",\"age\":30}";
36+
}
37+
38+
return "INVALID";
39+
}

‎ballerina/tests/test_values.bal‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
2+
type Blog record {
3+
string title;
4+
string content;
5+
};
6+
7+
type Review record {|
8+
int rating;
9+
string comment;
10+
|};
11+
12+
final readonly & Blog blog1 = {
13+
// Generated.
14+
title: "Tips for Growing a Beautiful Garden",
15+
content: string `Spring is the perfect time to start your garden.
16+
Begin by preparing your soil with organic compost and ensure proper drainage.
17+
Choose plants suitable for your climate zone, and remember to water them regularly.
18+
Don't forget to mulch to retain moisture and prevent weeds.`
19+
};
20+
21+
final readonly & Blog blog2 = {
22+
// Generated.
23+
title: "Essential Tips for Sports Performance",
24+
content: string `Success in sports requires dedicated preparation and training.
25+
Begin by establishing a proper warm-up routine and maintaining good form.
26+
Choose the right equipment for your sport, and stay consistent with training.
27+
Don't forget to maintain proper hydration and nutrition for optimal performance.`
28+
};
29+
30+
final readonly & Review review2 = {
31+
rating: 8,
32+
comment: "Talks about essential aspects of sports performance including warm-up, form, equipment, and nutrition."
33+
};
34+
35+
final string expectedPromptStringForRateBlog = string `Rate this blog out of 10.
36+
Title: ${blog1.title}
37+
Content: ${blog1.content}.
38+
The output should be a JSON value that satisfies the following JSON schema,
39+
returned within a markdown snippet enclosed within ${"```json"} and ${"```"}
40+
41+
Schema:
42+
{"type":"integer"}`;
43+
44+
final string expectedPromptStringForRateBlog2 = string `Please rate this blog out of 10.
45+
Title: ${blog2.title}
46+
Content: ${blog2.content}.
47+
The output should be a JSON value that satisfies the following JSON schema,
48+
returned within a markdown snippet enclosed within ${"```json"} and ${"```"}
49+
50+
Schema:
51+
{"$schema":"https://json-schema.org/draft/2020-12/schema", "type":"object", "properties":{"rating":{"type":"integer"}, "comment":{"type":"string"}}, "required":["rating", "comment"]}`;
52+
53+
final string expectedPromptStringForRateBlog3 = string `What is 1 + 1?.
54+
The output should be a JSON value that satisfies the following JSON schema,
55+
returned within a markdown snippet enclosed within ${"```json"} and ${"```"}
56+
57+
Schema:
58+
{"type":"boolean"}`;
59+
60+
final string expectedPromptStringForRateBlog4 = string `Tell me name and the age of the top 10 world class cricketers.
61+
The output should be a JSON value that satisfies the following JSON schema,
62+
returned within a markdown snippet enclosed within ${"```json"} and ${"```"}
63+
64+
Schema:
65+
{"$schema":"https://json-schema.org/draft/2020-12/schema", "type":"array", "items":{"$schema":"https://json-schema.org/draft/2020-12/schema", "type":"object", "properties":{"name":{"type":"string"}}, "required":["name"]}}`;

‎ballerina/tests/tests.bal‎

Lines changed: 15 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -14,33 +14,9 @@
1414
// specific language governing permissions and limitations
1515
// under the License.
1616

17-
import ballerina/http;
1817
import ballerina/test;
19-
import ballerinax/azure.openai.chat as azureOpenAIChat;
20-
import ballerinax/openai.chat as openAIChat;
2118

22-
final readonly & Blog blog1 = {
23-
// Generated.
24-
title: "Tips for Growing a Beautiful Garden",
25-
content: string `Spring is the perfect time to start your garden.
26-
Begin by preparing your soil with organic compost and ensure proper drainage.
27-
Choose plants suitable for your climate zone, and remember to water them regularly.
28-
Don't forget to mulch to retain moisture and prevent weeds.`
29-
};
30-
31-
final readonly & Blog blog2 = {
32-
// Generated.
33-
title: "Essential Tips for Sports Performance",
34-
content: string `Success in sports requires dedicated preparation and training.
35-
Begin by establishing a proper warm-up routine and maintaining good form.
36-
Choose the right equipment for your sport, and stay consistent with training.
37-
Don't forget to maintain proper hydration and nutrition for optimal performance.`
38-
};
39-
40-
final readonly & Review review2 = {
41-
rating: 8,
42-
comment: "Talks about essential aspects of sports performance including warm-up, form, equipment, and nutrition."
43-
};
19+
const ERROR_MESSAGE = "Error occurred while converting the LLM response to the given type. Please refined your prompt to get a better result.";
4420

4521
@test:Config
4622
function testPromptAsCodeFunctionWithSimpleExpectedTypeWithDefaultAzureOpenAIClient() returns error? {
@@ -58,90 +34,24 @@ function testPromptAsCodeFunctionWithStructuredExpectedTypeWithOpenAIClient() re
5834
},
5935
serviceUrl: "http://localhost:8080/llm/openai"
6036
}, "gpt4o");
61-
Review review = check callLlm(`Rate this blog out of 10.
37+
Review review = check callLlm(`Please rate this blog out of 10.
6238
Title: ${blog2.title}
6339
Content: ${blog2.content}`, {model});
6440
test:assertEquals(review, review2);
6541
}
6642

67-
type Blog record {
68-
string title;
69-
string content;
70-
};
71-
72-
type Review record {|
73-
int rating;
74-
string comment;
75-
|};
76-
77-
service /llm on new http:Listener(8080) {
78-
resource function post azureopenai/deployments/gpt4onew/chat/completions(
79-
string api\-version, azureOpenAIChat:CreateChatCompletionRequest payload)
80-
returns json {
81-
test:assertEquals(api\-version, "2023-08-01-preview");
82-
string expectedPromptString = string `Rate this blog out of 10.
83-
Title: ${blog1.title}
84-
Content: ${blog1.content}.
85-
The output should be a JSON value that satisfies the following JSON schema,
86-
returned within a markdown snippet enclosed within ${"```json"} and ${"```"}
87-
88-
Schema:
89-
{"type":"integer"}`;
90-
azureOpenAIChat:ChatCompletionRequestMessage[]? messages = payload.messages;
91-
if messages is () {
92-
test:assertFail("Expected messages in the payload");
93-
}
94-
azureOpenAIChat:ChatCompletionRequestMessage message = messages[0];
95-
test:assertEquals(message.role, "user");
96-
test:assertEquals(message["content"], expectedPromptString);
97-
return {
98-
'object: "chat.completion",
99-
created: 0,
100-
model: "",
101-
id: "",
102-
choices: [
103-
{
104-
message: {
105-
content: "4"
106-
}
107-
}
108-
]
109-
};
110-
}
111-
112-
resource function post openai/chat/completions(openAIChat:CreateChatCompletionRequest payload)
113-
returns json {
114-
string expectedPromptString = string `Rate this blog out of 10.
115-
Title: ${blog2.title}
116-
Content: ${blog2.content}.
117-
The output should be a JSON value that satisfies the following JSON schema,
118-
returned within a markdown snippet enclosed within ${"```json"} and ${"```"}
119-
120-
Schema:
121-
{"$schema":"https://json-schema.org/draft/2020-12/schema", "type":"object", "properties":{"rating":{"type":"integer"}, "comment":{"type":"string"}}, "required":["rating", "comment"]}`;
122-
azureOpenAIChat:ChatCompletionRequestMessage message = payload.messages[0];
123-
test:assertEquals(message.role, "user");
124-
test:assertEquals(message["content"], expectedPromptString);
125-
126-
test:assertEquals(payload.model, "gpt4o");
127-
return {
128-
'object: "chat.completion",
129-
created: 0,
130-
model: "",
131-
id: "",
132-
choices: [
133-
{
134-
finish_reason: "stop",
135-
index: 0,
136-
logprobs: (),
137-
message: {
138-
role: "assistant",
139-
content: review2.toJsonString(),
140-
refusal: ()
141-
}
142-
}
143-
]
144-
};
145-
}
43+
@test:Config
44+
function testJsonConversionError() {
45+
boolean|error rating = callLlm(`What is 1 + 1?`);
46+
test:assertTrue(rating is error);
47+
test:assertTrue((<error> rating).message().includes(ERROR_MESSAGE));
14648
}
14749

50+
@test:Config
51+
function testJsonConversionError2() {
52+
record{|
53+
string name;
54+
|}[]|error rating = callLlm(`Tell me name and the age of the top 10 world class cricketers`);
55+
test:assertTrue(rating is error);
56+
test:assertTrue((<error> rating).message().includes(ERROR_MESSAGE));
57+
}

0 commit comments

Comments
 (0)