Skip to content

Commit 3d240c4

Browse files
kermanxkermanx
committed
feat: add diff as prompt to agent
1 parent 5aba5d6 commit 3d240c4

File tree

4 files changed

+167
-5
lines changed

4 files changed

+167
-5
lines changed

‎pnpm-lock.yaml‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tasks/test262/agent.mts‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,22 @@ import { generateObject } from '@xsai/generate-object'
33
import { shakeSingleModule } from 'jsshaker'
44
import { appendFile, readFile } from 'node:fs/promises'
55
import * as v from 'valibot'
6+
import { diffLines } from 'diff'
7+
8+
function generateDiff(original: string, optimized: string): string {
9+
const changes = diffLines(original, optimized)
10+
const lines: string[] = []
11+
12+
for (const change of changes) {
13+
const prefix = change.added ? '+' : change.removed ? '-' : ' '
14+
const text = change.value.replace(/\n$/, '')
15+
for (const line of text.split('\n')) {
16+
lines.push(prefix + ' ' + line)
17+
}
18+
}
19+
20+
return lines.join('\n')
21+
}
622

723
async function readLines(path: string): Promise<Set<string>> {
824
try {
@@ -36,14 +52,20 @@ Return "reason" in Chinese, max 30 characters, should be informative.`
3652

3753
const ResponseSchema = v.object({
3854
reason: v.string(),
39-
verdict: v.picklist(['illegal_testcase', 'definitely_correct', 'uncertain', 'incorrect_optimization',]),
55+
verdict: v.picklist(['illegal_testcase', 'uncertain', 'incorrect_optimization',]),
4056
})
4157

4258
const TIMEOUT_MS = 5_000
4359
const SKIP_LLM = process.env.SKIP_LLM === '1' || process.argv.includes('--skip-llm')
4460

4561
async function worker(path: string): Promise<boolean> {
4662
const original = await readFile('./test262/test/' + path, 'utf-8')
63+
64+
const normalized = shakeSingleModule(original, {
65+
preset: 'disabled',
66+
minify: false,
67+
}).output.code
68+
4769
let optimized: string
4870
try {
4971
optimized = shakeSingleModule(original, {
@@ -56,7 +78,7 @@ async function worker(path: string): Promise<boolean> {
5678
}
5779

5880
if (SKIP_LLM) {
59-
const entry = `## ${path}\n\n**Verdict**: incorrect_optimization\n**Reason**: (skipped LLM)\n\n### Original\n\n\`\`\`javascript\n${original}\n\`\`\`\n\n### Optimized\n\n\`\`\`javascript\n${optimized}\n\`\`\`\n\n---\n\n`
81+
const entry = `## ${path}\n\n**Verdict**: incorrect_optimization\n**Reason**: (skipped LLM)\n\n### Original\n\n\`\`\`javascript\n${normalized}\n\`\`\`\n\n### Optimized\n\n\`\`\`javascript\n${optimized}\n\`\`\`\n\n---\n\n`
6082
await appendFile('./agent-declined.md', entry, 'utf-8')
6183
await appendFile('./agent-processed.txt', path + '\n', 'utf-8')
6284
console.log(`[DECLINE] ${path} - incorrect_optimization: (skipped LLM)`)
@@ -76,7 +98,7 @@ async function worker(path: string): Promise<boolean> {
7698
role: 'system',
7799
},
78100
{
79-
content: `## ORIGINAL CODE:\n\`\`\`javascript\n${original}\n\`\`\`\n\n## OPTIMIZED CODE:\n\`\`\`javascript\n${optimized}\n\`\`\``,
101+
content: `## ORIGINAL CODE:\n\`\`\`javascript\n${original}\n\`\`\`\n\n## OPTIMIZED CODE:\n\`\`\`javascript\n${optimized}\n\`\`\`\n\n## DIFF (- removed, + added):\n\`\`\`diff\n${generateDiff(normalized, optimized)}\n\`\`\``,
80102
role: 'user',
81103
},
82104
],
@@ -89,11 +111,11 @@ async function worker(path: string): Promise<boolean> {
89111

90112
const { verdict, reason } = object
91113

92-
if (verdict === 'definitely_correct' || verdict === 'illegal_testcase') {
114+
if (verdict === 'illegal_testcase') {
93115
await appendFile('./agent-accept.txt', path + '\t|\t' + reason + '\n', 'utf-8')
94116
console.log(`[ACCEPT] ${path} - ${verdict}: ${reason}`)
95117
} else {
96-
const entry = `## ${path}\n\n**Verdict**: ${verdict}\n**Reason**: ${reason}\n\n### Original\n\n\`\`\`javascript\n${original}\n\`\`\`\n\n### Optimized\n\n\`\`\`javascript\n${optimized}\n\`\`\`\n\n---\n\n`
118+
const entry = `## ${path}\n\n**Verdict**: ${verdict}\n**Reason**: ${reason}\n\n### Original\n\n\`\`\`javascript\n${normalized}\n\`\`\`\n\n### Optimized\n\n\`\`\`javascript\n${optimized}\n\`\`\`\n\n---\n\n`
97119
await appendFile('./agent-declined.md', entry, 'utf-8')
98120
console.log(`[DECLINE] ${path} - ${verdict}: ${reason}`)
99121
}

‎tasks/test262/ignored2.txt‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1296,3 +1296,133 @@ language/expressions/dynamic-import/eval-export-dflt-expr-gen-named.js | 函数
12961296
language/eval-code/indirect/var-env-func-init-global-new.js | eval
12971297
language/module-code/eval-export-dflt-expr-fn-named.js | 函数名称
12981298
built-ins/Function/prototype/bind/15.3.4.5-6-6.js | 修改原型链
1299+
1300+
built-ins/Promise/all/invoke-resolve-get-once-no-calls.js | 非法测试用例
1301+
staging/sm/Math/20.2.2.ToNumber.js | 非法用例
1302+
built-ins/Promise/any/invoke-resolve-get-once-no-calls.js | 原代码依赖getCount/callCount
1303+
built-ins/Promise/any/invoke-resolve-get-once-multiple-calls.js | 依赖异常行为
1304+
staging/sm/generators/delegating-yield-3.js | 原型修改
1305+
built-ins/Promise/all/invoke-resolve-get-error.js | 依赖异常行为
1306+
built-ins/Promise/any/invoke-resolve-get-error.js | 依赖异常行为
1307+
built-ins/Promise/any/invoke-resolve-get-error-reject.js | 改变了异常抛出行为
1308+
built-ins/Promise/allSettled/invoke-resolve-get-once-no-calls.js | 非法用例
1309+
built-ins/Promise/race/invoke-resolve-get-error-reject.js | 依赖异常抛出行为
1310+
built-ins/Array/prototype/some/15.4.4.17-7-c-i-21.js | 非法用例
1311+
built-ins/Promise/allSettled/invoke-resolve-get-error.js | 依赖异常行为
1312+
built-ins/Array/prototype/some/15.4.4.17-3-23.js | 原型链修改
1313+
built-ins/Promise/allSettled/invoke-resolve-get-once-multiple-calls.js | 非法测试用例
1314+
built-ins/Array/prototype/indexOf/15.4.4.14-3-23.js | 原型链修改不被支持
1315+
built-ins/Promise/race/invoke-resolve-get-once-no-calls.js | 修改了原型链
1316+
built-ins/Promise/race/invoke-resolve-get-once-multiple-calls.js | 原始代码依赖原型修改和计数
1317+
built-ins/Array/prototype/indexOf/15.4.4.14-5-25.js | 原型链修改
1318+
built-ins/Array/prototype/reduceRight/15.4.4.22-8-b-iii-1-15.js | 未被使用的参数改名
1319+
built-ins/Array/prototype/every/15.4.4.16-7-c-i-7.js | 非法测试用例
1320+
built-ins/Array/prototype/every/15.4.4.16-7-c-i-21.js | 非法测试用例
1321+
built-ins/Array/prototype/reduceRight/15.4.4.22-3-23.js | 语义一致
1322+
built-ins/Array/prototype/reduceRight/15.4.4.22-2-6.js | illegal_testcase
1323+
built-ins/Array/prototype/reduceRight/15.4.4.22-9-c-i-7.js | 原型鏈屬性訪問
1324+
built-ins/Array/prototype/map/15.4.4.19-1-10.js | illegal_testcase
1325+
built-ins/Array/prototype/map/15.4.4.19-8-c-i-3.js | 原始语义未改变
1326+
built-ins/Array/prototype/lastIndexOf/15.4.4.15-1-10.js | 非法测试用例
1327+
built-ins/Array/prototype/map/15.4.4.19-3-23.js | definitely_correct
1328+
built-ins/Array/prototype/map/15.4.4.19-8-c-i-11.js | definitely_correct
1329+
built-ins/Array/prototype/reduce/15.4.4.21-9-c-i-15.js | definitely_correct
1330+
built-ins/Array/prototype/forEach/15.4.4.18-1-10.js | definitely_correct
1331+
built-ins/Array/length/S15.4.5.1_A1.3_T1.js | 測試案例非法
1332+
built-ins/Array/prototype/forEach/15.4.4.18-2-10.js | 非法测试案例
1333+
built-ins/FinalizationRegistry/target-not-callable-throws.js | 依赖于异常抛出行为
1334+
built-ins/Array/prototype/forEach/15.4.4.18-3-23.js | definitely_correct
1335+
built-ins/Math/hypot/Math.hypot_ToNumberErr.js | 依赖异常抛出行为
1336+
built-ins/Object/create/15.2.3.5-4-251.js | 依赖Math对象修改
1337+
built-ins/Object/create/15.2.3.5-4-234.js | 非法用例
1338+
built-ins/Object/create/15.2.3.5-4-170.js | 修改内置对象Math
1339+
built-ins/Object/create/15.2.3.5-4-33.js | 原型属性修改
1340+
built-ins/Object/create/15.2.3.5-4-190.js | 原型链修改
1341+
built-ins/Object/hasOwn/toobject_null.js | 依賴異常行為
1342+
built-ins/Object/prototype/isPrototypeOf/this-value-is-in-prototype-chain-of-arg.js | 原型链判断结果一致
1343+
built-ins/Array/prototype/lastIndexOf/15.4.4.15-5-25.js | unsupported_f
1344+
built-ins/Array/prototype/reduce/15.4.4.21-1-10.js | 修改原型链
1345+
built-ins/Promise/all/invoke-resolve-get-error-reject.js | 依賴異常拋出行爲
1346+
built-ins/Array/prototype/reduce/15.4.4.21-3-23.js | 原型链修改
1347+
built-ins/Array/prototype/indexOf/15.4.4.14-1-10.js | 修改内置对象
1348+
built-ins/Object/create/15.2.3.5-4-196.js | 原型修改不被支持
1349+
built-ins/Object/create/15.2.3.5-4-181.js | 原型链修改
1350+
built-ins/Object/defineProperties/15.2.3.7-5-b-77.js | 修改内置对象
1351+
built-ins/Array/prototype/filter/15.4.4.20-9-c-i-7.js | 非法测试用例
1352+
built-ins/Object/defineProperties/15.2.3.7-5-b-209.js | 測試用例依賴Math.get函數
1353+
built-ins/Object/defineProperties/15.2.3.7-5-b-24.js | 非法修改内置对象
1354+
built-ins/Object/defineProperties/15.2.3.7-5-a-12.js | 非法测试用例
1355+
built-ins/Object/getPrototypeOf/15.2.3.2-1-2.js | 依赖异常行为
1356+
built-ins/Object/create/15.2.3.5-4-117.js | 修改原型链
1357+
built-ins/Object/defineProperties/15.2.3.7-5-b-9.js | 原型链修改非法
1358+
built-ins/Object/getPrototypeOf/15.2.3.2-0-3.js | 依赖异常行为
1359+
built-ins/Object/defineProperties/15.2.3.7-5-b-130.js | 修改内置对象行为
1360+
built-ins/Date/UTC/coercion-order.js | 依赖副作用
1361+
built-ins/Object/freeze/15.2.3.9-2-2.js | 非法用例
1362+
built-ins/Object/defineProperty/15.2.3.6-3-170.js | 不合法的测试用例，修改内置对象Math
1363+
built-ins/Object/defineProperties/15.2.3.7-2-11.js | 非法测试用例
1364+
built-ins/Promise/all/invoke-resolve-get-once-multiple-calls.js | 修改原型链
1365+
built-ins/BigInt/asUintN/bits-toindex-errors.js | 不合法测试用例
1366+
built-ins/Object/defineProperties/15.2.3.7-5-b-244.js | 违规测试用例
1367+
built-ins/Math/max/Math.max_each-element-coerced.js | 非法测试用例
1368+
built-ins/Object/create/15.2.3.5-4-64.js | 修改内置对象Math
1369+
built-ins/String/prototype/split/instance-is-math.js | 原代码依赖原型链修改
1370+
built-ins/BigInt/asIntN/bigint-tobigint-errors.js | 測試內容依賴錯誤行為
1371+
language/expressions/async-generator/generator-created-after-decl-inst.js | 依赖异常行为
1372+
built-ins/NativeErrors/ReferenceError/constructor.js | 不合法的测试用例
1373+
built-ins/String/prototype/trim/15.5.4.20-2-43.js | illegal_testcase
1374+
language/expressions/generators/generator-created-after-decl-inst.js | illegal_testcase
1375+
language/expressions/assignment/11.13.1-4-29gs.js | 依赖异常行为
1376+
built-ins/Object/defineProperties/15.2.3.7-5-b-194.js | 非法测试用例
1377+
language/statements/async-generator/generator-created-after-decl-inst.js | incorrect_optimization
1378+
built-ins/Proxy/setPrototypeOf/trap-is-undefined-or-null.js | 非法测试用例
1379+
language/expressions/assignment/11.13.1-4-28gs.js | 依赖于异常抛出行为
1380+
built-ins/TypedArrayConstructors/BigUint64Array/prototype/not-typedarray-object.js | 測試依賴於異常行為
1381+
built-ins/Array/prototype/reduceRight/15.4.4.22-8-b-iii-1-7.js | 非法测试用例
1382+
staging/sm/class/boundFunctionSubclassing.js | 非法测试用例
1383+
built-ins/Promise/race/invoke-resolve-get-error.js | 依赖异常抛出行为
1384+
built-ins/Array/prototype/reduceRight/15.4.4.22-8-b-iii-1-11.js | 原型链修改
1385+
built-ins/Array/prototype/some/15.4.4.17-1-10.js | 非法测试用例
1386+
built-ins/Array/prototype/every/15.4.4.16-3-23.js | 非法测试用例
1387+
built-ins/Array/prototype/map/15.4.4.19-8-c-i-15.js | definitely_correct
1388+
built-ins/Array/prototype/filter/15.4.4.20-3-23.js | 非法测试用例
1389+
built-ins/Array/prototype/map/15.4.4.19-8-c-i-13.js | 非法测试用例
1390+
built-ins/Array/prototype/forEach/15.4.4.18-7-c-i-15.js | 原型链修改未支持
1391+
built-ins/Math/min/Math.min_each-element-coerced.js | 依赖valueOf副作用
1392+
built-ins/Object/create/15.2.3.5-4-155.js | 非法测试用例
1393+
built-ins/Object/seal/object-seal-inherited-accessor-properties-are-ignored.js | 禁止修改prototype链
1394+
built-ins/Object/getOwnPropertyDescriptor/15.2.3.3-2-47.js | 非法测试用例
1395+
built-ins/Object/hasOwn/toobject_undefined.js | 依赖异常行为
1396+
built-ins/Object/defineProperty/15.2.3.6-3-253.js | 非法测试用例
1397+
built-ins/Object/defineProperty/15.2.3.6-4-151.js | 原型链修改
1398+
built-ins/Object/assign/Target-Boolean.js | 非法替换内置方法
1399+
built-ins/Object/freeze/15.2.3.9-2-3.js | 非法测试用例
1400+
language/expressions/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js | 非法測試用例
1401+
language/statements/for-in/S12.6.4_A6.1.js | 原型链修改不合规
1402+
built-ins/Object/create/15.2.3.5-4-10.js | 依赖异常行为
1403+
built-ins/Object/defineProperties/15.2.3.7-5-b-115.js | 非法测试用例
1404+
built-ins/Array/prototype/reduceRight/15.4.4.22-8-b-iii-1-21.js | 原型链修改影响
1405+
built-ins/Promise/allSettled/invoke-resolve-get-error-reject.js | 依赖于异常行为
1406+
built-ins/Array/prototype/filter/15.4.4.20-1-10.js | 依赖非法原型修改
1407+
built-ins/Array/prototype/forEach/15.4.4.18-2-6.js | length属性未定义
1408+
built-ins/Object/defineProperty/15.2.3.6-2-48.js | 原型鏈修改
1409+
built-ins/Object/assign/Target-Number.js | 不合法的测例
1410+
built-ins/Array/prototype/filter/15.4.4.20-1-10.js | 依赖原型方法
1411+
built-ins/Array/prototype/every/15.4.4.16-1-10.js | Illegal_testcase: The original code sets Math.length and Math[0] to test Array.prototype.every behavior on Math object
1412+
built-ins/Object/assign/Target-Boolean.js | 非法测试用例
1413+
built-ins/Math/min/Math.min_each-element-coerced.js | 违反侧效应
1414+
built-ins/Array/prototype/some/15.4.4.17-1-10.js | 修改Math.length可能引发错误
1415+
language/statements/for-of/array-expand.js | 非法测试用例
1416+
language/statements/for-of/iterator-next-result-type.js | 依赖异常行为
1417+
language/statements/class/elements/field-definition-accessor-no-line-terminator.js | 非法特性
1418+
language/statements/generators/generator-created-after-decl-inst.js | 依赖原型修改
1419+
language/expressions/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js | illegal_testcase
1420+
built-ins/Promise/allSettled/invoke-resolve-get-error-reject.js | 依赖异常行为
1421+
built-ins/Object/create/15.2.3.5-4-284.js | 依赖异常的副作用
1422+
built-ins/Object/hasOwn/toobject_undefined.js | 依赖异常抛出行为
1423+
language/statements/class/subclass/superclass-bound-function.js | 绑定函数原型修改
1424+
built-ins/Object/seal/object-seal-inherited-data-properties-are-ignored.js | 非法依赖删除操作
1425+
language/statements/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js | 非法用例
1426+
language/expressions/async-generator/default-proto.js | 非法测试用例
1427+
built-ins/String/prototype/padStart/normal-operation.js | OXC
1428+
built-ins/String/prototype/padEnd/normal-operation.js | OXC

‎tasks/test262/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"@xsai/generate-object": "^0.4.1",
99
"@xsai/generate-text": "^0.4.1",
1010
"cli-highlight": "^2.1.11",
11+
"diff": "^8.0.3",
1112
"dotenv": "^17.2.3",
1213
"ink": "^6.6.0",
1314
"ink-task-list": "^2.0.0",

0 commit comments

Comments
 (0)