Add comment trigger tests for issues and PRs

.github/workflows/e2e-tests-workflow.yml

@@ -28,13 +28,26 @@ jobs:
       - name: Install dependencies
         run: bun install
 
-      - name: Run Integration Tests
+      - name: Run Integration Tests in Parallel
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           TEST_ORG: melotria
+          CURRENT_BRANCH: ${{ github.head_ref || github.ref_name }}
         run: |
           mkdir -p test-results
-          bun test test/integration/*.test.ts
+
+          exit_code=0
+          for test_file in test/integration/*.test.ts; do
+            echo "Starting $test_file"
+            bun test "$test_file" &
+          done
+
+          for job in $(jobs -p); do
+            wait $job || exit_code=$?
+          done
+
+          echo "All tests completed"
+          exit $exit_code
 
       - name: Upload Test Report
         if: always()
@@ -43,4 +56,4 @@ jobs:
           name: integration-test-report
           if-no-files-found: ignore
           path: ./test-results/
-          retention-days: 2
+          retention-days: 2

.gitignore

@@ -9,4 +9,6 @@ yarn-error.log*
 .pnp.js
 .idea
 
-.vscode/*
+.vscode/*
+
+.env

test/client/client.ts

@@ -9,6 +9,8 @@ import {
 import {RestEndpointMethodTypes} from "@octokit/rest";
 
 type PullRequest = RestEndpointMethodTypes["pulls"]["list"]["response"]["data"][number];
+type Comment = RestEndpointMethodTypes["issues"]["listComments"]["response"]["data"][number];
+type Reaction = RestEndpointMethodTypes["reactions"]["listForIssueComment"]["response"]["data"][number];
 type GitHubFile =
     | RestEndpointMethodTypes["pulls"]["listFiles"]["response"]["data"][number]
     | NonNullable<RestEndpointMethodTypes["repos"]["getCommit"]["response"]["data"]["files"]>[number]
@@ -21,7 +23,7 @@ export const TEST_WORKFLOW_FILE_PATHS = {
 
 export class Client {
     private octokit: Octokit;
-    private org: string;
+    public readonly org: string;
     public currentRepo: string = "";
 
     constructor() {
@@ -37,6 +39,7 @@ export class Client {
             org: this.org,
             name: repoName,
             auto_init: true,
+            private: true,
         });
 
         this.currentRepo = repoName;
@@ -49,8 +52,10 @@ export class Client {
         workflowFilePathInTestDirectory: string = TEST_WORKFLOW_FILE_PATHS.workflowFilePathInTestDirectory
     ): Promise<void> {
         const workflowPath = path.join(process.cwd(), workflowFilePathInTestDirectory);
-        const workflowContent = fs.readFileSync(workflowPath, "utf-8");
+        let workflowContent = fs.readFileSync(workflowPath, "utf-8");
+        const currentBranch = process.env.CURRENT_BRANCH || "v0";
 
+        workflowContent = workflowContent.replace(/@v0/g, `@${currentBranch}`);
         await this.createOrUpdateFileContents(
             repoName,
             Buffer.from(workflowContent).toString("base64"),
@@ -67,46 +72,70 @@ export class Client {
         });
     }
 
-    async waitForJunieComment(issueNumber: number, message: string): Promise<void> {
-        console.log(`Waiting for Junie to post comment containing "${message}" in issue #${issueNumber} in ${this.currentRepo}...`);
-
+    async waitForJunieComment(issueOrPRNumber: number, message: string): Promise<Comment> {
+        console.log(`Waiting for Junie to post comment containing "${message}" in issue #${issueOrPRNumber} in ${this.currentRepo}...`);
+        let foundComment: Comment | undefined;
         await startPoll(
-            `Junie didn't post comment containing "${message}" in issue #${issueNumber}`,
+            `Junie didn't post comment containing "${message}" in issue #${issueOrPRNumber}`,
             {},
             async () => {
-                const { data: comments } = await this.getAllIssueComments(issueNumber);
+                const { data: comments } = await this.getAllIssueOrPRComments(issueOrPRNumber);
                 const junieComment = comments.find(c => c.body?.includes(message));
 
                 if (junieComment) {
+                    foundComment = junieComment;
                     console.log(`Found comment with message: "${message}"`);
                     return true;
                 }
                 return false;
             }
         );
+        return foundComment!;
+    }
+
+    async waitForCommentReaction(commentId: number, reactionType: string = "+1"): Promise<Reaction> {
+        console.log(`Waiting for reaction "${reactionType}" on comment #${commentId} in ${this.currentRepo}...`);
+        let foundReaction: Reaction | undefined;
+        await startPoll(
+            `Reaction "${reactionType}" not found on comment #${commentId}`,
+            {},
+            async () => {
+                const { data: reactions } = await this.getAllCommentReactions(commentId);
+
+                const hasReaction = reactions.some(r => r.content === reactionType);
+                if (hasReaction) {
+                    foundReaction = reactions.find(r => r.content === reactionType);
+                    console.log(`Found "${reactionType}" reaction on comment #${commentId}`);
+                    return true;
+                }
+                return false;
+            }
+        );
+        return foundReaction!;
     }
 
     async waitForPR(
-        condition: (pr: PullRequest) => boolean | Promise<boolean>,
-        fileContentChecks: { [filename: string]: string }
-    ): Promise<void> {
+        condition: (pr: PullRequest) => boolean | Promise<boolean>
+    ): Promise<PullRequest> {
         console.log(`Waiting for Junie to create a PR in ${this.currentRepo}...`);
-
+        let foundPR: PullRequest | undefined;
         await startPoll(
-            `Junie didn't create a PR in ${this.currentRepo} with expected files/content`,
+            `Junie didn't create a PR in ${this.currentRepo}`,
             {},
             async () => {
                 const { data: pulls } = await this.getAllPRs();
                 for (const pull of pulls) {
                     if (await condition(pull)) {
                         console.log(`PR found: ${pull.html_url}`);
-                        await this.checkPRFiles(pull, fileContentChecks);
+                        foundPR = pull;
                         return true;
                     }
                 }
                 return false;
             }
         );
+
+        return foundPR!;
     }
 
     createIssue(issueTitle: string, issueBody: string, repoName?: string) {
@@ -118,30 +147,48 @@ export class Client {
         });
     }
 
-    private async checkPRFiles(
+    async checkPRFiles(
         pr: PullRequest,
-        fileContentChecks: { [filename: string]: string }
+        condition: (files: GitHubFile[], pr: PullRequest) => boolean | Promise<boolean>
     ): Promise<boolean> {
         const { data: files } = await this.getAllPRFiles(pr);
+        return condition(files, pr);
+    }
 
-        for (const [filename, expectedSnippet] of Object.entries(fileContentChecks)) {
-            const file = files.find(f => f.filename.includes(filename));
-            if (!file) {
-                console.log(`PR found but missing file for content check: ${filename}`);
-                return false;
-            }
+    conditionPRFilesInclude(fileContentChecks: { [filename: string]: string }) {
+        return async (files: GitHubFile[], pr: PullRequest) => {
+            for (const [filename, expectedSnippet] of Object.entries(fileContentChecks)) {
+                const file = files.find(f => f.filename.includes(filename));
+                if (!file) {
+                    console.log(`PR found but missing file for content check: ${filename}`);
+                    return false;
+                }
 
-            const { data: contentData } = await this.getFileContent(pr.head.sha, file);
+                const {data: contentData} = await this.getFileContent(pr.head.sha, file);
 
-            if ("content" in contentData && typeof contentData.content === "string") {
-                const decodedContent = Buffer.from(contentData.content, "base64").toString("utf-8");
-                if (!decodedContent.includes(expectedSnippet)) {
-                    console.log(`Content of ${file.filename} doesn't match expected snippet.`);
-                    return false;
+                if ("content" in contentData && typeof contentData.content === "string") {
+                    const decodedContent = Buffer.from(contentData.content, "base64").toString("utf-8");
+                    if (!decodedContent.includes(expectedSnippet)) {
+                        console.log(`Content of ${file.filename} doesn't match expected snippet.`);
+                        return false;
+                    }
                 }
             }
+            return true;
+        };
+    }
+
+    conditionPRNumberEquals(prNumber: number) {
+        console.log(`Checking PR number is ${prNumber}`);
+        return async (pr: PullRequest): Promise<boolean> => {
+            return pr.number === prNumber;
+        }
+    }
+
+    conditionPRFilesCountIncrease(filesCount: number) {
+        return async (files: GitHubFile[]): Promise<boolean> => {
+            return files.length > filesCount;
         }
-        return true;
     }
 
     private async getAllPRs() {
@@ -169,26 +216,28 @@ export class Client {
         });
     }
 
-    private async getAllIssueComments(issueNumber: number) {
+    private async getAllIssueOrPRComments(issueOrPRNumber: number) {
         return this.octokit.issues.listComments({
             owner: this.org,
             repo: this.currentRepo,
-            issue_number: issueNumber,
+            issue_number: issueOrPRNumber,
         });
     }
 
-    private async createOrUpdateFileContents(
+    async createOrUpdateFileContents(
         repoName: string,
         content: string,
         path: string,
-        message: string
+        message: string,
+        branch: string = "main"
     ) {
         return this.octokit.repos.createOrUpdateFileContents({
             owner: this.org,
             repo: repoName,
             path: path,
             message: message,
             content: content,
+            branch: branch
         });
     }
 
@@ -197,6 +246,51 @@ export class Client {
             return titles.some(title => pr.title.includes(title));
         };
     }
+
+    async getBranch(repoName: string) {
+        return this.octokit.repos.getBranch({
+            owner: this.org,
+            repo: repoName,
+            branch: "main",
+        });
+    }
+
+    async createRef(repoName: string, branchName: string, sha: string) {
+        return this.octokit.git.createRef({
+            owner: this.org,
+            repo: repoName,
+            ref: `refs/heads/${branchName}`,
+            sha: sha,
+        });
+    }
+
+    async createPullRequest(repoName: string, branchName: string, title: string, body: string, base: string = "main") {
+        return this.octokit.pulls.create({
+            owner: this.org,
+            repo: repoName,
+            title: title,
+            head: branchName,
+            base: base,
+            body: body,
+        });
+    }
+
+    async createCommentToPROrIssue(repoName: string, issueOrPRNumber: number, commentBody: string) {
+        return this.octokit.issues.createComment({
+            owner: this.org,
+            repo: repoName,
+            issue_number: issueOrPRNumber,
+            body: commentBody,
+        });
+    }
+
+    private async getAllCommentReactions(commentId: number) {
+        return this.octokit.reactions.listForIssueComment({
+            owner: this.org,
+            repo: this.currentRepo,
+            comment_id: commentId,
+        });
+    }
 }
 
 export const testClient = new Client();

test/integration/issue_comment_trigger.test.ts

@@ -0,0 +1,55 @@
+import {describe, test, beforeAll, afterAll} from "bun:test";
+import {INIT_COMMENT_BODY, SUCCESS_FEEDBACK_COMMENT} from "../../src/constants/github";
+import {e2eConfig} from "../config/test-config";
+import {testClient} from "../client/client";
+
+describe("Trigger Junie in Issue Comment", () => {
+    let repoName: string;
+    let testPassed = false;
+
+    beforeAll(async () => {
+        repoName = await testClient.createTestRepo();
+        await testClient.setupWorkflow(repoName);
+    });
+
+    afterAll(async () => {
+        if (repoName && testPassed) {
+            await testClient.deleteTestRepo(repoName);
+        } else if (repoName) {
+            console.log(`⚠️ Keeping failed test repo: ${e2eConfig.org}/${repoName}`);
+        }
+    });
+
+    test("create PR based on comment in issue", async () => {
+        const issueTitle = "Feature Request: Math Utilities";
+        const issueBody = "We need some basic math utilities in this project.";
+        const filename = "math_ops.ts";
+        const functionName = "calculate_factorial(n)";
+
+        console.log(`Creating issue: "${issueTitle}" in ${e2eConfig.org}/${repoName}`);
+        const {data: issue} = await testClient.createIssue(issueTitle, issueBody, repoName);
+        const issueNumber = issue.number;
+        console.log(`Issue created: #${issueNumber}`);
+
+        const commentBody = `@junie-agent please implement a function ${functionName} in a new file ${filename}. The function should return the factorial of n. Also add a README.md file.`;
+        console.log(`Commenting on Issue #${issueNumber}: "${commentBody}"`);
+
+        const {data: comment} = await testClient.createCommentToPROrIssue(repoName, issueNumber, commentBody);
+
+        await testClient.waitForCommentReaction(comment.id);
+
+        await testClient.waitForJunieComment(issueNumber, INIT_COMMENT_BODY);
+
+        const titleKeywords = ["factorial", "math", "README"];
+        const foundPR = await testClient.waitForPR(testClient.conditionIncludes(titleKeywords));
+
+        await testClient.checkPRFiles(foundPR, testClient.conditionPRFilesInclude({
+            [filename]: "calculate_factorial",
+            "README.md": ""
+        }));
+
+        await testClient.waitForJunieComment(issueNumber, SUCCESS_FEEDBACK_COMMENT);
+
+        testPassed = true;
+    }, 900000);
+});