Add admin approval workflow and overrides

.github/workflows/admin-approve-submission.yml

@@ -0,0 +1,143 @@
+name: Admin Approve Submission
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  approve:
+    # Only run on issues (not PRs) with submission label when admin comments /approve
+    if: |
+      !github.event.issue.pull_request &&
+      contains(github.event.issue.labels.*.name, 'submission') &&
+      startsWith(github.event.comment.body, '/approve')
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Check if user is admin/maintainer
+        id: check-permission
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              username: context.actor
+            });
+
+            const allowed = ['admin', 'maintain', 'write'].includes(permission.permission);
+            console.log(`User ${context.actor} has ${permission.permission} permission. Allowed: ${allowed}`);
+
+            if (!allowed) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: `❌ @${context.actor} You don't have permission to approve submissions.`
+              });
+            }
+
+            return allowed;
+          result-encoding: string
+
+      - name: Parse approval command
+        if: steps.check-permission.outputs.result == 'true'
+        id: parse
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const comment = context.payload.comment.body.trim();
+            // Match /approve or /approve 2 or /approve 3
+            const match = comment.match(/^\/approve(?:\s+(\d))?/);
+
+            if (match) {
+              const scoreOverride = match[1] ? parseInt(match[1]) : null;
+              console.log(`Score override: ${scoreOverride}`);
+              core.setOutput('score_override', scoreOverride || '');
+              core.setOutput('valid', 'true');
+            } else {
+              core.setOutput('valid', 'false');
+            }
+
+      - name: Checkout
+        if: steps.check-permission.outputs.result == 'true' && steps.parse.outputs.valid == 'true'
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        if: steps.check-permission.outputs.result == 'true' && steps.parse.outputs.valid == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        if: steps.check-permission.outputs.result == 'true' && steps.parse.outputs.valid == 'true'
+        run: |
+          pip install requests beautifulsoup4 anthropic
+
+      - name: Re-evaluate with admin override
+        if: steps.check-permission.outputs.result == 'true' && steps.parse.outputs.valid == 'true'
+        id: evaluate
+        env:
+          ISSUE_BODY: ${{ github.event.issue.body }}
+          ISSUE_NUMBER: ${{ github.event.issue.number }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          ADMIN_SCORE_OVERRIDE: ${{ steps.parse.outputs.score_override }}
+          ADMIN_APPROVED: 'true'
+        run: |
+          python scripts/evaluate_submission.py
+
+      - name: Create PR
+        if: steps.check-permission.outputs.result == 'true' && steps.parse.outputs.valid == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ISSUE_NUMBER: ${{ github.event.issue.number }}
+        run: |
+          if [ -f submission_data.json ]; then
+            python scripts/create_submission_pr.py
+          else
+            echo "No submission data found"
+            exit 1
+          fi
+
+      - name: Update labels and comment
+        if: steps.check-permission.outputs.result == 'true' && steps.parse.outputs.valid == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            // Remove old labels
+            const labelsToRemove = ['needs-review', 'rejected'];
+            for (const label of labelsToRemove) {
+              try {
+                await github.rest.issues.removeLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  name: label
+                });
+              } catch (e) {
+                // Label might not exist
+              }
+            }
+
+            // Add approved label
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ['approved']
+            });
+
+            // Add success comment
+            const scoreOverride = '${{ steps.parse.outputs.score_override }}';
+            const scoreMsg = scoreOverride ? ` with score override to ${scoreOverride}/3` : '';
+
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: `✅ **Approved by @${context.actor}**${scoreMsg}\n\nA PR has been created to add this service to the directory.`
+            });

.github/workflows/evaluate-submission.yml

@@ -69,9 +69,11 @@ jobs:
 
             let label = 'needs-review';
             if (score === 3) {
-              label = 'ready-to-merge';
-            } else if (score <= 1) {
-              label = 'rejected';
+              label = 'auto-approved';
+            } else if (score === 2) {
+              label = 'needs-review';
+            } else {
+              label = 'needs-review';  // Even rejected ones go to needs-review for admin decision
             }
 
             await github.rest.issues.addLabels({
@@ -81,6 +83,24 @@ jobs:
               labels: [label]
             });
 
+            // Add info comment for low scores
+            if (score < 2) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: `⚠️ **Admin Review Required**\n\nThis submission scored ${score}/3 and needs manual verification.\n\n**To approve:** Comment \`/approve\` or \`/approve 3\` (to override score)\n**To reject:** Add the \`rejected\` label and close the issue`
+              });
+            }
+
+      - name: Upload submission data as artifact
+        if: steps.read_outputs.outputs.has_data == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: submission-data-${{ github.event.issue.number }}
+          path: submission_data.json
+          retention-days: 90
+
       - name: Create PR if score >= 2
         if: steps.read_outputs.outputs.score >= 2 && steps.read_outputs.outputs.has_data == 'true'
         env:

scripts/evaluate_submission.py

@@ -513,6 +513,15 @@ def main():
     issue_body = os.environ.get('ISSUE_BODY', '')
     issue_number = os.environ.get('ISSUE_NUMBER', '')
 
+    # Check for admin override
+    admin_approved = os.environ.get('ADMIN_APPROVED', '').lower() == 'true'
+    admin_score_override = os.environ.get('ADMIN_SCORE_OVERRIDE', '')
+
+    if admin_approved:
+        print("=== ADMIN APPROVAL MODE ===")
+        if admin_score_override:
+            print(f"Score override: {admin_score_override}")
+
     if not issue_body:
         print("Error: ISSUE_BODY environment variable not set")
         sys.exit(1)
@@ -544,8 +553,19 @@ def main():
         # Evaluate against criteria
         result = evaluate_service(url)
 
-        # If score >= 2, use Claude to generate metadata
-        if result['score'] >= 2:
+        # Apply admin score override if provided
+        if admin_approved and admin_score_override:
+            original_score = result['score']
+            result['score'] = int(admin_score_override)
+            result['admin_override'] = True
+            print(f"Admin override: {original_score}/3 -> {result['score']}/3")
+
+        # Determine if service should pass
+        # In admin mode: always pass (admin explicitly approved)
+        # In normal mode: pass if score >= 2
+        should_pass = admin_approved or result['score'] >= 2
+
+        if should_pass:
             ai_metadata = generate_metadata_with_claude(url, page_content)
             if ai_metadata:
                 result['company_name'] = ai_metadata['name']
@@ -556,7 +576,8 @@ def main():
                     'description': ai_metadata['description'],
                     'category': ai_metadata['category'],
                     'score': result['score'],
-                    'needs_manual_review': result.get('needs_manual_review', False),
+                    'needs_manual_review': False if admin_approved else result.get('needs_manual_review', False),
+                    'admin_approved': admin_approved,
                 })
 
         results_list.append(result)
@@ -565,9 +586,10 @@ def main():
     # Calculate max score for labeling
     max_score = max(r['score'] for r in results_list) if results_list else 0
 
-    # Write results
-    with open('evaluation_results.md', 'w') as f:
-        f.write(generate_markdown_results(results_list))
+    # Write results (only in non-admin mode to avoid duplicate comments)
+    if not admin_approved:
+        with open('evaluation_results.md', 'w') as f:
+            f.write(generate_markdown_results(results_list))
 
     with open('evaluation_score.txt', 'w') as f:
         f.write(str(max_score))