Update dependencies and improve static code analysis tests

changelog.md

@@ -9,10 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - [Add static code rules](https://github.com/ballerina-platform/ballerina-library/issues/7283)
 
-
 ### Changed
 
 - [Change the listener configuration as an included parameter](https://github.com/ballerina-platform/ballerina-library/issues/7494)
+- [Update the static analysis tests to use scan tool's test API](https://github.com/ballerina-platform/ballerina-library/issues/8249)
 
 ## [1.10.0] - 2024-08-20
 

compiler-plugin-test/build.gradle

@@ -48,6 +48,9 @@ dependencies {
     testImplementation group: 'org.ballerinalang', name: 'ballerina-lang', version: "${ballerinaLangVersion}"
     testImplementation group: 'org.ballerinalang', name: 'ballerina-tools-api', version: "${ballerinaLangVersion}"
     testImplementation group: 'org.ballerinalang', name: 'ballerina-parser', version: "${ballerinaLangVersion}"
+    testImplementation group: 'io.ballerina.scan', name: 'scan-command', version: "${balScanVersion}"
+    testImplementation group: 'io.ballerina.scan', name: 'scan-command-test-utils', version: "${balScanVersion}"
+    testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jacksonDatabindVersion}"
     testImplementation group: 'org.testng', name: 'testng', version: "${testngVersion}"
     implementation project(":file-compiler-plugin")
 }

compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/StaticCodeAnalyzerTest.java

@@ -18,105 +18,157 @@
 
 package io.ballerina.stdlib.file.compiler.staticcodeanalyzer;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import io.ballerina.projects.Project;
+import io.ballerina.projects.ProjectEnvironmentBuilder;
+import io.ballerina.projects.directory.BuildProject;
+import io.ballerina.projects.environment.Environment;
+import io.ballerina.projects.environment.EnvironmentBuilder;
+import io.ballerina.scan.Issue;
+import io.ballerina.scan.Rule;
+import io.ballerina.scan.Source;
+import io.ballerina.scan.test.Assertions;
+import io.ballerina.scan.test.TestOptions;
+import io.ballerina.scan.test.TestRunner;
 import org.testng.Assert;
-import org.testng.annotations.BeforeSuite;
 import org.testng.annotations.Test;
-import org.testng.internal.ExitCode;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static io.ballerina.scan.RuleKind.VULNERABILITY;
+import static io.ballerina.stdlib.file.compiler.staticcodeanalyzer.FileRule.AVOID_INSECURE_DIRECTORY_ACCESS;
+import static io.ballerina.stdlib.file.compiler.staticcodeanalyzer.FileRule.AVOID_PATH_INJECTION;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public class StaticCodeAnalyzerTest {
     private static final Path RESOURCE_PACKAGES_DIRECTORY = Paths
             .get("src", "test", "resources", "static_code_analyzer", "ballerina_packages").toAbsolutePath();
-    private static final Path EXPECTED_JSON_OUTPUT_DIRECTORY = Paths
+    private static final Path EXPECTED_OUTPUT_DIRECTORY = Paths
             .get("src", "test", "resources", "static_code_analyzer", "expected_output").toAbsolutePath();
-    private static final Path BALLERINA_PATH = getBalCommandPath();
     private static final Path JSON_RULES_FILE_PATH = Paths
             .get("../", "compiler-plugin", "src", "main", "resources", "rules.json").toAbsolutePath();
-    private static final String SCAN_COMMAND = "scan";
-
-    private static Path getBalCommandPath() {
-        String balCommand = isWindows() ? "bal.bat" : "bal";
-        return Paths.get("../", "target", "ballerina-runtime", "bin", balCommand).toAbsolutePath();
-    }
-
-    @BeforeSuite
-    public void pullScanTool() throws IOException, InterruptedException {
-        ProcessBuilder processBuilder = new ProcessBuilder(BALLERINA_PATH.toString(), "tool", "pull", SCAN_COMMAND);
-        ProcessOutputGobbler output = getOutput(processBuilder.start());
-        if (Pattern.compile("tool 'scan:.+\\..+\\..+' successfully set as the active version\\.")
-                .matcher(output.getOutput()).find() || Pattern.compile("tool 'scan:.+\\..+\\..+' is already active\\.")
-                .matcher(output.getOutput()).find()) {
-            return;
-        }
-        Assert.assertFalse(ExitCode.hasFailure(output.getExitCode()));
-    }
+    private static final Path DISTRIBUTION_PATH = Paths.get("../", "target", "ballerina-runtime");
+    private static final String MODULE_BALLERINA_FilE = "module-ballerina-file";
 
     @Test
     public void validateRulesJson() throws IOException {
         String expectedRules = "[" + Arrays.stream(FileRule.values())
                 .map(FileRule::toString).collect(Collectors.joining(",")) + "]";
         String actualRules = Files.readString(JSON_RULES_FILE_PATH);
-        assertJsonEqual(normalizeJson(actualRules), normalizeJson(expectedRules));
+        assertJsonEqual(actualRules, expectedRules);
     }
 
     @Test
-    public void testStaticCodeRules() throws IOException, InterruptedException {
+    public void testStaticCodeRulesWithAPI() throws IOException {
+        ByteArrayOutputStream console = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(console, true, UTF_8);
+
         for (FileRule rule : FileRule.values()) {
-            String targetPackageName = "rule" + rule.getId();
-            String actualJsonReport = StaticCodeAnalyzerTest.executeScanProcess(targetPackageName);
-            String expectedJsonReport = Files
-                    .readString(EXPECTED_JSON_OUTPUT_DIRECTORY.resolve(targetPackageName + ".json"));
-            assertJsonEqual(actualJsonReport, expectedJsonReport);
+            testIndividualRule(rule, console, printStream);
         }
     }
 
-    private static String executeScanProcess(String targetPackage) throws IOException, InterruptedException {
-        ProcessBuilder processBuilder = new ProcessBuilder(BALLERINA_PATH.toString(), SCAN_COMMAND);
-        processBuilder.directory(RESOURCE_PACKAGES_DIRECTORY.resolve(targetPackage).toFile());
-        ProcessOutputGobbler output = getOutput(processBuilder.start());
-        Assert.assertFalse(ExitCode.hasFailure(output.getExitCode()));
-        return Files.readString(RESOURCE_PACKAGES_DIRECTORY.resolve(targetPackage)
-                .resolve("target").resolve("report").resolve("scan_results.json"));
+    private void testIndividualRule(FileRule rule, ByteArrayOutputStream console, PrintStream printStream)
+            throws IOException {
+        String targetPackageName = "rule" + rule.getId();
+        Path targetPackagePath = RESOURCE_PACKAGES_DIRECTORY.resolve(targetPackageName);
+
+        TestRunner testRunner = setupTestRunner(targetPackagePath, printStream);
+        testRunner.performScan();
+
+        validateRules(testRunner.getRules());
+        validateIssues(rule, testRunner.getIssues());
+        validateOutput(console, targetPackageName);
+
+        console.reset();
+    }
+
+    private TestRunner setupTestRunner(Path targetPackagePath, PrintStream printStream) {
+        Project project = BuildProject.load(getEnvironmentBuilder(), targetPackagePath);
+        TestOptions options = TestOptions.builder(project).setOutputStream(printStream).build();
+        return new TestRunner(options);
     }
 
-    private static ProcessOutputGobbler getOutput(Process process) throws InterruptedException {
-        ProcessOutputGobbler outputGobbler = new ProcessOutputGobbler(process.getInputStream());
-        ProcessOutputGobbler errorGobbler = new ProcessOutputGobbler(process.getErrorStream());
-        Thread outputThread = new Thread(outputGobbler);
-        Thread errorThread = new Thread(errorGobbler);
-        outputThread.start();
-        errorThread.start();
-        int exitCode = process.waitFor();
-        outputGobbler.setExitCode(exitCode);
-        errorGobbler.setExitCode(exitCode);
-        outputThread.join();
-        errorThread.join();
-        return outputGobbler;
+    private void validateRules(List<Rule> rules) {
+        Assertions.assertRule(
+                rules,
+                "ballerina/file:1",
+                AVOID_INSECURE_DIRECTORY_ACCESS.getDescription(),
+                VULNERABILITY);
+        Assertions.assertRule(
+                rules,
+                "ballerina/file:2",
+                AVOID_PATH_INJECTION.getDescription(),
+                VULNERABILITY);
+    }
+
+    private void validateIssues(FileRule rule, List<Issue> issues) {
+        switch (rule) {
+            case AVOID_INSECURE_DIRECTORY_ACCESS:
+                Assert.assertEquals(issues.size(), 2);
+                Assertions.assertIssue(issues, 0, "ballerina/file:1", "main.bal",
+                        21, 21, Source.BUILT_IN);
+                Assertions.assertIssue(issues, 1, "ballerina/file:2", "main.bal",
+                        22, 22, Source.BUILT_IN);
+                break;
+            case AVOID_PATH_INJECTION:
+                Assert.assertEquals(issues.size(), 1);
+                Assertions.assertIssue(issues, 0, "ballerina/file:2", "main.bal",
+                        21, 21, Source.BUILT_IN);
+                break;
+            default:
+                Assert.fail("Unhandled rule in validateIssues: " + rule);
+                break;
+        }
+    }
+
+    private void validateOutput(ByteArrayOutputStream console, String targetPackageName) throws IOException {
+        String output = console.toString(UTF_8);
+        String jsonOutput = extractJson(output);
+        String expectedOutput = Files.readString(EXPECTED_OUTPUT_DIRECTORY.resolve(targetPackageName + ".json"));
+        assertJsonEqual(jsonOutput, expectedOutput);
+    }
+
+    private static ProjectEnvironmentBuilder getEnvironmentBuilder() {
+        Environment environment = EnvironmentBuilder.getBuilder().setBallerinaHome(DISTRIBUTION_PATH).build();
+        return ProjectEnvironmentBuilder.getBuilder(environment);
+    }
+
+    private String extractJson(String consoleOutput) {
+        int startIndex = consoleOutput.indexOf("[");
+        int endIndex = consoleOutput.lastIndexOf("]");
+        if (startIndex == -1 || endIndex == -1) {
+            return "";
+        }
+        return consoleOutput.substring(startIndex, endIndex + 1);
     }
 
     private void assertJsonEqual(String actual, String expected) {
-        Assert.assertEquals(normalizeJson(actual), normalizeJson(expected));
+        Assert.assertEquals(normalizeString(actual), normalizeString(expected));
     }
 
-    private static String normalizeJson(String json) {
-        String normalizedJson = json.replaceAll("\\s*\"\\s*", "\"")
-                .replaceAll("\\s*:\\s*", ":")
-                .replaceAll("\\s*,\\s*", ",")
-                .replaceAll("\\s*\\{\\s*", "{")
-                .replaceAll("\\s*}\\s*", "}")
-                .replaceAll("\\s*\\[\\s*", "[")
-                .replaceAll("\\s*]\\s*", "]")
-                .replaceAll("\n", "")
-                .replaceAll(":\".*module-ballerina-file", ":\"module-ballerina-file");
-        return isWindows() ? normalizedJson.replaceAll("/", "\\\\\\\\") : normalizedJson;
+    private static String normalizeString(String json) {
+        try {
+            ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true);
+            JsonNode node = mapper.readTree(json);
+            String normalizedJson = mapper.writeValueAsString(node)
+                    .replaceAll(":\".*" + MODULE_BALLERINA_FilE, ":\"" + MODULE_BALLERINA_FilE);
+            return isWindows() ? normalizedJson.replace("/", "\\\\") : normalizedJson;
+        } catch (JsonProcessingException  ignore) {
+            return json;
+        }
     }
 
     private static boolean isWindows() {