diff --git a/changelog.md b/changelog.md index a23f50e..bb33fd9 100644 --- a/changelog.md +++ b/changelog.md @@ -9,10 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [Add static code rules](https://github.com/ballerina-platform/ballerina-library/issues/7283) - ### Changed - [Change the listener configuration as an included parameter](https://github.com/ballerina-platform/ballerina-library/issues/7494) +- [Update the static analysis tests to use scan tool's test API](https://github.com/ballerina-platform/ballerina-library/issues/8249) ## [1.10.0] - 2024-08-20 diff --git a/compiler-plugin-test/build.gradle b/compiler-plugin-test/build.gradle index c358e09..4a77957 100644 --- a/compiler-plugin-test/build.gradle +++ b/compiler-plugin-test/build.gradle @@ -48,6 +48,9 @@ dependencies { testImplementation group: 'org.ballerinalang', name: 'ballerina-lang', version: "${ballerinaLangVersion}" testImplementation group: 'org.ballerinalang', name: 'ballerina-tools-api', version: "${ballerinaLangVersion}" testImplementation group: 'org.ballerinalang', name: 'ballerina-parser', version: "${ballerinaLangVersion}" + testImplementation group: 'io.ballerina.scan', name: 'scan-command', version: "${balScanVersion}" + testImplementation group: 'io.ballerina.scan', name: 'scan-command-test-utils', version: "${balScanVersion}" + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jacksonDatabindVersion}" testImplementation group: 'org.testng', name: 'testng', version: "${testngVersion}" implementation project(":file-compiler-plugin") } diff --git a/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/ProcessOutputGobbler.java b/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/ProcessOutputGobbler.java deleted file mode 100644 index 54eb2ac..0000000 --- a/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/ProcessOutputGobbler.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.org) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.stdlib.file.compiler.staticcodeanalyzer; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; - -/** - * Helper class to consume the process streams. - */ -class ProcessOutputGobbler implements Runnable { - private final InputStream inputStream; - private final StringBuilder output; - private int exitCode; - - public ProcessOutputGobbler(InputStream inputStream) { - this.inputStream = inputStream; - this.output = new StringBuilder(); - } - - @Override - public void run() { - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - output.append(line).append("\n"); - } - } catch (IOException e) { - this.output.append(e.getMessage()); - } - } - - public String getOutput() { - return output.toString(); - } - - public int getExitCode() { - return exitCode; - } - - public void setExitCode(int exitCode) { - this.exitCode = exitCode; - } -} diff --git a/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/StaticCodeAnalyzerTest.java b/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/StaticCodeAnalyzerTest.java index ffdd228..cd7f33c 100644 --- a/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/StaticCodeAnalyzerTest.java +++ b/compiler-plugin-test/src/test/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/StaticCodeAnalyzerTest.java @@ -18,105 +18,157 @@ package io.ballerina.stdlib.file.compiler.staticcodeanalyzer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import io.ballerina.projects.Project; +import io.ballerina.projects.ProjectEnvironmentBuilder; +import io.ballerina.projects.directory.BuildProject; +import io.ballerina.projects.environment.Environment; +import io.ballerina.projects.environment.EnvironmentBuilder; +import io.ballerina.scan.Issue; +import io.ballerina.scan.Rule; +import io.ballerina.scan.Source; +import io.ballerina.scan.test.Assertions; +import io.ballerina.scan.test.TestOptions; +import io.ballerina.scan.test.TestRunner; import org.testng.Assert; -import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; -import org.testng.internal.ExitCode; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.List; import java.util.Locale; -import java.util.regex.Pattern; import java.util.stream.Collectors; +import static io.ballerina.scan.RuleKind.VULNERABILITY; +import static io.ballerina.stdlib.file.compiler.staticcodeanalyzer.FileRule.AVOID_INSECURE_DIRECTORY_ACCESS; +import static io.ballerina.stdlib.file.compiler.staticcodeanalyzer.FileRule.AVOID_PATH_INJECTION; +import static java.nio.charset.StandardCharsets.UTF_8; + public class StaticCodeAnalyzerTest { private static final Path RESOURCE_PACKAGES_DIRECTORY = Paths .get("src", "test", "resources", "static_code_analyzer", "ballerina_packages").toAbsolutePath(); - private static final Path EXPECTED_JSON_OUTPUT_DIRECTORY = Paths + private static final Path EXPECTED_OUTPUT_DIRECTORY = Paths .get("src", "test", "resources", "static_code_analyzer", "expected_output").toAbsolutePath(); - private static final Path BALLERINA_PATH = getBalCommandPath(); private static final Path JSON_RULES_FILE_PATH = Paths .get("../", "compiler-plugin", "src", "main", "resources", "rules.json").toAbsolutePath(); - private static final String SCAN_COMMAND = "scan"; - - private static Path getBalCommandPath() { - String balCommand = isWindows() ? "bal.bat" : "bal"; - return Paths.get("../", "target", "ballerina-runtime", "bin", balCommand).toAbsolutePath(); - } - - @BeforeSuite - public void pullScanTool() throws IOException, InterruptedException { - ProcessBuilder processBuilder = new ProcessBuilder(BALLERINA_PATH.toString(), "tool", "pull", SCAN_COMMAND); - ProcessOutputGobbler output = getOutput(processBuilder.start()); - if (Pattern.compile("tool 'scan:.+\\..+\\..+' successfully set as the active version\\.") - .matcher(output.getOutput()).find() || Pattern.compile("tool 'scan:.+\\..+\\..+' is already active\\.") - .matcher(output.getOutput()).find()) { - return; - } - Assert.assertFalse(ExitCode.hasFailure(output.getExitCode())); - } + private static final Path DISTRIBUTION_PATH = Paths.get("../", "target", "ballerina-runtime"); + private static final String MODULE_BALLERINA_FilE = "module-ballerina-file"; @Test public void validateRulesJson() throws IOException { String expectedRules = "[" + Arrays.stream(FileRule.values()) .map(FileRule::toString).collect(Collectors.joining(",")) + "]"; String actualRules = Files.readString(JSON_RULES_FILE_PATH); - assertJsonEqual(normalizeJson(actualRules), normalizeJson(expectedRules)); + assertJsonEqual(actualRules, expectedRules); } @Test - public void testStaticCodeRules() throws IOException, InterruptedException { + public void testStaticCodeRulesWithAPI() throws IOException { + ByteArrayOutputStream console = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(console, true, UTF_8); + for (FileRule rule : FileRule.values()) { - String targetPackageName = "rule" + rule.getId(); - String actualJsonReport = StaticCodeAnalyzerTest.executeScanProcess(targetPackageName); - String expectedJsonReport = Files - .readString(EXPECTED_JSON_OUTPUT_DIRECTORY.resolve(targetPackageName + ".json")); - assertJsonEqual(actualJsonReport, expectedJsonReport); + testIndividualRule(rule, console, printStream); } } - private static String executeScanProcess(String targetPackage) throws IOException, InterruptedException { - ProcessBuilder processBuilder = new ProcessBuilder(BALLERINA_PATH.toString(), SCAN_COMMAND); - processBuilder.directory(RESOURCE_PACKAGES_DIRECTORY.resolve(targetPackage).toFile()); - ProcessOutputGobbler output = getOutput(processBuilder.start()); - Assert.assertFalse(ExitCode.hasFailure(output.getExitCode())); - return Files.readString(RESOURCE_PACKAGES_DIRECTORY.resolve(targetPackage) - .resolve("target").resolve("report").resolve("scan_results.json")); + private void testIndividualRule(FileRule rule, ByteArrayOutputStream console, PrintStream printStream) + throws IOException { + String targetPackageName = "rule" + rule.getId(); + Path targetPackagePath = RESOURCE_PACKAGES_DIRECTORY.resolve(targetPackageName); + + TestRunner testRunner = setupTestRunner(targetPackagePath, printStream); + testRunner.performScan(); + + validateRules(testRunner.getRules()); + validateIssues(rule, testRunner.getIssues()); + validateOutput(console, targetPackageName); + + console.reset(); + } + + private TestRunner setupTestRunner(Path targetPackagePath, PrintStream printStream) { + Project project = BuildProject.load(getEnvironmentBuilder(), targetPackagePath); + TestOptions options = TestOptions.builder(project).setOutputStream(printStream).build(); + return new TestRunner(options); } - private static ProcessOutputGobbler getOutput(Process process) throws InterruptedException { - ProcessOutputGobbler outputGobbler = new ProcessOutputGobbler(process.getInputStream()); - ProcessOutputGobbler errorGobbler = new ProcessOutputGobbler(process.getErrorStream()); - Thread outputThread = new Thread(outputGobbler); - Thread errorThread = new Thread(errorGobbler); - outputThread.start(); - errorThread.start(); - int exitCode = process.waitFor(); - outputGobbler.setExitCode(exitCode); - errorGobbler.setExitCode(exitCode); - outputThread.join(); - errorThread.join(); - return outputGobbler; + private void validateRules(List rules) { + Assertions.assertRule( + rules, + "ballerina/file:1", + AVOID_INSECURE_DIRECTORY_ACCESS.getDescription(), + VULNERABILITY); + Assertions.assertRule( + rules, + "ballerina/file:2", + AVOID_PATH_INJECTION.getDescription(), + VULNERABILITY); + } + + private void validateIssues(FileRule rule, List issues) { + switch (rule) { + case AVOID_INSECURE_DIRECTORY_ACCESS: + Assert.assertEquals(issues.size(), 2); + Assertions.assertIssue(issues, 0, "ballerina/file:1", "main.bal", + 21, 21, Source.BUILT_IN); + Assertions.assertIssue(issues, 1, "ballerina/file:2", "main.bal", + 22, 22, Source.BUILT_IN); + break; + case AVOID_PATH_INJECTION: + Assert.assertEquals(issues.size(), 1); + Assertions.assertIssue(issues, 0, "ballerina/file:2", "main.bal", + 21, 21, Source.BUILT_IN); + break; + default: + Assert.fail("Unhandled rule in validateIssues: " + rule); + break; + } + } + + private void validateOutput(ByteArrayOutputStream console, String targetPackageName) throws IOException { + String output = console.toString(UTF_8); + String jsonOutput = extractJson(output); + String expectedOutput = Files.readString(EXPECTED_OUTPUT_DIRECTORY.resolve(targetPackageName + ".json")); + assertJsonEqual(jsonOutput, expectedOutput); + } + + private static ProjectEnvironmentBuilder getEnvironmentBuilder() { + Environment environment = EnvironmentBuilder.getBuilder().setBallerinaHome(DISTRIBUTION_PATH).build(); + return ProjectEnvironmentBuilder.getBuilder(environment); + } + + private String extractJson(String consoleOutput) { + int startIndex = consoleOutput.indexOf("["); + int endIndex = consoleOutput.lastIndexOf("]"); + if (startIndex == -1 || endIndex == -1) { + return ""; + } + return consoleOutput.substring(startIndex, endIndex + 1); } private void assertJsonEqual(String actual, String expected) { - Assert.assertEquals(normalizeJson(actual), normalizeJson(expected)); + Assert.assertEquals(normalizeString(actual), normalizeString(expected)); } - private static String normalizeJson(String json) { - String normalizedJson = json.replaceAll("\\s*\"\\s*", "\"") - .replaceAll("\\s*:\\s*", ":") - .replaceAll("\\s*,\\s*", ",") - .replaceAll("\\s*\\{\\s*", "{") - .replaceAll("\\s*}\\s*", "}") - .replaceAll("\\s*\\[\\s*", "[") - .replaceAll("\\s*]\\s*", "]") - .replaceAll("\n", "") - .replaceAll(":\".*module-ballerina-file", ":\"module-ballerina-file"); - return isWindows() ? normalizedJson.replaceAll("/", "\\\\\\\\") : normalizedJson; + private static String normalizeString(String json) { + try { + ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + JsonNode node = mapper.readTree(json); + String normalizedJson = mapper.writeValueAsString(node) + .replaceAll(":\".*" + MODULE_BALLERINA_FilE, ":\"" + MODULE_BALLERINA_FilE); + return isWindows() ? normalizedJson.replace("/", "\\\\") : normalizedJson; + } catch (JsonProcessingException ignore) { + return json; + } } private static boolean isWindows() { diff --git a/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule1.json b/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule1.json index 3025f77..ba4839d 100644 --- a/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule1.json +++ b/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule1.json @@ -1,58 +1,42 @@ -[ { - "location" : { - "filePath" : "main.bal", - "startLine" : 22, - "endLine" : 22, - "startColumn" : 20, - "endColumn" : 68, - "startOffset" : 800, - "length" : 48 +[ + { + "location": { + "filePath": "main.bal", + "startLine": 21, + "endLine": 21, + "startColumn": 28, + "endColumn": 44, + "startOffset": 762, + "length": 16 + }, + "rule": { + "id": "ballerina/file:1", + "numericId": 1, + "description": "Avoid using publicly writable directories for file operations without proper access controls", + "ruleKind": "VULNERABILITY" + }, + "source": "BUILT_IN", + "fileName": "rule1/main.bal", + "filePath": "/home/nureka/Documents/GitHub/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule1/main.bal" }, - "rule" : { - "id" : "ballerina/file:1", - "numericId" : 1, - "description" : "Avoid using publicly writable directories for file operations without proper access controls", - "ruleKind" : "VULNERABILITY" - }, - "source" : "BUILT_IN", - "fileName" : "rule1/main.bal", - "filePath" : "/Users/sachink/Desktop/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule1/main.bal" -}, { - "location" : { - "filePath" : "main.bal", - "startLine" : 25, - "endLine" : 25, - "startColumn" : 8, - "endColumn" : 57, - "startOffset" : 886, - "length" : 49 - }, - "rule" : { - "id" : "ballerina/file:1", - "numericId" : 1, - "description" : "Avoid using publicly writable directories for file operations without proper access controls", - "ruleKind" : "VULNERABILITY" - }, - "source" : "BUILT_IN", - "fileName" : "rule1/main.bal", - "filePath" : "/Users/sachink/Desktop/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule1/main.bal" -}, { - "location" : { - "filePath" : "main.bal", - "startLine" : 27, - "endLine" : 27, - "startColumn" : 8, - "endColumn" : 87, - "startOffset" : 958, - "length" : 79 - }, - "rule" : { - "id" : "ballerina/file:1", - "numericId" : 1, - "description" : "Avoid using publicly writable directories for file operations without proper access controls", - "ruleKind" : "VULNERABILITY" - }, - "source" : "BUILT_IN", - "fileName" : "rule1/main.bal", - "filePath" : "/Users/sachink/Desktop/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule1/main.bal" -} ] + { + "location": { + "filePath": "main.bal", + "startLine": 22, + "endLine": 22, + "startColumn": 20, + "endColumn": 68, + "startOffset": 800, + "length": 48 + }, + "rule": { + "id": "ballerina/file:2", + "numericId": 2, + "description": "File function calls should not be vulnerable to path injection attacks", + "ruleKind": "VULNERABILITY" + }, + "source": "BUILT_IN", + "fileName": "rule1/main.bal", + "filePath": "/home/nureka/Documents/GitHub/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule1/main.bal" + } +] diff --git a/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule2.json b/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule2.json index edb5f35..5db01ef 100644 --- a/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule2.json +++ b/compiler-plugin-test/src/test/resources/static_code_analyzer/expected_output/rule2.json @@ -1,20 +1,42 @@ -[ { - "location" : { - "filePath" : "main.bal", - "startLine" : 21, - "endLine" : 21, - "startColumn" : 28, - "endColumn" : 44, - "startOffset" : 762, - "length" : 16 +[ + { + "location": { + "filePath": "main.bal", + "startLine": 19, + "endLine": 19, + "startColumn": 0, + "endColumn": 6, + "startOffset": 707, + "length": 6 + }, + "rule": { + "id": "ballerina:3", + "numericId": 3, + "description": "Non isolated public function", + "ruleKind": "CODE_SMELL" + }, + "source": "BUILT_IN", + "fileName": "rule2/main.bal", + "filePath": "/home/nureka/Documents/GitHub/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule2/main.bal" }, - "rule" : { - "id" : "ballerina/file:2", - "numericId" : 2, - "description" : "File function calls should not be vulnerable to path injection attacks", - "ruleKind" : "VULNERABILITY" - }, - "source" : "BUILT_IN", - "fileName" : "rule2/main.bal", - "filePath" : "/Users/sachink/Desktop/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule2/main.bal" -} ] + { + "location": { + "filePath": "main.bal", + "startLine": 21, + "endLine": 21, + "startColumn": 25, + "endColumn": 52, + "startOffset": 856, + "length": 27 + }, + "rule": { + "id": "ballerina/file:2", + "numericId": 2, + "description": "File function calls should not be vulnerable to path injection attacks", + "ruleKind": "VULNERABILITY" + }, + "source": "BUILT_IN", + "fileName": "rule2/main.bal", + "filePath": "/home/nureka/Documents/GitHub/module-ballerina-file/compiler-plugin-test/src/test/resources/static_code_analyzer/ballerina_packages/rule2/main.bal" + } +] diff --git a/compiler-plugin/src/main/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/FileRule.java b/compiler-plugin/src/main/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/FileRule.java index 5c9e7e3..8bad271 100644 --- a/compiler-plugin/src/main/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/FileRule.java +++ b/compiler-plugin/src/main/java/io/ballerina/stdlib/file/compiler/staticcodeanalyzer/FileRule.java @@ -42,8 +42,8 @@ public int getId() { return this.rule.numericId(); } - public Rule getRule() { - return this.rule; + public String getDescription() { + return this.rule.description(); } @Override diff --git a/gradle.properties b/gradle.properties index bc4af7d..7ba705c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,4 +21,5 @@ observeVersion=1.5.0 observeInternalVersion=1.5.0 jacocoVersion=0.8.10 -balScanVersion=0.5.0 +balScanVersion=0.11.0 +jacksonDatabindVersion=2.17.2