Ensure predictable Ballerina dependency resolution in workflows

.github/workflows/build-master.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Set Up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Cache Ballerina dependencies
         uses: actions/cache@v4
@@ -70,7 +70,7 @@ jobs:
       - name: Set Up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Cache Ballerina dependencies
         uses: actions/cache@v4

.github/workflows/daily-build.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Set Up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Cache Ballerina dependencies
         uses: actions/cache@v4
@@ -63,7 +63,7 @@ jobs:
       - name: Set Up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Cache Ballerina dependencies
         uses: actions/cache@v4

.github/workflows/publish-release.yml

@@ -99,7 +99,7 @@ jobs:
       - name: Set up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Build with Gradle
         env:

.github/workflows/pull-request.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Set Up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Cache Ballerina dependencies
         uses: actions/cache@v4
@@ -67,7 +67,7 @@ jobs:
       - name: Set Up Ballerina
         uses: ballerina-platform/setup-ballerina@v1.1.1
         with:
-          version: 2201.12.3
+          version: 2201.12.9
 
       - name: Cache Ballerina dependencies
         uses: actions/cache@v4

build-config/ballerina_dependencies/Ballerina.toml

@@ -0,0 +1,123 @@
+[package]
+org = "wso2"
+name = "ballerina_dependencies"
+version = "0.1.0"
+
+[build-options]
+sticky = true
+
+[[dependency]]
+org = "ballerinax"
+name = "kafka"
+version = "4.5.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "rabbitmq"
+version = "3.2.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "trigger.github"
+version = "0.10.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "np"
+version = "0.8.1"
+
+[[dependency]]
+org = "ballerinax"
+name = "redis"
+version = "3.1.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "postgresql"
+version = "1.15.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "postgresql.driver"
+version = "1.6.1"
+
+[[dependency]]
+org = "ballerinax"
+name = "trigger.salesforce"
+version = "0.10.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "github"
+version = "5.1.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "snowflake"
+version = "2.2.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "docusign.dsadmin"
+version = "2.0.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "mysql"
+version = "1.16.1"
+
+[[dependency]]
+org = "ballerinax"
+name = "wso2.controlplane"
+version = "1.0.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.pinecone"
+version = "1.1.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.devant"
+version = "1.0.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.openai"
+version = "1.1.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.ollama"
+version = "1.0.1"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.azure"
+version = "1.0.1"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.anthropic"
+version = "1.0.1"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai.deepseek"
+version = "1.0.1"
+
+# Tracked with https://github.com/wso2/product-ballerina-integrator/issues/1249
+#[[dependency]]
+#org = "ballerinax"
+#name = "ai.mistral"
+#version = "1.0.1"
+
+[[dependency]]
+org = "ballerina"
+name = "ai"
+version = "1.5.0"
+
+[[dependency]]
+org = "ballerinax"
+name = "ai"
+version = "1.0.1"

build-config/ballerina_dependencies/main.bal

@@ -0,0 +1,24 @@
+import ballerinax/kafka as _;
+import ballerinax/rabbitmq as _;
+import ballerinax/trigger.github as _;
+import ballerinax/np as _;
+import ballerinax/redis as _;
+import ballerinax/postgresql as _;
+import ballerinax/postgresql.driver as _;
+import ballerinax/trigger.salesforce as _;
+import ballerinax/github as _;
+import ballerinax/snowflake as _;
+import ballerinax/docusign.dsadmin as _;
+import ballerinax/mysql as _;
+import ballerinax/wso2.controlplane as _;
+import ballerinax/ai as _;
+import ballerina/ai as _;
+import ballerinax/ai.pinecone as _;
+import ballerinax/ai.devant as _;
+import ballerinax/ai.openai as _;
+import ballerinax/ai.ollama as _;
+import ballerinax/ai.azure as _;
+import ballerinax/ai.anthropic as _;
+import ballerinax/ai.deepseek as _;
+// Tracked with https://github.com/wso2/product-ballerina-integrator/issues/1249
+// import ballerinax/ai.mistral as _;

build.gradle

@@ -83,8 +83,8 @@ subprojects {
         ballerinaStdLibs "io.ballerina.stdlib:os-ballerina:${stdlibOsVersion}"
         ballerinaStdLibs "io.ballerina.stdlib:uuid-ballerina:${stdlibUuidVersion}"
         ballerinaStdLibs "io.ballerina.stdlib:auth-ballerina:${stdlibAuthVersion}"
-        ballerinaStdLibs "io.ballerina.stdlib:oauth2-ballerina:${stdlibOAuth2Version}"
-        ballerinaStdLibs "io.ballerina.lib:data.jsondata-ballerina:${stdlibJsonDataVersion}"
+        ballerinaStdLibs "io.ballerina.stdlib:oauth2-ballerina:${stdlibOauth2Version}"
+        ballerinaStdLibs "io.ballerina.lib:data.jsondata-ballerina:${stdlibJsondataVersion}"
         ballerinaStdLibs "io.ballerina.stdlib:http-ballerina:${stdlibHttpVersion}"
         ballerinaStdLibs "io.ballerina.stdlib:graphql-ballerina:${stdlibGraphqlVersion}"
         ballerinaStdLibs "io.ballerina.stdlib:sql-ballerina:${stdLibSqlVersion}"
@@ -103,46 +103,89 @@ if (moduleVersion.indexOf('-') != -1) {
     moduleVersion = moduleVersion.substring(0, moduleVersion.indexOf('-'))
 }
 
-// TODO: Remove this once the workspace manager is refactored to import modules where necessary.
-def pullBallerinaModule(String packageName, String version = null, String orgName = "ballerinax") {
-    tasks.register("pull${orgName.capitalize()}Module_${packageName.replace('/', '_')}") {
-        doLast {
-            def errOutput = new ByteArrayOutputStream()
-            def result = exec {
-                ignoreExitValue = true
-
-                // Check if the package exists in the ballerina user home
-                def centralRepoDir = new File(System.getProperty("user.home"), ".ballerina/repositories/central.ballerina.io/bala/${orgName}")
-                if (centralRepoDir.exists()) {
-                    def pkgDir = new File(centralRepoDir, packageName)
-                    if (pkgDir.exists()) {
-                        if (version != null) {
-                            // Check if the specific version exists
-                            def versionDir = new File(pkgDir, version)
-                            if (versionDir.exists()) {
-                                commandLine 'echo', "Package ${packageName} version ${version} exists"
-                                return
-                            }
-                        } else {
-                            // If no version is specified, check for any version
-                            commandLine 'echo', "Package ${packageName} exists"
-                            return
-                        }
-                    }
-                }
-
-                def balPullCommand = "bal pull ${orgName}/${packageName}${version ? ':' + version : ''} --sticky=false"
-                if (org.gradle.internal.os.OperatingSystem.current().isWindows()) {
-                    commandLine 'cmd', '/c', balPullCommand
-                } else {
-                    commandLine '/bin/sh', '-c', balPullCommand
+task resolveBallerinaDependencies {
+    description 'Resolves ballerina_dependencies by copying to build, adding stdlib dependencies, and executing bal build'
+    group 'build'
+
+    def sourceDir = file("build-config/ballerina_dependencies")
+    def targetDir = file("$buildDir/ballerina_dependencies")
+
+    inputs.dir sourceDir
+    inputs.file 'gradle.properties'
+    outputs.dir targetDir
+
+    doLast {
+        // Copy the directory to build location
+        copy {
+            from sourceDir
+            into targetDir
+        }
+
+        // Read gradle.properties and extract stdlib versions
+        def gradleProps = new Properties()
+        gradleProps.load(file('gradle.properties').newInputStream())
+
+        def ballerinaTomlFile = file("$targetDir/Ballerina.toml")
+        def ballerinaTomlContent = ballerinaTomlFile.text
+
+        // Extract stdlib dependencies and add them to Ballerina.toml
+        gradleProps.each { key, value ->
+            if (key.toString().startsWith('stdlib') && key.toString().endsWith('Version')) {
+                // Extract module name: stdlibHttpVersion -> http
+                def moduleName = key.toString().replaceFirst('stdlib', '').replaceFirst('Version', '')
+                // Convert first character to lowercase: Http -> http
+                moduleName = moduleName[0].toLowerCase() + moduleName.substring(1)
+
+                // Check if dependency already exists
+                def dependencyPattern = ~/(?s).*\[\[dependency\]\]\s*org\s*=\s*"ballerina"\s*name\s*=\s*"${moduleName}".*/
+                if (!ballerinaTomlContent.matches(dependencyPattern)) {
+                    // Add the dependency
+                    ballerinaTomlContent += """
+[[dependency]]
+org = "ballerina"
+name = "${moduleName}"
+version = "${value}"
+
+"""
                 }
-                errorOutput = errOutput
             }
-            if (result.exitValue != 0) {
-                println errOutput
-                throw new GradleException("Failed to pull Ballerina module: ${packageName}")
+        }
+
+        // Write the updated Ballerina.toml
+        ballerinaTomlFile.text = ballerinaTomlContent
+
+        // Display the updated Ballerina.toml content
+        println "Updated Ballerina.toml content:"
+        def catResult = exec {
+            ignoreExitValue = true
+            workingDir targetDir
+
+            if (org.gradle.internal.os.OperatingSystem.current().isWindows()) {
+                commandLine 'cmd', '/c', 'type Ballerina.toml'
+            } else {
+                commandLine '/bin/sh', '-c', 'cat Ballerina.toml'
+            }
+            standardOutput = System.out
+        }
+
+        // Execute bal build in the copied directory
+        def errOutput = new ByteArrayOutputStream()
+        def result = exec {
+            ignoreExitValue = true
+            workingDir targetDir
+
+            def balBuildCommand = "bal build"
+            if (org.gradle.internal.os.OperatingSystem.current().isWindows()) {
+                commandLine 'cmd', '/c', balBuildCommand
+            } else {
+                commandLine '/bin/sh', '-c', balBuildCommand
             }
+            errorOutput = errOutput
+        }
+
+        if (result.exitValue != 0) {
+            println errOutput
+            throw new GradleException("Failed to build Ballerina dependencies")
         }
     }
 }
@@ -332,16 +375,8 @@ task pack(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
     }
 }
 
-task pullBallerinaModules {
-    dependsOn pullBallerinaModule('ai', ballerinaxAiAgent)
-    dependsOn pullBallerinaModule('kafka', ballerinaxKafkaVersion)
-    dependsOn pullBallerinaModule('rabbitmq', ballerinaxRabbitmqVersion)
-    dependsOn pullBallerinaModule('trigger.github', ballerinaxGithubTriggerVersion)
-    dependsOn pullBallerinaModule('np', ballerinaxNp)
-}
-
 test {
-    dependsOn pullBallerinaModules
+    dependsOn resolveBallerinaDependencies
 }
 
 task buildFlow {

flow-model-generator/modules/flow-model-generator-ls-extension/build.gradle

@@ -102,29 +102,6 @@ task copyStdlibs() {
     }
 }
 
-test.dependsOn rootProject.pullBallerinaModule('redis', ballerinaxRedisVersion)
-test.dependsOn rootProject.pullBallerinaModule('postgresql', ballerinaxPostgresqlVersion)
-test.dependsOn rootProject.pullBallerinaModule('postgresql.driver', ballerinaxPostgresqlDriverVersion)
-test.dependsOn rootProject.pullBallerinaModule('trigger.salesforce', ballerinaxTriggerSalesforceVersion)
-test.dependsOn rootProject.pullBallerinaModule('github', ballerinaxGithubVersion)
-test.dependsOn rootProject.pullBallerinaModule('snowflake', ballerinaxSnowflakeVersion)
-test.dependsOn rootProject.pullBallerinaModule('docusign.dsadmin', ballerinaxDocusignDsadminVersion)
-test.dependsOn rootProject.pullBallerinaModule('mysql', ballerinaxMysqlVersion)
-test.dependsOn rootProject.pullBallerinaModule('wso2.controlplane', ballerinaxWso2ControlplaneVersion)
-
-test.dependsOn rootProject.pullBallerinaModule('ai.pinecone', ballerinaxAiPineconeVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.devant', ballerinaxAiDevantVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.openai', ballerinaxAiOpenaiVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.ollama', ballerinaxAiOllamaVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.azure', ballerinaxAiAzureVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.anthropic', ballerinaxAiAnthropicVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.deepseek', ballerinaxAiDeepseekVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai.mistral', ballerinaxAiMistralVersion)
-test.dependsOn rootProject.pullBallerinaModule('ai', ballerinaAiVersion, "ballerina")
-
-
-// TODO: Re-enable this once the `np` module is available on central
-//test.dependsOn rootProject.pullBallerinaModule('np')
 
 test {
     dependsOn {