From 75318f0c6ebf7b3a4bbab8b0e16e3bfeb7443d1f Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 10:33:19 -0500 Subject: [PATCH 01/19] updated elastic4s to support 7.10.x --- common/scala/build.gradle | 6 +++- .../ElasticSearchActivationStore.scala | 28 +++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/common/scala/build.gradle b/common/scala/build.gradle index ccdfc8a567a..fdf3eb73fff 100644 --- a/common/scala/build.gradle +++ b/common/scala/build.gradle @@ -90,7 +90,11 @@ dependencies { compile "io.reactivex:rxjava-reactive-streams:1.2.1" compile "com.microsoft.azure:azure-cosmosdb:2.6.2" - compile "com.sksamuel.elastic4s:elastic4s-http_${gradle.scala.depVersion}:6.7.4" + //compile "com.sksamuel.elastic4s:elastic4s-http_${gradle.scala.depVersion}:6.7.4" + //compile "com.sksamuel.elastic4s:elastic4s-core_${gradle.scala.depVersion}:7.10.3" + compile "com.sksamuel.elastic4s:elastic4s-client-esjava_${gradle.scala.depVersion}:7.10.3" + //compile "org.elasticsearch.client:elasticsearch-rest-client:7.16.0" + //for mongo compile "org.mongodb.scala:mongo-scala-driver_${gradle.scala.depVersion}:2.7.0" diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala index 50504263f90..296defc0cea 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala @@ -19,17 +19,17 @@ package org.apache.openwhisk.core.database.elasticsearch import java.time.Instant import java.util.concurrent.TimeUnit - import scala.language.postfixOps import akka.actor.ActorSystem import akka.event.Logging.ErrorLevel import akka.http.scaladsl.model._ import akka.stream.scaladsl.Flow -import com.sksamuel.elastic4s.http.search.SearchHit -import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} -import com.sksamuel.elastic4s.indexes.IndexRequest -import com.sksamuel.elastic4s.searches.queries.RangeQuery -import com.sksamuel.elastic4s.searches.queries.matches.MatchPhrase +import com.sksamuel.elastic4s.http.{JavaClient, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties} +import com.sksamuel.elastic4s.requests.indexes.IndexRequest +import com.sksamuel.elastic4s.requests.searches.SearchHit +import com.sksamuel.elastic4s.requests.searches.queries.RangeQuery +import com.sksamuel.elastic4s.requests.searches.queries.matches.MatchPhrase import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder @@ -60,18 +60,16 @@ class ElasticSearchActivationStore( useBatching: Boolean = false)(implicit actorSystem: ActorSystem, override val logging: Logging) extends ActivationStore { - import com.sksamuel.elastic4s.http.ElasticDsl._ + import com.sksamuel.elastic4s.ElasticDsl._ import ElasticSearchActivationStore.{generateIndex, httpClientCallback} private implicit val executionContextExecutor: ExecutionContextExecutor = actorSystem.dispatcher + private val javaClient = JavaClient(ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), + NoOpRequestConfigCallback, + httpClientCallback) - private val client = - ElasticClient( - ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), - NoOpRequestConfigCallback, - httpClientCallback) + private val client = ElasticClient(javaClient) - private val esType = "_doc" private val maxOpenDbRequests = actorSystem.settings.config .getInt("akka.http.host-connection-pool.max-connections") / 2 private val batcher: Batcher[IndexRequest, Either[ArtifactStoreException, DocInfo]] = @@ -111,7 +109,7 @@ class ElasticSearchActivationStore( "response" -> response)) val index = generateIndex(activation.namespace.namespace) - val op = indexInto(index, esType).doc(payload.toString).id(activation.docid.asString) + val op = indexInto(index).doc(payload.toString).id(activation.docid.asString) // always use batching val res = batcher.put(op).map { @@ -227,7 +225,7 @@ class ElasticSearchActivationStore( val res = client .execute { - deleteByQuery(index, esType, termQuery("_id", activationId.asString)) + deleteByQuery(index, termQuery("_id", activationId.asString)) } .map { res => if (res.status == StatusCodes.OK.intValue) { From f3690a0243ea9042d7eb6732d6361973469335cb Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 10:34:57 -0500 Subject: [PATCH 02/19] removed comments --- common/scala/build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/scala/build.gradle b/common/scala/build.gradle index fdf3eb73fff..30f41f873ed 100644 --- a/common/scala/build.gradle +++ b/common/scala/build.gradle @@ -89,11 +89,7 @@ dependencies { compile "io.reactivex:rxjava:1.3.8" compile "io.reactivex:rxjava-reactive-streams:1.2.1" compile "com.microsoft.azure:azure-cosmosdb:2.6.2" - - //compile "com.sksamuel.elastic4s:elastic4s-http_${gradle.scala.depVersion}:6.7.4" - //compile "com.sksamuel.elastic4s:elastic4s-core_${gradle.scala.depVersion}:7.10.3" compile "com.sksamuel.elastic4s:elastic4s-client-esjava_${gradle.scala.depVersion}:7.10.3" - //compile "org.elasticsearch.client:elasticsearch-rest-client:7.16.0" //for mongo compile "org.mongodb.scala:mongo-scala-driver_${gradle.scala.depVersion}:2.7.0" From a2dacf7c02b546df1def0e59fe6c593ca0826def Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 13:26:10 -0500 Subject: [PATCH 03/19] adding gradle workflow --- .github/workflows/gradle-build.yml | 26 +++++++++++++++++++ .../ElasticSearchActivationStore.scala | 2 +- tools/macos/README.md | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/gradle-build.yml diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml new file mode 100644 index 00000000000..7c5573c9082 --- /dev/null +++ b/.github/workflows/gradle-build.yml @@ -0,0 +1,26 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'adopt' + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b + - name: Build with Gradle + uses: gradle/gradle-build-action@0d13054264b0bb894ded474f08ebb30921341cee + with: + arguments: build \ No newline at end of file diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala index 296defc0cea..4b551e86312 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala @@ -183,7 +183,7 @@ class ElasticSearchActivationStore( } .map { res => if (res.status == StatusCodes.OK.intValue) { - if (res.result.hits.total == 0) { + if (res.result.hits.total.value == 0) { transid.finished(this, start, s"[GET] 'activations', document: '$activationId'; not found.") throw NoDocumentException("not found on 'get'") } else { diff --git a/tools/macos/README.md b/tools/macos/README.md index e669a538955..989ae336157 100644 --- a/tools/macos/README.md +++ b/tools/macos/README.md @@ -55,7 +55,7 @@ brew install gnu-tar # install pip sudo easy_install pip # install script prerequisites -pip install docker==5.0.0 ansible==4.1.0 jinja2==3.0.1 couchdb==1.2 httplib2==0.19.1 requests==2.25.1 six=1.16.0 +pip install docker==5.0.0 ansible==4.1.0 jinja2==3.0.1 couchdb==1.2 httplib2==0.19.1 requests==2.25.1 six==1.16.0 ``` Make sure you correctly configure the environment variable $JAVA_HOME. From 23bb9df091ec99bcc71e14cb7697834a4245d9a6 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 13:27:52 -0500 Subject: [PATCH 04/19] no idea what im doing --- .github/workflows/gradle-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index 7c5573c9082..f53e0bfd7cf 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -23,4 +23,4 @@ jobs: - name: Build with Gradle uses: gradle/gradle-build-action@0d13054264b0bb894ded474f08ebb30921341cee with: - arguments: build \ No newline at end of file + arguments: distDocker \ No newline at end of file From be2c10136db233487271869682eb06d1ee423ea5 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 14:42:49 -0500 Subject: [PATCH 05/19] updated the elastic search activation store plugin --- .../queue/ElasticSearchDurationChecker.scala | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala index 88e1f2ff177..dcc3ab4e998 100644 --- a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala +++ b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala @@ -17,9 +17,10 @@ package org.apache.openwhisk.core.scheduler.queue import akka.actor.ActorSystem -import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} -import com.sksamuel.elastic4s.searches.queries.Query +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.http.{JavaClient, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties} +import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.{ElasticDate, ElasticDateMath, Seconds} import org.apache.openwhisk.common.Logging import org.apache.openwhisk.core.ConfigKeys @@ -28,11 +29,10 @@ import org.apache.openwhisk.spi.Spi import pureconfig.loadConfigOrThrow import spray.json.{JsArray, JsNumber, JsValue, RootJsonFormat, deserializationError, _} -import scala.concurrent.Future +import scala.concurrent.{ExecutionContextExecutor, Future} import scala.concurrent.duration.FiniteDuration import scala.language.implicitConversions import scala.util.{Failure, Try} -import pureconfig.generic.auto._ trait DurationChecker { def checkAverageDuration(invocationNamespace: String, actionMetaData: WhiskActionMetaData)( @@ -45,7 +45,7 @@ object ElasticSearchDurationChecker { val FilterAggregationName = "filterAggregation" val AverageAggregationName = "averageAggregation" - implicit val serde = new ElasticSearchDurationCheckResultFormat() + implicit val serde: ElasticSearchDurationCheckResultFormat = new ElasticSearchDurationCheckResultFormat() def getFromDate(timeWindow: FiniteDuration): ElasticDateMath = ElasticDate.now minus (timeWindow.toSeconds.toInt, Seconds) @@ -58,7 +58,7 @@ class ElasticSearchDurationChecker(private val client: ElasticClient, val timeWi import ElasticSearchDurationChecker._ import org.apache.openwhisk.core.database.elasticsearch.ElasticSearchActivationStore.generateIndex - implicit val ec = actorSystem.getDispatcher + implicit val ec: ExecutionContextExecutor = actorSystem.getDispatcher override def checkAverageDuration(invocationNamespace: String, actionMetaData: WhiskActionMetaData)( callback: DurationCheckResult => DurationCheckResult): Future[DurationCheckResult] = { @@ -104,7 +104,7 @@ class ElasticSearchDurationChecker(private val client: ElasticClient, val timeWi .map(callback(_)) .andThen { case Failure(t) => - logging.error(this, s"failed to check the average duration: ${t}") + logging.error(this, s"failed to check the average duration: $t") } } } @@ -116,11 +116,11 @@ object ElasticSearchDurationCheckerProvider extends DurationCheckerProvider { implicit val as: ActorSystem = actorSystem implicit val logging: Logging = log - val elasticClient = - ElasticClient( - ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), - NoOpRequestConfigCallback, - httpClientCallback) + val javaClient = JavaClient(ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), + NoOpRequestConfigCallback, + httpClientCallback) + + val elasticClient = ElasticClient(javaClient) new ElasticSearchDurationChecker(elasticClient, durationCheckerConfig.timeWindow) } @@ -187,7 +187,7 @@ class ElasticSearchDurationCheckResultFormat extends RootJsonFormat[DurationChec "took": 0 } */ - implicit def read(json: JsValue) = { + implicit def read(json: JsValue): DurationCheckResult = { val jsObject = json.asJsObject jsObject.getFields("aggregations", "took", "hits") match { From 57077261df2def9dae83ed9d5b8f62a95e438dc8 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 14:46:35 -0500 Subject: [PATCH 06/19] added back an import that i guess we needed? --- .../core/scheduler/queue/ElasticSearchDurationChecker.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala index dcc3ab4e998..7041d28ef33 100644 --- a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala +++ b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala @@ -33,6 +33,7 @@ import scala.concurrent.{ExecutionContextExecutor, Future} import scala.concurrent.duration.FiniteDuration import scala.language.implicitConversions import scala.util.{Failure, Try} +import pureconfig.generic.auto._ trait DurationChecker { def checkAverageDuration(invocationNamespace: String, actionMetaData: WhiskActionMetaData)( From 51e8ab2f86b22220f1ba322ea185f7b5085a2ccf Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 15:15:00 -0500 Subject: [PATCH 07/19] try get these images into a repo --- .github/workflows/gradle-build.yml | 25 ++++++++++++++++++++++++- tools/jenkins/apache/dockerhub.groovy | 27 +++++++++++++-------------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index f53e0bfd7cf..33947321061 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -23,4 +23,27 @@ jobs: - name: Build with Gradle uses: gradle/gradle-build-action@0d13054264b0bb894ded474f08ebb30921341cee with: - arguments: distDocker \ No newline at end of file + arguments: distDocker + + - name: Set the image repo name and tag + id: image_vars + run: | + # Construct an image id and includes the repo. + IMAGE_ID=ghcr.io/${{ github.repository }} + VERSION=latest + + # Set variables that can be used in other steps. + echo ::set-output name=repo::$(echo $IMAGE_ID | awk '{print tolower($0)}') + echo ::set-output name=tag::$(echo $VERSION | awk '{print tolower($0)}') + + - name: Login to github docker Registry + run: docker login ghcr.io -u dylanturn -p ${{ secrets.GITHUB_TOKEN }} + + - name: Push the Docker image + run: docker push ${{steps.image_vars.outputs.repo}}/controller:${{steps.image_vars.outputs.tag}} + + - name: Push the Docker image + run: docker push ${{steps.image_vars.outputs.repo}}/invoker:${{steps.image_vars.outputs.tag}} + + - name: Push the Docker image + run: docker push ${{steps.image_vars.outputs.repo}}/scheduler:${{steps.image_vars.outputs.tag}} \ No newline at end of file diff --git a/tools/jenkins/apache/dockerhub.groovy b/tools/jenkins/apache/dockerhub.groovy index 6ff1c591d9d..5ef556ecaa8 100644 --- a/tools/jenkins/apache/dockerhub.groovy +++ b/tools/jenkins/apache/dockerhub.groovy @@ -39,18 +39,17 @@ node('ubuntu') { } } - stage("Clean") { - sh "docker images" - sh 'docker rmi -f $(docker images -f "reference=openwhisk/*" -q) || true' - sh "docker images" - sh "docker logout" - sh "rm -rf $WORKSPACE/local-docker-cfg" - } - - stage("Notify") { - withCredentials([string(credentialsId: 'openwhisk_slack_token', variable: 'OPENWHISK_SLACK_TOKEN')]) { - sh "curl -X POST --data-urlencode 'payload={\"channel\": \"#dev\", \"username\": \"whiskbot\", \"text\": \"OpenWhisk Docker Images build and posted to https://hub.docker.com/u/openwhisk by Jenkins job ${BUILD_URL}\", \"icon_emoji\": \":openwhisk:\"}' https://hooks.slack.com/services/${OPENWHISK_SLACK_TOKEN}" - } - - } + // stage("Clean") { + // sh "docker images" + // sh 'docker rmi -f $(docker images -f "reference=openwhisk/*" -q) || true' + // sh "docker images" + // sh "docker logout" + // sh "rm -rf $WORKSPACE/local-docker-cfg" + // } + // + // stage("Notify") { + // withCredentials([string(credentialsId: 'openwhisk_slack_token', variable: 'OPENWHISK_SLACK_TOKEN')]) { + // sh "curl -X POST --data-urlencode 'payload={\"channel\": \"#dev\", \"username\": \"whiskbot\", \"text\": \"OpenWhisk Docker Images build and posted to https://hub.docker.com/u/openwhisk by Jenkins job ${BUILD_URL}\", \"icon_emoji\": \":openwhisk:\"}' https://hooks.slack.com/services/${OPENWHISK_SLACK_TOKEN}" + // } + // } } From 3799f4928e963c4e01436953c4ce52b69a7368db Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 16:05:42 -0500 Subject: [PATCH 08/19] trying to get image pushing to work without tricks --- .github/workflows/gradle-build.yml | 44 ++++++++++-------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index 33947321061..1d81faa93bf 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -12,38 +12,24 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'adopt' - - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b - - name: Build with Gradle - uses: gradle/gradle-build-action@0d13054264b0bb894ded474f08ebb30921341cee - with: - arguments: distDocker - - - name: Set the image repo name and tag - id: image_vars - run: | - # Construct an image id and includes the repo. - IMAGE_ID=ghcr.io/${{ github.repository }} - VERSION=latest - - # Set variables that can be used in other steps. - echo ::set-output name=repo::$(echo $IMAGE_ID | awk '{print tolower($0)}') - echo ::set-output name=tag::$(echo $VERSION | awk '{print tolower($0)}') + - uses: actions/checkout@v2 - name: Login to github docker Registry run: docker login ghcr.io -u dylanturn -p ${{ secrets.GITHUB_TOKEN }} - - name: Push the Docker image - run: docker push ${{steps.image_vars.outputs.repo}}/controller:${{steps.image_vars.outputs.tag}} + - name: Set the image tag + id: image_vars + run: | + TAG=latest + echo ::set-output name=tag::$(echo $TAG | awk '{print tolower($0)}') + + - uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: 11 - - name: Push the Docker image - run: docker push ${{steps.image_vars.outputs.repo}}/invoker:${{steps.image_vars.outputs.tag}} + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 - - name: Push the Docker image - run: docker push ${{steps.image_vars.outputs.repo}}/scheduler:${{steps.image_vars.outputs.tag}} \ No newline at end of file + - name: Execute Gradle build + run: ./gradlew distDocker -PdockerRegistry=ghcr.io -PdockerImagePrefix=${{ github.repository }} -PdockerImageTag=${{steps.image_vars.outputs.tag}} \ No newline at end of file From 4d1090d5b6e4aeb60567d01d58215ac426eade8a Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Mon, 4 Apr 2022 16:07:14 -0500 Subject: [PATCH 09/19] undid change to dockerhub.groovy --- tools/jenkins/apache/dockerhub.groovy | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/jenkins/apache/dockerhub.groovy b/tools/jenkins/apache/dockerhub.groovy index 5ef556ecaa8..f0238c6cabe 100644 --- a/tools/jenkins/apache/dockerhub.groovy +++ b/tools/jenkins/apache/dockerhub.groovy @@ -39,17 +39,17 @@ node('ubuntu') { } } - // stage("Clean") { - // sh "docker images" - // sh 'docker rmi -f $(docker images -f "reference=openwhisk/*" -q) || true' - // sh "docker images" - // sh "docker logout" - // sh "rm -rf $WORKSPACE/local-docker-cfg" - // } - // - // stage("Notify") { - // withCredentials([string(credentialsId: 'openwhisk_slack_token', variable: 'OPENWHISK_SLACK_TOKEN')]) { - // sh "curl -X POST --data-urlencode 'payload={\"channel\": \"#dev\", \"username\": \"whiskbot\", \"text\": \"OpenWhisk Docker Images build and posted to https://hub.docker.com/u/openwhisk by Jenkins job ${BUILD_URL}\", \"icon_emoji\": \":openwhisk:\"}' https://hooks.slack.com/services/${OPENWHISK_SLACK_TOKEN}" - // } - // } + stage("Clean") { + sh "docker images" + sh 'docker rmi -f $(docker images -f "reference=openwhisk/*" -q) || true' + sh "docker images" + sh "docker logout" + sh "rm -rf $WORKSPACE/local-docker-cfg" + } + + stage("Notify") { + withCredentials([string(credentialsId: 'openwhisk_slack_token', variable: 'OPENWHISK_SLACK_TOKEN')]) { + sh "curl -X POST --data-urlencode 'payload={\"channel\": \"#dev\", \"username\": \"whiskbot\", \"text\": \"OpenWhisk Docker Images build and posted to https://hub.docker.com/u/openwhisk by Jenkins job ${BUILD_URL}\", \"icon_emoji\": \":openwhisk:\"}' https://hooks.slack.com/services/${OPENWHISK_SLACK_TOKEN}" + } + } } From 08fc4a3f489e5bbe5cccee6ed10fdbf20ee5ccb4 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 5 Apr 2022 12:48:40 -0500 Subject: [PATCH 10/19] configure the action to run manually --- .github/workflows/gradle-build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index 1d81faa93bf..d3c4a4d620b 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -5,8 +5,7 @@ name: Java CI -on: [push] - +on: [workflow_dispatch] jobs: build: runs-on: ubuntu-latest @@ -15,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - name: Login to github docker Registry - run: docker login ghcr.io -u dylanturn -p ${{ secrets.GITHUB_TOKEN }} + run: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - name: Set the image tag id: image_vars From f50c70d1fb8f52b58902365a7abe0d6e6fb309ea Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 5 Apr 2022 12:50:04 -0500 Subject: [PATCH 11/19] added space --- .github/workflows/gradle-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index d3c4a4d620b..90dd03e92d1 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -6,6 +6,7 @@ name: Java CI on: [workflow_dispatch] + jobs: build: runs-on: ubuntu-latest From cc082552b9937df70ddf850df41fa02995447302 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 5 Apr 2022 12:51:21 -0500 Subject: [PATCH 12/19] trying to get this workflow_dispatch to work --- .github/workflows/gradle-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index 90dd03e92d1..f3a5d5a73f0 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -5,7 +5,8 @@ name: Java CI -on: [workflow_dispatch] +on: + workflow_dispatch: jobs: build: From d30998612ae42c4880eec0ba2e08bcac2aa22c4c Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 18:34:37 +0000 Subject: [PATCH 13/19] updated the ElasticSearchDurationCheckerTests test --- .../test/ElasticSearchDurationCheckerTests.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala index 4f6d33e2756..5c3ff3da5f2 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala @@ -17,8 +17,9 @@ package org.apache.openwhisk.core.scheduler.queue.test -import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.http.{JavaClient, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties} import common._ import common.rest.WskRestOperations import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} @@ -91,11 +92,11 @@ class ElasticSearchDurationCheckerTests } } - private val client = - ElasticClient( - ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), + val javaClient = JavaClient(ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), NoOpRequestConfigCallback, httpClientCallback) + + val client = ElasticClient(javaClient) private val elasticSearchDurationChecker = new ElasticSearchDurationChecker(client, defaultDurationCheckWindow) From 0676879f1b869eea951e36a860075eca645d47bc Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 13:41:57 -0500 Subject: [PATCH 14/19] update the MemoryQueueFlowTests elastic4s searches import --- .../core/scheduler/queue/test/MemoryQueueFlowTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala index 16250661218..bdbf4eb0577 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala @@ -3,7 +3,7 @@ package org.apache.openwhisk.core.scheduler.queue.test import akka.actor.ActorRef import akka.actor.FSM.{CurrentState, StateTimeout, SubscribeTransitionCallBack, Transition} import akka.testkit.{TestActor, TestFSMRef, TestProbe} -import com.sksamuel.elastic4s.http.{search => _} +import com.sksamuel.elastic4s.requests.{searches => _} import org.apache.openwhisk.common.GracefulShutdown import org.apache.openwhisk.core.connector.ContainerCreationError.{NonExecutableActionError, WhiskError} import org.apache.openwhisk.core.connector.ContainerCreationMessage From ab12b5b9387d5b1d47e3a3e24ff2b9f9e1ea8262 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 13:51:59 -0500 Subject: [PATCH 15/19] made some changes to fix the test --- .../queue/test/MemoryQueueTestsFixture.scala | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index 2d2ead20aed..86236d49b2f 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -1,14 +1,14 @@ package org.apache.openwhisk.core.scheduler.queue.test import java.time.Instant - import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{ImplicitSender, TestKit, TestProbe} -import com.sksamuel.elastic4s.http -import com.sksamuel.elastic4s.http.ElasticDsl.{avgAgg, boolQuery, matchQuery, rangeQuery, search} -import com.sksamuel.elastic4s.http._ -import com.sksamuel.elastic4s.http.search.{SearchHits, SearchResponse} -import com.sksamuel.elastic4s.searches.SearchRequest +import com.sksamuel.elastic4s.Executor +import com.sksamuel.elastic4s.ElasticDsl.{avgAgg, boolQuery, matchQuery, rangeQuery, search} +import com.sksamuel.elastic4s._ +import com.sksamuel.elastic4s.requests.common.Shards +import com.sksamuel.elastic4s.requests.searches.ScoreMode.Total +import com.sksamuel.elastic4s.requests.searches.{SearchHits, SearchRequest, SearchResponse, Total} import common.StreamLogging import org.apache.kafka.clients.producer.RecordMetadata import org.apache.kafka.common.TopicPartition @@ -24,20 +24,9 @@ import org.apache.openwhisk.core.entity.{WhiskActivation, _} import org.apache.openwhisk.core.etcd.EtcdKV.{ContainerKeys, QueueKeys, ThrottlingKeys} import org.apache.openwhisk.core.scheduler.SchedulerEndpoints import org.apache.openwhisk.core.scheduler.grpc.GetActivation -import org.apache.openwhisk.core.scheduler.queue.ElasticSearchDurationChecker.{getFromDate, AverageAggregationName} +import org.apache.openwhisk.core.scheduler.queue.ElasticSearchDurationChecker.{AverageAggregationName, getFromDate} import org.apache.openwhisk.core.scheduler.queue._ -import org.apache.openwhisk.core.service.{ - AlreadyExist, - DeleteEvent, - Done, - InitialDataStorageResults, - PutEvent, - RegisterData, - RegisterInitialData, - UnregisterData, - UnwatchEndpoint, - WatchEndpoint -} +import org.apache.openwhisk.core.service.{AlreadyExist, DeleteEvent, Done, InitialDataStorageResults, PutEvent, RegisterData, RegisterInitialData, UnregisterData, UnwatchEndpoint, WatchEndpoint} import org.scalamock.scalatest.MockFactory import scala.concurrent.duration.DurationInt @@ -218,7 +207,7 @@ class MemoryQueueTestsFixture (mockEsClient .execute[SearchRequest, SearchResponse, Future](_: SearchRequest)( _: Functor[Future], - _: http.Executor[Future], + _: Executor[Future], _: Handler[SearchRequest, SearchResponse], _: Manifest[SearchResponse])) .expects(searchRequest, *, *, *, *) @@ -227,7 +216,7 @@ class MemoryQueueTestsFixture 200, None, Map.empty, - SearchResponse(1, false, false, Map.empty, Shards(0, 0, 0), None, Map.empty, SearchHits(0, 0, Array.empty))))) + SearchResponse(1, false, false, Map.empty, Shards(0, 0, 0), None, Map.empty, SearchHits(Total(0, "eq"), 0, Array.empty))))) .once() } From a742486972753371512b4b50fa73553589b05b1d Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 13:52:46 -0500 Subject: [PATCH 16/19] updated an elastic import to fix a test --- .../openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala index 55ca7dc4777..4b8de2afe64 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala @@ -15,7 +15,7 @@ import com.ibm.etcd.api._ import com.ibm.etcd.client.kv.KvClient.Watch import com.ibm.etcd.client.kv.WatchUpdate import com.ibm.etcd.client.{EtcdClient => Client} -import com.sksamuel.elastic4s.http.ElasticClient +import com.sksamuel.elastic4s.ElasticClient import common.StreamLogging import org.apache.openwhisk.common.{GracefulShutdown, TransactionId} import org.apache.openwhisk.core.ack.ActiveAck From 74b14c5fdfc06b31749cf62ade4fde1d85017b73 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 14:05:27 -0500 Subject: [PATCH 17/19] removed ambiguous import --- .../core/scheduler/queue/test/MemoryQueueTestsFixture.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index 86236d49b2f..5a6373ea7df 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -7,7 +7,6 @@ import com.sksamuel.elastic4s.Executor import com.sksamuel.elastic4s.ElasticDsl.{avgAgg, boolQuery, matchQuery, rangeQuery, search} import com.sksamuel.elastic4s._ import com.sksamuel.elastic4s.requests.common.Shards -import com.sksamuel.elastic4s.requests.searches.ScoreMode.Total import com.sksamuel.elastic4s.requests.searches.{SearchHits, SearchRequest, SearchResponse, Total} import common.StreamLogging import org.apache.kafka.clients.producer.RecordMetadata From 97734cb6a1a832f9fdab46da78be805109188f59 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 14:22:03 -0500 Subject: [PATCH 18/19] trying to fix a test --- .../core/scheduler/queue/test/MemoryQueueTestsFixture.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index 5a6373ea7df..ead4c5e99fc 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -205,10 +205,11 @@ class MemoryQueueTestsFixture (mockEsClient .execute[SearchRequest, SearchResponse, Future](_: SearchRequest)( - _: Functor[Future], _: Executor[Future], + _: Functor[Future], _: Handler[SearchRequest, SearchResponse], - _: Manifest[SearchResponse])) + _: Manifest[SearchResponse], + _: CommonRequestOptions)) .expects(searchRequest, *, *, *, *) .returns( Future.successful(RequestSuccess( From db8bad0c5154859cf2b564aaf882acee6afee5d0 Mon Sep 17 00:00:00 2001 From: Dylan Turnbull Date: Tue, 12 Apr 2022 14:23:51 -0500 Subject: [PATCH 19/19] somehow fixed the test --- .../core/scheduler/queue/test/MemoryQueueTestsFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index ead4c5e99fc..751ee816ee1 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -210,7 +210,7 @@ class MemoryQueueTestsFixture _: Handler[SearchRequest, SearchResponse], _: Manifest[SearchResponse], _: CommonRequestOptions)) - .expects(searchRequest, *, *, *, *) + .expects(searchRequest, *, *, *, *, *) .returns( Future.successful(RequestSuccess( 200,