Skip to content

Commit aff489b

Browse files
ysaito1001ysaito1001
authored
Run canary as part of CI (#3525)
## Motivation and Context CI now runs [canary](https://github.com/smithy-lang/smithy-rs/tree/main/tools/ci-cdk/canary-lambda/src/latest) on pull requests. ## Description Historically, canary has been running only during releases and daily in the [aws-sdk-rust](https://github.com/awslabs/aws-sdk-rust/actions/workflows/canary.yaml) repository. This presents a problem of action at a distance where a potential bug that may hurt canary won't immediately be caught when a PR is merged to `main` in `smithy-rs` (such as code under code under `#[cfg(debug_assertions)]` was not used during canary's lambda execution and the definition of an item only used under that resulted in unused warning). This PR will address that problem. PRs from forked repositories cannot run the `Canary` job (it will be skipped). A maintainer can run it on their behalf within `smithy-lang:smithy-rs` using a newly added [manual workflow](https://github.com/smithy-lang/smithy-rs/pull/3525/files#diff-1b1b5b27850107cb97519c98de99d35a49b90277ccb52201b842e9b81feb5d47). After we merge this to main, we will update the branch protection so that the `Canary` job is required for merge, in addition to ` Matrix Success ` (this will not affect PRs from forked repositories since the `Canary` job will be skipped a skipped job will [report its status as Success](https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution), but a manual canary run should pass). ## Testing - Verified a PR in `smithy-lang:smithy-rs` ran the canary ([passed](https://github.com/smithy-lang/smithy-rs/actions/runs/8473703578/job/23219021189?pr=3525)). - Verified a PR in a `smithy-rs` fork cannot run the canary and [indicated that a maintainer needs to run it manually](https://github.com/smithy-lang/smithy-rs/actions/runs/8473799146/job/23218944122?pr=3528) - Verified a maintainer can manually invoke the canary using a PR from a fork ([passed](https://github.com/smithy-lang/smithy-rs/actions/runs/8474050953)) - Verified internal release pipeline passed with [the changes to canary-runner](https://github.com/smithy-lang/smithy-rs/pull/3525/files#diff-cc953ba68fec8cf937e1aa70181a901150fc5f1a74bf6ca1075d522c63d1eb6f) ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._
1 parent 879a8e6 commit aff489b

File tree

11 files changed

+223
-25
lines changed

11 files changed

+223
-25
lines changed

‎.github/workflows/canary.yml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎.github/workflows/ci-main.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,5 @@ jobs:
6868
secrets:
6969
ENCRYPTED_DOCKER_PASSWORD: ${{ needs.acquire-base-image.outputs.docker-login-password }}
7070
DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
71+
CANARY_GITHUB_ACTIONS_ROLE_ARN: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
72+
CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }}

‎.github/workflows/ci-merge-queue.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,5 @@ jobs:
9191
secrets:
9292
ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }}
9393
DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
94+
CANARY_GITHUB_ACTIONS_ROLE_ARN: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
95+
CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }}

‎.github/workflows/ci-pr-forks.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,5 @@ jobs:
4242
if: ${{ github.event.pull_request.head.repo.full_name != 'smithy-lang/smithy-rs' }}
4343
uses: ./.github/workflows/ci.yml
4444
with:
45+
run_canary: false
4546
run_sdk_examples: true

‎.github/workflows/ci-pr.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ jobs:
9393
secrets:
9494
ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }}
9595
DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
96+
CANARY_GITHUB_ACTIONS_ROLE_ARN: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
97+
CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }}
9698

9799
# The PR bot requires a Docker build image, so make it depend on the `acquire-base-image` job.
98100
pr_bot:

‎.github/workflows/ci.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ name: Test
99
on:
1010
workflow_call:
1111
inputs:
12+
run_canary:
13+
description: Whether to run the canary or not.
14+
required: false
15+
default: true
16+
type: boolean
1217
run_sdk_examples:
1318
description: Whether to run the SDK example checks or not.
1419
required: false
@@ -27,6 +32,10 @@ on:
2732
required: false
2833
DOCKER_LOGIN_TOKEN_PASSPHRASE:
2934
required: false
35+
CANARY_GITHUB_ACTIONS_ROLE_ARN:
36+
required: false
37+
CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME:
38+
required: false
3039

3140
env:
3241
rust_version: 1.74.1
@@ -315,6 +324,49 @@ jobs:
315324
shell: bash
316325
run: cross test --target ${{ matrix.target }} --manifest-path "aws/rust-runtime/Cargo.toml" ${{ matrix.test_aws_exclude }} --workspace
317326

327+
# Run the canary against generated SDKs
328+
#
329+
# In addition to Matrix Success, this job will also be required to pass for merge.
330+
# CI execution from forked repositories will skip this job, and when it does
331+
# this skipped job will report its status as "Success".
332+
# https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution#overview
333+
canary:
334+
name: Canary
335+
if: ${{ inputs.run_canary }}
336+
needs: generate
337+
runs-on: smithy_ubuntu-latest_8-core
338+
permissions:
339+
id-token: write
340+
contents: read
341+
steps:
342+
- uses: actions/checkout@v4
343+
with:
344+
path: smithy-rs
345+
ref: ${{ inputs.git_ref }}
346+
- name: Configure credentials
347+
id: creds
348+
uses: aws-actions/configure-aws-credentials@v4
349+
with:
350+
aws-region: us-west-2
351+
role-to-assume: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
352+
output-credentials: true
353+
- name: Run canary
354+
uses: ./smithy-rs/.github/actions/docker-build
355+
with:
356+
action: run-canary
357+
action-arguments: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} ${{ steps.creds.outputs.aws-access-key-id }} ${{ steps.creds.outputs.aws-secret-access-key }} ${{ steps.creds.outputs.aws-session-token }}
358+
359+
# This is always a failing job since forked repositories do not have necessary repository secrets
360+
# to run the PR bot workflow or the canary workflow
361+
ask-maintainer-to-run-pr-bot-and-canary:
362+
name: Ask maintainer to run the PR bot and canary workflows
363+
if: ${{ !inputs.run_canary }}
364+
runs-on: ubuntu-latest
365+
steps:
366+
- run: |
367+
echo "PR bot and canary cannot be invoked from a forked repository. Ask a maintainer to manually invoke them using your PR."
368+
exit 1
369+
318370
# This job is split out from the rest since it is not required to pass for merge
319371
check-sdk-examples:
320372
name: Check SDK Examples

‎.github/workflows/manual-canary.yml‎

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,98 @@ on:
88
description: The PR number to invoke the canary for.
99
required: true
1010
type: string
11+
run-name: ${{ github.workflow }} for Pull Request ${{ inputs.pull_request_number }}
12+
13+
# Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed
14+
concurrency:
15+
group: manual-canary-${{ inputs.pull_request_number }}
16+
cancel-in-progress: true
1117

1218
jobs:
19+
get-pr-info:
20+
name: Get PR info
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: Get PR info
24+
id: get-pr-info
25+
uses: actions/github-script@v7
26+
with:
27+
script: |
28+
const response = await github.rest.pulls.get({
29+
pull_number: ${{ inputs.pull_request_number }},
30+
owner: context.repo.owner,
31+
repo: context.repo.repo,
32+
});
33+
const data = {
34+
commit_sha: response.data.head.sha,
35+
};
36+
console.log("data:", data);
37+
return data;
38+
outputs:
39+
pull_data: ${{ steps.get-pr-info.outputs.result }}
40+
41+
acquire-base-image:
42+
name: Acquire Base Image
43+
needs:
44+
- get-pr-info
45+
runs-on: ubuntu-latest
46+
steps:
47+
- uses: actions/checkout@v4
48+
with:
49+
path: smithy-rs
50+
# The ref used needs to match the HEAD revision of the PR being diffed, or else
51+
# the `docker-build` action won't find the built Docker image.
52+
ref: ${{ fromJSON(needs.get-pr-info.outputs.pull_data).commit_sha }}
53+
fetch-depth: 0
54+
- name: Acquire base image
55+
id: acquire
56+
run: ./smithy-rs/.github/scripts/acquire-build-image
57+
- name: Upload base image
58+
uses: actions/upload-artifact@v4
59+
with:
60+
name: smithy-rs-base-image
61+
path: smithy-rs-base-image
62+
retention-days: 1
63+
64+
generate:
65+
name: Generate
66+
needs:
67+
- acquire-base-image
68+
- get-pr-info
69+
runs-on: smithy_ubuntu-latest_8-core
70+
steps:
71+
- uses: actions/checkout@v4
72+
with:
73+
path: smithy-rs
74+
ref: ${{ fromJSON(needs.get-pr-info.outputs.pull_data).commit_sha }}
75+
- name: Generate a subset of SDKs for the canary
76+
uses: ./smithy-rs/.github/actions/docker-build
77+
with:
78+
action: generate-aws-sdk-for-canary
79+
1380
canary:
1481
name: Canary
15-
runs-on: ubuntu-latest
82+
needs:
83+
- generate
84+
- get-pr-info
85+
runs-on: smithy_ubuntu-latest_8-core
86+
permissions:
87+
id-token: write
88+
contents: read
1689
steps:
17-
- name: Invoke canary
18-
run: echo "Hello World"
90+
- uses: actions/checkout@v4
91+
with:
92+
path: smithy-rs
93+
ref: ${{ fromJSON(needs.get-pr-info.outputs.pull_data).commit_sha }}
94+
- name: Configure credentials
95+
id: creds
96+
uses: aws-actions/configure-aws-credentials@v4
97+
with:
98+
aws-region: us-west-2
99+
role-to-assume: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
100+
output-credentials: true
101+
- name: Run canary
102+
uses: ./smithy-rs/.github/actions/docker-build
103+
with:
104+
action: run-canary
105+
action-arguments: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} ${{ steps.creds.outputs.aws-access-key-id }} ${{ steps.creds.outputs.aws-secret-access-key }} ${{ steps.creds.outputs.aws-session-token }}

‎tools/ci-build/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,12 +167,14 @@ RUN set -eux; \
167167
python3-pip \
168168
shadow-utils \
169169
cmake \
170-
tar; \
170+
tar \
171+
unzip; \
171172
yum clean all; \
172173
rm -rf /var/cache/yum; \
173174
groupadd build; \
174175
useradd -m -g build build; \
175176
chmod 775 /home/build;
177+
RUN set -eux; cd /tmp; curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o awscliv2.zip && unzip awscliv2.zip && ./aws/install
176178
COPY --chown=build:build --from=local_tools /opt/cargo /opt/cargo
177179
COPY --chown=build:build --from=cargo_deny /opt/cargo/bin/cargo-deny /opt/cargo/bin/cargo-deny
178180
COPY --chown=build:build --from=cargo_udeps /opt/cargo/bin/cargo-udeps /opt/cargo/bin/cargo-udeps

‎tools/ci-cdk/canary-runner/src/run.rs‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -153,15 +153,42 @@ impl Options {
153153
lambda_execution_role_arn: String,
154154
}
155155
#[derive(Deserialize)]
156-
struct Outer {
156+
enum Outer {
157157
#[serde(rename = "aws-sdk-rust-canary-stack")]
158-
inner: Inner,
158+
AwsSdkRust {
159+
#[serde(flatten)]
160+
aws_sdk_rust_canary_stack: Inner,
161+
},
162+
#[serde(rename = "smithy-rs-canary-stack")]
163+
SmithyRs {
164+
#[serde(flatten)]
165+
smithy_rs_canary_stack: Inner,
166+
},
167+
}
168+
impl Outer {
169+
fn into_inner(self) -> Inner {
170+
match self {
171+
Outer::AwsSdkRust {
172+
aws_sdk_rust_canary_stack,
173+
} => aws_sdk_rust_canary_stack,
174+
Outer::SmithyRs {
175+
smithy_rs_canary_stack,
176+
} => smithy_rs_canary_stack,
177+
}
178+
}
159179
}
160180

161181
let value: Outer = serde_json::from_reader(
162182
std::fs::File::open(cdk_output).context("open cdk output")?,
163183
)
164184
.context("read cdk output")?;
185+
let Inner {
186+
lambda_code_s3_bucket_name,
187+
lambda_test_s3_bucket_name,
188+
lambda_test_s3_mrap_bucket_arn,
189+
lambda_test_s3_express_bucket_name,
190+
lambda_execution_role_arn,
191+
} = value.into_inner();
165192
Ok(Options {
166193
rust_version: run_opt.rust_version,
167194
sdk_release_tag: run_opt.sdk_release_tag,
@@ -171,11 +198,11 @@ impl Options {
171198
lambda_function_memory_size_in_mb: run_opt
172199
.lambda_function_memory_size_in_mb
173200
.unwrap_or(DEFAULT_LAMBDA_FUNCTION_MEMORY_SIZE_IN_MB),
174-
lambda_code_s3_bucket_name: value.inner.lambda_code_s3_bucket_name,
175-
lambda_test_s3_bucket_name: value.inner.lambda_test_s3_bucket_name,
176-
lambda_test_s3_mrap_bucket_arn: value.inner.lambda_test_s3_mrap_bucket_arn,
177-
lambda_test_s3_express_bucket_name: value.inner.lambda_test_s3_express_bucket_name,
178-
lambda_execution_role_arn: value.inner.lambda_execution_role_arn,
201+
lambda_code_s3_bucket_name,
202+
lambda_test_s3_bucket_name,
203+
lambda_test_s3_mrap_bucket_arn,
204+
lambda_test_s3_express_bucket_name,
205+
lambda_execution_role_arn,
179206
})
180207
} else {
181208
Ok(Options {

‎tools/ci-scripts/generate-aws-sdk-for-canary‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
#
3+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
set -eux
8+
9+
cd smithy-rs
10+
11+
# Generate only SDKs used by canary (see BASE_MANIFEST in build_bundle.rs of canary-runner)
12+
./gradlew -Paws.services=+ec2,+s3,+sso,+ssooidc,+sts,+transcribestreaming :aws:sdk:assemble
13+
14+
mv aws/sdk/build/aws-sdk ../artifacts/

0 commit comments

Comments
 (0)