Skip to content

Commit f4914c3

Browse files
cataphractcataphract
committed
Move helper-rust to dd-trace-php
1 parent cb826e6 commit f4914c3

38 files changed

+9685
-31
lines changed

‎.gitlab/generate-appsec.php‎

Lines changed: 197 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@
9494
-DENABLE_ASAN=$ASAN_FLAG"
9595
- ASAN_OPTIONS=malloc_context_size=0 make -j 4 xtest
9696

97-
"appsec integration tests":
97+
.appsec_integration_tests:
9898
stage: test
9999
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal # TODO: use a proper docker image with java pre-installed?
100100
tags: [ "docker-in-docker:amd64" ]
@@ -103,6 +103,40 @@
103103
KUBERNETES_MEMORY_REQUEST: 24Gi
104104
KUBERNETES_MEMORY_LIMIT: 30Gi
105105
ARCH: amd64
106+
HELPER_RUST_FLAG: ""
107+
before_script:
108+
<?php echo $ecrLoginSnippet, "\n"; ?>
109+
<?php dockerhub_login() ?>
110+
script:
111+
- apt update && apt install -y openjdk-17-jre
112+
- find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true
113+
- |
114+
cd appsec/tests/integration
115+
CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
116+
if [ -f "$CACHE_PATH" ]; then
117+
echo "Loading cache from $CACHE_PATH"
118+
TERM=dumb ./gradlew loadCaches --info
119+
fi
120+
121+
TERM=dumb ./gradlew $targets --info -Pbuildscan --scan $HELPER_RUST_FLAG
122+
TERM=dumb ./gradlew saveCaches --info
123+
after_script:
124+
- mkdir -p "${CI_PROJECT_DIR}/artifacts"
125+
- find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
126+
- .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec"
127+
artifacts:
128+
reports:
129+
junit: "artifacts/**/test-results/**/TEST-*.xml"
130+
paths:
131+
- "artifacts/"
132+
when: "always"
133+
cache:
134+
- key: "appsec int test cache"
135+
paths:
136+
- appsec/tests/integration/build/*.tar.gz
137+
138+
"appsec integration tests":
139+
extends: .appsec_integration_tests
106140
parallel:
107141
matrix:
108142
- targets:
@@ -128,32 +162,189 @@
128162
- test8.4-release-zts
129163
- test8.5-release
130164
- test8.5-release-zts
165+
166+
"appsec integration tests (helper-rust)":
167+
extends: .appsec_integration_tests
168+
variables:
169+
HELPER_RUST_FLAG: "-PuseHelperRust"
170+
parallel:
171+
matrix:
172+
- targets:
173+
- test7.4-release
174+
- test8.1-release
175+
- test8.3-debug
176+
- test8.4-release-zts
177+
178+
"helper-rust build and test":
179+
stage: test
180+
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
181+
tags: [ "docker-in-docker:amd64" ]
182+
interruptible: true
183+
rules:
184+
- if: $CI_COMMIT_BRANCH == "master"
185+
interruptible: false
186+
- when: on_success
187+
variables:
188+
KUBERNETES_CPU_REQUEST: 4
189+
KUBERNETES_MEMORY_REQUEST: 8Gi
190+
KUBERNETES_MEMORY_LIMIT: 10Gi
191+
ARCH: amd64
131192
before_script:
132193
<?php echo $ecrLoginSnippet, "\n"; ?>
133194
<?php dockerhub_login() ?>
134195
script:
135196
- apt update && apt install -y openjdk-17-jre
136-
- find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true
137197
- |
138198
cd appsec/tests/integration
139199
CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
140200
if [ -f "$CACHE_PATH" ]; then
141201
echo "Loading cache from $CACHE_PATH"
142202
TERM=dumb ./gradlew loadCaches --info
143203
fi
204+
# Build and test helper-rust (includes formatting check and cargo test)
205+
TERM=dumb ./gradlew testHelperRust --info -Pbuildscan --scan
206+
TERM=dumb ./gradlew saveCaches --info
207+
cache:
208+
- key: "appsec int test cache"
209+
paths:
210+
- appsec/tests/integration/build/*.tar.gz
144211

145-
TERM=dumb ./gradlew $targets --info -Pbuildscan --scan
212+
"helper-rust code coverage":
213+
stage: test
214+
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
215+
tags: [ "docker-in-docker:amd64" ]
216+
interruptible: true
217+
rules:
218+
- if: $CI_COMMIT_BRANCH == "master"
219+
interruptible: false
220+
- when: on_success
221+
variables:
222+
KUBERNETES_CPU_REQUEST: 4
223+
KUBERNETES_MEMORY_REQUEST: 8Gi
224+
KUBERNETES_MEMORY_LIMIT: 10Gi
225+
ARCH: amd64
226+
before_script:
227+
<?php echo $ecrLoginSnippet, "\n"; ?>
228+
<?php dockerhub_login() ?>
229+
script:
230+
- apt update && apt install -y openjdk-17-jre
231+
- |
232+
echo "Installing codecov CLI"
233+
curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import
234+
CODECOV_VERSION=0.6.1
235+
curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov
236+
curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM
237+
curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig
238+
gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
239+
shasum -a 256 -c codecov.SHA256SUM
240+
rm codecov.SHA256SUM.sig codecov.SHA256SUM
241+
chmod +x codecov
242+
mv codecov /usr/local/bin/codecov
243+
- |
244+
echo "Installing vault for codecov token"
245+
curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip
246+
unzip vault.zip
247+
mv vault /usr/local/bin/vault
248+
rm vault.zip
249+
- |
250+
cd appsec/tests/integration
251+
CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
252+
if [ -f "$CACHE_PATH" ]; then
253+
echo "Loading cache from $CACHE_PATH"
254+
TERM=dumb ./gradlew loadCaches --info
255+
fi
256+
# Run unit tests with coverage instrumentation
257+
TERM=dumb ./gradlew coverageHelperRust --info -Pbuildscan --scan
146258
TERM=dumb ./gradlew saveCaches --info
259+
- |
260+
echo "Extracting coverage data from Docker volume"
261+
mkdir -p "$CI_PROJECT_DIR"/appsec/helper-rust
262+
docker run --rm -v php-helper-rust-coverage:/vol alpine cat /vol/coverage-unit.lcov > "$CI_PROJECT_DIR"/appsec/helper-rust/coverage-unit.lcov
263+
- |
264+
echo "Uploading helper-rust unit test coverage to codecov"
265+
cd "$CI_PROJECT_DIR"
266+
CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token)
267+
codecov -t "$CODECOV_TOKEN" -n helper-rust-unit -F helper-rust-unit -v -f appsec/helper-rust/coverage-unit.lcov
268+
artifacts:
269+
paths:
270+
- appsec/helper-rust/coverage-unit.lcov
271+
when: always
272+
cache:
273+
- key: "appsec int test cache"
274+
paths:
275+
- appsec/tests/integration/build/*.tar.gz
276+
277+
"helper-rust integration coverage":
278+
stage: test
279+
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
280+
tags: [ "docker-in-docker:amd64" ]
281+
interruptible: true
282+
rules:
283+
- if: $CI_COMMIT_BRANCH == "master"
284+
interruptible: false
285+
- when: on_success
286+
variables:
287+
KUBERNETES_CPU_REQUEST: 8
288+
KUBERNETES_MEMORY_REQUEST: 24Gi
289+
KUBERNETES_MEMORY_LIMIT: 30Gi
290+
ARCH: amd64
291+
before_script:
292+
<?php echo $ecrLoginSnippet, "\n"; ?>
293+
<?php dockerhub_login() ?>
294+
script:
295+
- apt update && apt install -y openjdk-17-jre
296+
- |
297+
echo "Installing codecov CLI"
298+
curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import
299+
CODECOV_VERSION=0.6.1
300+
curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov
301+
curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM
302+
curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig
303+
gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
304+
shasum -a 256 -c codecov.SHA256SUM
305+
rm codecov.SHA256SUM.sig codecov.SHA256SUM
306+
chmod +x codecov
307+
mv codecov /usr/local/bin/codecov
308+
- |
309+
echo "Installing vault for codecov token"
310+
curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip
311+
unzip vault.zip
312+
mv vault /usr/local/bin/vault
313+
rm vault.zip
314+
- |
315+
cd appsec/tests/integration
316+
CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
317+
if [ -f "$CACHE_PATH" ]; then
318+
echo "Loading cache from $CACHE_PATH"
319+
TERM=dumb ./gradlew loadCaches --info
320+
fi
321+
# Build helper-rust with coverage instrumentation
322+
TERM=dumb ./gradlew buildHelperRustWithCoverage --info -Pbuildscan --scan
323+
# Run integration tests with coverage-instrumented binary
324+
TERM=dumb ./gradlew test8.3-debug --info -Pbuildscan --scan -PuseHelperRust
325+
# Generate coverage report from profraw files
326+
TERM=dumb ./gradlew generateHelperRustIntegrationCoverage --info -Pbuildscan --scan
327+
TERM=dumb ./gradlew saveCaches --info
328+
- |
329+
echo "Extracting coverage data from Docker volume"
330+
mkdir -p "$CI_PROJECT_DIR"/appsec/helper-rust
331+
docker run --rm -v php-helper-rust-coverage:/vol alpine cat /vol/coverage-integration.lcov > "$CI_PROJECT_DIR"/appsec/helper-rust/coverage-integration.lcov
332+
- |
333+
echo "Uploading helper-rust integration test coverage to codecov"
334+
cd "$CI_PROJECT_DIR"
335+
CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token)
336+
codecov -t "$CODECOV_TOKEN" -n helper-rust-integration -F helper-rust-integration -v -f appsec/helper-rust/coverage-integration.lcov
147337
after_script:
148338
- mkdir -p "${CI_PROJECT_DIR}/artifacts"
149-
- find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
150-
- .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec"
339+
- find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \; || true
340+
- .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec" || true
151341
artifacts:
152342
reports:
153343
junit: "artifacts/**/test-results/**/TEST-*.xml"
154344
paths:
155345
- "artifacts/"
156-
when: "always"
346+
- appsec/helper-rust/coverage-integration.lcov
347+
when: always
157348
cache:
158349
- key: "appsec int test cache"
159350
paths:

‎.gitmodules‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,7 @@
1414
[submodule "tea/benchmarks/google-benchmark"]
1515
path = tea/benchmarks/google-benchmark
1616
url = https://github.com/google/benchmark.git
17+
[submodule "appsec/third_party/libddwaf-rust"]
18+
path = appsec/third_party/libddwaf-rust
19+
url = https://github.com/DataDog/libddwaf-rust.git
20+
branch = glopes/v2

‎appsec/helper-rust/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/target

0 commit comments

Comments
 (0)