diff --git a/README.md b/README.md index 03b09bc..9c48240 100644 --- a/README.md +++ b/README.md @@ -38,39 +38,41 @@ jobs: ## Inputs -| Variable | Description | -|----------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| `version` | ew-cli version to use | -| `api-token` | API token for emulator.wtf. We recommend using a secret for this. | -| `app` | Path to application apk file | -| `test` | Path to test apk file | -| `library-test` | Path to library test apk file | -| `outputs-dir` | Location to store test outputs in | -| `outputs` | Comma-separated list to specify what to download to output-dir. Defaults to `merged_results_xml,coverage,pulled_dirs`. | -| `record-video` | Set to true to record a video of the test run. Defaults to false. | -| `devices` | Device configurations to use, in the form of `model=X,version=Y` per line | -| `timeout` | Timeout for the test run, number with unit (`h`, `m` or `s`). Defaults to 15m. | -| `test-targets` | Test targets to run, e.g. `class foo.bar.Baz` will only run tests in that class | -| `use-orchestrator` | Whether to use the Android Test Orchestrator | -| `clear-package-data` | Whether to clear app data between every test (requires `use-orchestrator`) | -| `with-coverage` | Set to true to collect coverage files and save them to `outputs-dir` | -| `additional-apks` | Additional apks to install, one per line | -| `environment-variables` | Environment variables to pass to AndroidJUnitRunner, one per line in the form of `key=value` | -| `num-uniform-shards` | Set to a number larger than 1 to randomly split your tests into multiple shards to be executed in parallel | -| `num-shards` | Set to a number larger than 1 to split your tests into multiple shards based on test counts to be executed in parallel | -| `num-balanced-shards` | Set to a number larger than 1 to split your tests into multiple shards based on test execution time to be executed in parallel | -| `shard-target-runtime` | Target a specific runtime (in minutes), this will spawn as many shards as needed to hit the target runtime. | -| `directories-to-pull` | Directories to pull from device and store in `outputs-dir`, one per line | -| `side-effects` | Whether the test has any side effects, like hitting external APIs. Prevents any test caching and retries. | -| `num-flaky-test-attempts` | Number of times to retry flaky tests. Defaults to 0. | -| `flaky-test-repeat-mode` | Whether to repeat only flaky tests (failed_only) or full shards (all) with flaky test attempts. Defaults to failed_only. | -| `file-cache` | Whether to cache files between test runs. Defaults to true. | -| `file-cache-ttl` | How long to cache test files for. Defaults to 1h. | -| `test-cache` | Whether to cache test results between test runs. Defaults to true. | -| `proxy-host` | Configure a proxy host to use for all requests when making emulator.wtf API calls. | -| `proxy-port` | Configure a proxy port to use for all requests when making emulator.wtf API calls. | -| `proxy-user` | Set the proxy user to use for authentication. | -| `proxy-password` | Set the proxy password to use for authentication. | +| Variable | Description | +|----------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| `version` | ew-cli version to use | +| `api-token` | API token for emulator.wtf. We recommend using a secret for this. | +| `app` | Path to application apk file | +| `test` | Path to test apk file | +| `library-test` | Path to library test apk file | +| `additional-apks` | Additional apks to install, one per line | +| `file-cache-ttl` | How long to cache test files for. Defaults to 1h. | +| `file-cache` | Whether to cache files between test runs. Defaults to true. | +| `display-name` | Display name for the test run | +| `devices` | Device configurations to use, in the form of `model=X,version=Y` per line | +| `use-orchestrator` | Whether to use the Android Test Orchestrator | +| `clear-package-data` | Whether to clear app data between every test (requires `use-orchestrator`) | +| `num-flaky-test-attempts` | Number of times to retry flaky tests. Defaults to 0. | +| `flaky-test-repeat-mode` | Whether to repeat only flaky tests (failed_only) or full shards (all) with flaky test attempts. Defaults to failed_only. | +| `timeout` | Timeout for the test run, number with unit (`h`, `m` or `s`). Defaults to 15m. | +| `test-targets` | Test targets to run, e.g. `class foo.bar.Baz` will only run tests in that class | +| `environment-variables` | Environment variables to pass to AndroidJUnitRunner, one per line in the form of `key=value` | +| `test-cache` | Whether to cache test results between test runs. Defaults to true. | +| `side-effects` | Whether the test has any side effects, like hitting external APIs. Prevents any test caching and retries. | +| `num-balanced-shards` | Set to a number larger than 1 to split your tests into multiple shards based on test execution time to be executed in parallel | +| `shard-target-runtime` | Target a specific runtime (in minutes), this will spawn as many shards as needed to hit the target runtime. | +| `num-uniform-shards` | Set to a number larger than 1 to randomly split your tests into multiple shards to be executed in parallel | +| `num-shards` | Set to a number larger than 1 to split your tests into multiple shards based on test counts to be executed in parallel | +| `record-video` | Set to true to record a video of the test run. Defaults to false. | +| `with-coverage` | Set to true to collect coverage files and save them to `outputs-dir` | +| `directories-to-pull` | Directories to pull from device and store in `outputs-dir`, one per line | +| `outputs` | Comma-separated list to specify what to download to output-dir. Defaults to `merged_results_xml,coverage,pulled_dirs`. | +| `outputs-dir` | Location to store test outputs in | +| `proxy-host` | Configure a proxy host to use for all requests when making emulator.wtf API calls. | +| `proxy-port` | Configure a proxy port to use for all requests when making emulator.wtf API calls. | +| `proxy-user` | Set the proxy user to use for authentication. | +| `proxy-password` | Set the proxy password to use for authentication. | +| `async` | Run the test asynchronously, without waiting for the results. Useful if you're using the GitHub integration with check statuses. | ## Common examples diff --git a/action.yml b/action.yml index 61ca6d4..8c350f0 100644 --- a/action.yml +++ b/action.yml @@ -2,13 +2,16 @@ name: 'Run Android instrumentation tests' author: 'emulator.wtf' description: 'Run tests with emulator.wtf' inputs: + # ew-cli version: description: 'ew-cli version to use' required: true default: '0.12.6' + # Authentication api-token: description: 'Api token for emulator.wtf' required: false + # File options app: description: 'Application apk file' required: false @@ -18,78 +21,80 @@ inputs: library-test: description: 'Library test apk file' required: false - outputs-dir: - description: 'Location to store test outputs in' + additional-apks: + description: 'Additional apks to install, one per line' required: false - outputs: - description: "Comma-separated list to specify what to download to output-dir. Defaults to merged_results_xml,coverage,pulled_dirs" + file-cache-ttl: + description: "Max time to keep cached files in the backend followed by a unit (d, h, m or s), with the maximum value being 90d and minimum value being 5m (default: 1h)" required: false - record-video: - description: 'Set to true to record video of the test run' + file-cache: + description: "Set to false to disable remote file cache - APK and test-data will always be uploaded, even if they were uploaded before" + required: false + # Test configuration + display-name: + description: "Display name for the test run, this is used to identify the test run in the GitHub integration." required: false devices: description: 'Device configurations to use, in the form of model=X,version=Y per line' required: false - timeout: - description: 'Max time until this test run is cancelled, expressed by a number followed by an unit (h, m or s), with the maximum value being 3h and minimum value being 60s (default: 15m)' - required: false - test-targets: - description: 'Test targets to run, one operator-target per line, e.g. class foo.bar.Foobar' - required: false use-orchestrator: description: 'Set to true to use the Android Test Orchestrator' required: false clear-package-data: description: 'Set to true to clear app data between every test (only works with orchestrator)' required: false - with-coverage: - description: 'Set to true to collect coverage files and save them to ' + num-flaky-test-attempts: + description: "Number of times to re-run shards or jobs that had test failures. Repeat attempts will run in parallel. Default: 0." required: false - additional-apks: - description: 'Additional apks to install, one per line' + flaky-test-repeat-mode: + description: "Whether to repeat the whole failed shard (all) or only the failed tests (failed_only) in case of flaky tests. Default: failed_only." + required: false + timeout: + description: 'Max time until this test run is cancelled, expressed by a number followed by a unit (h, m or s), with the maximum value being 3h and minimum value being 60s (default: 15m)' + required: false + test-targets: + description: 'Test targets to run, one operator-target per line, e.g. class foo.bar.Foobar' required: false environment-variables: description: 'Environment variables to pass to AndroidJUnitRunner, one per line in the form of key=value' required: false - num-uniform-shards: - description: 'Set to a number larger than 1 to randomly split your tests into multiple shards.' + test-cache: + description: "Set to false to disable remote test cache - tests will always re-run, even if they were run before" required: false - num-shards: - description: 'Set to a number larger than 1 to split your tests into even shards based on test count.' + side-effects: + description: "Indicates that the test run has side effects, i.e. it hits external resources and might be a part of a bigger test suite. Adding this flag means that the test will not be automatically retried in case of errors. Default: false." required: false + # Test sharding num-balanced-shards: description: 'Set to a number larger than 1 to split your tests into even shards based on historical runtime.' required: false shard-target-runtime: - description: 'Target runtime for each shard in seconds, tests will be split based to shards targeting this runtime for each shard based on historical data on a best effort basis.' + description: 'Target runtime for each shard in minutes, tests will be split into shards targeting this runtime for each shard based on historical data on a best effort basis.' required: false - directories-to-pull: - description: 'Directories to pull from device and store in , one per line' - required: false - side-effects: - description: "Indicates that the test run has side effects, i.e. it hits external resources and might be a part of a bigger test suite. Adding this flag means that the test will not be automatically retried in case of errors. Default: false." - required: false - num-flaky-test-attempts: - description: "Number of times to re-run shards or jobs that had test failures. Repeat attempts will run in parallel. Default: 0." + num-uniform-shards: + description: 'Set to a number larger than 1 to randomly split your tests into multiple shards.' required: false - flaky-test-repeat-mode: - description: "Whether to repeat the whole failed shard (all) or only the failed tests (failed_only) in case of flaky tests. Default: failed_only." + num-shards: + description: 'Set to a number larger than 1 to split your tests into even shards based on test count.' required: false - file-cache: - description: "Set to false to disable remote file cache - APK and test-data will always be uploaded, even if they were uploaded before" + # Test outputs + record-video: + description: 'Set to true to record video of the test run' required: false - file-cache-ttl: - description: "Max time to keep cached files in the backend followed by an unit (d,h, m or s), with the maximum value being 90d and minimum value being 5m (default: 1h)" + with-coverage: + description: 'Set to true to collect coverage files and save them to ' required: false - test-cache: - description: "Set to false to disable remote test cache - tests will always re-run, even if they were run before" + directories-to-pull: + description: 'Directories to pull from device and store in , one per line' required: false - async: - description: "Run the test asynchronously, without waiting for the results. This shines when used together with GitHub integration." + outputs: + description: "Comma-separated list to specify what to download to output-dir. Defaults to merged_results_xml,coverage,pulled_dirs" required: false - display-name: - description: "Display name for the test run, this is used to identify the test run in the GitHub integration." + outputs-dir: + description: 'Location to store test outputs in' required: false + # Network options (emulator) + # Network options (action) proxy-host: description: "Configure a proxy host to use for all requests when making emulator.wtf API calls." required: false @@ -102,6 +107,10 @@ inputs: proxy-password: description: "Set the proxy password to use for authentication." required: false + # Operation + async: + description: "Run the test asynchronously, without waiting for the results. This shines when used together with GitHub integration." + required: false branding: color: orange icon: smartphone @@ -115,33 +124,43 @@ runs: - name: Run instrumented tests uses: emulator-wtf/invoke@v0.9.7 with: + # Authentication api-token: ${{ inputs.api-token }} + # File options app: ${{ inputs.app }} test: ${{ inputs.test }} library-test: ${{ inputs.library-test }} - outputs-dir: ${{ inputs.outputs-dir }} - outputs: ${{ inputs.outputs }} - record-video: ${{ inputs.record-video }} + additional-apks: ${{ inputs.additional-apks }} + file-cache-ttl: ${{ inputs.file-cache-ttl }} + file-cache: ${{ inputs.file-cache }} + # Test configuration + display-name: ${{ inputs.display-name }} devices: ${{ inputs.devices }} - timeout: ${{ inputs.timeout }} - test-targets: ${{ inputs.test-targets }} use-orchestrator: ${{ inputs.use-orchestrator }} clear-package-data: ${{ inputs.clear-package-data }} - with-coverage: ${{ inputs.with-coverage }} - additional-apks: ${{ inputs.additional-apks }} + num-flaky-test-attempts: ${{ inputs.num-flaky-test-attempts }} + flaky-test-repeat-mode: ${{ inputs.flaky-test-repeat-mode }} + timeout: ${{ inputs.timeout }} + test-targets: ${{ inputs.test-targets }} environment-variables: ${{ inputs.environment-variables }} + test-cache: ${{ inputs.test-cache }} + side-effects: ${{ inputs.side-effects }} + # Test Sharding + shard-target-runtime: ${{ inputs.shard-target-runtime }} + num-balanced-shards: ${{ inputs.num-balanced-shards }} num-uniform-shards: ${{ inputs.num-uniform-shards }} num-shards: ${{ inputs.num-shards }} - num-balanced-shards: ${{ inputs.num-balanced-shards }} + # Test outputs + record-video: ${{ inputs.record-video }} + with-coverage: ${{ inputs.with-coverage }} directories-to-pull: ${{ inputs.directories-to-pull }} - side-effects: ${{ inputs.side-effects }} - num-flaky-test-attempts: ${{ inputs.num-flaky-test-attempts }} - file-cache: ${{ inputs.file-cache }} - file-cache-ttl: ${{ inputs.file-cache-ttl }} - test-cache: ${{ inputs.test-cache }} - async: ${{ inputs.async }} - display-name: ${{ inputs.display-name }} + outputs: ${{ inputs.outputs }} + outputs-dir: ${{ inputs.outputs-dir }} + # Network options (emulator) + # Network options (action) proxy-host: ${{ inputs.proxy-host }} proxy-port: ${{ inputs.proxy-port }} proxy-user: ${{ inputs.proxy-user }} proxy-password: ${{ inputs.proxy-password }} + # Operation + async: ${{ inputs.async }}