Skip to content

Commit 6c481b1

Browse files
marcellodesalesmarcellodesales
committed
🔨 Workflows: build paralle docker images BUILDKIT
1 parent adf787b commit 6c481b1

File tree

3 files changed

+74
-48
lines changed

3 files changed

+74
-48
lines changed

‎.github/workflows/develop.yaml‎

Lines changed: 50 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -50,58 +50,75 @@ jobs:
5050
name: cloner.dockerimage
5151
path: dist/cloner-dependencies.dockerimage
5252

53-
build:
54-
name: Build CLI Binaries
55-
runs-on: ubuntu-latest
53+
# https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
54+
compile:
55+
name: Compile CLI as Binaries
56+
runs-on: ${{ matrix.os }}
57+
needs: dependencies
58+
strategy:
59+
matrix:
60+
os: [darwin, linux, windows]
61+
include:
62+
- os: darwin
63+
extension:
64+
- os: linux
65+
extension:
66+
- os: windows
67+
extension: .exe
5668
steps:
5769
- uses: actions/checkout@v2
5870
with:
5971
# https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version
6072
fetch-depth: 0
6173

62-
- name: Dockerized Cross-compile Build
63-
run: make build
74+
- name: Download Docker Image for Dependencies as cache
75+
uses: actions/download-artifact@v2
76+
with:
77+
name: cloner-dependencies.dockerimage
6478

65-
# Produces the binaries at the directory ./dist
66-
- name: Dockerized Binary Distribution
67-
run: make dist
79+
- name: Load Docker Image Dependencies for cache to compile
80+
run: |
81+
ls -la ./cloner.dockerimage
82+
docker load -i ./cloner-dependencies.dockerimage
6883
69-
# https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9
70-
# Produces the docker image at the directory ./dist/docker-image.raw
71-
- name: Save Raw Docker Image for Reuse
72-
run: make save-docker-image
84+
# Compiles for OS specific dependencies through Dockerkit
85+
- name: Compile for ${{matrix.os}}
86+
env:
87+
OS_NAME: ${{ matrix.os }}
88+
run: make compile-${OS_NAME}
7389

74-
# https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#passing-data-between-jobs-in-a-workflow
75-
- name: Upload MacOS Binary
76-
uses: actions/upload-artifact@v2
77-
with:
78-
name: cloner-darwin-amd64
79-
path: dist/cloner-darwin-amd64
90+
# Compiles for OS specific dependencies through Dockerkit
91+
- name: Get distribution binaries for ${{matrix.os}}
92+
env:
93+
OS_NAME: ${{ matrix.os }}
94+
run: make dist-${OS_NAME}
8095

81-
- name: Upload Linux Binary
96+
- name: Upload ${{ matrix.os }} Binary
8297
uses: actions/upload-artifact@v2
8398
with:
84-
name: cloner-linux-amd64
85-
path: dist/cloner-linux-amd64
99+
name: cloner-${{ matrix.os }}-amd64.${{ matrix.extension }}
100+
path: dist/cloner-${{ matrix.os }}-amd64.${{ matrix.extension }}
86101

87-
- name: Upload Windows Binary
88-
uses: actions/upload-artifact@v2
89-
with:
90-
name: cloner-windows-amd64.exe
91-
path: dist/cloner-windows-amd64.exe
102+
# Compiles for OS specific dependencies through Dockerkit
103+
- name: Save docker runtime image for ${{matrix.os}}
104+
if: matrix.os == 'linux'
105+
env:
106+
OS_NAME: ${{ matrix.os }}
107+
run: make save-docker-image
108+
# output: docker image saved at ./dist/cloner.dockerimage
92109

93-
# Local cache of docker images
94-
- name: Upload Docker Image
110+
- name: Upload runtime docker image
95111
uses: actions/upload-artifact@v2
112+
if: matrix.os == 'linux'
96113
with:
97114
name: cloner.dockerimage
98-
path: dist/cloner.dockerimage
115+
path: ./dist/cloner.dockerimage
99116

100117
# https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
101118
verify:
102119
name: Verify CLI Binaries
103120
runs-on: ${{ matrix.os }}
104-
needs: build
121+
needs: compile
105122
strategy:
106123
matrix:
107124
os: [ubuntu-latest, macOS-latest, windows-latest]
@@ -133,7 +150,7 @@ jobs:
133150
if: matrix.os == 'windows-latest'
134151
env:
135152
BIN_NAME: ${{ matrix.bin_name }}
136-
# https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070
153+
# start-process -nonewwindow https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070
137154
run: |
138155
dir
139156
echo $pwd\$env:BIN_NAME
@@ -142,7 +159,7 @@ jobs:
142159
e2e:
143160
name: Verify Dockerized E2E Test
144161
runs-on: ubuntu-latest
145-
needs: build
162+
needs: compile
146163
steps:
147164
- uses: actions/checkout@v2
148165
with:
@@ -173,7 +190,7 @@ jobs:
173190
push:
174191
name: Push CLI Docker Images
175192
runs-on: ubuntu-latest
176-
needs: build
193+
needs: compile
177194
steps:
178195
- uses: actions/checkout@v2
179196
with:

‎Makefile‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,22 +44,31 @@ build-dependencies: clean ## Builds the docker image only with dependencies usin
4444
@echo "Building dependencies for version $(BIN_VERSION) - Dependencies ONLY"
4545
DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) docker-compose build dependencies
4646

47-
compile-linux: build-dependencies ## Compiles for Linux
48-
@echo "Compiling version $(BIN_VERSION) for linux"
49-
DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build binaries
50-
51-
compile-darwin: build-dependencies ## Compiles for MacOS
52-
@echo "Compiling version $(BIN_VERSION) for darwin"
53-
DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=darwin docker-compose build binaries
47+
save-dependencies-docker-image: ## Saves the raw docker image of dependencies for cache
48+
ifndef GITHUB_ACTION
49+
$(error GITHUB_ACTION is undefined. This must run only by Github Actions)
50+
endif
51+
$(eval BUILD_IMAGE_TAG=$(shell BIN_VERSION=$(BIN_VERSION) docker-compose config | grep image: | grep dependencies | awk '{print $$2}'))
52+
docker save -o ./dist/$(APP_NAME)-dependencies.dockerimage $(BUILD_IMAGE_TAG)
53+
ls -la ./dist/$(APP_NAME)-dependencies.dockerimage
5454

55-
compile-windows: build-dependencies ## Compiles for Windows
56-
@echo "Compiling version $(BIN_VERSION) for windows"
57-
DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=windows docker-compose build binaries
55+
compile-%: build-dependencies ## Compiles for (darwin, linux, windows)
56+
$(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}'))
57+
@echo "Compiling version $(BIN_VERSION) for $(PLATFORM)"
58+
DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose build binaries
5859

5960
build-docker-runtime:
6061
@echo "Building linux runtime for version $(BIN_VERSION)"
6162
DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build runtime
6263

64+
dist-%: ## Makes the dir ./dist with binaries from docker image
65+
$(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}'))
66+
@echo "$(PLATFORM) Distribution binary for version $(BIN_VERSION)"
67+
$(eval DIST_IMAGE=$(shell DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose config | grep image: | grep binaries | awk '{print $$2}'))
68+
docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64s /bins" || true
69+
docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64.exe /bins" || true
70+
ls -la $(PWD)/$(DIST_DIR)
71+
6372
dist: build ## Makes the dir ./dist with binaries from docker image
6473
@echo "Distribution libraries for version $(BIN_VERSION)"
6574
docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(ORG)/$(APP_NAME):$(BIN_VERSION) -c "cp /usr/local/bin/$(APP_NAME)-darwin-amd64 /bins"

‎docker-compose.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,22 @@ services:
44

55
# Usually takes time to download, and can be reused for all platforms
66
dependencies:
7-
image: marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0}
7+
image: marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0}
88
build:
99
context: .
1010
target: dependencies
1111

1212
# Reuses the dependencies image as cache so we can parallelize binary builds
1313
binaries:
14-
image: marcellodesales/cloner-binaries-${PLATFORMS}:${BIN_VERSION:-0.1.0}
14+
image: marcellodesales/cloner/binaries-${PLATFORMS:-all}:${BIN_VERSION:-0.1.0}
1515
build:
1616
context: .
1717
args:
1818
- BIN_NAME=cloner
1919
- BIN_VERSION=${BIN_VERSION:-0.1.0}
20-
- PLATFORMS=${PLATFORMS}
20+
- PLATFORMS=${PLATFORMS:-darwin linux windows}
2121
cache_from:
22-
- marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0}
22+
- marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0}
2323
target: compiler
2424

2525
# Reuses the dependencies image as cache so we can parallelize binary builds
@@ -32,5 +32,5 @@ services:
3232
- BIN_VERSION=${BIN_VERSION:-0.1.0}
3333
- PLATFORMS=linux
3434
cache_from:
35-
- marcellodesales/cloner-binaries-linux:${BIN_VERSION:-0.1.0}
35+
- marcellodesales/cloner/binaries-linux:${BIN_VERSION:-0.1.0}
3636
target: runtime

0 commit comments

Comments
 (0)