Skip to content

Commit ae7edb4

Browse files
maleadtmaleadt
authored
Refactor CI pipeline a little. (#1805)
1 parent a2b5ec2 commit ae7edb4

File tree

1 file changed

+57
-108
lines changed

1 file changed

+57
-108
lines changed

‎.buildkite/pipeline.yml

Lines changed: 57 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ steps:
33
- group: ":julia: Julia"
44
key: "julia"
55
steps:
6-
- label: "Julia {{matrix.version}}"
6+
- label: "Julia {{matrix.julia}}"
77
plugins:
88
- JuliaCI/julia#v1:
9-
version: "{{matrix.version}}"
9+
version: "{{matrix.julia}}"
1010
- JuliaCI/julia-test#v1:
1111
test_args: "--quickfail"
1212
- JuliaCI/julia-coverage#v1:
@@ -24,15 +24,15 @@ steps:
2424
timeout_in_minutes: 120
2525
matrix:
2626
setup:
27-
version:
27+
julia:
2828
- "1.6"
2929
- "1.7"
3030
- "1.8"
3131
- "1.9-nightly"
3232
#- "nightly"
3333
adjustments:
3434
- with:
35-
version: "1.9-nightly"
35+
julia: "1.9-nightly"
3636
soft_fail: true
3737

3838
# then, test supported CUDA toolkits (installed through the artifact system)
@@ -43,10 +43,10 @@ steps:
4343
# NOTE: we support those CUDA versions for which the latest cuDNN is available
4444
# https://developer.nvidia.com/rdp/cudnn-archive
4545

46-
- label: "CUDA {{matrix}}"
46+
- label: "CUDA {{matrix.cuda}}"
4747
plugins:
4848
- JuliaCI/julia#v1:
49-
version: 1.6
49+
version: 1.8
5050
- JuliaCI/julia-test#v1:
5151
test_args: "--thorough"
5252
- JuliaCI/julia-coverage#v1:
@@ -61,68 +61,42 @@ steps:
6161
if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
6262
timeout_in_minutes: 120
6363
matrix:
64-
- "12.1"
65-
- "12.0"
66-
- "11.8"
67-
- "11.7"
68-
- "11.6"
69-
- "11.5"
70-
- "11.4"
71-
- "11.3"
72-
- "11.2"
73-
- "11.1"
74-
- "11.0"
64+
setup:
65+
cuda:
66+
- "12.1"
67+
- "12.0"
68+
- "11.8"
69+
- "11.7"
70+
- "11.6"
71+
- "11.5"
72+
- "11.4"
73+
- "11.3"
74+
- "11.2"
75+
- "11.1"
76+
- "11.0"
7577
commands: |
76-
julia --project -e 'using CUDA; CUDA.set_runtime_version!(v"{{matrix}}")'
78+
julia --project -e 'using CUDA; CUDA.set_runtime_version!(v"{{matrix.cuda}}")'
7779
7880
- group: ":nesting_dolls: Subpackages"
7981
depends_on: "cuda"
8082
steps:
81-
- label: "{{matrix}} on CUDA 11"
82-
matrix:
83-
- "cuDNN"
84-
- "cuTENSOR"
85-
- "cuStateVec"
86-
- "cuTensorNet"
87-
plugins:
88-
- JuliaCI/julia#v1:
89-
version: 1.6
90-
- JuliaCI/julia-coverage#v1:
91-
codecov: true
92-
dirs:
93-
- src
94-
- lib
95-
- examples
96-
agents:
97-
queue: "juliagpu"
98-
cuda: "*"
99-
if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
100-
timeout_in_minutes: 120
101-
commands: |
102-
julia -e '
103-
using Pkg
104-
105-
println("--- :julia: Instantiating project")
106-
Pkg.activate(joinpath(pwd(), "lib", lowercase("{{matrix}}")))
107-
if "{{matrix}}" == "cuTensorNet"
108-
# HACK: cuTensorNet depends on a development version of cuTENSOR
109-
Pkg.develop(path=joinpath(pwd(), "lib", "cutensor"))
110-
end
111-
Pkg.develop(path=pwd())
112-
Pkg.instantiate()
113-
114-
using CUDA
115-
CUDA.set_runtime_version!(v"11.8")
116-
117-
println("+++ :julia: Running tests")
118-
Pkg.test()'
119-
120-
- label: "{{matrix}} on CUDA 12"
83+
- label: "{{matrix.package}} on CUDA {{matrix.cuda}}"
12184
matrix:
122-
- "cuDNN"
85+
setup:
86+
cuda:
87+
- "11.0"
88+
package:
89+
- "cuDNN"
90+
- "cuTENSOR"
91+
- "cuStateVec"
92+
- "cuTensorNet"
93+
adjustments:
94+
- with:
95+
cuda: "12.0"
96+
package: "cuDNN"
12397
plugins:
12498
- JuliaCI/julia#v1:
125-
version: 1.6
99+
version: "1.8"
126100
- JuliaCI/julia-coverage#v1:
127101
codecov: true
128102
dirs:
@@ -139,16 +113,16 @@ steps:
139113
using Pkg
140114
141115
println("--- :julia: Instantiating project")
142-
Pkg.activate(joinpath(pwd(), "lib", lowercase("{{matrix}}")))
143-
if "{{matrix}}" == "cuTensorNet"
116+
Pkg.activate(joinpath(pwd(), "lib", lowercase("{{matrix.package}}")))
117+
if "{{matrix.package}}" == "cuTensorNet"
144118
# HACK: cuTensorNet depends on a development version of cuTENSOR
145119
Pkg.develop(path=joinpath(pwd(), "lib", "cutensor"))
146120
end
147121
Pkg.develop(path=pwd())
148122
Pkg.instantiate()
149123
150124
using CUDA
151-
CUDA.set_runtime_version!(v"12.0")
125+
CUDA.set_runtime_version!(v"{{matrix.cuda}}")
152126
153127
println("+++ :julia: Running tests")
154128
Pkg.test()'
@@ -159,7 +133,7 @@ steps:
159133
- label: "NNlibCUDA.jl on CUDA 12"
160134
plugins:
161135
- JuliaCI/julia#v1:
162-
version: 1.6
136+
version: 1.8
163137
- JuliaCI/julia-coverage#v1:
164138
codecov: true
165139
dirs:
@@ -193,7 +167,7 @@ steps:
193167
- label: "GPU-less environment"
194168
plugins:
195169
- JuliaCI/julia#v1:
196-
version: 1.6
170+
version: 1.8
197171
- JuliaCI/julia-coverage#v1:
198172
codecov: true
199173
dirs:
@@ -242,7 +216,7 @@ steps:
242216
- label: "Compute sanitizer"
243217
plugins:
244218
- JuliaCI/julia#v1:
245-
version: 1.6
219+
version: 1.8
246220
- JuliaCI/julia-test#v1:
247221
julia_args: "-g2"
248222
test_args: "--sanitize --quickfail --jobs=2"
@@ -269,22 +243,14 @@ steps:
269243

270244
- group: ":racehorse: Benchmarks"
271245
steps:
272-
# if we will submit results, use the benchmark queue so that we will
273-
# be running on the same system each time
274-
- label: "Benchmarks on 1.6"
246+
# benchmarks outside of the master branch don't submit their results,
247+
# so they can run on any system in the juliagpu queue.
248+
- label: "Benchmarks (dry run)"
275249
plugins:
276250
- JuliaCI/julia#v1:
277-
version: 1.6
278-
env:
279-
BENCHMARKS: "true"
280-
CODESPEED_PROJECT: "$BUILDKITE_PIPELINE_NAME"
281-
CODESPEED_BRANCH: "$BUILDKITE_BRANCH"
282-
CODESPEED_COMMIT: "$BUILDKITE_COMMIT"
283-
CODESPEED_EXECUTABLE: "Julia 1.6"
251+
version: 1.8
284252
command: |
285253
julia --project -e '
286-
ENV["CODESPEED_ENVIRONMENT"] = ENV["BUILDKITE_AGENT_NAME"]
287-
288254
println("--- :julia: Instantiating project")
289255
using Pkg
290256
Pkg.instantiate()
@@ -295,22 +261,25 @@ steps:
295261
println("+++ :julia: Benchmarking")
296262
include("perf/runbenchmarks.jl")'
297263
agents:
298-
queue: "benchmark"
264+
queue: "juliagpu"
299265
cuda: "*"
300266
if: build.message !~ /\[skip benchmarks\]/ &&
301-
build.branch =~ /^master$$/
267+
build.branch !~ /^master$$/ &&
268+
!build.pull_request.draft
302269
timeout_in_minutes: 30
303270

304-
- label: "Benchmarks on 1.7"
271+
# if we will submit results, use the benchmark queue so that we will
272+
# be running on the same system each time
273+
- label: "Benchmarks on Julia {{matrix.julia}}"
305274
plugins:
306275
- JuliaCI/julia#v1:
307-
version: 1.7
276+
version: "{{matrix.julia}}"
308277
env:
309278
BENCHMARKS: "true"
310279
CODESPEED_PROJECT: "$BUILDKITE_PIPELINE_NAME"
311280
CODESPEED_BRANCH: "$BUILDKITE_BRANCH"
312281
CODESPEED_COMMIT: "$BUILDKITE_COMMIT"
313-
CODESPEED_EXECUTABLE: "Julia 1.7"
282+
CODESPEED_EXECUTABLE: "Julia {{matrix.julia}}"
314283
command: |
315284
julia --project -e '
316285
ENV["CODESPEED_ENVIRONMENT"] = ENV["BUILDKITE_AGENT_NAME"]
@@ -329,31 +298,11 @@ steps:
329298
cuda: "*"
330299
if: build.message !~ /\[skip benchmarks\]/ &&
331300
build.branch =~ /^master$$/
332-
timeout_in_minutes: 30
333-
334-
# benchmarks outside of the master branch don't submit their results,
335-
# so they can run on any system in the juliagpu queue.
336-
- label: "Benchmarks (dry run)"
337-
plugins:
338-
- JuliaCI/julia#v1:
339-
version: 1.6
340-
command: |
341-
julia --project -e '
342-
println("--- :julia: Instantiating project")
343-
using Pkg
344-
Pkg.instantiate()
345-
Pkg.activate("perf")
346-
Pkg.instantiate()
347-
push!(LOAD_PATH, @__DIR__)
348-
349-
println("+++ :julia: Benchmarking")
350-
include("perf/runbenchmarks.jl")'
351-
agents:
352-
queue: "juliagpu"
353-
cuda: "*"
354-
if: build.message !~ /\[skip benchmarks\]/ &&
355-
build.branch !~ /^master$$/ &&
356-
!build.pull_request.draft
301+
matrix:
302+
setup:
303+
julia:
304+
- "1.6"
305+
- "1.7"
357306
timeout_in_minutes: 30
358307

359308
env:

0 commit comments

Comments
 (0)