🌱 Improve build/test times in Github actions

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ env:
   # sometimes we need to explicitly override the Go version, ex. CVEs,
   # when it is not possible to update the go.mod version yet, ex. the
   # internal builds do not yet support that version of Go.
-  GO_VERSION:
+  GO_VERSION: ""
 
 on:
   pull_request:
@@ -61,12 +61,24 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Run go mod tidy
       run: make modules
     - name: Verify go modules have not changed
@@ -88,12 +100,24 @@ jobs:
         cache: 'npm'
         cache-dependency-path: 'hack/quicktype/package-lock.json'
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Verify codegen
       run: make verify-codegen
 
@@ -111,12 +135,24 @@ jobs:
         cache: 'npm'
         cache-dependency-path: 'hack/quicktype/package-lock.json'
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Verify unfocus
       run: make verify-unfocus
 
@@ -128,12 +164,24 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Create kind cluster
       id: create_kind_cluster
       run: make kind-up
@@ -153,19 +201,31 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
     - name: Setup the cache for golangci-lint
       uses: actions/cache@v4
       with:
-        key: golangci-lint-${{ runner.os }}-go${{ env.GO_VERSION }}-${{ hashFiles('go.sum', 'hack/tools/go.sum') }}
+        key: golangci-lint-${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-${{ hashFiles('go.sum', 'hack/tools/go.sum') }}
         path: |
           ~/.cache/golangci-lint
           hack/tools/bin/golangci-lint
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Lint Go
       run: make lint-go-full
 
@@ -177,18 +237,29 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
     - name: Setup the cache for govulncheck
       uses: actions/cache@v4
       with:
-        key: govulncheck-${{ runner.os }}-go${{ env.GO_VERSION }}-${{ hashFiles('go.sum', 'hack/tools/go.sum') }}
-        path: |
-          hack/tools/bin/govulncheck
+        key: govulncheck-${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-${{ hashFiles('go.sum', 'hack/tools/go.sum') }}
+        path: hack/tools/bin/govulncheck
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Vulncheck Go
       run: make vulncheck-go
 
@@ -200,12 +271,24 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Build Image
       run: GOOS=linux GOARCH=amd64 make image-build
 
@@ -217,12 +300,24 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
+      id:   install-go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
         go-version-file: 'go.mod'
-        cache: true
-        cache-dependency-path: '**/go.sum'
+        cache: false
+    - name: Go mod cache
+      uses: actions/cache@v4
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-mod-cache-${{ hashFiles('**/go.sum') }}
+    - name: Go build cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build
+        key: ${{ runner.os }}-go-${{ steps.install-go.outputs.go-version }}-build-cache-${{ hashFiles('**/go.sum', '**/*.go') }}
+    - name: Restore go source mtime
+      run: make restore-mtime
     - name: Test
       run: make test
     - name: Convert coverage to XML report

Makefile

@@ -92,6 +92,7 @@ KUBECTL            := $(TOOLS_BIN_DIR)/kubectl
 ETCD               := $(TOOLS_BIN_DIR)/etcd
 GOVULNCHECK        := $(TOOLS_BIN_DIR)/govulncheck
 KIND               := $(TOOLS_BIN_DIR)/kind
+MTIMEHASH          := $(TOOLS_BIN_DIR)/mtimehash
 
 # Allow overriding manifest generation destination directory
 MANIFEST_ROOT     ?= config
@@ -241,6 +242,10 @@ coverage-xml:
 coverage: ## Show test coverage in browser
 	go tool cover -html="$(COVERAGE_FILE)"
 
+# Used with GitHub actions
+.PHONY: restore-mtime
+restore-mtime: | $(MTIMEHASH)
+	find . -type f -name '*.go' | $(MTIMEHASH)
 
 ## --------------------------------------
 ## Binaries
@@ -274,7 +279,8 @@ $(VMCLASS): cmd/vmclass/main.go
 TOOLING_BINARIES := $(CRD_REF_DOCS) $(CONTROLLER_GEN) $(CONVERSION_GEN) \
                     $(GOLANGCI_LINT) $(KUSTOMIZE) \
                     $(KUBE_APISERVER) $(KUBEBUILDER) $(KUBECTL) $(ETCD) \
-                    $(GINKGO) $(GOCOV) $(GOCOV_XML) $(GOVULNCHECK) $(KIND)
+                    $(GINKGO) $(GOCOV) $(GOCOV_XML) $(GOVULNCHECK) $(KIND) \
+                    $(MTIMEHASH)
 tools: $(TOOLING_BINARIES) ## Build tooling binaries
 $(TOOLING_BINARIES):
 	make -C $(TOOLS_DIR) $(@F)

hack/tools/Makefile

@@ -42,6 +42,7 @@ GOCOV             := $(BIN_DIR)/$(GOHOSTOSARCH)/gocov
 GOCOV_XML         := $(BIN_DIR)/$(GOHOSTOSARCH)/gocov-xml
 GOVULNCHECK       := $(BIN_DIR)/$(GOHOSTOSARCH)/govulncheck
 GOIMPORTS         := $(BIN_DIR)/$(GOHOSTOSARCH)/goimports
+MTIMEHASH         := $(BIN_DIR)/$(GOHOSTOSARCH)/mtimehash
 
 ## --------------------------------------
 ## Help
@@ -130,6 +131,10 @@ $(ETCD) $(KUBE_APISERVER) $(KUBECTL): k8s-envtest  ## Install envtest related bi
 	@mkdir -p $(@D)
 	@cp -f "$$($(SETUP_ENVTEST) use -p path $(K8S_VERSION))/$(@F)" $(@)
 
+mtimehash: $(MTIMEHASH) ## Install mtimehash
+$(MTIMEHASH):
+	GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \
+	go build -tags=vmop_tools -o $(@) github.com/slsyy/mtimehash/cmd/mtimehash
 
 ## --------------------------------------
 ## Generate

hack/tools/go.mod

@@ -1,8 +1,8 @@
 module github.com/vmware-tanzu/vm-operator/hack/tools
 
-go 1.23.0
+go 1.23.4
 
-toolchain go1.24.4
+toolchain go1.24.7
 
 // The version of Ginkgo must match the one from ../../go.mod.
 require github.com/onsi/ginkgo/v2 v2.23.4
@@ -12,6 +12,7 @@ require (
 	github.com/axw/gocov v1.1.0
 	github.com/elastic/crd-ref-docs v0.0.12
 	github.com/golangci/golangci-lint/v2 v2.1.6
+	github.com/slsyy/mtimehash v1.0.0
 	golang.org/x/tools v0.32.0
 	golang.org/x/vuln v1.1.1
 	k8s.io/code-generator v0.31.0
@@ -68,6 +69,7 @@ require (
 	github.com/charmbracelet/x/term v0.2.1 // indirect
 	github.com/chavacava/garif v0.1.0 // indirect
 	github.com/ckaznocha/intrange v0.3.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
 	github.com/curioswitch/go-reassign v0.3.0 // indirect
 	github.com/daixiang0/gci v0.13.6 // indirect
 	github.com/dave/dst v0.27.3 // indirect
@@ -197,6 +199,7 @@ require (
 	github.com/raeperd/recvcheck v0.2.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryancurrah/gomodguard v1.4.1 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
 	github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect
@@ -207,6 +210,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sivchari/containedctx v1.0.3 // indirect
 	github.com/sonatard/noctx v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/sourcegraph/go-diff v0.7.0 // indirect
 	github.com/spf13/afero v1.14.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
@@ -227,12 +231,14 @@ require (
 	github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
 	github.com/ultraware/funlen v0.2.0 // indirect
 	github.com/ultraware/whitespace v0.2.0 // indirect
+	github.com/urfave/cli/v2 v2.27.5 // indirect
 	github.com/uudashr/gocognit v1.2.0 // indirect
 	github.com/uudashr/iface v1.3.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xen0n/gosmopolitan v1.3.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect
 	github.com/yeya24/promlinter v0.3.0 // indirect
 	github.com/ykadowak/zerologlint v0.1.5 // indirect

hack/tools/go.sum

@@ -99,6 +99,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs=
 github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk=
+github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs=
@@ -438,6 +439,7 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryancurrah/gomodguard v1.4.1 h1:eWC8eUMNZ/wM/PWuZBv7JxxqT5fiIKSIyTvjb7Elr+g=
 github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I=
@@ -461,8 +463,12 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE=
 github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4=
+github.com/slsyy/mtimehash v1.0.0 h1:Wo9873vX4bX6wyIy16jUFWENXP9wjC56Cuhe4DlM2x0=
+github.com/slsyy/mtimehash v1.0.0/go.mod h1:hLHlOIQ5hRqTQ7M5Pmf/Dx4m53GV5sE57rRHEzfefOM=
 github.com/sonatard/noctx v0.1.0 h1:JjqOc2WN16ISWAjAk8M5ej0RfExEXtkEyExl2hLW+OM=
 github.com/sonatard/noctx v0.1.0/go.mod h1:0RvBxqY8D4j9cTTTWE8ylt2vqj2EPI8fHmrxHdsaZ2c=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0=
 github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
 github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA=
@@ -520,6 +526,8 @@ github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLk
 github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA=
 github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g=
 github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/uudashr/gocognit v1.2.0 h1:3BU9aMr1xbhPlvJLSydKwdLN3tEUUrzPSSM8S4hDYRA=
 github.com/uudashr/gocognit v1.2.0/go.mod h1:k/DdKPI6XBZO1q7HgoV2juESI2/Ofj9AcHPZhBBdrTU=
 github.com/uudashr/iface v1.3.1 h1:bA51vmVx1UIhiIsQFSNq6GZ6VPTk3WNMZgRiCe9R29U=
@@ -532,6 +540,8 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=
 github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk=
 github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs=