feat(test reports): test_report key functionality

cmd/vela-worker/exec.go

@@ -10,6 +10,7 @@
 	"sync"
 	"time"
 
+	"github.com/go-vela/server/storage"
 	"github.com/sirupsen/logrus"
 
 	"github.com/go-vela/sdk-go/vela"
@@ -25,9 +26,11 @@
 // exec is a helper function to poll the queue
 // and execute Vela pipelines for the Worker.
 //
-//nolint:nilerr,funlen // ignore returning nil - don't want to crash worker
+//nolint:funlen // ignore returning nil - don't want to crash worker
 func (w *Worker) exec(index int, config *api.Worker) error {
 	var err error
+	var execStorage storage.Storage
+	var _executor executor.Engine
 
 	// setup the version
 	v := version.New()
@@ -156,6 +159,13 @@
 	execOutputCtn := *w.Config.Executor.OutputCtn
 	execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID)
 
+	if w.Storage != nil {
+		execStorage = w.Storage
+		logrus.Debugf("executor storage is available, setting up storage")
+	} else {
+		logrus.Debugf("executor storage is nil, skipping storage setup")
+	}
+
 	// create logger with extra metadata
 	//
 	// https://pkg.go.dev/github.com/sirupsen/logrus#WithFields
@@ -229,11 +239,10 @@
 	if err != nil {
 		return err
 	}
-
 	// setup the executor
 	//
 	// https://pkg.go.dev/github.com/go-vela/worker/executor#New
-	_executor, err := executor.New(&executor.Setup{
+	setup := &executor.Setup{
 		Logger:              logger,
 		Mock:                w.Config.Mock,
 		Driver:              w.Config.Executor.Driver,
@@ -248,8 +257,17 @@
 		Pipeline:            p.Sanitize(w.Config.Runtime.Driver),
 		Version:             v.Semantic(),
 		OutputCtn:           &execOutputCtn,
-	})
+	}
 
+	if execStorage != nil {
+		fmt.Printf("setting up executor storage\n")
+		setup.Storage = execStorage
+	}
+	_executor, err = executor.New(setup)
+	if err != nil {
+		logger.Errorf("unable to setup executor: %v", err)
+		return err
+	}
 	// add the executor to the worker
 	w.Executors[index] = _executor
 

cmd/vela-worker/flags.go

@@ -11,6 +11,7 @@ import (
 	"github.com/urfave/cli/v3"
 
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 )
@@ -120,5 +121,8 @@ func flags() []cli.Flag {
 
 	f = append(f, runtime.Flags...)
 
+	// Storage Flags
+	f = append(f, storage.Flags...)
+
 	return f
 }

cmd/vela-worker/operate.go

@@ -6,6 +6,7 @@
 	"context"
 	"time"
 
+	"github.com/go-vela/server/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
@@ -19,7 +20,7 @@
 // queue and execute Vela pipelines.
 //
 //nolint:funlen // refactor candidate
 func (w *Worker) operate(ctx context.Context) error {
 	var err error
 	// create the errgroup for managing operator subprocesses
 	//
@@ -31,7 +32,7 @@
 	registryWorker.SetHostname(w.Config.API.Address.Hostname())
 	registryWorker.SetAddress(w.Config.API.Address.String())
 	registryWorker.SetActive(true)
 	registryWorker.SetBuildLimit(int32(w.Config.Build.Limit))
 
 	// set routes from config if set or defaulted to `vela`
 	if (len(w.Config.Queue.Routes) > 0) && (w.Config.Queue.Routes[0] != "NONE" && w.Config.Queue.Routes[0] != "") {
@@ -77,6 +78,56 @@
 		w.updateWorkerStatus(registryWorker, constants.WorkerStatusError)
 	}
 
+	// getting storage creds
+	logrus.Trace("getting storage s3 creds..")
+	// fetching queue credentials using registration token
+	stCreds, _, err := w.VelaClient.Storage.GetInfo()
+	if err != nil {
+		logrus.Tracef("error getting storage creds: %v", err)
+		return err
+	}
+
+	if stCreds.GetEnabled() {
+		logrus.Trace("storage enabled")
+		// if an address was given at start up, use that — else use what is returned from server
+		if len(w.Config.Storage.Endpoint) == 0 {
+			w.Config.Storage.Endpoint = stCreds.GetStorageAddress()
+		}
+
+		// set access key in storage config
+		w.Config.Storage.AccessKey = stCreds.GetAccessKey()
+
+		// set secret key in storage config
+		w.Config.Storage.SecretKey = stCreds.GetSecretKey()
+
+		// set bucket name in storage config
+		w.Config.Storage.Bucket = stCreds.GetStorageBucket()
+
+		// set storage enabled to true
+		w.Config.Storage.Enable = stCreds.GetEnabled()
+
+		s, err := storage.New(w.Config.Storage)
+		if err != nil {
+			logrus.Error("storage setup failed")
+			// set to error as storage setup fails
+			w.updateWorkerStatus(registryWorker, constants.WorkerStatusError)
+			return err
+		}
+		w.Storage = s
+		logrus.WithFields(logrus.Fields{
+			"driver":   w.Config.Storage.Driver,
+			"bucket":   w.Config.Storage.Bucket,
+			"endpoint": w.Config.Storage.Endpoint,
+		}).Debug("storage initialized")
+
+	} else {
+		logrus.Trace("storage not enabled")
+		// storage disabled; nothing to validate
+		w.Config.Storage.Enable = false
+		w.Storage = nil
+		logrus.Debug("storage disabled: worker storage unset")
+	}
+
 	// spawn goroutine for phoning home
 	executors.Go(func() error {
 		// five second ticker for signal handling

cmd/vela-worker/run.go

@@ -8,6 +8,7 @@
 	"net/url"
 
 	"github.com/gin-gonic/gin"
+	"github.com/go-vela/server/storage"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v3"
 
@@ -96,15 +97,15 @@
 			},
 			// build configuration
 			Build: &Build{
-				Limit:   int(c.Int("build.limit")),
+				Limit:   c.Int("build.limit"),
 				Timeout: c.Duration("build.timeout"),
 			},
 			// build configuration
 			CheckIn: c.Duration("checkIn"),
 			// executor configuration
 			Executor: &executor.Setup{
 				Driver:              c.String("executor.driver"),
-				MaxLogSize:          uint(c.Uint("executor.max_log_size")),
+				MaxLogSize:          c.Uint("executor.max_log_size"),
 				LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"),
 				EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"),
 				OutputCtn:           outputsCtn,
@@ -138,6 +139,13 @@
 				Address: c.String("server.addr"),
 				Secret:  c.String("server.secret"),
 			},
+			Storage: &storage.Setup{
+				Driver:    c.String("storage.driver"),
+				Endpoint:  c.String("storage.endpoint.name"),
+				AccessKey: c.String("storage.access.key"),
+				SecretKey: c.String("storage.secret.key"),
+				Bucket:    c.String("storage.bucket.name"),
+			},
 			// Certificate configuration
 			Certificate: &Certificate{
 				Cert: c.String("server.cert"),
@@ -172,5 +180,6 @@
 	}
 
 	// start the worker
+	//nolint: contextcheck // not using ctx yet
 	return w.Start()
 }

cmd/vela-worker/start.go

@@ -47,17 +47,21 @@ func (w *Worker) Start() error {
 		select {
 		case sig := <-signalChannel:
 			logrus.Infof("Received signal: %s", sig)
+
 			err := server.Shutdown(ctx)
 			if err != nil {
 				logrus.Error(err)
 			}
+
 			done()
 		case <-gctx.Done():
 			logrus.Info("Closing signal goroutine")
+
 			err := server.Shutdown(ctx)
 			if err != nil {
 				logrus.Error(err)
 			}
+
 			return gctx.Err()
 		}
 
@@ -67,7 +71,9 @@ func (w *Worker) Start() error {
 	// spawn goroutine for starting the server
 	g.Go(func() error {
 		var err error
+
 		logrus.Info("starting worker server")
+
 		if tlsCfg != nil {
 			if err := server.ListenAndServeTLS(w.Config.Certificate.Cert, w.Config.Certificate.Key); !errors.Is(err, http.ErrServerClosed) {
 				// log a message indicating the start of the server

cmd/vela-worker/worker.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-vela/sdk-go/vela"
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 )
@@ -57,6 +58,7 @@ type (
 		Server        *Server
 		Certificate   *Certificate
 		TLSMinVersion string
+		Storage       *storage.Setup
 	}
 
 	// Worker represents all configuration and
@@ -72,5 +74,6 @@ type (
 		RunningBuilds      []*api.Build
 		QueueCheckedIn     bool
 		RunningBuildsMutex sync.Mutex
+		Storage            storage.Storage
 	}
 )

executor/executor_test.go

@@ -16,6 +16,7 @@
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/mock/server"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor/linux"
 	"github.com/go-vela/worker/executor/local"
 	"github.com/go-vela/worker/runtime/docker"
@@ -37,6 +38,36 @@
 		t.Errorf("unable to create runtime engine: %v", err)
 	}
 
+	_storageT := &storage.Setup{
+		Enable:    true,
+		Driver:    "minio",
+		Endpoint:  "http://localhost:9000",
+		AccessKey: "ad",
+		SecretKey: "asd",
+		Bucket:    "vela",
+		Region:    "",
+		Secure:    false,
+	}
+	_sT, err := storage.New(_storageT)
+	if err != nil {
+		t.Errorf("unable to create storage engine: %v", err)
+	}
+
+	_storageF := &storage.Setup{
+		Enable:    false,
+		Driver:    "",
+		Endpoint:  "",
+		AccessKey: "",
+		SecretKey: "",
+		Bucket:    "",
+		Region:    "",
+		Secure:    false,
+	}
+	_sF, err := storage.New(_storageF)
+	if err != nil {
+		t.Errorf("unable to create storage engine: %v", err)
+	}
+
 	_linux, err := linux.New(
 		linux.WithBuild(_build),
 		linux.WithHostname("localhost"),
@@ -45,6 +76,7 @@
 		linux.WithRuntime(_runtime),
 		linux.WithVelaClient(_client),
 		linux.WithVersion("v1.0.0"),
+		linux.WithStorage(_sT),
 	)
 	if err != nil {
 		t.Errorf("unable to create linux engine: %v", err)
@@ -57,6 +89,7 @@
 		local.WithRuntime(_runtime),
 		local.WithVelaClient(_client),
 		local.WithVersion("v1.0.0"),
+		local.WithStorage(_sT),
 	)
 	if err != nil {
 		t.Errorf("unable to create local engine: %v", err)
@@ -80,6 +113,7 @@
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sF,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,
@@ -95,6 +129,7 @@
 				Pipeline:   _pipeline,
 				Runtime:    _runtime,
 				Version:    "v1.0.0",
+				Storage:    _sT,
 			},
 			want:  _linux,
 			equal: linux.Equal,
@@ -109,6 +144,7 @@
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  _local,
 			equal: local.Equal,
@@ -123,6 +159,7 @@
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,
@@ -137,6 +174,7 @@
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,
@@ -151,6 +189,7 @@
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,