diff --git a/agent/job_runner.go b/agent/job_runner.go index 35fb1984a2..6a84efbe59 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -423,6 +423,16 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) { } } + // Wrap setting values in env, so that when any that were already present in + // supplied Job env are overwritten, they can be added to ignoredEnv. + var ignoredEnv []string + setEnv := func(name, value string) { + if _, exists := env[name]; exists { + ignoredEnv = append(ignoredEnv, name) + } + env[name] = value + } + // Write out the job environment to file: // - envShellFile: in k="v" format, with newlines escaped. If the // propagate-agent-vars experiment is enabled, the names of several agent @@ -488,29 +498,15 @@ BUILDKITE_AGENT_JWKS_KEY_ID` // Now that the env files have been written, we can add their corresponding // paths to the job env. if r.envShellFile != nil { - env["BUILDKITE_ENV_FILE"] = r.envShellFile.Name() + setEnv("BUILDKITE_ENV_FILE", r.envShellFile.Name()) } if r.envJSONFile != nil { - env["BUILDKITE_ENV_JSON_FILE"] = r.envJSONFile.Name() - } - - var ignoredEnv []string - - // Check if the user has defined any protected env - for k := range envutil.ProtectedEnv { - if _, exists := r.conf.Job.Env[k]; exists { - ignoredEnv = append(ignoredEnv, k) - } + setEnv("BUILDKITE_ENV_JSON_FILE", r.envJSONFile.Name()) } cache := r.conf.Job.Step.Cache if cache != nil && len(cache.Paths) > 0 { - env["BUILDKITE_AGENT_CACHE_PATHS"] = strings.Join(cache.Paths, ",") - } - - // Set BUILDKITE_IGNORED_ENV so the bootstrap can show warnings - if len(ignoredEnv) > 0 { - env["BUILDKITE_IGNORED_ENV"] = strings.Join(ignoredEnv, ",") + setEnv("BUILDKITE_AGENT_CACHE_PATHS", strings.Join(cache.Paths, ",")) } // Set BUILDKITE_SECRETS_CONFIG so bootstrap can access secrets configuration @@ -521,14 +517,14 @@ BUILDKITE_AGENT_JWKS_KEY_ID` return nil, err } - env["BUILDKITE_SECRETS_CONFIG"] = string(secretsJSON) + setEnv("BUILDKITE_SECRETS_CONFIG", string(secretsJSON)) } // Add the API configuration apiConfig := r.apiClient.Config() - env["BUILDKITE_AGENT_ENDPOINT"] = apiConfig.Endpoint - env["BUILDKITE_AGENT_ACCESS_TOKEN"] = apiConfig.Token - env["BUILDKITE_NO_HTTP2"] = fmt.Sprint(apiConfig.DisableHTTP2) + setEnv("BUILDKITE_AGENT_ENDPOINT", apiConfig.Endpoint) + setEnv("BUILDKITE_AGENT_ACCESS_TOKEN", apiConfig.Token) + setEnv("BUILDKITE_NO_HTTP2", fmt.Sprint(apiConfig.DisableHTTP2)) // ... including any server-specified request headers, so that sub-processes such as // buildkite-agent annotate etc can respect them. @@ -543,9 +539,9 @@ BUILDKITE_AGENT_JWKS_KEY_ID` } // Add agent environment variables - env["BUILDKITE_AGENT_DEBUG"] = fmt.Sprint(r.conf.Debug) - env["BUILDKITE_AGENT_DEBUG_HTTP"] = fmt.Sprint(r.conf.DebugHTTP) - env["BUILDKITE_AGENT_PID"] = strconv.Itoa(os.Getpid()) + setEnv("BUILDKITE_AGENT_DEBUG", fmt.Sprint(r.conf.Debug)) + setEnv("BUILDKITE_AGENT_DEBUG_HTTP", fmt.Sprint(r.conf.DebugHTTP)) + setEnv("BUILDKITE_AGENT_PID", strconv.Itoa(os.Getpid())) // We know the BUILDKITE_BIN_PATH dir, because it's the path to the // currently running file (there is only 1 binary) @@ -554,77 +550,80 @@ BUILDKITE_AGENT_JWKS_KEY_ID` if err != nil { return nil, err } - env["BUILDKITE_BIN_PATH"] = filepath.Dir(exePath) + + setEnv("BUILDKITE_BIN_PATH", filepath.Dir(exePath)) // Add options from the agent configuration - env["BUILDKITE_CONFIG_PATH"] = r.conf.AgentConfiguration.ConfigPath - env["BUILDKITE_BUILD_PATH"] = r.conf.AgentConfiguration.BuildPath - env["BUILDKITE_SOCKETS_PATH"] = r.conf.AgentConfiguration.SocketsPath - env["BUILDKITE_GIT_MIRRORS_PATH"] = r.conf.AgentConfiguration.GitMirrorsPath - env["BUILDKITE_GIT_MIRRORS_SKIP_UPDATE"] = fmt.Sprint(r.conf.AgentConfiguration.GitMirrorsSkipUpdate) - env["BUILDKITE_HOOKS_PATH"] = r.conf.AgentConfiguration.HooksPath - env["BUILDKITE_ADDITIONAL_HOOKS_PATHS"] = strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ",") - env["BUILDKITE_PLUGINS_PATH"] = r.conf.AgentConfiguration.PluginsPath - env["BUILDKITE_SSH_KEYSCAN"] = fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan) - env["BUILDKITE_GIT_SUBMODULES"] = fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules) - env["BUILDKITE_COMMAND_EVAL"] = fmt.Sprint(r.conf.AgentConfiguration.CommandEval) - env["BUILDKITE_PLUGINS_ENABLED"] = fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled) - env["BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH"] = fmt.Sprint(r.conf.AgentConfiguration.PluginsAlwaysCloneFresh) - env["BUILDKITE_LOCAL_HOOKS_ENABLED"] = fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled) - env["BUILDKITE_GIT_CHECKOUT_FLAGS"] = r.conf.AgentConfiguration.GitCheckoutFlags - env["BUILDKITE_GIT_CLONE_FLAGS"] = r.conf.AgentConfiguration.GitCloneFlags - env["BUILDKITE_GIT_FETCH_FLAGS"] = r.conf.AgentConfiguration.GitFetchFlags - env["BUILDKITE_GIT_CLONE_MIRROR_FLAGS"] = r.conf.AgentConfiguration.GitCloneMirrorFlags - env["BUILDKITE_GIT_CLEAN_FLAGS"] = r.conf.AgentConfiguration.GitCleanFlags - env["BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT"] = strconv.Itoa(r.conf.AgentConfiguration.GitMirrorsLockTimeout) - env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell - env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(ctx), ",") - env["BUILDKITE_REDACTED_VARS"] = strings.Join(r.conf.AgentConfiguration.RedactedVars, ",") - env["BUILDKITE_STRICT_SINGLE_HOOKS"] = fmt.Sprint(r.conf.AgentConfiguration.StrictSingleHooks) - env["BUILDKITE_CANCEL_GRACE_PERIOD"] = strconv.Itoa(r.conf.AgentConfiguration.CancelGracePeriod) - env["BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS"] = strconv.Itoa(int(r.conf.AgentConfiguration.SignalGracePeriod / time.Second)) - env["BUILDKITE_TRACE_CONTEXT_ENCODING"] = r.conf.AgentConfiguration.TraceContextEncoding + setEnv("BUILDKITE_CONFIG_PATH", r.conf.AgentConfiguration.ConfigPath) + setEnv("BUILDKITE_BUILD_PATH", r.conf.AgentConfiguration.BuildPath) + setEnv("BUILDKITE_SOCKETS_PATH", r.conf.AgentConfiguration.SocketsPath) + setEnv("BUILDKITE_GIT_MIRRORS_PATH", r.conf.AgentConfiguration.GitMirrorsPath) + setEnv("BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", fmt.Sprint(r.conf.AgentConfiguration.GitMirrorsSkipUpdate)) + setEnv("BUILDKITE_HOOKS_PATH", r.conf.AgentConfiguration.HooksPath) + setEnv("BUILDKITE_ADDITIONAL_HOOKS_PATHS", strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ",")) + setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath) + setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan)) + setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules)) + setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval)) + setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled)) + setEnv("BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", fmt.Sprint(r.conf.AgentConfiguration.PluginsAlwaysCloneFresh)) + setEnv("BUILDKITE_LOCAL_HOOKS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled)) + + setEnv("BUILDKITE_GIT_CHECKOUT_FLAGS", r.conf.AgentConfiguration.GitCheckoutFlags) + setEnv("BUILDKITE_GIT_CLONE_FLAGS", r.conf.AgentConfiguration.GitCloneFlags) + setEnv("BUILDKITE_GIT_FETCH_FLAGS", r.conf.AgentConfiguration.GitFetchFlags) + setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags) + setEnv("BUILDKITE_GIT_CLEAN_FLAGS", r.conf.AgentConfiguration.GitCleanFlags) + setEnv("BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", strconv.Itoa(r.conf.AgentConfiguration.GitMirrorsLockTimeout)) + + setEnv("BUILDKITE_SHELL", r.conf.AgentConfiguration.Shell) + setEnv("BUILDKITE_AGENT_EXPERIMENT", strings.Join(experiments.Enabled(ctx), ",")) + setEnv("BUILDKITE_REDACTED_VARS", strings.Join(r.conf.AgentConfiguration.RedactedVars, ",")) + setEnv("BUILDKITE_STRICT_SINGLE_HOOKS", fmt.Sprint(r.conf.AgentConfiguration.StrictSingleHooks)) + setEnv("BUILDKITE_CANCEL_GRACE_PERIOD", strconv.Itoa(r.conf.AgentConfiguration.CancelGracePeriod)) + setEnv("BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS", strconv.Itoa(int(r.conf.AgentConfiguration.SignalGracePeriod/time.Second))) + setEnv("BUILDKITE_TRACE_CONTEXT_ENCODING", r.conf.AgentConfiguration.TraceContextEncoding) if r.conf.KubernetesExec { - env["BUILDKITE_KUBERNETES_EXEC"] = "true" + setEnv("BUILDKITE_KUBERNETES_EXEC", "true") } if !r.conf.AgentConfiguration.AllowMultipartArtifactUpload { - env["BUILDKITE_NO_MULTIPART_ARTIFACT_UPLOAD"] = "true" + setEnv("BUILDKITE_NO_MULTIPART_ARTIFACT_UPLOAD", "true") } // propagate CancelSignal to bootstrap, unless it's the default SIGTERM if r.conf.CancelSignal != process.SIGTERM { - env["BUILDKITE_CANCEL_SIGNAL"] = r.conf.CancelSignal.String() + setEnv("BUILDKITE_CANCEL_SIGNAL", r.conf.CancelSignal.String()) } // Whether to enable profiling in the bootstrap if r.conf.AgentConfiguration.Profile != "" { - env["BUILDKITE_AGENT_PROFILE"] = r.conf.AgentConfiguration.Profile + setEnv("BUILDKITE_AGENT_PROFILE", r.conf.AgentConfiguration.Profile) } // PTY-mode is enabled by default in `start` and `bootstrap`, so we only need // to propagate it if it's explicitly disabled. if !r.conf.AgentConfiguration.RunInPty { - env["BUILDKITE_PTY"] = "false" + setEnv("BUILDKITE_PTY", "false") } // pass through the KMS key ID for signing if r.conf.AgentConfiguration.SigningAWSKMSKey != "" { - env["BUILDKITE_AGENT_AWS_KMS_KEY"] = r.conf.AgentConfiguration.SigningAWSKMSKey + setEnv("BUILDKITE_AGENT_AWS_KMS_KEY", r.conf.AgentConfiguration.SigningAWSKMSKey) } // Pass signing details through to the executor - any pipelines uploaded by this agent will be signed if r.conf.AgentConfiguration.SigningJWKSFile != "" { - env["BUILDKITE_AGENT_JWKS_FILE"] = r.conf.AgentConfiguration.SigningJWKSFile + setEnv("BUILDKITE_AGENT_JWKS_FILE", r.conf.AgentConfiguration.SigningJWKSFile) } if r.conf.AgentConfiguration.SigningJWKSKeyID != "" { - env["BUILDKITE_AGENT_JWKS_KEY_ID"] = r.conf.AgentConfiguration.SigningJWKSKeyID + setEnv("BUILDKITE_AGENT_JWKS_KEY_ID", r.conf.AgentConfiguration.SigningJWKSKeyID) } if r.conf.AgentConfiguration.DebugSigning { - env["BUILDKITE_AGENT_DEBUG_SIGNING"] = "true" + setEnv("BUILDKITE_AGENT_DEBUG_SIGNING", "true") } enablePluginValidation := r.conf.AgentConfiguration.PluginValidation @@ -633,28 +632,30 @@ BUILDKITE_AGENT_JWKS_KEY_ID` if pluginValidation, ok := env["BUILDKITE_PLUGIN_VALIDATION"]; ok { switch pluginValidation { case "true", "1", "on": + // Skip ignoredEnv by pretending it wasn't set by the job. + delete(env, "BUILDKITE_PLUGIN_VALIDATION") enablePluginValidation = true } } - env["BUILDKITE_PLUGIN_VALIDATION"] = fmt.Sprint(enablePluginValidation) + setEnv("BUILDKITE_PLUGIN_VALIDATION", fmt.Sprint(enablePluginValidation)) if r.conf.AgentConfiguration.TracingBackend != "" { - env["BUILDKITE_TRACING_BACKEND"] = r.conf.AgentConfiguration.TracingBackend - env["BUILDKITE_TRACING_SERVICE_NAME"] = r.conf.AgentConfiguration.TracingServiceName + setEnv("BUILDKITE_TRACING_BACKEND", r.conf.AgentConfiguration.TracingBackend) + setEnv("BUILDKITE_TRACING_SERVICE_NAME", r.conf.AgentConfiguration.TracingServiceName) // Buildkite backend can provide a traceparent property on the job // which can be propagated to the job tracing if OpenTelemetry is used // // https://www.w3.org/TR/trace-context/#traceparent-header if r.conf.Job.TraceParent != "" { - env["BUILDKITE_TRACING_TRACEPARENT"] = r.conf.Job.TraceParent + setEnv("BUILDKITE_TRACING_TRACEPARENT", r.conf.Job.TraceParent) } if r.conf.AgentConfiguration.TracingPropagateTraceparent { - env["BUILDKITE_TRACING_PROPAGATE_TRACEPARENT"] = "true" + setEnv("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT", "true") } } - env["BUILDKITE_AGENT_DISABLE_WARNINGS_FOR"] = strings.Join(r.conf.AgentConfiguration.DisableWarningsFor, ",") + setEnv("BUILDKITE_AGENT_DISABLE_WARNINGS_FOR", strings.Join(r.conf.AgentConfiguration.DisableWarningsFor, ",")) // see documentation for BuildkiteMessageMax if err := truncateEnv(r.agentLogger, env, BuildkiteMessageName, BuildkiteMessageMax); err != nil { @@ -662,6 +663,11 @@ BUILDKITE_AGENT_JWKS_KEY_ID` // attempt to continue anyway } + // Finally, set BUILDKITE_IGNORED_ENV so the bootstrap can show warnings. + if len(ignoredEnv) > 0 { + env["BUILDKITE_IGNORED_ENV"] = strings.Join(ignoredEnv, ",") + } + // Convert the env map into a slice (which is what the script gear // needs) envSlice := []string{}