HIVE-2302, HIVE-2644: Destroyers use metadata.json

An earlier commit ensures that ClusterDeployments have an associated
Secret containing the metadata.json emitted by the installer.

This change adds a new generic destroyer via the (existing) `hiveutil
deprovision` command that consumes this metadata.json to deprovision the
cluster.

This new behavior is the default, but we also include an escape hatch to
run the platform-specific legacy destroyer by setting the following
annotation on the ClusterDeployment:

`hive.openshift.io/legacy-deprovision: "true"`

Author: 2uasimojo

apis/hive/v1/clusterdeprovision_types.go

@@ -24,6 +24,10 @@ type ClusterDeprovisionSpec struct {
 	// BaseDomain is the DNS base domain.
 	BaseDomain string `json:"baseDomain,omitempty"`
 
+	// MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
+	// installer, potentially scrubbed for sensitive data.
+	MetadataJSONSecretRef *corev1.LocalObjectReference `json:"metadataJSONSecretRef,omitempty"`
+
 	// Platform contains platform-specific configuration for a ClusterDeprovision
 	Platform ClusterDeprovisionPlatform `json:"platform,omitempty"`
 }

apis/hive/v1/zz_generated.deepcopy.go

@@ -68,6 +68,22 @@ spec:
                 infraID:
                   description: InfraID is the identifier generated during installation for a cluster. It is used for tagging/naming resources in cloud providers.
                   type: string
+                metadataJSONSecretRef:
+                  description: |-
+                    MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
+                    installer, potentially scrubbed for sensitive data.
+                  properties:
+                    name:
+                      default: ""
+                      description: |-
+                        Name of the referent.
+                        This field is effectively required, but due to backwards compatibility is
+                        allowed to be empty. Instances of this type with an empty value here are
+                        almost certainly wrong.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      type: string
+                  type: object
+                  x-kubernetes-map-type: atomic
                 platform:
                   description: Platform contains platform-specific configuration for a ClusterDeprovision
                   properties:

config/crds/hive.openshift.io_clusterdeprovisions.yaml

@@ -13,7 +13,6 @@ import (
 )
 
 // NewDeprovisionAWSWithTagsCommand is the entrypoint to create the 'aws-tag-deprovision' subcommand
-// TODO: Port to a sub-command of deprovision.
 func NewDeprovisionAWSWithTagsCommand() *cobra.Command {
 	opt := &aws.ClusterUninstaller{}
 	var credsDir string

contrib/pkg/deprovision/awstagdeprovision.go

@@ -23,8 +23,10 @@ type AzureOptions struct {
 }
 
 // NewDeprovisionAzureCommand is the entrypoint to create the azure deprovision subcommand
-func NewDeprovisionAzureCommand() *cobra.Command {
-	opt := &AzureOptions{}
+func NewDeprovisionAzureCommand(logLevel string) *cobra.Command {
+	opt := &AzureOptions{
+		logLevel: logLevel,
+	}
 	cmd := &cobra.Command{
 		Use:   "azure INFRAID [--azure-cloud-name CLOUDNAME] [--azure-resource-group-name RG] [--azure-base-domain-resource-group-name BDRG]",
 		Short: "Deprovision Azure assets (as created by openshift-installer)",
@@ -46,7 +48,6 @@ func NewDeprovisionAzureCommand() *cobra.Command {
 		},
 	}
 	flags := cmd.Flags()
-	flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
 	flags.StringVar(&opt.cloudName, "azure-cloud-name", installertypesazure.PublicCloud.Name(), "The name of the Azure cloud environment used to configure the Azure SDK")
 	flags.StringVar(&opt.resourceGroupName, "azure-resource-group-name", "", "The name of the custom Azure resource group in which the cluster was created when not using the default installer-created resource group")
 	flags.StringVar(&opt.baseDomainResourceGroupName, "azure-base-domain-resource-group-name", "", "The name of the custom Azure resource group in which the cluster's DNS records were created when not using the default installer-created resource group or custom resource group")

contrib/pkg/deprovision/azure.go

@@ -1,27 +1,154 @@
 package deprovision
 
 import (
+	"encoding/json"
+	"log"
+	"os"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openshift/installer/pkg/destroy/providers"
+	"github.com/openshift/installer/pkg/types"
+	"github.com/openshift/installer/pkg/types/aws"
+	"github.com/openshift/installer/pkg/types/azure"
+	"github.com/openshift/installer/pkg/types/gcp"
+	"github.com/openshift/installer/pkg/types/ibmcloud"
+	"github.com/openshift/installer/pkg/types/nutanix"
+	"github.com/openshift/installer/pkg/types/openstack"
+	"github.com/openshift/installer/pkg/types/ovirt"
+	"github.com/openshift/installer/pkg/types/vsphere"
+
+	"github.com/openshift/hive/contrib/pkg/utils"
+	awsutil "github.com/openshift/hive/contrib/pkg/utils/aws"
+	azureutil "github.com/openshift/hive/contrib/pkg/utils/azure"
+	gcputil "github.com/openshift/hive/contrib/pkg/utils/gcp"
+	ibmcloudutil "github.com/openshift/hive/contrib/pkg/utils/ibmcloud"
+	nutanixutil "github.com/openshift/hive/contrib/pkg/utils/nutanix"
+	openstackutil "github.com/openshift/hive/contrib/pkg/utils/openstack"
+	ovirtutil "github.com/openshift/hive/contrib/pkg/utils/ovirt"
+	vsphereutil "github.com/openshift/hive/contrib/pkg/utils/vsphere"
+	"github.com/openshift/hive/pkg/constants"
+
 	"github.com/spf13/cobra"
 )
 
 // NewDeprovisionCommand is the entrypoint to create the 'deprovision' subcommand
 func NewDeprovisionCommand() *cobra.Command {
 	var credsDir string
+	var mjSecretName string
+	var logLevel string
 	cmd := &cobra.Command{
 		Use:   "deprovision",
 		Short: "Deprovision clusters in supported cloud providers",
+		Long: `Platform subcommands use a legacy code path and are deprecated. \
+To run the generic destroyer, use the --metadata-json-secret-name parameter.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			cmd.Usage()
+			if mjSecretName == "" {
+				cmd.Usage()
+				return
+			}
+
+			// Generic deprovision flow using metadata.json
+			logger, err := utils.NewLogger(logLevel)
+			if err != nil {
+				log.Fatalf("failed to create logger: %s", err)
+			}
+
+			c, err := utils.GetClient("hiveutil-deprovision-generic")
+			if err != nil {
+				logger.WithError(err).Fatal("failed to create kube client")
+			}
+
+			// TODO: Refactor LoadSecretOrDie to avoid this setenv/getenv cycle
+			k := "METADATA_JSON_SECRET_NAME"
+			os.Setenv(k, mjSecretName)
+			mjSecret := utils.LoadSecretOrDie(c, k)
+			if mjSecret == nil {
+				// This should not be reachable -- we should have Fatal()ed in LoadSecretOrDie()
+				logger.WithField("secretName", mjSecretName).Fatal("failed to load metadata.json Secret")
+			}
+
+			mjBytes, ok := mjSecret.Data[constants.MetadataJSONSecretKey]
+			if !ok {
+				logger.Fatalf("metadata.json Secret did not contain %q key", constants.MetadataJSONSecretKey)
+			}
+
+			var metadata *types.ClusterMetadata
+			if err = json.Unmarshal(mjBytes, &metadata); err != nil {
+				logger.WithError(err).Fatal("failed to unmarshal metadata.json")
+			}
+
+			platform := metadata.Platform()
+			if platform == "" {
+				logger.Fatal("no platform configured in metadata.json")
+			}
+
+			// TODO: Make a registry or interface for this
+			var ConfigureCreds func(client.Client)
+			switch platform {
+			case aws.Name:
+				ConfigureCreds = awsutil.ConfigureCreds
+			case azure.Name:
+				ConfigureCreds = azureutil.ConfigureCreds
+			case gcp.Name:
+				ConfigureCreds = gcputil.ConfigureCreds
+			case ibmcloud.Name:
+				ConfigureCreds = ibmcloudutil.ConfigureCreds
+			case nutanix.Name:
+				// Snowflake! We need to inject the creds into the metadata.
+				// If env vars are unset, the destroyer will fail organically.
+				ConfigureCreds = func(c client.Client) {
+					nutanixutil.ConfigureCreds(c)
+					metadata.Nutanix.Username = os.Getenv(constants.NutanixUsernameEnvVar)
+					metadata.Nutanix.Password = os.Getenv(constants.NutanixPasswordEnvVar)
+				}
+			case openstack.Name:
+				ConfigureCreds = openstackutil.ConfigureCreds
+			case ovirt.Name:
+				ConfigureCreds = ovirtutil.ConfigureCreds
+			case vsphere.Name:
+				// Snowflake! We need to inject the creds into the metadata.
+				// If env vars are unset, the destroyer will fail organically.
+				ConfigureCreds = func(c client.Client) {
+					vsphereutil.ConfigureCreds(c)
+					metadata.VSphere.VCenter = os.Getenv(constants.VSphereVCenterEnvVar)
+					metadata.VSphere.Username = os.Getenv(constants.VSphereUsernameEnvVar)
+					metadata.VSphere.Password = os.Getenv(constants.VSpherePasswordEnvVar)
+				}
+			}
+			ConfigureCreds(c)
+
+			destroyerBuilder, ok := providers.Registry[platform]
+			if !ok {
+				logger.WithField("platform", platform).Fatal("no destroyers registered for platform")
+			}
+
+			destroyer, err := destroyerBuilder(logger, metadata)
+			if err != nil {
+				logger.WithError(err).Fatal("failed to create destroyer")
+			}
+
+			// Ignore quota return
+			_, err = destroyer.Run()
+			if err != nil {
+				logger.WithError(err).Fatal("destroyer returned an error")
+			}
 		},
 	}
 	flags := cmd.PersistentFlags()
+	// TODO: Unused -- remove from here and generate.go
 	flags.StringVar(&credsDir, "creds-dir", "", "directory of the creds. Changes in the creds will cause the program to terminate")
-	cmd.AddCommand(NewDeprovisionAzureCommand())
-	cmd.AddCommand(NewDeprovisionGCPCommand())
-	cmd.AddCommand(NewDeprovisionIBMCloudCommand())
-	cmd.AddCommand(NewDeprovisionOpenStackCommand())
-	cmd.AddCommand(NewDeprovisionvSphereCommand())
-	cmd.AddCommand(NewDeprovisionOvirtCommand())
-	cmd.AddCommand(NewDeprovisionNutanixCommand())
+	// TODO: Make this more useful to CLI users by accepting a path to a metadata.json file in the file system
+	flags.StringVar(&mjSecretName, "metadata-json-secret-name", "", "name of a Secret in the current namespace containing `metadata.json` from the installer")
+	flags.StringVar(&logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
+
+	// Legacy destroyers
+	cmd.AddCommand(NewDeprovisionAzureCommand(logLevel))
+	cmd.AddCommand(NewDeprovisionGCPCommand(logLevel))
+	cmd.AddCommand(NewDeprovisionIBMCloudCommand(logLevel))
+	cmd.AddCommand(NewDeprovisionOpenStackCommand(logLevel))
+	cmd.AddCommand(NewDeprovisionvSphereCommand(logLevel))
+	cmd.AddCommand(NewDeprovisionOvirtCommand(logLevel))
+	cmd.AddCommand(NewDeprovisionNutanixCommand(logLevel))
 	return cmd
 }

contrib/pkg/deprovision/deprovision.go

@@ -26,8 +26,10 @@ type gcpOptions struct {
 }
 
 // NewDeprovisionGCPCommand is the entrypoint to create the GCP deprovision subcommand
-func NewDeprovisionGCPCommand() *cobra.Command {
-	opt := &gcpOptions{}
+func NewDeprovisionGCPCommand(logLevel string) *cobra.Command {
+	opt := &gcpOptions{
+		logLevel: logLevel,
+	}
 	cmd := &cobra.Command{
 		Use:   "gcp INFRAID --region=REGION",
 		Short: "Deprovision GCP assets (as created by openshift-installer)",
@@ -45,7 +47,6 @@ func NewDeprovisionGCPCommand() *cobra.Command {
 		},
 	}
 	flags := cmd.Flags()
-	flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
 	flags.StringVar(&opt.region, "region", "", "GCP region where the cluster is installed")
 	flags.StringVar(&opt.networkProjectID, "network-project-id", "", "For shared VPC setups")
 	return cmd

contrib/pkg/deprovision/gcp.go

@@ -30,8 +30,10 @@ type ibmCloudDeprovisionOptions struct {
 }
 
 // NewDeprovisionIBMCloudCommand is the entrypoint to create the IBM Cloud deprovision subcommand
-func NewDeprovisionIBMCloudCommand() *cobra.Command {
-	opt := &ibmCloudDeprovisionOptions{}
+func NewDeprovisionIBMCloudCommand(logLevel string) *cobra.Command {
+	opt := &ibmCloudDeprovisionOptions{
+		logLevel: logLevel,
+	}
 	cmd := &cobra.Command{
 		Use:   "ibmcloud INFRAID --region=us-east --base-domain=BASE_DOMAIN --cluster-name=CLUSTERNAME",
 		Short: "Deprovision IBM Cloud assets (as created by openshift-installer)",
@@ -50,7 +52,6 @@ func NewDeprovisionIBMCloudCommand() *cobra.Command {
 	}
 
 	flags := cmd.Flags()
-	flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
 
 	// Required flags
 	flags.StringVar(&opt.baseDomain, "base-domain", "", "cluster's base domain")

contrib/pkg/deprovision/ibmcloud.go

@@ -26,8 +26,10 @@ type nutanixOptions struct {
 	password string
 }
 
-func NewDeprovisionNutanixCommand() *cobra.Command {
-	opt := &nutanixOptions{}
+func NewDeprovisionNutanixCommand(logLevel string) *cobra.Command {
+	opt := &nutanixOptions{
+		logLevel: logLevel,
+	}
 
 	cmd := &cobra.Command{
 		Use:   "nutanix INFRAID",
@@ -46,7 +48,6 @@ func NewDeprovisionNutanixCommand() *cobra.Command {
 		},
 	}
 	flags := cmd.Flags()
-	flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
 	flags.StringVar(&opt.endpoint, constants.CliNutanixPcAddressOpt, "", "Domain name or IP address of the Nutanix Prism Central endpoint")
 	flags.StringVar(&opt.port, constants.CliNutanixPcPortOpt, "", "Port of the Nutanix Prism Central endpoint")
 	return cmd

contrib/pkg/deprovision/nutanix.go

@@ -23,8 +23,10 @@ type openStackOptions struct {
 }
 
 // NewDeprovisionOpenStackCommand is the entrypoint to create the OpenStack deprovision subcommand
-func NewDeprovisionOpenStackCommand() *cobra.Command {
-	opt := &openStackOptions{}
+func NewDeprovisionOpenStackCommand(logLevel string) *cobra.Command {
+	opt := &openStackOptions{
+		logLevel: logLevel,
+	}
 	cmd := &cobra.Command{
 		Use:   "openstack INFRAID --cloud=OS_CLOUD",
 		Short: "Deprovision OpenStack assets (as created by openshift-installer)",
@@ -42,7 +44,6 @@ func NewDeprovisionOpenStackCommand() *cobra.Command {
 		},
 	}
 	flags := cmd.Flags()
-	flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
 	flags.StringVar(&opt.cloud, "cloud", "", "OpenStack cloud entry name from clouds.yaml for access/authentication")
 	return cmd
 }