From 50e2ba20935a02636a12b876018fe4974cfb832a Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Wed, 7 May 2025 17:02:37 -0300 Subject: [PATCH 1/5] Implement vc session manager client --- pkg/common/vclib/vc_session_manager.go | 87 ++++++++++ pkg/common/vclib/vc_session_manager_test.go | 170 ++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 pkg/common/vclib/vc_session_manager.go create mode 100644 pkg/common/vclib/vc_session_manager_test.go diff --git a/pkg/common/vclib/vc_session_manager.go b/pkg/common/vclib/vc_session_manager.go new file mode 100644 index 000000000..bb5cb55d2 --- /dev/null +++ b/pkg/common/vclib/vc_session_manager.go @@ -0,0 +1,87 @@ +package vclib + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "net/http" + "time" +) + +// SharedSessionResponse is the expected structure for a session manager valid +// token response +type SharedSessionResponse struct { + Token string `json:"token"` +} + +// SharedTokenOptions represents the options that can be used when calling vc session manager +type SharedTokenOptions struct { + // URL is the session manager URL. Eg.: https://my-session-manager/session) + URL string + // Token is the authorization token that should be passed to session manager + Token string + // TrustedCertificates contains the certpool of certificates trusted by the client + TrustedCertificates *x509.CertPool + // InsecureSkipVerify defines if bad certificates requests should be ignored + InsecureSkipVerify bool + // Timeout defines the client timeout. Defaults to 5 seconds + Timeout time.Duration +} + +// GetSharedToken executes an http request on session manager and gets the session manager +// token that can be reused on govmomi sessions +func GetSharedToken(ctx context.Context, options SharedTokenOptions) (string, error) { + if options.URL == "" { + return "", fmt.Errorf("URL of session manager cannot be empty") + } + if options.Token == "" { + return "", fmt.Errorf("token of session manager cannot be empty") + } + + timeout := 5 * time.Second + if options.Timeout != 0 { + timeout = options.Timeout + } + + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: options.TrustedCertificates, + InsecureSkipVerify: options.InsecureSkipVerify, + }, + } + + client := &http.Client{ + Timeout: timeout, + Transport: transport, + } + + request, err := http.NewRequest(http.MethodGet, options.URL, nil) + if err != nil { + return "", fmt.Errorf("failed creating new http client: %w", err) + } + authToken := fmt.Sprintf("Bearer %s", options.Token) + request.Header.Add("Authorization", authToken) + + resp, err := client.Do(request) + if err != nil { + return "", fmt.Errorf("failed calling vc session manager: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid vc session manager response: %s", resp.Status) + } + + token := &SharedSessionResponse{} + defer resp.Body.Close() + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(token); err != nil { + return "", fmt.Errorf("failed decoding vc session manager response: %w", err) + } + + if token.Token == "" { + return "", fmt.Errorf("returned vc session token is empty") + } + return token.Token, nil +} diff --git a/pkg/common/vclib/vc_session_manager_test.go b/pkg/common/vclib/vc_session_manager_test.go new file mode 100644 index 000000000..023f1eb35 --- /dev/null +++ b/pkg/common/vclib/vc_session_manager_test.go @@ -0,0 +1,170 @@ +package vclib_test + +import ( + "context" + "crypto/x509" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "k8s.io/cloud-provider-vsphere/pkg/common/vclib" +) + +const ( + validToken = "validtoken" + validResponse = "a-valid-response" +) + +var ( + handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authZHdr := r.Header.Get("Authorization") + if authZHdr != fmt.Sprintf("Bearer %s", validToken) { + w.WriteHeader(http.StatusForbidden) + return + } + if r.URL.Path == "/timeout" { + time.Sleep(15 * time.Millisecond) + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Path == "/invalid-token" { + w.WriteHeader(http.StatusOK) + w.Write([]byte("not a json")) + return + } + if r.URL.Path == "/session" { + token := vclib.SharedSessionResponse{ + Token: validResponse, + } + response, err := json.Marshal(&token) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Write(response) + return + } + if r.URL.Path == "/empty" { + token := vclib.SharedSessionResponse{ + Token: "", + } + response, err := json.Marshal(&token) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Write(response) + return + } + w.WriteHeader(http.StatusNotFound) + }) +) + +func TestGetSharedToken(t *testing.T) { + ctx := context.Background() + t.Run("when options are invalid", func(t *testing.T) { + t.Run("should fail when no URL is sent", func(t *testing.T) { + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{}) + assert.ErrorContains(t, err, "URL of session manager cannot be empty") + }) + + t.Run("should fail when no token is passed", func(t *testing.T) { + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: "https://some-session-manager.tld/session", + }) + assert.ErrorContains(t, err, "token of session manager cannot be empty") + }) + + t.Run("should fail when passed URL is invalid", func(t *testing.T) { + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: "https://some-session-manager.tld:xxxxx/session", + Token: "anything", + }) + assert.ErrorContains(t, err, "invalid port") + }) + }) + + t.Run("when using a valid session manager", func(t *testing.T) { + server := httptest.NewTLSServer(handler) + + certpool := x509.NewCertPool() + certpool.AddCert(server.Certificate()) + t.Cleanup(server.Close) + + t.Run("should respect the timeout", func(t *testing.T) { + reqURL := fmt.Sprintf("%s/timeout", server.URL) + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + TrustedCertificates: certpool, + Token: validToken, + Timeout: 5 * time.Millisecond, + }) + assert.ErrorContains(t, err, "context deadline exceeded") + }) + t.Run("should fail when calling an invalid path", func(t *testing.T) { + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: server.URL, + TrustedCertificates: certpool, + Token: validToken, + }) + assert.ErrorContains(t, err, "404 Not Found") + }) + t.Run("should fail when an empty token is returned", func(t *testing.T) { + reqURL := fmt.Sprintf("%s/empty", server.URL) + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + TrustedCertificates: certpool, + Token: validToken, + }) + assert.ErrorContains(t, err, "returned vc session token is empty") + }) + + t.Run("should fail when an invalid json is returned", func(t *testing.T) { + reqURL := fmt.Sprintf("%s/invalid-token", server.URL) + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + TrustedCertificates: certpool, + Token: validToken, + }) + assert.ErrorContains(t, err, "failed decoding vc session manager response") + }) + + t.Run("should fail when no cert is passed and insecureskipverify is false", func(t *testing.T) { + reqURL := fmt.Sprintf("%s/session", server.URL) + _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + Token: validToken, + }) + assert.ErrorContains(t, err, "tls: failed to verify certificate: x509") + }) + + t.Run("should return a valid token for the right request and insecureskip=true", func(t *testing.T) { + reqURL := fmt.Sprintf("%s/session", server.URL) + token, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + InsecureSkipVerify: true, + Token: validToken, + }) + assert.NoError(t, err) + assert.Equal(t, validResponse, token) + }) + + t.Run("should return a valid token for the right request and cert", func(t *testing.T) { + reqURL := fmt.Sprintf("%s/session", server.URL) + token, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + TrustedCertificates: certpool, + Token: validToken, + }) + assert.NoError(t, err) + assert.Equal(t, validResponse, token) + }) + }) + +} From d2782792fccc4641024369f4a2ac7d1d03a89589 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Wed, 7 May 2025 17:30:28 -0300 Subject: [PATCH 2/5] Implement vc session manager on credential manager --- .../credentialmanager/credentialmanager.go | 29 +++++- .../credentialmanager_test.go | 99 ++++++++++++++++--- pkg/common/credentialmanager/types.go | 3 + 3 files changed, 111 insertions(+), 20 deletions(-) diff --git a/pkg/common/credentialmanager/credentialmanager.go b/pkg/common/credentialmanager/credentialmanager.go index 6c7121a8f..273be7267 100644 --- a/pkg/common/credentialmanager/credentialmanager.go +++ b/pkg/common/credentialmanager/credentialmanager.go @@ -193,19 +193,35 @@ func parseConfig(data map[string][]byte, config map[string]*Credential) error { } unknownKeys := map[string][]byte{} for credentialKey, credentialValue := range data { - if strings.HasSuffix(credentialKey, "password") { + switch { + case strings.HasSuffix(credentialKey, "password"): vcServer := strings.Split(credentialKey, ".password")[0] if _, ok := config[vcServer]; !ok { config[vcServer] = &Credential{} } config[vcServer].Password = strings.TrimSuffix(string(credentialValue), "\n") - } else if strings.HasSuffix(credentialKey, "username") { + + case strings.HasSuffix(credentialKey, "username"): vcServer := strings.Split(credentialKey, ".username")[0] if _, ok := config[vcServer]; !ok { config[vcServer] = &Credential{} } config[vcServer].User = strings.TrimSuffix(string(credentialValue), "\n") - } else { + + case strings.HasSuffix(credentialKey, "vc-session-manager-url"): + vcServer := strings.Split(credentialKey, ".vc-session-manager-url")[0] + if _, ok := config[vcServer]; !ok { + config[vcServer] = &Credential{} + } + config[vcServer].VCSessionManagerURL = strings.TrimSuffix(string(credentialValue), "\n") + + case strings.HasSuffix(credentialKey, "vc-session-manager-token"): + vcServer := strings.Split(credentialKey, ".vc-session-manager-token")[0] + if _, ok := config[vcServer]; !ok { + config[vcServer] = &Credential{} + } + config[vcServer].VCSessionManagerToken = strings.TrimSuffix(string(credentialValue), "\n") + default: unknownKeys[credentialKey] = credentialValue } } @@ -287,10 +303,13 @@ func parseConfig(data map[string][]byte, config map[string]*Credential) error { } for vcServer, credential := range config { - if credential.User == "" || credential.Password == "" { - klog.Errorf("Username/Password is missing for server %s", vcServer) + if (credential.User == "" || credential.Password == "") && + (credential.VCSessionManagerURL == "" || credential.VCSessionManagerToken == "") { + + klog.Errorf("Username/Password or shared session manager URL/Token directives are missing for server %s", vcServer) return ErrCredentialMissing } + } return nil } diff --git a/pkg/common/credentialmanager/credentialmanager_test.go b/pkg/common/credentialmanager/credentialmanager_test.go index 7ea890d85..828d73483 100644 --- a/pkg/common/credentialmanager/credentialmanager_test.go +++ b/pkg/common/credentialmanager/credentialmanager_test.go @@ -29,16 +29,20 @@ import ( func TestSecretCredentialManagerK8s_GetCredential(t *testing.T) { var ( - userKey = "username" - passwordKey = "password" - testUser = "user" - testPassword = "password" - testServer = "0.0.0.0" - testServer2 = "0.0.1.1" - testIPv6Server = "fd01::1" - testUserServer2 = "user1" - testPasswordServer2 = "password1" - testIncorrectServer = "1.1.1.1" + userKey = "username" + passwordKey = "password" + vcSessionURL = "vc-session-manager-url" + vcSessionToken = "vc-session-manager-token" + testUser = "user" + testPassword = "password" + testServer = "0.0.0.0" + testServer2 = "0.0.1.1" + testIPv6Server = "fd01::1" + testUserServer2 = "user1" + testPasswordServer2 = "password1" + testIncorrectServer = "1.1.1.1" + testSessionManagerURL = "https://somemanager.tld/session" + testSessionManagerToken = "token" ) var ( secretName = "vsconf" @@ -50,10 +54,12 @@ func TestSecretCredentialManagerK8s_GetCredential(t *testing.T) { deleteSecretOp = "DELETE_SECRET_OP" ) type GetCredentialsTest struct { - server string - username string - password string - err error + server string + username string + password string + vcSessionURL string + vcSessionToken string + err error } type OpSecretTest struct { secret *corev1.Secret @@ -88,6 +94,16 @@ func TestSecretCredentialManagerK8s_GetCredential(t *testing.T) { }, } + multiVCSecretMixedWithSessionManager := &corev1.Secret{ + ObjectMeta: metaObj, + Data: map[string][]byte{ + testServer + "." + userKey: []byte(testUser), + testServer + "." + passwordKey: []byte(testPassword), + testServer2 + "." + vcSessionURL: []byte(testSessionManagerURL), + testServer2 + "." + vcSessionToken: []byte(testSessionManagerToken), + }, + } + ipv6CompatSecret := &corev1.Secret{ ObjectMeta: metaObj, Data: map[string][]byte{ @@ -194,6 +210,20 @@ func TestSecretCredentialManagerK8s_GetCredential(t *testing.T) { }, }, }, + { + testName: "GetCredential for multi-vc with session manager", + ops: []string{addSecretOp, getCredentialsOp}, + expectedValues: []interface{}{ + OpSecretTest{ + secret: multiVCSecretMixedWithSessionManager, + }, + GetCredentialsTest{ + server: testServer2, + vcSessionURL: testSessionManagerURL, + vcSessionToken: testSessionManagerToken, + }, + }, + }, { testName: "GetCredential for alternative IPv6 server address compatable format", ops: []string{addSecretOp, getCredentialsOp}, @@ -259,7 +289,9 @@ func TestSecretCredentialManagerK8s_GetCredential(t *testing.T) { } if expected.err == nil { if expected.username != credential.User || - expected.password != credential.Password { + expected.password != credential.Password || + expected.vcSessionToken != credential.VCSessionManagerToken || + expected.vcSessionURL != credential.VCSessionManagerURL { t.Fatalf("Received credentials %v "+ "are different than actual credential user:%s password:%s", credential, expected.username, expected.password) @@ -352,6 +384,43 @@ func TestParseSecretConfig(t *testing.T) { }, expectedError: ErrCredentialMissing, }, + { + testName: "Missing session manager token", + data: map[string][]byte{ + "10.20.30.40.vc-session-manager-url": []byte("https://something.tld/session"), + }, + config: map[string]*Credential{ + testIP: { + VCSessionManagerURL: "https://something.tld/session", + }, + }, + expectedError: ErrCredentialMissing, + }, + { + testName: "Missing session manager url", + data: map[string][]byte{ + "10.20.30.40.vc-session-manager-token": []byte("token"), + }, + config: map[string]*Credential{ + testIP: { + VCSessionManagerToken: "token", + }, + }, + expectedError: ErrCredentialMissing, + }, + { + testName: "Valid session manager configuration", + data: map[string][]byte{ + "10.20.30.40.vc-session-manager-url": []byte("https://something.tld/session"), + "10.20.30.40.vc-session-manager-token": []byte("token"), + }, + config: map[string]*Credential{ + testIP: { + VCSessionManagerURL: "https://something.tld/session", + VCSessionManagerToken: "token", + }, + }, + }, { testName: "IP with unknown key", data: map[string][]byte{ diff --git a/pkg/common/credentialmanager/types.go b/pkg/common/credentialmanager/types.go index b9daebda9..0fe2917b0 100644 --- a/pkg/common/credentialmanager/types.go +++ b/pkg/common/credentialmanager/types.go @@ -36,6 +36,9 @@ type SecretCache struct { type Credential struct { User string `gcfg:"user"` Password string `gcfg:"password"` + // VC shared session manager directives + VCSessionManagerURL string `gcfg:"vc-session-manager-url"` + VCSessionManagerToken string `gcfg:"vc-session-manager-token"` } // CredentialManager is used to manage vCenter credentials stored as From c1876430b2351526ee552987ec58f19d0fd32115 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Wed, 7 May 2025 17:42:02 -0300 Subject: [PATCH 3/5] Implement vc session manager on vc client --- .../connectionmanager/connectionmanager.go | 2 +- pkg/common/connectionmanager/zones.go | 10 +++++ pkg/common/vclib/connection.go | 42 ++++++++++++++----- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pkg/common/connectionmanager/connectionmanager.go b/pkg/common/connectionmanager/connectionmanager.go index 5974d3641..2f130bc68 100644 --- a/pkg/common/connectionmanager/connectionmanager.go +++ b/pkg/common/connectionmanager/connectionmanager.go @@ -161,7 +161,7 @@ func (connMgr *ConnectionManager) Connect(ctx context.Context, vcInstance *VSphe klog.Error("Failed to get credentials from Secret Credential Manager with err:", err) return err } - vcInstance.Conn.UpdateCredentials(credentials.User, credentials.Password) + vcInstance.Conn.UpdateCredentials(credentials.User, credentials.Password, credentials.VCSessionManagerURL, credentials.VCSessionManagerToken) return vcInstance.Conn.Connect(ctx) } diff --git a/pkg/common/connectionmanager/zones.go b/pkg/common/connectionmanager/zones.go index 40ba1b2b5..df20b8bf5 100644 --- a/pkg/common/connectionmanager/zones.go +++ b/pkg/common/connectionmanager/zones.go @@ -292,6 +292,11 @@ func (cm *ConnectionManager) getDIFromMultiVCorDC(ctx context.Context, func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f func(c *rest.Client) error) error { c := rest.NewClient(connection.Client) + if connection.SessionManagerURL != "" && connection.SessionManagerToken != "" { + c.SessionID(connection.Client.SessionCookie().Value) + return nil + } + signer, err := connection.Signer(ctx, connection.Client) if err != nil { return err @@ -307,6 +312,11 @@ func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f } defer func() { + // When using shared session manager we don't need to logout + if connection.SessionManagerURL != "" && connection.SessionManagerToken != "" { + return + } + if err := c.Logout(ctx); err != nil { klog.Errorf("failed to logout: %v", err) } diff --git a/pkg/common/vclib/connection.go b/pkg/common/vclib/connection.go index a2a1c9230..8dccae9c2 100644 --- a/pkg/common/vclib/connection.go +++ b/pkg/common/vclib/connection.go @@ -37,16 +37,18 @@ const ( // VSphereConnection contains information for connecting to vCenter type VSphereConnection struct { - Client *vim25.Client - Username string - Password string - Hostname string - Port string - CACert string - Thumbprint string - Insecure bool - RoundTripperCount uint - credentialsLock sync.Mutex + Client *vim25.Client + Username string + Password string + Hostname string + Port string + CACert string + Thumbprint string + Insecure bool + SessionManagerURL string + SessionManagerToken string + RoundTripperCount uint + credentialsLock sync.Mutex } var ( @@ -132,6 +134,22 @@ func (connection *VSphereConnection) login(ctx context.Context, client *vim25.Cl connection.credentialsLock.Lock() defer connection.credentialsLock.Unlock() + if connection.SessionManagerURL != "" && connection.SessionManagerToken != "" { + token, err := GetSharedToken(ctx, SharedTokenOptions{ + URL: connection.SessionManagerURL, + Token: connection.SessionManagerToken, + }) + if err != nil { + klog.Errorf("error getting shared session token: %s", err) + return err + } + if err := m.CloneSession(ctx, token); err != nil { + klog.Errorf("error getting shared cloned session token: %s", err) + return err + } + return nil + } + signer, err := connection.Signer(ctx, client) if err != nil { return err @@ -196,9 +214,11 @@ func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Clie // UpdateCredentials updates username and password. // Note: Updated username and password will be used when there is no session active -func (connection *VSphereConnection) UpdateCredentials(username string, password string) { +func (connection *VSphereConnection) UpdateCredentials(username string, password string, sessionmgrURL string, sessionmgrToken string) { connection.credentialsLock.Lock() defer connection.credentialsLock.Unlock() connection.Username = username connection.Password = password + connection.SessionManagerURL = sessionmgrURL + connection.SessionManagerToken = sessionmgrToken } From 1aadbdd41cf111e6fe2405bee46ea8be20a0f302 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Thu, 8 May 2025 09:24:07 -0300 Subject: [PATCH 4/5] Support using serviceaccount on session auth --- pkg/common/connectionmanager/zones.go | 4 ++-- .../credentialmanager/credentialmanager.go | 3 +-- .../credentialmanager_test.go | 2 +- pkg/common/vclib/connection.go | 2 +- pkg/common/vclib/vc_session_manager.go | 19 ++++++++++++++- pkg/common/vclib/vc_session_manager_test.go | 24 ++++++++++++++++--- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pkg/common/connectionmanager/zones.go b/pkg/common/connectionmanager/zones.go index df20b8bf5..01401604e 100644 --- a/pkg/common/connectionmanager/zones.go +++ b/pkg/common/connectionmanager/zones.go @@ -292,7 +292,7 @@ func (cm *ConnectionManager) getDIFromMultiVCorDC(ctx context.Context, func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f func(c *rest.Client) error) error { c := rest.NewClient(connection.Client) - if connection.SessionManagerURL != "" && connection.SessionManagerToken != "" { + if connection.SessionManagerURL != "" { c.SessionID(connection.Client.SessionCookie().Value) return nil } @@ -313,7 +313,7 @@ func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f defer func() { // When using shared session manager we don't need to logout - if connection.SessionManagerURL != "" && connection.SessionManagerToken != "" { + if connection.SessionManagerURL != "" { return } diff --git a/pkg/common/credentialmanager/credentialmanager.go b/pkg/common/credentialmanager/credentialmanager.go index 273be7267..e68338813 100644 --- a/pkg/common/credentialmanager/credentialmanager.go +++ b/pkg/common/credentialmanager/credentialmanager.go @@ -303,8 +303,7 @@ func parseConfig(data map[string][]byte, config map[string]*Credential) error { } for vcServer, credential := range config { - if (credential.User == "" || credential.Password == "") && - (credential.VCSessionManagerURL == "" || credential.VCSessionManagerToken == "") { + if (credential.User == "" || credential.Password == "") && credential.VCSessionManagerURL == "" { klog.Errorf("Username/Password or shared session manager URL/Token directives are missing for server %s", vcServer) return ErrCredentialMissing diff --git a/pkg/common/credentialmanager/credentialmanager_test.go b/pkg/common/credentialmanager/credentialmanager_test.go index 828d73483..cd22af4ba 100644 --- a/pkg/common/credentialmanager/credentialmanager_test.go +++ b/pkg/common/credentialmanager/credentialmanager_test.go @@ -394,7 +394,7 @@ func TestParseSecretConfig(t *testing.T) { VCSessionManagerURL: "https://something.tld/session", }, }, - expectedError: ErrCredentialMissing, + expectedError: nil, }, { testName: "Missing session manager url", diff --git a/pkg/common/vclib/connection.go b/pkg/common/vclib/connection.go index 8dccae9c2..71e03f8fa 100644 --- a/pkg/common/vclib/connection.go +++ b/pkg/common/vclib/connection.go @@ -134,7 +134,7 @@ func (connection *VSphereConnection) login(ctx context.Context, client *vim25.Cl connection.credentialsLock.Lock() defer connection.credentialsLock.Unlock() - if connection.SessionManagerURL != "" && connection.SessionManagerToken != "" { + if connection.SessionManagerURL != "" { token, err := GetSharedToken(ctx, SharedTokenOptions{ URL: connection.SessionManagerURL, Token: connection.SessionManagerToken, diff --git a/pkg/common/vclib/vc_session_manager.go b/pkg/common/vclib/vc_session_manager.go index bb5cb55d2..4506c9cfa 100644 --- a/pkg/common/vclib/vc_session_manager.go +++ b/pkg/common/vclib/vc_session_manager.go @@ -7,9 +7,14 @@ import ( "encoding/json" "fmt" "net/http" + "os" "time" ) +const ( + saFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" +) + // SharedSessionResponse is the expected structure for a session manager valid // token response type SharedSessionResponse struct { @@ -28,6 +33,8 @@ type SharedTokenOptions struct { InsecureSkipVerify bool // Timeout defines the client timeout. Defaults to 5 seconds Timeout time.Duration + // TokenFile defines a file with token content. Defaults to Kubernetes Service Account file + TokenFile string } // GetSharedToken executes an http request on session manager and gets the session manager @@ -36,8 +43,18 @@ func GetSharedToken(ctx context.Context, options SharedTokenOptions) (string, er if options.URL == "" { return "", fmt.Errorf("URL of session manager cannot be empty") } + + if options.TokenFile == "" { + options.TokenFile = saFile + } + + // If the token is empty, we should use service account from the Pod instead if options.Token == "" { - return "", fmt.Errorf("token of session manager cannot be empty") + saValue, err := os.ReadFile(options.TokenFile) + if err != nil { + return "", fmt.Errorf("failed reading token from service account: %w", err) + } + options.Token = string(saValue) } timeout := 5 * time.Second diff --git a/pkg/common/vclib/vc_session_manager_test.go b/pkg/common/vclib/vc_session_manager_test.go index 023f1eb35..07c7e685a 100644 --- a/pkg/common/vclib/vc_session_manager_test.go +++ b/pkg/common/vclib/vc_session_manager_test.go @@ -7,10 +7,12 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/cloud-provider-vsphere/pkg/common/vclib" ) @@ -74,11 +76,11 @@ func TestGetSharedToken(t *testing.T) { assert.ErrorContains(t, err, "URL of session manager cannot be empty") }) - t.Run("should fail when no token is passed", func(t *testing.T) { + t.Run("should fail when no token is passed and SA token cannot be read", func(t *testing.T) { _, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ - URL: "https://some-session-manager.tld/session", + URL: "http://something.tld/lala", }) - assert.ErrorContains(t, err, "token of session manager cannot be empty") + assert.ErrorContains(t, err, "failed reading token from service account: open /var/run/secrets/kubernetes.io/serviceaccount/token: no such file or directory") }) t.Run("should fail when passed URL is invalid", func(t *testing.T) { @@ -165,6 +167,22 @@ func TestGetSharedToken(t *testing.T) { assert.NoError(t, err) assert.Equal(t, validResponse, token) }) + + t.Run("should return a valid token when using a file as a token", func(t *testing.T) { + tokenFile, err := os.CreateTemp("", "") + require.NoError(t, err) + require.NoError(t, tokenFile.Close()) + require.NoError(t, os.WriteFile(tokenFile.Name(), []byte(validToken), 0755)) + + reqURL := fmt.Sprintf("%s/session", server.URL) + token, err := vclib.GetSharedToken(ctx, vclib.SharedTokenOptions{ + URL: reqURL, + TrustedCertificates: certpool, + TokenFile: tokenFile.Name(), + }) + assert.NoError(t, err) + assert.Equal(t, validResponse, token) + }) }) } From 2f0981edc6d2edee4e3a08deab63943ac32d0139 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Wed, 14 May 2025 10:07:24 -0300 Subject: [PATCH 5/5] Add docs on shared session implementation --- docs/book/vc_shared_sessions.md | 117 ++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 docs/book/vc_shared_sessions.md diff --git a/docs/book/vc_shared_sessions.md b/docs/book/vc_shared_sessions.md new file mode 100644 index 000000000..1ce6b8c75 --- /dev/null +++ b/docs/book/vc_shared_sessions.md @@ -0,0 +1,117 @@ +# vSphere Shared Session capability + +One problem that can be found when provisioning a large amount of clusters using +vSphere Cloud Provider is vCenter session exhaustion. This happens because every +workload cluster needs to request a new session to vSphere to do proper reconciliation. + +vSphere 8.0U3 and up uses a new approach of session management, that allows the +creation and sharing of the sessions among different clusters. + +A cluster admin can implement a rest API that, once called, requests a new vCenter +session and shares with CPI. This session will not count on the total generated +sessions of vSphere, and instead will be a child derived session. + +This configuration can be applied on vSphere Cloud Provider with the usage of +the following secret/credentials, instead of vSphere Username/password: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + namespace: kube-system + name: vsphere-cloud-secret +stringData: + your-vcenter-host.vc-session-manager-url: "https://shared-session-service.tld/session" + your-vcenter-host.vc-session-manager-token: "authenticationtoken" +``` + +The configuration above will make CPI call the shared session rest API and use the +provided token to authenticate against vSphere, instead of using a username/password. + +The parameter provider at `vc-session-manager-token` is sent as a `Authorization: Bearer` token +to the session manager, and in case this directive is not configured CPI will send the +Pod Service Account token instead. + +Below is an example implementation of a shared session manager rest API. Starting the +program below and calling `http://127.0.0.1:18080/session` should return a JSON that is expected +by CPI using session manager to work: + +```shell +$ curl 127.0.0.1:18080/session +{"token":"cst-VCT-52f8d061-aace-4506-f4e6-fca78293a93f-....."} +``` + +**NOTE**: Below implementation is **NOT PRODUCTION READY** and does not implement +any kind of authentication! + +```go +package main + +import ( + "context" + "encoding/json" + "log" + "net/http" + "net/url" + + "github.com/vmware/govmomi" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" +) + +const ( + vcURL = "https://my-vc.tld" + vcUsername = "Administrator@vsphere.local" + vcPassword = "somepassword" +) + +var ( + userPassword = url.UserPassword(vcUsername, vcPassword) +) + +// SharedSessionResponse is the expected response of CPI when using Shared session manager +type SharedSessionResponse struct { + Token string `json:"token"` +} + +func main() { + ctx := context.Background() + vcURL, err := soap.ParseURL(vcURL) + if err != nil { + panic(err) + } + soapClient := soap.NewClient(vcURL, false) + c, err := vim25.NewClient(ctx, soapClient) + if err != nil { + panic(err) + } + client := &govmomi.Client{ + Client: c, + SessionManager: session.NewManager(c), + } + if err := client.SessionManager.Login(ctx, userPassword); err != nil { + panic(err) + } + + vcsession := func(w http.ResponseWriter, r *http.Request) { + clonedtoken, err := client.SessionManager.AcquireCloneTicket(ctx) + if err != nil { + w.WriteHeader(http.StatusForbidden) + return + } + token := &SharedSessionResponse{Token: clonedtoken} + jsonT, err := json.Marshal(token) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Write(jsonT) + } + + http.HandleFunc("/session", vcsession) + log.Printf("starting webserver on port 18080") + http.ListenAndServe(":18080", nil) +} +```