Fix issue where server pool could fail empty due to haproxy being alive but not fully up.

Author: arlaneenalra

.github/workflows/goreleaser.yaml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.21
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2

.goreleaser.yml

@@ -18,11 +18,7 @@ builds:
       - arm64
 
 archives:
-  - replacements:
-      darwin: Darwin
-      linux: Linux
-      amd64: x86_64
-    name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}"
+  - name_template: '{{ .ProjectName }}_{{- title  .Os -}}_{{- if (or (eq .Arch "amd64") (eq .Arch "amd64_v1")) }}x86_64{{- else -}}{{ .Arch }}{{ end -}}'
 
 checksum:
   name_template: 'checksums.txt'

internal/haproxy/info.go

@@ -1,6 +1,8 @@
 package haproxy
 
 import (
+  "errors"
+
 	log "github.com/sirupsen/logrus"
 )
 
@@ -28,6 +30,10 @@ func (c *Client) GetBackendNames() ([]string, error) {
 		backends = append(backends, backend.Name)
 	}
 
+  if len(backends) == 0 {
+    return nil, errors.New("no backends found") 
+  }
+
 	return backends, nil
 }
 
@@ -49,6 +55,13 @@ func (c *Client) GetServerNames(backend string) ([]string, error) {
 		servers = append(servers, server.Name)
 	}
 
+  if len(servers) == 0 {
+    log.WithFields(log.Fields{
+      "backend": backend,
+    }).Error("no servers for backend")
+    return nil, errors.New("no servers for backend") 
+  }
+
 	return servers, nil
 }
 

internal/mapper/mapper.go

@@ -28,20 +28,46 @@ func New(backend string, haProxyClient *haproxy.Client, useExternal bool) (Backe
 	return mapper, nil
 }
 
-func (m *Mapper) nextServer() (string, error) {
+func (m *Mapper) getServer() (string, error) {
 	var next string
 	if len(m.serverPool) == 0 {
 		log.WithFields(log.Fields{
 			"backend": m.backend,
 		}).Warn("server pool exhausted")
-		return "", fmt.Errorf("server pool exhausted")
+	
+    return "", fmt.Errorf("server pool exhausted")
 	}
 
 	next, m.serverPool = m.serverPool[0], m.serverPool[1:]
 
 	return next, nil
 }
 
+func (m* Mapper) nextServer() (string, error) {
+  next, err  := m.getServer()
+  if err == nil {
+    return next, nil 
+  }
+
+  // Attempt to reset the server pool since we don't seem to have any
+  if len(m.serverMap) == 0 {
+    log.WithFields(log.Fields{
+      "backend": m.backend,
+    }).Info("no servers configured, attempting to reset server pool")
+
+    if err := m.resetServerPool(); err != nil {
+      log.WithFields(log.Fields{
+        "backend": m.backend,
+      }).Warn("unble to reset server pool")
+
+      return "", err
+    }
+  } 
+
+  // try again.
+  return m.getServer()
+}
+
 func (m *Mapper) releaseServer(server string) {
 	m.serverPool = append(m.serverPool, server)
 }

internal/mapper/sync_all.go

@@ -1,10 +1,28 @@
 package mapper
 
 import (
+  "errors"
+  "time"
+
 	log "github.com/sirupsen/logrus"
 )
 
 func (m *Mapper) resetServerPool() error {
+  // HAProxy can take a moment to start up
+  for retries := 0; retries < 5; retries++ {
+    if err := m.tryResetServerPool(); err == nil {
+      return nil
+    }
+    log.WithFields(log.Fields{
+      "retries": retries,
+    }).Warn("unable to load servers for backend")
+    time.Sleep(5 * time.Second)
+  }
+
+  return errors.New("could not load servers for backend, failing")
+}
+
+func (m *Mapper) tryResetServerPool() error {
 	// Populate pool of servers
 	serverPool, err := m.haProxyClient.GetServerNames(m.backend)
 	if err != nil {