Merge pull request #16336 from influxdata/ready-manual-override

feat(kit/check): Manual Overrides for Readiness Endpoint

CHANGELOG.md

@@ -22,6 +22,7 @@
 1. [16340](https://github.com/influxdata/influxdb/pull/16340): Add last run status to tasks
 1. [16341](https://github.com/influxdata/influxdb/pull/16341): Extend pkger apply functionality with ability to provide secrets outside of pkg
 1. [16345](https://github.com/influxdata/influxdb/pull/16345): Add hide headers flag to influx cli task find cmd
+1. [16336](https://github.com/influxdata/influxdb/pull/16336): Manual Overrides for Readiness Endpoint
 
 ### Bug Fixes
 

kit/check/check.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
-	"sync/atomic"
+	"sync"
 )
 
 // Status string to indicate the overall status of the check.
@@ -26,10 +26,10 @@ const (
 
 // Check wraps a map of service names to status checkers.
 type Check struct {
-	healthChecks      []Checker
+	healthChecks   []Checker
-	readyChecks       []Checker
+	readyChecks    []Checker
-	manualOverride    atomic.Value
+	healthOverride override
-	manualHealthState atomic.Value
+	readyOverride  override
 
 	passthroughHandler http.Handler
 }
@@ -42,8 +42,8 @@ type Checker interface {
 // NewCheck returns a Health with a default checker.
 func NewCheck() *Check {
 	ch := &Check{}
-	ch.manualOverride.Store(false)
+	ch.healthOverride.disable()
-	ch.manualHealthState.Store(false)
+	ch.readyOverride.disable()
 	return ch
 }
 
@@ -74,13 +74,10 @@ func (c *Check) CheckHealth(ctx context.Context) Response {
 		Status: StatusPass,
 		Checks: make(Responses, len(c.healthChecks)),
 	}
-	override := c.manualOverride.Load().(bool)
+
-	if override {
+	status, overriding := c.healthOverride.get()
-		if c.manualHealthState.Load().(bool) {
+	if overriding {
-			response.Status = StatusPass
+		response.Status = status
-		} else {
-			response.Status = StatusFail
-		}
 		overrideResponse := Response{
 			Name:    "manual-override",
 			Message: "health manually overridden",
@@ -89,7 +86,7 @@ func (c *Check) CheckHealth(ctx context.Context) Response {
 	}
 	for i, ch := range c.healthChecks {
 		resp := ch.Check(ctx)
-		if resp.Status != StatusPass && !override {
+		if resp.Status != StatusPass && !overriding {
 			response.Status = resp.Status
 		}
 		response.Checks[i] = resp
@@ -105,9 +102,19 @@ func (c *Check) CheckReady(ctx context.Context) Response {
 		Status: StatusPass,
 		Checks: make(Responses, len(c.readyChecks)),
 	}
+
+	status, overriding := c.readyOverride.get()
+	if overriding {
+		response.Status = status
+		overrideResponse := Response{
+			Name:    "manual-override",
+			Message: "ready manually overridden",
+		}
+		response.Checks = append(response.Checks, overrideResponse)
+	}
 	for i, c := range c.readyChecks {
 		resp := c.Check(ctx)
-		if resp.Status != StatusPass {
+		if resp.Status != StatusPass && !overriding {
 			response.Status = resp.Status
 		}
 		response.Checks[i] = resp
@@ -124,54 +131,108 @@ func (c *Check) SetPassthrough(h http.Handler) {
 
 // ServeHTTP serves /ready and /health requests with the respective checks.
 func (c *Check) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// allow requests not intended for checks to pass through.
+	const (
-	if r.URL.Path != "/ready" && r.URL.Path != "/health" {
+		pathReady  = "/ready"
+		pathHealth = "/health"
+		queryForce = "force"
+	)
+
+	path := r.URL.Path
+
+	// Allow requests not intended for checks to pass through.
+	if path != pathReady && path != pathHealth {
 		if c.passthroughHandler != nil {
 			c.passthroughHandler.ServeHTTP(w, r)
 			return
 		}
 
-		// We cant handle this request.
+		// We can't handle this request.
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	msg := ""
+	ctx := r.Context()
-	status := http.StatusOK
+	query := r.URL.Query()
 
-	var resp Response
+	switch path {
-	switch r.URL.Path {
+	case pathReady:
-	case "/ready":
+		switch query.Get(queryForce) {
-		resp = c.CheckReady(r.Context())
-	case "/health":
-		query := r.URL.Query()
-		switch query.Get("force") {
 		case "true":
-			c.manualOverride.Store(true)
+			switch query.Get("ready") {
-			switch query.Get("healthy") {
 			case "true":
-				c.manualHealthState.Store(true)
+				c.readyOverride.enable(StatusPass)
 			case "false":
-				c.manualHealthState.Store(false)
+				c.readyOverride.enable(StatusFail)
 			}
 		case "false":
-			c.manualOverride.Store(false)
+			c.readyOverride.disable()
 		}
-		resp = c.CheckHealth(r.Context())
+		writeResponse(w, c.CheckReady(ctx))
+	case pathHealth:
+		switch query.Get(queryForce) {
+		case "true":
+			switch query.Get("healthy") {
+			case "true":
+				c.healthOverride.enable(StatusPass)
+			case "false":
+				c.healthOverride.enable(StatusFail)
+			}
+		case "false":
+			c.healthOverride.disable()
+		}
+		writeResponse(w, c.CheckHealth(ctx))
 	}
+}
 
-	// Set the HTTP status if the check failed
+// writeResponse writes a Response to the wire as JSON. The HTTP status code
+// accompanying the payload is the primary means for signaling the status of the
+// checks. The possible status codes are:
+//
+// - 200 OK: All checks pass.
+// - 503 Service Unavailable: Some checks are failing.
+// - 500 Internal Server Error: There was a problem serializing the Response.
+func writeResponse(w http.ResponseWriter, resp Response) {
+	status := http.StatusOK
 	if resp.Status == StatusFail {
-		// Normal state, the HTTP response status reflects the status-reported health.
 		status = http.StatusServiceUnavailable
 	}
 
-	b, err := json.MarshalIndent(resp, "", "  ")
+	msg, err := json.MarshalIndent(resp, "", "  ")
 	if err != nil {
-		b = []byte(`{"message": "error marshaling response", "status": "fail"}`)
+		msg = []byte(`{"message": "error marshaling response", "status": "fail"}`)
 		status = http.StatusInternalServerError
 	}
-	msg = string(b)
 	w.WriteHeader(status)
-	fmt.Fprintln(w, msg)
+	fmt.Fprintln(w, string(msg))
+}
+
+// override is a manual override for an entire group of checks.
+type override struct {
+	mtx    sync.Mutex
+	status Status
+	active bool
+}
+
+// get returns the Status of an override as well as whether or not an override
+// is currently active.
+func (m *override) get() (Status, bool) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	return m.status, m.active
+}
+
+// disable disables the override.
+func (m *override) disable() {
+	m.mtx.Lock()
+	m.active = false
+	m.status = StatusFail
+	m.mtx.Unlock()
+}
+
+// enable turns on the override and establishes a specific Status for which to.
+func (m *override) enable(s Status) {
+	m.mtx.Lock()
+	m.active = true
+	m.status = s
+	m.mtx.Unlock()
 }

kit/check/check_test.go

@@ -165,7 +165,67 @@ func TestHealthSorting(t *testing.T) {
 	}
 }
 
-func TestForceHealth(t *testing.T) {
+func TestForceHealthy(t *testing.T) {
+	c, ts := buildCheckWithServer()
+	defer ts.Close()
+
+	c.AddHealthCheck(mockFail("a"))
+
+	_, err := http.Get(ts.URL + "/health?force=true&healthy=true")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err := http.Get(ts.URL + "/health")
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual, err := respBuilder(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := &Response{
+		Name:   "Health",
+		Status: "pass",
+		Checks: Responses{
+			Response{Name: "manual-override", Message: "health manually overridden"},
+			Response{Name: "a", Status: "fail"},
+		},
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("unexpected response. expected %v, actual %v", expected, actual)
+	}
+
+	_, err = http.Get(ts.URL + "/health?force=false")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected = &Response{
+		Name:   "Health",
+		Status: "fail",
+		Checks: Responses{
+			Response{Name: "a", Status: "fail"},
+		},
+	}
+
+	resp, err = http.Get(ts.URL + "/health")
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual, err = respBuilder(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("unexpected response. expected %v, actual %v", expected, actual)
+	}
+}
+
+func TestForceUnhealthy(t *testing.T) {
 	c, ts := buildCheckWithServer()
 	defer ts.Close()
 
@@ -225,6 +285,126 @@ func TestForceHealth(t *testing.T) {
 	}
 }
 
+func TestForceReady(t *testing.T) {
+	c, ts := buildCheckWithServer()
+	defer ts.Close()
+
+	c.AddReadyCheck(mockFail("a"))
+
+	_, err := http.Get(ts.URL + "/ready?force=true&ready=true")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err := http.Get(ts.URL + "/ready")
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual, err := respBuilder(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := &Response{
+		Name:   "Ready",
+		Status: "pass",
+		Checks: Responses{
+			Response{Name: "manual-override", Message: "ready manually overridden"},
+			Response{Name: "a", Status: "fail"},
+		},
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("unexpected response. expected %v, actual %v", expected, actual)
+	}
+
+	_, err = http.Get(ts.URL + "/ready?force=false")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected = &Response{
+		Name:   "Ready",
+		Status: "fail",
+		Checks: Responses{
+			Response{Name: "a", Status: "fail"},
+		},
+	}
+
+	resp, err = http.Get(ts.URL + "/ready")
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual, err = respBuilder(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("unexpected response. expected %v, actual %v", expected, actual)
+	}
+}
+
+func TestForceNotReady(t *testing.T) {
+	c, ts := buildCheckWithServer()
+	defer ts.Close()
+
+	c.AddReadyCheck(mockPass("a"))
+
+	_, err := http.Get(ts.URL + "/ready?force=true&ready=false")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err := http.Get(ts.URL + "/ready")
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual, err := respBuilder(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := &Response{
+		Name:   "Ready",
+		Status: "fail",
+		Checks: Responses{
+			Response{Name: "manual-override", Message: "ready manually overridden"},
+			Response{Name: "a", Status: "pass"},
+		},
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("unexpected response. expected %v, actual %v", expected, actual)
+	}
+
+	_, err = http.Get(ts.URL + "/ready?force=false")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected = &Response{
+		Name:   "Ready",
+		Status: "pass",
+		Checks: Responses{
+			Response{Name: "a", Status: "pass"},
+		},
+	}
+
+	resp, err = http.Get(ts.URL + "/ready")
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual, err = respBuilder(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		t.Errorf("unexpected response. expected %v, actual %v", expected, actual)
+	}
+}
+
 func TestNoCrossOver(t *testing.T) {
 	c, ts := buildCheckWithServer()
 	defer ts.Close()