From b98bb3308c0fe77af7f36706e5b631802a63dd30 Mon Sep 17 00:00:00 2001 From: Matthew Scerri <32162885+scermat@users.noreply.github.com> Date: Wed, 18 Sep 2024 19:30:41 +0200 Subject: [PATCH 1/3] updated health checks --- gateway/Dockerfile | 2 +- gateway/go.mod | 2 +- gateway/go.sum | 4 ++-- gateway/proxy/health.go | 45 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 82022a9a..06f1e232 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -12,4 +12,4 @@ RUN go build -o /go/bin/porters EXPOSE 9000 -CMD ["porters", "gateway"] +CMD ["sh", "-c", "ulimit -n 65536 && exec /go/bin/porters gateway"] diff --git a/gateway/go.mod b/gateway/go.mod index 316fc774..bb46b99a 100644 --- a/gateway/go.mod +++ b/gateway/go.mod @@ -9,6 +9,7 @@ require ( github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.4.0 + golang.org/x/sys v0.25.0 ) require ( @@ -18,6 +19,5 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - golang.org/x/sys v0.16.0 // indirect google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/gateway/go.sum b/gateway/go.sum index 507fba6f..f5b554e3 100644 --- a/gateway/go.sum +++ b/gateway/go.sum @@ -34,8 +34,8 @@ github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwy github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/gateway/proxy/health.go b/gateway/proxy/health.go index 1d4c27cb..9457dc7b 100644 --- a/gateway/proxy/health.go +++ b/gateway/proxy/health.go @@ -2,13 +2,52 @@ package proxy import ( "io" + "log/slog" "net/http" + "porters/common" "porters/db" + + "golang.org/x/sys/unix" ) -// TODO other healthchecks should be added +type HealthStatus struct { + CacheHealth *common.HealthCheckStatus `json:"cache_health"` // Use the correct type here +} + +// New function to check file descriptor health and log the results using slog +func checkFileDescriptorHealth() { + var rLimit unix.rLimit + + // Get the max number of allowed file descriptors + //err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) // Corrected package usage + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) + if err != nil { + slog.Error("Error retrieving file descriptor limit", "error", err) + return + } + + // Log the file descriptor information using slog (mocking file descriptor usage for now) + fdUsage := uint64(0) // Placeholder, replace with actual logic if available + slog.Info("File Descriptor Usage", + "max", rLimit.Cur, + "used", fdUsage, + "healthy", fdUsage < rLimit.Cur*80/100) +} + +// Update your healthHandler to log file descriptor health but not expose it func healthHandler(resp http.ResponseWriter, req *http.Request) { - hc := (&db.Cache{}).Healthcheck() + // Existing cache health check + cacheHealth := (&db.Cache{}).Healthcheck() + + // Log file descriptor health (do not expose it in the response) + checkFileDescriptorHealth() + + // Only return the non-sensitive health information (cache health) + healthStatus := HealthStatus{ + CacheHealth: cacheHealth, + } + + // Marshal the health status into JSON resp.Header().Set("Content-Type", "application/json") - io.WriteString(resp, hc.ToJson()) + io.WriteString(resp, healthStatus.CacheHealth.ToJson()) } From 46f9ecdb8ce21f8a1160be39197efb02ff2488ee Mon Sep 17 00:00:00 2001 From: Matthew Scerri <32162885+scermat@users.noreply.github.com> Date: Wed, 18 Sep 2024 19:33:40 +0200 Subject: [PATCH 2/3] fixed type references --- gateway/proxy/health.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gateway/proxy/health.go b/gateway/proxy/health.go index 9457dc7b..19e66e1c 100644 --- a/gateway/proxy/health.go +++ b/gateway/proxy/health.go @@ -16,10 +16,9 @@ type HealthStatus struct { // New function to check file descriptor health and log the results using slog func checkFileDescriptorHealth() { - var rLimit unix.rLimit + var rLimit unix.Rlimit // Correct capitalization // Get the max number of allowed file descriptors - //err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) // Corrected package usage err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) if err != nil { slog.Error("Error retrieving file descriptor limit", "error", err) From 015709a4b9254da0d468aedef486de93a98c8cf0 Mon Sep 17 00:00:00 2001 From: Matthew Scerri <32162885+scermat@users.noreply.github.com> Date: Thu, 26 Sep 2024 00:56:08 +0200 Subject: [PATCH 3/3] updated health endpoint to check for file descriptors --- gateway/proxy/health.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/gateway/proxy/health.go b/gateway/proxy/health.go index 19e66e1c..30652a38 100644 --- a/gateway/proxy/health.go +++ b/gateway/proxy/health.go @@ -1,9 +1,11 @@ package proxy import ( + "fmt" "io" "log/slog" "net/http" + "os" "porters/common" "porters/db" @@ -15,22 +17,33 @@ type HealthStatus struct { } // New function to check file descriptor health and log the results using slog -func checkFileDescriptorHealth() { - var rLimit unix.Rlimit // Correct capitalization +func checkFileDescriptorHealth() bool { + var rLimit unix.Rlimit // Get the max number of allowed file descriptors err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit) if err != nil { slog.Error("Error retrieving file descriptor limit", "error", err) - return + return false } - // Log the file descriptor information using slog (mocking file descriptor usage for now) - fdUsage := uint64(0) // Placeholder, replace with actual logic if available + // Get the actual number of file descriptors in use + files, err := os.ReadDir("/proc/self/fd") + if err != nil { + slog.Error("Error reading /proc/self/fd", "error", err) + return false + } + fdUsage := uint64(len(files)) + + healthy := fdUsage < rLimit.Cur*80/100 + + // Log the file descriptor information using slog slog.Info("File Descriptor Usage", "max", rLimit.Cur, "used", fdUsage, - "healthy", fdUsage < rLimit.Cur*80/100) + "healthy", healthy) + + return healthy } // Update your healthHandler to log file descriptor health but not expose it @@ -38,8 +51,8 @@ func healthHandler(resp http.ResponseWriter, req *http.Request) { // Existing cache health check cacheHealth := (&db.Cache{}).Healthcheck() - // Log file descriptor health (do not expose it in the response) - checkFileDescriptorHealth() + // Log file descriptor health and determine if the server is healthy + fdHealthy := checkFileDescriptorHealth() // Only return the non-sensitive health information (cache health) healthStatus := HealthStatus{ @@ -48,5 +61,11 @@ func healthHandler(resp http.ResponseWriter, req *http.Request) { // Marshal the health status into JSON resp.Header().Set("Content-Type", "application/json") - io.WriteString(resp, healthStatus.CacheHealth.ToJson()) + + if fdHealthy { + io.WriteString(resp, healthStatus.CacheHealth.ToJson()) + } else { + resp.WriteHeader(http.StatusInternalServerError) + io.WriteString(resp, fmt.Sprintf(`{"error": "file descriptor limit exceeded"}`)) + } }