observability: add prometheus variable labels; remove collector

* when re-initializing backends (new-aws() and friends): - register the corresponding metrics only once * part two, prev. commit: 118a821 Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Dec 24, 2024 · 38bc48a · 38bc48a
1 parent 118a821
commit 38bc48a
Show file tree

Hide file tree

Showing 11 changed files with 46 additions and 34 deletions.
diff --git a/ais/backend/aws.go b/ais/backend/aws.go
@@ -66,17 +66,21 @@ var _ core.Backend = (*s3bp)(nil)
 
 // environment variables => static defaults that can still be overridden via bck.Props.Extra.AWS
 // in addition to these two (below), default bucket region = env.AwsDefaultRegion()
-func NewAWS(t core.TargetPut, tstats stats.Tracker) (core.Backend, error) {
+func NewAWS(t core.TargetPut, tstats stats.Tracker, startingUp bool) (core.Backend, error) {
 	s3Endpoint = os.Getenv(env.AWS.Endpoint)
 	awsProfile = os.Getenv(env.AWS.Profile)
 	bp := &s3bp{
 		t:    t,
 		mm:   t.PageMM(),
 		base: base{provider: apc.AWS},
 	}
-	bp.base.init(t.Snode(), tstats)
-	// reset clients map to recreate and reload credentials
-	clients.Clear()
+	if startingUp {
+		// register metrics only once
+		bp.base.init(t.Snode(), tstats)
+	} else {
+		// reset clients map to recreate and reload credentials
+		clients.Clear()
+	}
 	return bp, nil
 }
 

diff --git a/ais/backend/azure.go b/ais/backend/azure.go
@@ -98,7 +98,7 @@ func asEndpoint() string {
 	}
 }
 
-func NewAzure(t core.TargetPut, tstats stats.Tracker) (core.Backend, error) {
+func NewAzure(t core.TargetPut, tstats stats.Tracker, startingUp bool) (core.Backend, error) {
 	blurl := asEndpoint()
 
 	// NOTE: NewSharedKeyCredential requires account name and its primary or secondary key
@@ -112,7 +112,10 @@ func NewAzure(t core.TargetPut, tstats stats.Tracker) (core.Backend, error) {
 		u:     blurl,
 		base:  base{provider: apc.Azure},
 	}
-	bp.base.init(t.Snode(), tstats)
+	if startingUp {
+		// register metrics only once
+		bp.base.init(t.Snode(), tstats)
+	}
 	return bp, nil
 }
 

diff --git a/ais/backend/gcp.go b/ais/backend/gcp.go
@@ -63,7 +63,7 @@ var (
 	_ core.Backend = (*gsbp)(nil)
 )
 
-func NewGCP(t core.TargetPut, tstats stats.Tracker) (_ core.Backend, err error) {
+func NewGCP(t core.TargetPut, tstats stats.Tracker, startingUp bool) (_ core.Backend, err error) {
 	var (
 		projectID     string
 		credProjectID = readCredFile()
@@ -89,8 +89,10 @@ func NewGCP(t core.TargetPut, tstats stats.Tracker) (_ core.Backend, err error)
 		projectID: projectID,
 		base:      base{provider: apc.GCP},
 	}
-	bp.base.init(t.Snode(), tstats)
-
+	if startingUp {
+		// register metrics only once
+		bp.base.init(t.Snode(), tstats)
+	}
 	gctx = context.Background()
 	gcpClient, err = bp.createClient(gctx)
 

diff --git a/ais/backend/mock_aws.go b/ais/backend/mock_aws.go
@@ -18,7 +18,7 @@ import (
 	"github.com/NVIDIA/aistore/stats"
 )
 
-func NewAWS(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) {
+func NewAWS(core.TargetPut, stats.Tracker, bool) (core.Backend, error) {
 	return nil, &cmn.ErrInitBackend{Provider: apc.AWS}
 }
 

diff --git a/ais/backend/mock_azure.go b/ais/backend/mock_azure.go
@@ -13,6 +13,6 @@ import (
 	"github.com/NVIDIA/aistore/stats"
 )
 
-func NewAzure(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) {
+func NewAzure(core.TargetPut, stats.Tracker, bool) (core.Backend, error) {
 	return nil, &cmn.ErrInitBackend{Provider: apc.Azure}
 }
diff --git a/ais/backend/mock_gcp.go b/ais/backend/mock_gcp.go
@@ -13,6 +13,6 @@ import (
 	"github.com/NVIDIA/aistore/stats"
 )
 
-func NewGCP(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) {
+func NewGCP(core.TargetPut, stats.Tracker, bool) (core.Backend, error) {
 	return nil, &cmn.ErrInitBackend{Provider: apc.GCP}
 }
diff --git a/ais/backend/mock_oci.go b/ais/backend/mock_oci.go
@@ -13,6 +13,6 @@ import (
 	"github.com/NVIDIA/aistore/stats"
 )
 
-func NewOCI(core.TargetPut, stats.Tracker) (core.Backend, error) {
+func NewOCI(core.TargetPut, stats.Tracker, bool) (core.Backend, error) {
 	return nil, &cmn.ErrInitBackend{Provider: apc.GCP}
 }
diff --git a/ais/backend/oci.go b/ais/backend/oci.go
@@ -74,7 +74,7 @@ type ocibp struct {
 	base
 }
 
-func NewOCI(t core.TargetPut, tstats stats.Tracker) (core.Backend, error) {
+func NewOCI(t core.TargetPut, tstats stats.Tracker, startingUp bool) (core.Backend, error) {
 	bp := &ocibp{
 		t:    t,
 		base: base{provider: apc.AWS},
@@ -128,7 +128,10 @@ func NewOCI(t core.TargetPut, tstats stats.Tracker) (core.Backend, error) {
 	}
 	bp.namespace = *resp.Value
 
-	bp.base.init(t.Snode(), tstats)
+	if startingUp {
+		// register metrics only once
+		bp.base.init(t.Snode(), tstats)
+	}
 
 	return bp, nil
 }

diff --git a/ais/target.go b/ais/target.go
@@ -114,14 +114,14 @@ func (t *target) initBackends(tstats *stats.Trunner) {
 		}
 	}
 
-	if err := t.initBuiltTagged(tstats, config); err != nil {
+	if err := t.initBuiltTagged(tstats, config, true /*starting up*/); err != nil {
 		cos.ExitLog(err)
 	}
 }
 
 // - remote (e.g. cloud) backends  w/ empty stubs unless populated via build tags
 // - enabled/disabled via config.Backend
-func (t *target) initBuiltTagged(tstats *stats.Trunner, config *cmn.Config) error {
+func (t *target) initBuiltTagged(tstats *stats.Trunner, config *cmn.Config, startingUp bool) error {
 	var enabled, disabled, notlinked []string
 
 	for provider := range apc.Providers {
@@ -131,13 +131,13 @@ func (t *target) initBuiltTagged(tstats *stats.Trunner, config *cmn.Config) erro
 		)
 		switch provider {
 		case apc.AWS:
-			add, err = backend.NewAWS(t, tstats)
+			add, err = backend.NewAWS(t, tstats, startingUp)
 		case apc.GCP:
-			add, err = backend.NewGCP(t, tstats)
+			add, err = backend.NewGCP(t, tstats, startingUp)
 		case apc.Azure:
-			add, err = backend.NewAzure(t, tstats)
+			add, err = backend.NewAzure(t, tstats, startingUp)
 		case apc.OCI:
-			add, err = backend.NewOCI(t, tstats)
+			add, err = backend.NewOCI(t, tstats, startingUp)
 		case apc.HT:
 			add, err = backend.NewHT(t, config, tstats)
 		case apc.AIS:

diff --git a/ais/test/object_test.go b/ais/test/object_test.go
@@ -922,13 +922,13 @@ func TestValidateOnWarmGetRemoteBucket(t *testing.T) {
 	var mockBackend core.Backend
 	switch m.bck.Provider {
 	case apc.AWS:
-		mockBackend, _ = backend.NewAWS(tMock, mock.NewStatsTracker())
+		mockBackend, _ = backend.NewAWS(tMock, mock.NewStatsTracker(), false /*starting up*/)
 	case apc.GCP:
-		mockBackend, _ = backend.NewGCP(tMock, mock.NewStatsTracker())
+		mockBackend, _ = backend.NewGCP(tMock, mock.NewStatsTracker(), false /*starting up*/)
 	case apc.Azure:
-		mockBackend, _ = backend.NewAzure(tMock, mock.NewStatsTracker())
+		mockBackend, _ = backend.NewAzure(tMock, mock.NewStatsTracker(), false /*starting up*/)
 	case apc.OCI:
-		mockBackend, _ = backend.NewOCI(tMock, mock.NewStatsTracker())
+		mockBackend, _ = backend.NewOCI(tMock, mock.NewStatsTracker(), false /*starting up*/)
 	default:
 		t.Fatalf("unexpected backend provider %q", m.bck.Provider)
 	}

diff --git a/ais/tgtcp.go b/ais/tgtcp.go
@@ -188,7 +188,7 @@ func (t *target) daeputMsg(w http.ResponseWriter, r *http.Request) {
 			provider = msg.Name
 		)
 		if provider == "" { // all
-			if err := t.initBuiltTagged(t.statsT.(*stats.Trunner), cmn.GCO.Get()); err != nil {
+			if err := t.initBuiltTagged(t.statsT.(*stats.Trunner), cmn.GCO.Get(), false); err != nil {
 				t.writeErr(w, r, err)
 			}
 			return
@@ -198,13 +198,13 @@ func (t *target) daeputMsg(w http.ResponseWriter, r *http.Request) {
 		var add core.Backend
 		switch provider {
 		case apc.AWS:
-			add, err = backend.NewAWS(t, tstats)
+			add, err = backend.NewAWS(t, tstats, false /*starting up*/)
 		case apc.GCP:
-			add, err = backend.NewGCP(t, tstats)
+			add, err = backend.NewGCP(t, tstats, false)
 		case apc.Azure:
-			add, err = backend.NewAzure(t, tstats)
+			add, err = backend.NewAzure(t, tstats, false)
 		case apc.OCI:
-			add, err = backend.NewOCI(t, tstats)
+			add, err = backend.NewOCI(t, tstats, false)
 		}
 		if err != nil {
 			t.writeErr(w, r, err)
@@ -301,13 +301,13 @@ func (t *target) enableBackend(w http.ResponseWriter, r *http.Request, items []s
 		var err error
 		switch provider {
 		case apc.AWS:
-			bp, err = backend.NewAWS(t, t.statsT)
+			bp, err = backend.NewAWS(t, t.statsT, false /*starting up*/)
 		case apc.GCP:
-			bp, err = backend.NewGCP(t, t.statsT)
+			bp, err = backend.NewGCP(t, t.statsT, false /*starting up*/)
 		case apc.Azure:
-			bp, err = backend.NewAzure(t, t.statsT)
+			bp, err = backend.NewAzure(t, t.statsT, false /*starting up*/)
 		case apc.OCI:
-			bp, err = backend.NewOCI(t, t.statsT)
+			bp, err = backend.NewOCI(t, t.statsT, false /*starting up*/)
 		}
 		if err != nil {
 			debug.AssertNoErr(err) // (unlikely)