tls cert loader: rewrite from scratch; intra-cluster clients

* from scratch, prev. commit 8107a3b
  - original ticket: [NGNSDS-632]
* separately, introduce `intra-cluster` to differentiate clients
  - `NewTLS(..., intra-cluster)`

Signed-off-by: Alex Aizman <[email protected]>

.gitignore

@@ -5,6 +5,7 @@
 .*.swp
 *.key
 *.crt
+*.csr
 
 ## PYTHON ##
 

ais/daemon.go

@@ -21,7 +21,6 @@ import (
 	"github.com/NVIDIA/aistore/cmn/debug"
 	"github.com/NVIDIA/aistore/cmn/k8s"
 	"github.com/NVIDIA/aistore/cmn/nlog"
-	"github.com/NVIDIA/aistore/cmn/tls"
 	"github.com/NVIDIA/aistore/core/meta"
 	"github.com/NVIDIA/aistore/fs"
 	"github.com/NVIDIA/aistore/hk"
@@ -202,13 +201,6 @@ func initDaemon(version, buildTime string) cos.Runner {
 	// declared xactions, as per xact/api.go
 	xreg.Init()
 
-	if config.Net.HTTP.UseHTTPS {
-		err = tls.Init(config.Net.HTTP.Certificate, config.Net.HTTP.CertKey)
-		if err != nil {
-			cos.ExitLogf("failed to initialize Certificate Manager: %v", err)
-		}
-	}
-
 	// primary 'host[:port]' endpoint or URL from the environment
 	if daemon.EP = os.Getenv(env.AIS.PrimaryEP); daemon.EP != "" {
 		scheme := "http"

ais/htcommon.go

@@ -588,21 +588,19 @@ func newTLS(conf *cmn.HTTPConf) (tlsConf *tls.Config, err error) {
 	}
 	if clientAuth > tls.RequestClientCert {
 		if caCert, err = os.ReadFile(conf.ClientCA); err != nil {
-			return
+			return nil, fmt.Errorf("new-tls: failed to read PEM %q, err: %w", conf.ClientCA, err)
 		}
 		pool = x509.NewCertPool()
 		if ok := pool.AppendCertsFromPEM(caCert); !ok {
-			return nil, fmt.Errorf("tls: failed to append CA certs from PEM: %q", conf.ClientCA)
+			return nil, fmt.Errorf("new-tls: failed to append CA certs from PEM %q", conf.ClientCA)
 		}
 		tlsConf.ClientCAs = pool
 	}
 	if conf.Certificate != "" && conf.CertKey != "" {
-		if !aistls.IsLoaderSet() {
-			return nil, errors.New("tls: certificate manager not set")
-		}
-		tlsConf.GetCertificate = aistls.GetCert()
+		tlsConf.GetCertificate, err = aistls.GetCert()
+		debug.AssertNoErr(err)
 	}
-	return
+	return tlsConf, err
 }
 
 func (server *netServer) connStateListener(c net.Conn, cs http.ConnState) {

ais/htrun.go

@@ -33,6 +33,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn/k8s"
 	"github.com/NVIDIA/aistore/cmn/mono"
 	"github.com/NVIDIA/aistore/cmn/nlog"
+	aistls "github.com/NVIDIA/aistore/cmn/tls"
 	"github.com/NVIDIA/aistore/core"
 	"github.com/NVIDIA/aistore/core/meta"
 	"github.com/NVIDIA/aistore/memsys"
@@ -258,6 +259,13 @@ func (h *htrun) regNetHandlers(networkHandlers []networkHandler) {
 }
 
 func (h *htrun) init(config *cmn.Config) {
+	// before newTLS() below and before clients
+	if config.Net.HTTP.UseHTTPS {
+		if err := aistls.Init(config.Net.HTTP.Certificate, config.Net.HTTP.CertKey); err != nil {
+			cos.ExitLog(err)
+		}
+	}
+
 	initCtrlClient(config)
 	initDataClient(config)
 
@@ -511,6 +519,7 @@ func (h *htrun) run(config *cmn.Config) error {
 		}
 		tlsConf = c
 	}
+
 	if config.HostNet.UseIntraControl {
 		go func() {
 			_ = g.netServ.control.listen(h.si.ControlNet.TCPEndpoint(), logger, tlsConf, config)

ais/prxrev.go

@@ -113,7 +113,7 @@ func rpTransport(config *cmn.Config) *http.Transport {
 		transport = cmn.NewTransport(cmn.TransportArgs{Timeout: config.Client.Timeout.D()})
 	)
 	if config.Net.HTTP.UseHTTPS {
-		transport.TLSClientConfig, err = cmn.NewTLS(config.Net.HTTP.ToTLS())
+		transport.TLSClientConfig, err = cmn.NewTLS(config.Net.HTTP.ToTLS(), true /*intra-cluster*/)
 		if err != nil {
 			cos.ExitLog(err)
 		}

bench/tools/aisloader/client.go

@@ -35,7 +35,7 @@ var (
 	cargs = cmn.TransportArgs{
 		UseHTTPProxyEnv: true,
 	}
-	// NOTE: client X509 certificate and other `cmn.TLSArgs` variables can be provided via (os.Getenv) environment.
+	// NOTE: client X.509 certificate and other `cmn.TLSArgs` variables can be provided via (os.Getenv) environment.
 	// See also:
 	// - docs/aisloader.md, section "Environment variables"
 	// - AIS_ENDPOINT and aisEndpoint
@@ -261,7 +261,7 @@ func newTraceCtx(proxyURL string) *traceCtx {
 		err       error
 	)
 	if cos.IsHTTPS(proxyURL) {
-		transport.TLSClientConfig, err = cmn.NewTLS(sargs)
+		transport.TLSClientConfig, err = cmn.NewTLS(sargs, false /*intra-cluster*/)
 		cos.AssertNoErr(err)
 	}
 	tctx.tr = &traceableTransport{

bench/tools/aisloader/run.go

@@ -920,7 +920,7 @@ func _init(p *params) (err error) {
 	if useHTTPS {
 		// environment to override client config
 		cmn.EnvToTLS(&sargs)
-		p.bp.Client = cmn.NewClientTLS(cargs, sargs)
+		p.bp.Client = cmn.NewClientTLS(cargs, sargs, false /*intra-cluster*/)
 	} else {
 		p.bp.Client = cmn.NewClient(cargs)
 	}

cmd/cli/cli/init.go

@@ -56,8 +56,8 @@ func Init(args []string) (err error) {
 		UA:    ua,
 	}
 	if cos.IsHTTPS(clusterURL) {
-		cfg.WarnTLS("aistore at " + clusterURL)
-		clientTLS = cmn.NewClientTLS(cargs, sargs)
+		// TODO -- FIXME: cfg.WarnTLS("aistore at " + clusterURL)
+		clientTLS = cmn.NewClientTLS(cargs, sargs, false /*intra-cluster*/)
 		apiBP.Client = clientTLS
 	} else {
 		clientH = cmn.NewClient(cargs)
@@ -72,8 +72,8 @@ func Init(args []string) (err error) {
 		}
 		if cos.IsHTTPS(authnURL) {
 			if clientTLS == nil {
-				cfg.WarnTLS("AuthN at " + authnURL)
-				clientTLS = cmn.NewClientTLS(cargs, sargs)
+				// TODO -- FIXME: cfg.WarnTLS("AuthN at " + authnURL)
+				clientTLS = cmn.NewClientTLS(cargs, sargs, false /*intra-cluster*/)
 			}
 			authParams.Client = clientTLS
 		} else {

cmd/cli/config/config.go

@@ -36,8 +36,8 @@ type (
 		DefaultAISHost    string `json:"default_ais_host"`
 		DefaultDockerHost string `json:"default_docker_host"`
 		// TLS
-		Certificate   string `json:"client_crt"`     // X509 certificate
-		CertKey       string `json:"client_crt_key"` // X509 key
+		Certificate   string `json:"client_crt"`     // X.509 certificate
+		CertKey       string `json:"client_crt_key"` // X.509 key
 		ClientCA      string `json:"client_ca_tls"`  // #6410
 		SkipVerifyCrt bool   `json:"skip_verify_crt"`
 	}

cmd/cli/go.mod

@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.22.3
 
 require (
-	github.com/NVIDIA/aistore v1.3.24-0.20240821225139-c434fb653803
+	github.com/NVIDIA/aistore v1.3.24-0.20240826235310-8c273cfa0d36
 	github.com/fatih/color v1.17.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.20.0

cmd/cli/go.sum

@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.24-0.20240821225139-c434fb653803 h1:eRCEK36yW0HZSLD5ZEV7G/m/dSv+POq9YB4byuZXo5Y=
-github.com/NVIDIA/aistore v1.3.24-0.20240821225139-c434fb653803/go.mod h1:si83S9r29vwIC0f0CE2Mk+25bFiaN6mmVlmuBpP4hHM=
+github.com/NVIDIA/aistore v1.3.24-0.20240826235310-8c273cfa0d36 h1:6WbWE3vqkTVP4i1hnHqye3yktBQaD4KDtJ0MUdcmc64=
+github.com/NVIDIA/aistore v1.3.24-0.20240826235310-8c273cfa0d36/go.mod h1:si83S9r29vwIC0f0CE2Mk+25bFiaN6mmVlmuBpP4hHM=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

cmn/client.go

@@ -93,7 +93,7 @@ func NewTransport(cargs TransportArgs) *http.Transport {
 	return transport
 }
 
-func NewTLS(sargs TLSArgs) (tlsConf *tls.Config, _ error) {
+func NewTLS(sargs TLSArgs, intra bool) (tlsConf *tls.Config, err error) {
 	var pool *x509.CertPool
 	if sargs.ClientCA != "" {
 		cert, err := os.ReadFile(sargs.ClientCA)
@@ -106,30 +106,37 @@ func NewTLS(sargs TLSArgs) (tlsConf *tls.Config, _ error) {
 		}
 	}
 	tlsConf = &tls.Config{RootCAs: pool, InsecureSkipVerify: sargs.SkipVerify}
-	if sargs.Certificate != "" && sargs.Key != "" {
-		if aistls.IsLoaderSet() {
-			// Certificate Manager initiated as part of service startup
-			tlsConf.GetClientCertificate = aistls.GetClientCert()
-		} else {
-			// One-shot client probably
-			cert, err := tls.LoadX509KeyPair(sargs.Certificate, sargs.Key)
-			if err != nil {
-				var hint string
-				if os.IsNotExist(err) {
-					hint = "\n(hint: check the two filenames for existence/accessibility)"
-				}
-				return nil, fmt.Errorf("client tls: failed to load public/private key pair: (%q, %q)%s",
-					sargs.Certificate, sargs.Key, hint)
-			}
-			tlsConf.Certificates = []tls.Certificate{cert}
-		}
+
+	if sargs.Certificate == "" && sargs.Key == "" {
+		return tlsConf, nil
+	}
+
+	// intra-cluster client
+	if intra {
+		tlsConf.GetClientCertificate, err = aistls.GetClientCert()
+		return tlsConf, err
+	}
+
+	// external client
+	var (
+		cert tls.Certificate
+		hint string
+	)
+	if cert, err = tls.LoadX509KeyPair(sargs.Certificate, sargs.Key); err == nil {
+		tlsConf.Certificates = []tls.Certificate{cert}
+		return tlsConf, nil
+	}
+
+	if os.IsNotExist(err) {
+		hint = "\n(hint: check the two filenames for existence/accessibility)"
 	}
-	return tlsConf, nil
+	return nil, fmt.Errorf("client tls: failed to load public/private key pair: (%q, %q)%s", sargs.Certificate, sargs.Key, hint)
 }
 
+// TODO -- FIXME: this call must get cert file and key to be used for the `clientTLS`
 func NewDefaultClients(timeout time.Duration) (clientH, clientTLS *http.Client) {
 	clientH = NewClient(TransportArgs{Timeout: timeout})
-	clientTLS = NewClientTLS(TransportArgs{Timeout: timeout}, TLSArgs{SkipVerify: true})
+	clientTLS = NewClientTLS(TransportArgs{Timeout: timeout}, TLSArgs{SkipVerify: true}, false /*intra-cluster*/)
 	return
 }
 
@@ -139,15 +146,15 @@ func NewClient(cargs TransportArgs) *http.Client {
 }
 
 func NewIntraClientTLS(cargs TransportArgs, config *Config) *http.Client {
-	return NewClientTLS(cargs, config.Net.HTTP.ToTLS())
+	return NewClientTLS(cargs, config.Net.HTTP.ToTLS(), true /*intra-cluster*/)
 }
 
 // https client (ditto)
-func NewClientTLS(cargs TransportArgs, sargs TLSArgs) *http.Client {
+func NewClientTLS(cargs TransportArgs, sargs TLSArgs, intra bool) *http.Client {
 	transport := NewTransport(cargs)
 
 	// initialize TLS config
-	tlsConfig, err := NewTLS(sargs)
+	tlsConfig, err := NewTLS(sargs, intra)
 	if err != nil {
 		cos.ExitLog(err) // FATAL
 	}

cmn/config.go

@@ -445,15 +445,15 @@ type (
 
 	HTTPConf struct {
 		Proto           string `json:"-"`                 // http or https (set depending on `UseHTTPS`)
-		Certificate     string `json:"server_crt"`        // HTTPS: X509 certificate
-		CertKey         string `json:"server_key"`        // HTTPS: X509 key
+		Certificate     string `json:"server_crt"`        // HTTPS: X.509 certificate
+		CertKey         string `json:"server_key"`        // HTTPS: X.509 key
 		ServerNameTLS   string `json:"domain_tls"`        // #6410
 		ClientCA        string `json:"client_ca_tls"`     // #6410
 		ClientAuthTLS   int    `json:"client_auth_tls"`   // #6410 tls.ClientAuthType enum
 		WriteBufferSize int    `json:"write_buffer_size"` // http.Transport.WriteBufferSize; zero defaults to 4KB
 		ReadBufferSize  int    `json:"read_buffer_size"`  // http.Transport.ReadBufferSize; ditto
 		UseHTTPS        bool   `json:"use_https"`         // use HTTPS
-		SkipVerifyCrt   bool   `json:"skip_verify"`       // skip X509 cert verification (used with self-signed certs)
+		SkipVerifyCrt   bool   `json:"skip_verify"`       // skip X.509 cert verification (used with self-signed certs)
 		Chunked         bool   `json:"chunked_transfer"`  // (https://tools.ietf.org/html/rfc7230#page-36; not used since 02/23)
 	}
 	HTTPConfToSet struct {

cmn/cos/node_state.go

@@ -35,14 +35,14 @@ const (
 func (f NodeStateFlags) IsOK() bool { return f == NodeStarted|ClusterStarted }
 
 func (f NodeStateFlags) IsRed() bool {
-	return f.IsSet(OOS) || f.IsSet(OOM) || f.IsSet(LowCapacity) || f.IsSet(LowMemory) || f.IsSet(DiskFault) ||
-		f.IsSet(NoMountpaths) || f.IsSet(NumGoroutines)
+	return f.IsSet(OOS) || f.IsSet(OOM) || f.IsSet(DiskFault) || f.IsSet(NoMountpaths) || f.IsSet(NumGoroutines)
 }
 
 func (f NodeStateFlags) IsWarn() bool {
 	return f.IsSet(Rebalancing) || f.IsSet(RebalanceInterrupted) ||
 		f.IsSet(Resilvering) || f.IsSet(ResilverInterrupted) ||
-		f.IsSet(Restarted) || f.IsSet(MaintenanceMode) || f.IsSet(LowCapacity)
+		f.IsSet(Restarted) || f.IsSet(MaintenanceMode) ||
+		f.IsSet(LowCapacity) || f.IsSet(LowMemory)
 }
 
 func (f NodeStateFlags) IsSet(flag NodeStateFlags) bool { return BitFlags(f).IsSet(BitFlags(flag)) }