diff --git a/.gitignore b/.gitignore index 07c444b8c9..abbe1b7da7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .*.swp *.key *.crt +*.csr ## PYTHON ## diff --git a/ais/daemon.go b/ais/daemon.go index 373af8a101..7918b1b670 100644 --- a/ais/daemon.go +++ b/ais/daemon.go @@ -21,7 +21,6 @@ import ( "github.com/NVIDIA/aistore/cmn/debug" "github.com/NVIDIA/aistore/cmn/k8s" "github.com/NVIDIA/aistore/cmn/nlog" - "github.com/NVIDIA/aistore/cmn/tls" "github.com/NVIDIA/aistore/core/meta" "github.com/NVIDIA/aistore/fs" "github.com/NVIDIA/aistore/hk" @@ -202,13 +201,6 @@ func initDaemon(version, buildTime string) cos.Runner { // declared xactions, as per xact/api.go xreg.Init() - if config.Net.HTTP.UseHTTPS { - err = tls.Init(config.Net.HTTP.Certificate, config.Net.HTTP.CertKey) - if err != nil { - cos.ExitLogf("failed to initialize Certificate Manager: %v", err) - } - } - // primary 'host[:port]' endpoint or URL from the environment if daemon.EP = os.Getenv(env.AIS.PrimaryEP); daemon.EP != "" { scheme := "http" diff --git a/ais/htcommon.go b/ais/htcommon.go index e58d04414f..a6d0629ad8 100644 --- a/ais/htcommon.go +++ b/ais/htcommon.go @@ -588,21 +588,19 @@ func newTLS(conf *cmn.HTTPConf) (tlsConf *tls.Config, err error) { } if clientAuth > tls.RequestClientCert { if caCert, err = os.ReadFile(conf.ClientCA); err != nil { - return + return nil, fmt.Errorf("new-tls: failed to read PEM %q, err: %w", conf.ClientCA, err) } pool = x509.NewCertPool() if ok := pool.AppendCertsFromPEM(caCert); !ok { - return nil, fmt.Errorf("tls: failed to append CA certs from PEM: %q", conf.ClientCA) + return nil, fmt.Errorf("new-tls: failed to append CA certs from PEM %q", conf.ClientCA) } tlsConf.ClientCAs = pool } if conf.Certificate != "" && conf.CertKey != "" { - if !aistls.IsLoaderSet() { - return nil, errors.New("tls: certificate manager not set") - } - tlsConf.GetCertificate = aistls.GetCert() + tlsConf.GetCertificate, err = aistls.GetCert() + debug.AssertNoErr(err) } - return + return tlsConf, err } func (server *netServer) connStateListener(c net.Conn, cs http.ConnState) { diff --git a/ais/htrun.go b/ais/htrun.go index f086b5288d..ebfc58a4cd 100644 --- a/ais/htrun.go +++ b/ais/htrun.go @@ -33,6 +33,7 @@ import ( "github.com/NVIDIA/aistore/cmn/k8s" "github.com/NVIDIA/aistore/cmn/mono" "github.com/NVIDIA/aistore/cmn/nlog" + aistls "github.com/NVIDIA/aistore/cmn/tls" "github.com/NVIDIA/aistore/core" "github.com/NVIDIA/aistore/core/meta" "github.com/NVIDIA/aistore/memsys" @@ -258,6 +259,13 @@ func (h *htrun) regNetHandlers(networkHandlers []networkHandler) { } func (h *htrun) init(config *cmn.Config) { + // before newTLS() below and before clients + if config.Net.HTTP.UseHTTPS { + if err := aistls.Init(config.Net.HTTP.Certificate, config.Net.HTTP.CertKey); err != nil { + cos.ExitLog(err) + } + } + initCtrlClient(config) initDataClient(config) @@ -511,6 +519,7 @@ func (h *htrun) run(config *cmn.Config) error { } tlsConf = c } + if config.HostNet.UseIntraControl { go func() { _ = g.netServ.control.listen(h.si.ControlNet.TCPEndpoint(), logger, tlsConf, config) diff --git a/ais/prxrev.go b/ais/prxrev.go index 3b0b3c3f11..5bd548cc51 100644 --- a/ais/prxrev.go +++ b/ais/prxrev.go @@ -113,7 +113,7 @@ func rpTransport(config *cmn.Config) *http.Transport { transport = cmn.NewTransport(cmn.TransportArgs{Timeout: config.Client.Timeout.D()}) ) if config.Net.HTTP.UseHTTPS { - transport.TLSClientConfig, err = cmn.NewTLS(config.Net.HTTP.ToTLS()) + transport.TLSClientConfig, err = cmn.NewTLS(config.Net.HTTP.ToTLS(), true /*intra-cluster*/) if err != nil { cos.ExitLog(err) } diff --git a/bench/tools/aisloader/client.go b/bench/tools/aisloader/client.go index 1d98b7a4fd..1d2cd7a4d8 100644 --- a/bench/tools/aisloader/client.go +++ b/bench/tools/aisloader/client.go @@ -35,7 +35,7 @@ var ( cargs = cmn.TransportArgs{ UseHTTPProxyEnv: true, } - // NOTE: client X509 certificate and other `cmn.TLSArgs` variables can be provided via (os.Getenv) environment. + // NOTE: client X.509 certificate and other `cmn.TLSArgs` variables can be provided via (os.Getenv) environment. // See also: // - docs/aisloader.md, section "Environment variables" // - AIS_ENDPOINT and aisEndpoint @@ -261,7 +261,7 @@ func newTraceCtx(proxyURL string) *traceCtx { err error ) if cos.IsHTTPS(proxyURL) { - transport.TLSClientConfig, err = cmn.NewTLS(sargs) + transport.TLSClientConfig, err = cmn.NewTLS(sargs, false /*intra-cluster*/) cos.AssertNoErr(err) } tctx.tr = &traceableTransport{ diff --git a/bench/tools/aisloader/run.go b/bench/tools/aisloader/run.go index e4bd5bbd3f..91ad57926d 100644 --- a/bench/tools/aisloader/run.go +++ b/bench/tools/aisloader/run.go @@ -920,7 +920,7 @@ func _init(p *params) (err error) { if useHTTPS { // environment to override client config cmn.EnvToTLS(&sargs) - p.bp.Client = cmn.NewClientTLS(cargs, sargs) + p.bp.Client = cmn.NewClientTLS(cargs, sargs, false /*intra-cluster*/) } else { p.bp.Client = cmn.NewClient(cargs) } diff --git a/cmd/cli/cli/init.go b/cmd/cli/cli/init.go index e2aa9ba995..c66ff975ac 100644 --- a/cmd/cli/cli/init.go +++ b/cmd/cli/cli/init.go @@ -56,8 +56,8 @@ func Init(args []string) (err error) { UA: ua, } if cos.IsHTTPS(clusterURL) { - cfg.WarnTLS("aistore at " + clusterURL) - clientTLS = cmn.NewClientTLS(cargs, sargs) + // TODO -- FIXME: cfg.WarnTLS("aistore at " + clusterURL) + clientTLS = cmn.NewClientTLS(cargs, sargs, false /*intra-cluster*/) apiBP.Client = clientTLS } else { clientH = cmn.NewClient(cargs) @@ -72,8 +72,8 @@ func Init(args []string) (err error) { } if cos.IsHTTPS(authnURL) { if clientTLS == nil { - cfg.WarnTLS("AuthN at " + authnURL) - clientTLS = cmn.NewClientTLS(cargs, sargs) + // TODO -- FIXME: cfg.WarnTLS("AuthN at " + authnURL) + clientTLS = cmn.NewClientTLS(cargs, sargs, false /*intra-cluster*/) } authParams.Client = clientTLS } else { diff --git a/cmd/cli/config/config.go b/cmd/cli/config/config.go index 58a7a804f8..a5d2c716c4 100644 --- a/cmd/cli/config/config.go +++ b/cmd/cli/config/config.go @@ -36,8 +36,8 @@ type ( DefaultAISHost string `json:"default_ais_host"` DefaultDockerHost string `json:"default_docker_host"` // TLS - Certificate string `json:"client_crt"` // X509 certificate - CertKey string `json:"client_crt_key"` // X509 key + Certificate string `json:"client_crt"` // X.509 certificate + CertKey string `json:"client_crt_key"` // X.509 key ClientCA string `json:"client_ca_tls"` // #6410 SkipVerifyCrt bool `json:"skip_verify_crt"` } diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod index bbadcf8bbf..8f11b038ac 100644 --- a/cmd/cli/go.mod +++ b/cmd/cli/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli go 1.22.3 require ( - github.com/NVIDIA/aistore v1.3.24-0.20240821225139-c434fb653803 + github.com/NVIDIA/aistore v1.3.24-0.20240826235310-8c273cfa0d36 github.com/fatih/color v1.17.0 github.com/json-iterator/go v1.1.12 github.com/onsi/ginkgo/v2 v2.20.0 diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum index 4fe734ab1c..0897dfb9f0 100644 --- a/cmd/cli/go.sum +++ b/cmd/cli/go.sum @@ -1,7 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/NVIDIA/aistore v1.3.24-0.20240821225139-c434fb653803 h1:eRCEK36yW0HZSLD5ZEV7G/m/dSv+POq9YB4byuZXo5Y= -github.com/NVIDIA/aistore v1.3.24-0.20240821225139-c434fb653803/go.mod h1:si83S9r29vwIC0f0CE2Mk+25bFiaN6mmVlmuBpP4hHM= +github.com/NVIDIA/aistore v1.3.24-0.20240826235310-8c273cfa0d36 h1:6WbWE3vqkTVP4i1hnHqye3yktBQaD4KDtJ0MUdcmc64= +github.com/NVIDIA/aistore v1.3.24-0.20240826235310-8c273cfa0d36/go.mod h1:si83S9r29vwIC0f0CE2Mk+25bFiaN6mmVlmuBpP4hHM= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= diff --git a/cmn/client.go b/cmn/client.go index ff41bb49d2..8981bfdb10 100644 --- a/cmn/client.go +++ b/cmn/client.go @@ -93,7 +93,7 @@ func NewTransport(cargs TransportArgs) *http.Transport { return transport } -func NewTLS(sargs TLSArgs) (tlsConf *tls.Config, _ error) { +func NewTLS(sargs TLSArgs, intra bool) (tlsConf *tls.Config, err error) { var pool *x509.CertPool if sargs.ClientCA != "" { cert, err := os.ReadFile(sargs.ClientCA) @@ -106,30 +106,37 @@ func NewTLS(sargs TLSArgs) (tlsConf *tls.Config, _ error) { } } tlsConf = &tls.Config{RootCAs: pool, InsecureSkipVerify: sargs.SkipVerify} - if sargs.Certificate != "" && sargs.Key != "" { - if aistls.IsLoaderSet() { - // Certificate Manager initiated as part of service startup - tlsConf.GetClientCertificate = aistls.GetClientCert() - } else { - // One-shot client probably - cert, err := tls.LoadX509KeyPair(sargs.Certificate, sargs.Key) - if err != nil { - var hint string - if os.IsNotExist(err) { - hint = "\n(hint: check the two filenames for existence/accessibility)" - } - return nil, fmt.Errorf("client tls: failed to load public/private key pair: (%q, %q)%s", - sargs.Certificate, sargs.Key, hint) - } - tlsConf.Certificates = []tls.Certificate{cert} - } + + if sargs.Certificate == "" && sargs.Key == "" { + return tlsConf, nil + } + + // intra-cluster client + if intra { + tlsConf.GetClientCertificate, err = aistls.GetClientCert() + return tlsConf, err + } + + // external client + var ( + cert tls.Certificate + hint string + ) + if cert, err = tls.LoadX509KeyPair(sargs.Certificate, sargs.Key); err == nil { + tlsConf.Certificates = []tls.Certificate{cert} + return tlsConf, nil + } + + if os.IsNotExist(err) { + hint = "\n(hint: check the two filenames for existence/accessibility)" } - return tlsConf, nil + return nil, fmt.Errorf("client tls: failed to load public/private key pair: (%q, %q)%s", sargs.Certificate, sargs.Key, hint) } +// TODO -- FIXME: this call must get cert file and key to be used for the `clientTLS` func NewDefaultClients(timeout time.Duration) (clientH, clientTLS *http.Client) { clientH = NewClient(TransportArgs{Timeout: timeout}) - clientTLS = NewClientTLS(TransportArgs{Timeout: timeout}, TLSArgs{SkipVerify: true}) + clientTLS = NewClientTLS(TransportArgs{Timeout: timeout}, TLSArgs{SkipVerify: true}, false /*intra-cluster*/) return } @@ -139,15 +146,15 @@ func NewClient(cargs TransportArgs) *http.Client { } func NewIntraClientTLS(cargs TransportArgs, config *Config) *http.Client { - return NewClientTLS(cargs, config.Net.HTTP.ToTLS()) + return NewClientTLS(cargs, config.Net.HTTP.ToTLS(), true /*intra-cluster*/) } // https client (ditto) -func NewClientTLS(cargs TransportArgs, sargs TLSArgs) *http.Client { +func NewClientTLS(cargs TransportArgs, sargs TLSArgs, intra bool) *http.Client { transport := NewTransport(cargs) // initialize TLS config - tlsConfig, err := NewTLS(sargs) + tlsConfig, err := NewTLS(sargs, intra) if err != nil { cos.ExitLog(err) // FATAL } diff --git a/cmn/config.go b/cmn/config.go index 994afca270..f3eea7b9df 100644 --- a/cmn/config.go +++ b/cmn/config.go @@ -445,15 +445,15 @@ type ( HTTPConf struct { Proto string `json:"-"` // http or https (set depending on `UseHTTPS`) - Certificate string `json:"server_crt"` // HTTPS: X509 certificate - CertKey string `json:"server_key"` // HTTPS: X509 key + Certificate string `json:"server_crt"` // HTTPS: X.509 certificate + CertKey string `json:"server_key"` // HTTPS: X.509 key ServerNameTLS string `json:"domain_tls"` // #6410 ClientCA string `json:"client_ca_tls"` // #6410 ClientAuthTLS int `json:"client_auth_tls"` // #6410 tls.ClientAuthType enum WriteBufferSize int `json:"write_buffer_size"` // http.Transport.WriteBufferSize; zero defaults to 4KB ReadBufferSize int `json:"read_buffer_size"` // http.Transport.ReadBufferSize; ditto UseHTTPS bool `json:"use_https"` // use HTTPS - SkipVerifyCrt bool `json:"skip_verify"` // skip X509 cert verification (used with self-signed certs) + SkipVerifyCrt bool `json:"skip_verify"` // skip X.509 cert verification (used with self-signed certs) Chunked bool `json:"chunked_transfer"` // (https://tools.ietf.org/html/rfc7230#page-36; not used since 02/23) } HTTPConfToSet struct { diff --git a/cmn/cos/node_state.go b/cmn/cos/node_state.go index d632374269..f7c62a7ed6 100644 --- a/cmn/cos/node_state.go +++ b/cmn/cos/node_state.go @@ -35,14 +35,14 @@ const ( func (f NodeStateFlags) IsOK() bool { return f == NodeStarted|ClusterStarted } func (f NodeStateFlags) IsRed() bool { - return f.IsSet(OOS) || f.IsSet(OOM) || f.IsSet(LowCapacity) || f.IsSet(LowMemory) || f.IsSet(DiskFault) || - f.IsSet(NoMountpaths) || f.IsSet(NumGoroutines) + return f.IsSet(OOS) || f.IsSet(OOM) || f.IsSet(DiskFault) || f.IsSet(NoMountpaths) || f.IsSet(NumGoroutines) } func (f NodeStateFlags) IsWarn() bool { return f.IsSet(Rebalancing) || f.IsSet(RebalanceInterrupted) || f.IsSet(Resilvering) || f.IsSet(ResilverInterrupted) || - f.IsSet(Restarted) || f.IsSet(MaintenanceMode) || f.IsSet(LowCapacity) + f.IsSet(Restarted) || f.IsSet(MaintenanceMode) || + f.IsSet(LowCapacity) || f.IsSet(LowMemory) } func (f NodeStateFlags) IsSet(flag NodeStateFlags) bool { return BitFlags(f).IsSet(BitFlags(flag)) } diff --git a/cmn/tls/certloader.go b/cmn/tls/certloader.go new file mode 100644 index 0000000000..ac25a27523 --- /dev/null +++ b/cmn/tls/certloader.go @@ -0,0 +1,158 @@ +// Package tls provides support for TLS. +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + */ +package tls + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/NVIDIA/aistore/cmn/cos" + "github.com/NVIDIA/aistore/cmn/debug" + "github.com/NVIDIA/aistore/cmn/nlog" + "github.com/NVIDIA/aistore/hk" + "github.com/OneOfOne/xxhash" +) + +const name = "certificate-loader" + +const ( + hktime = time.Hour // TODO -- FIXME: revise and remove +) + +type ( + _cert struct { + // TODO -- FIXME: add mtime, ctime, size + tls.Certificate + notBefore time.Time + notAfter time.Time + digest uint64 + } + certLoader struct { + curr atomic.Pointer[_cert] + certFile string + keyFile string + } + + GetCertCB func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) + GetClientCertCB func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) +) + +var ( + loader *certLoader +) + +// (htrun only) +func Init(certFile, keyFile string) (err error) { + if certFile == "" && keyFile == "" { + return nil + } + + debug.Assert(loader == nil) + loader = &certLoader{certFile: certFile, keyFile: keyFile} + if err = loader.load(true); err != nil { + nlog.Errorln("FATAL:", err) + loader = nil + return err + } + hk.Reg(name, loader.hk, hktime) // TODO -- FIXME: revise - use notAfter + return nil +} + +func (c *certLoader) _get() *tls.Certificate { return &c.curr.Load().Certificate } + +func (c *certLoader) _hello(*tls.ClientHelloInfo) (*tls.Certificate, error) { return c._get(), nil } + +func GetCert() (GetCertCB, error) { + if loader == nil { + return nil, errors.New(name + " is ") + } + return loader._hello, nil +} + +func (c *certLoader) _info(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return c._get(), nil +} + +func GetClientCert() (GetClientCertCB, error) { + if loader == nil { + return nil, errors.New(name + " is ") + } + return loader._info, nil +} + +func (c *certLoader) hk() time.Duration { + if err := c.load(false); err != nil { + nlog.Errorln(err) + } + return hktime +} + +// TODO -- FIXME: use mtime, ctime, etc. +func (c *certLoader) load(first bool) (err error) { + var ( + parsed _cert + ) + parsed.Certificate, err = tls.LoadX509KeyPair(c.certFile, c.keyFile) + if err != nil { + return fmt.Errorf("%s: failed to load X.509, err: %w", name, err) + } + if err = parsed.init(); err != nil { + return err + } + + if !first { + curr := c.curr.Load() + if curr.digest == parsed.digest { + debug.Assert(curr.notAfter == parsed.notAfter, curr.notAfter, " vs ", parsed.notAfter) + return nil // nothing to do + } + } + + c.curr.Store(&parsed) + + // log + var sb strings.Builder + sb.WriteString(c.certFile) + parsed._str(&sb) + nlog.Infoln(sb.String()) + + return nil +} + +/////////// +// _cert // +/////////// + +func (parsed *_cert) _str(sb *strings.Builder) { + sb.WriteByte('[') + sb.WriteString(cos.FormatTime(parsed.notBefore, "")) + sb.WriteByte(',') + sb.WriteString(cos.FormatTime(parsed.notAfter, "")) + sb.WriteByte(',') + sb.WriteString(strconv.Itoa(int(parsed.digest >> 48))) + sb.WriteString("...") + sb.WriteByte(']') +} + +func (parsed *_cert) init() (err error) { + if parsed.Certificate.Leaf == nil { + parsed.Certificate.Leaf, err = x509.ParseCertificate(parsed.Certificate.Certificate[0]) + if err != nil { + return fmt.Errorf("%s: failed to parse X.509, err: %w", name, err) + } + } + { + parsed.digest = xxhash.Checksum64S(parsed.Certificate.Leaf.Raw, cos.MLCG32) + parsed.notBefore = parsed.Certificate.Leaf.NotBefore + parsed.notAfter = parsed.Certificate.Leaf.NotAfter + } + return nil +} diff --git a/cmn/tls/loader.go b/cmn/tls/loader.go deleted file mode 100644 index ff96b3e4e7..0000000000 --- a/cmn/tls/loader.go +++ /dev/null @@ -1,142 +0,0 @@ -// Package tls provides support for TLS. -/* - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - */ -package tls - -import ( - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "errors" - "sync/atomic" - "time" - - "github.com/NVIDIA/aistore/cmn/debug" - "github.com/NVIDIA/aistore/cmn/nlog" - "github.com/NVIDIA/aistore/hk" -) - -type certLoader struct { - cert atomic.Pointer[tls.Certificate] - certFile, keyFile string - retries int -} -type GetCertCB func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) -type GetClientCertCB func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) - -var ( - loader *certLoader -) - -const loadInterval = 1 * time.Hour -const loadRetries = 6 - -func Init(certFile, keyFile string) (err error) { - debug.Assertf(loader != nil, "TLS loader shouldn't be initialized more than once") - - // Allow creation of nil loader, that makes some check easier later. - if certFile != "" && keyFile != "" { - loader, err = startLoader(certFile, keyFile) - } - if err != nil { - nlog.Warningln("Fail to load TLS certificates at start up") - loader = nil - } - return err -} - -func GetCert() GetCertCB { - if loader == nil { - return nil - } - return func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { return loader.getCert(), nil } -} - -func GetClientCert() GetClientCertCB { - if loader == nil { - return nil - } - return func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) { return loader.getCert(), nil } -} - -func IsLoaderSet() bool { - return loader != nil -} - -// startLoader will monitor files of certPath and keyPath and reload certificates -// if any of the two was updated. -func startLoader(certPath, keyPath string) (c *certLoader, err error) { - c = &certLoader{ - certFile: certPath, - keyFile: keyPath, - retries: 0, - } - // Immediately try to load existing certs. - if err := c.load(); err != nil { - return nil, err - } - - hk.Reg("tlsloader", c.housekeep, loadInterval) - return -} - -func (c *certLoader) housekeep() time.Duration { - if err := c.load(); err != nil { - c.retries++ - if c.retries > loadRetries { - nlog.Errorf("unable to load TLS certificate: %v", err) - debug.AssertNoErr(err) - } - } else { - c.retries = 0 - } - return loadInterval -} - -func (c *certLoader) getCert() *tls.Certificate { - return c.cert.Load() -} - -func (c *certLoader) load() error { - cert, err := tls.LoadX509KeyPair(c.certFile, c.keyFile) - if err != nil { - nlog.Errorln("failed to load X509 key pair:", err) - return err - } - - // Compare fingerprints of previous and current certificates, and log if updated - if prevCert := c.cert.Load(); prevCert != nil { - var ( - prevFringerprint string - curFingerprint string - ) - if prevFringerprint, err = fingerprint(prevCert); err != nil { - nlog.Errorln(err) - } - if curFingerprint, err = fingerprint(&cert); err != nil { - nlog.Errorln(err) - } - if prevFringerprint != curFingerprint { - nlog.Infof("Certificate has changed. New fingerprint: %s", curFingerprint) - } - } - c.cert.Store(&cert) - return nil -} - -// Function to compute the fingerprint of a TLS certificate -func fingerprint(tlsCert *tls.Certificate) (string, error) { - if tlsCert.Leaf == nil { - // Parse the leaf certificate if not already parsed - var err error - tlsCert.Leaf, err = x509.ParseCertificate(tlsCert.Certificate[0]) - if err != nil { - return "", errors.New("error on parsing TLS certificate") - } - } - - hash := sha256.Sum256(tlsCert.Leaf.Raw) - return hex.EncodeToString(hash[:]), nil -} diff --git a/docs/aisloader.md b/docs/aisloader.md index d3a87428ba..c65b1d471f 100644 --- a/docs/aisloader.md +++ b/docs/aisloader.md @@ -250,10 +250,10 @@ In addition, environment can be used to specify client-side TLS (aka, HTTPS) con | var name | description | | -- | -- | -| `AIS_CRT` | X509 certificate | -| `AIS_CRT_KEY` | X509 certificate's private key | +| `AIS_CRT` | X.509 certificate | +| `AIS_CRT_KEY` | X.509 certificate's private key | | `AIS_CLIENT_CA` | Certificate authority that authorized (signed) the certificate | -| `AIS_SKIP_VERIFY_CRT` | when true, skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) | +| `AIS_SKIP_VERIFY_CRT` | when true, skip X.509 cert verification (usually enabled to circumvent limitations of self-signed certs) | * See also: [TLS: testing with self-signed certificates](/docs/getting_started.md#tls-testing-with-self-signed-certificates) diff --git a/docs/cli.md b/docs/cli.md index 945105aa71..8a49eb8614 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -240,10 +240,10 @@ In addition, environment can be used to **override** client-side TLS (aka, HTTPS | var name | description | the corresponding [CLI Config](#cli-config) | | -- | -- | -- | -| `AIS_CRT` | X509 certificate | "cluster.client_crt" | -| `AIS_CRT_KEY` | X509 certificate's private key | "cluster.client_crt_key"| +| `AIS_CRT` | X.509 certificate | "cluster.client_crt" | +| `AIS_CRT_KEY` | X.509 certificate's private key | "cluster.client_crt_key"| | `AIS_CLIENT_CA` | Certificate authority that authorized (signed) the certificate | "cluster.client_ca_tls" | -| `AIS_SKIP_VERIFY_CRT` | true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) | "cluster.skip_verify_crt" | +| `AIS_SKIP_VERIFY_CRT` | true: skip X.509 cert verification (usually enabled to circumvent limitations of self-signed certs) | "cluster.skip_verify_crt" | * See also: [TLS: testing with self-signed certificates](/docs/getting_started.md#tls-testing-with-self-signed-certificates) diff --git a/docs/development.md b/docs/development.md index 09db2caee5..1f35f08db6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -104,7 +104,7 @@ OPTIONS: --ht Build with ht:// backend (experimental) --loopback Loopback device size, e.g. 10G, 100M (default: 0). Zero size means emulated mountpaths (with no loopback devices). --dir The root directory of the aistore repository - --https Use HTTPS (note: X509 certificates may be required) + --https Use HTTPS (note: X.509 certificates may be required) --standby When starting up, do not join cluster - wait instead for admin request (advanced usage, target-only) --transient Do not store config changes, keep all the updates in memory -h, --help Show this help text diff --git a/docs/environment-vars.md b/docs/environment-vars.md index a5207877ad..de84b60c8c 100644 --- a/docs/environment-vars.md +++ b/docs/environment-vars.md @@ -99,10 +99,10 @@ See also: | name | comment | | ---- | ------- | | `AIS_USE_HTTPS` | tells aistore to run HTTPS transport (both public and intra-cluster networks); overrides the corresponding config; e.g. usage: 'export AIS_USE_HTTPS=true' | -| `AIS_CRT` | X509 certificate pathname (this and the rest variables in the table are ignored when aistore is AIS_USE_HTTPS==false | -| `AIS_CRT_KEY` | pathname that contains X509 certificate private key | +| `AIS_CRT` | X.509 certificate pathname (this and the rest variables in the table are ignored when aistore is AIS_USE_HTTPS==false | +| `AIS_CRT_KEY` | pathname that contains X.509 certificate private key | | `AIS_CLIENT_CA` | certificate authority that authorized (signed) the certificate | -| `AIS_SKIP_VERIFY_CRT` | when true will skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) | +| `AIS_SKIP_VERIFY_CRT` | when true will skip X.509 cert verification (usually enabled to circumvent limitations of self-signed certs) | ## Local Playground diff --git a/docs/getting_started.md b/docs/getting_started.md index 62e765a696..2fdb5536be 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -387,7 +387,7 @@ For developers, CLI `ais config cluster log.modules ec xs` (for instance) would > To list all log modules, type `ais config cluster` or `ais config node` and press ``. -Finally, there's also HTTPS configuration (including **X509** certificates and options), and the corresponding [environment](#tls-testing-with-self-signed-certificates). +Finally, there's also HTTPS configuration (including **X.509** certificates and options), and the corresponding [environment](#tls-testing-with-self-signed-certificates). For details, please see section [TLS: testing with self-signed certificates](#tls-testing-with-self-signed-certificates) below. @@ -473,27 +473,45 @@ This is still a so-called _local playground_ type deployment _from scratch_, whe ### Generate Certificates -Creating a self-signed certificate along with its private key and a Certificate Authority (CA) certificate using [OpenSSL](https://www.openssl.org/) involves several steps. First we create a self-signed Certificate Authority (CA) certificate (`ca.crt`). Then we create a Certificate Signing Request (CSR) and finally based on CSR and CA we create the server certs (`server.key` and `server.crt`). +Creating a self-signed certificate along with its private key and a Certificate Authority (CA) certificate using [OpenSSL](https://www.openssl.org/) involves steps: + +* First, we create a self-signed Certificate Authority (CA) certificate (`ca.crt`). +* Second, create a Certificate Signing Request (CSR). +* Finally, based on CSR and CA we create the server certs (`server.key` and `server.crt`) - as follows: ```console -$ openssl req -x509 -newkey rsa:2048 -keyout ca.key -out ca.crt -days 1024 -nodes -subj "/CN=localhost" -extensions v3_ca -config <(printf "[req]\ndistinguished_name=req\nx509_extensions=v3_ca\n[ v3_ca ]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1,IP:127.0.0.1\nbasicConstraints=CA:TRUE\n") -$ openssl req -new -newkey rsa:2048 -nodes -keyout server.key -out server.csr -subj "/C=US/ST=California/L=Santa Clara/O=NVIDIA/OU=AIStore/CN=localhost" -config <(printf "[req]\ndistinguished_name=req\nreq_extensions = v3_req\n[ v3_req ]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1,IP:127.0.0.1\n") -$ openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256 -extfile <(printf "[ext]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1,IP:127.0.0.1\nbasicConstraints=CA:FALSE\nkeyUsage=digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment\nextendedKeyUsage=serverAuth,clientAuth\n") -extensions ext +openssl req -x509 -newkey rsa:2048 -keyout ca.key -out ca.crt -days 1024 -nodes -subj "/CN=localhost" -extensions v3_ca -config <(printf "[req]\ndistinguished_name=req\nx509_extensions=v3_ca\n[ v3_ca ]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1,IP:127.0.0.1\nbasicConstraints=CA:TRUE\n") + +openssl req -new -newkey rsa:2048 -nodes -keyout server.key -out server.csr -subj "/C=US/ST=California/L=Santa Clara/O=NVIDIA/OU=AIStore/CN=localhost" -config <(printf "[req]\ndistinguished_name=req\nreq_extensions = v3_req\n[ v3_req ]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1,IP:127.0.0.1\n") + +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256 -extfile <(printf "[ext]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1,IP:127.0.0.1\nbasicConstraints=CA:FALSE\nkeyUsage=digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment\nextendedKeyUsage=serverAuth,clientAuth\n") -extensions ext ``` -**Important:** Ensure you specify the appropriate DNS relevant to your deployment context. For deployments intended to run locally, the domain name `localhost` and `127.0.0.1` should be provided for the certificates to be valid. +> **Important:** make sure to specify correct DNS. For local deployments, `localhost` and `127.0.0.1` domain must be provided - otherwise certificate validation will fail. -### Deploy Cluster (4 targets, 1 gateway, 6 mountpaths, AWS) +### Deploy Cluster: 4 targets, 1 gateway, 6 mountpaths, AWS backend ```console -$ # shutdown prev running AIS cluster +## shutdown previous running AIS cluster +## $ make kill -$ # delete smaps of prev AIS clusters -$ find ~/.ais* -type f -name ".ais.smap" | xargs rm -$ AIS_USE_HTTPS=true AIS_SKIP_VERIFY_CRT=true AIS_SERVER_CRT=/server.crt AIS_SERVER_KEY=/server.key make deploy <<< $'4\n1\n6\ny\nn\nn\n0\n' + +## cleanup +## +$ make clean + +## run cluster: 4 targets, 1 gateway, 6 mountpaths, AWS backend, HTTPS +## +$ TAGS=aws AIS_USE_HTTPS=true AIS_SKIP_VERIFY_CRT=true AIS_SERVER_CRT=/server.crt AIS_SERVER_KEY=/server.key make deploy <<< $'4\n1\n6' ``` > Notice environment variables above: **AIS_USE_HTTPS**, **AIS_SKIP_VERIFY_CRT**, **AIS_SERVER_CRT** and **AIS_SERVER_KEY**. +> Also note that `` (above) must not necessarily be absolute. Assuming, you have `server.*` in your **local directory**, the following will work as well: + +```console +$ TAGS=aws AIS_USE_HTTPS=true AIS_SKIP_VERIFY_CRT=true AIS_SERVER_CRT=server.crt AIS_SERVER_KEY=server.key make deploy <<< $'4\n1\n6' +``` + ### Accessing the Cluster To use CLI, try first any command with HTTPS-based cluster endpoint, for instance: @@ -518,12 +536,12 @@ In the previous example, the cluster is simply ignoring SSL certificate verifica | var name | description | the corresponding cluster configuration | | -- | -- | -- | | `AIS_USE_HTTPS` | when false, we use plain HTTP with all the TLS config (below) simply **ignored** | "net.http.use_https" | -| `AIS_SERVER_CRT` | aistore cluster X509 certificate | "net.http.server_crt" | +| `AIS_SERVER_CRT` | aistore cluster X.509 certificate | "net.http.server_crt" | | `AIS_SERVER_KEY` | certificate's private key | "net.http.server_key"| | `AIS_DOMAIN_TLS` | NOTE: not supported, must be empty (domain, hostname, or SAN registered with the certificate) | "net.http.domain_tls"| | `AIS_CLIENT_CA_TLS` | Certificate authority that authorized (signed) the certificate | "net.http.client_ca_tls" | | `AIS_CLIENT_AUTH_TLS` | Client authentication during TLS handshake: a range from 0 (no authentication) to 4 (request and validate client's certificate) | "net.http.client_auth_tls" | -| `AIS_SKIP_VERIFY_CRT` | when true: skip X509 cert verification (usually enabled to circumvent limitations of self-signed certs) | "net.http.skip_verify" | +| `AIS_SKIP_VERIFY_CRT` | when true: skip X.509 cert verification (usually enabled to circumvent limitations of self-signed certs) | "net.http.skip_verify" | > More info on [`AIS_CLIENT_AUTH_TLS`](https://pkg.go.dev/crypto/tls#ClientAuthType). diff --git a/tools/init.go b/tools/init.go index da6124e159..a3a248aa80 100644 --- a/tools/init.go +++ b/tools/init.go @@ -101,7 +101,7 @@ func init() { if cos.IsHTTPS(os.Getenv(env.AIS.Endpoint)) { // fill-in from env cmn.EnvToTLS(&tlsArgs) - gctx.Client = cmn.NewClientTLS(transportArgs, tlsArgs) + gctx.Client = cmn.NewClientTLS(transportArgs, tlsArgs, false /*intra-cluster*/) } else { gctx.Client = cmn.NewClient(transportArgs) } @@ -117,7 +117,7 @@ func NewClientWithProxy(proxyURL string) *http.Client { if parsedURL.Scheme == "https" { cos.AssertMsg(cos.IsHTTPS(proxyURL), proxyURL) - tlsConfig, err := cmn.NewTLS(tlsArgs) + tlsConfig, err := cmn.NewTLS(tlsArgs, false /*intra-cluster*/) cos.AssertNoErr(err) transport.TLSClientConfig = tlsConfig } diff --git a/transport/bundle/dmover.go b/transport/bundle/dmover.go index 93a6609a52..f029e10fda 100644 --- a/transport/bundle/dmover.go +++ b/transport/bundle/dmover.go @@ -181,7 +181,9 @@ func (dm *DataMover) Quiesce(d time.Duration) core.QuiRes { func (dm *DataMover) Close(err error) { if dm == nil { - nlog.Errorln("Warning: DM is ") + if cmn.Rom.FastV(5, cos.SmoduleTransport) { + nlog.Warningln("Warning: DM is ") // e.g., single-node cluster + } return } if !dm.stage.opened.CAS(true, false) { diff --git a/transport/client_fasthttp.go b/transport/client_fasthttp.go index c7fe3c0a44..bf78868f2a 100644 --- a/transport/client_fasthttp.go +++ b/transport/client_fasthttp.go @@ -52,7 +52,7 @@ func NewIntraDataClient() Client { WriteBufferSize: wbuf, } if config.Net.HTTP.UseHTTPS { - tlsConfig, err := cmn.NewTLS(config.Net.HTTP.ToTLS()) + tlsConfig, err := cmn.NewTLS(config.Net.HTTP.ToTLS(), true /*intra-cluster*/) // streams if err != nil { cos.ExitLog(err) } diff --git a/transport/client_nethttp.go b/transport/client_nethttp.go index 53bc68c373..98b5104f13 100644 --- a/transport/client_nethttp.go +++ b/transport/client_nethttp.go @@ -48,7 +48,7 @@ func NewIntraDataClient() (client *http.Client) { ReadBufferSize: rbuf, } if config.Net.HTTP.UseHTTPS { - client = cmn.NewClientTLS(cargs, config.Net.HTTP.ToTLS()) + client = cmn.NewClientTLS(cargs, config.Net.HTTP.ToTLS(), true /*intra-cluster*/) // streams } else { client = cmn.NewClient(cargs) }