diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7e8d848e..1b8168d0 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -59,7 +59,7 @@ jobs: --set extraArgs.loaderType=file \ --set extraArgs.loaderFilePath=/runconfig/checks.yaml \ --set image.tag=${{ steps.version.outputs.value }} \ - --set startupConfig.sparrowName=the-sparrow.com \ + --set startupConfig.name=the-sparrow.com \ chart - name: Check Pods run: | diff --git a/.gitignore b/.gitignore index d41d1df3..e3f116e8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ gen # Temporary directory .tmp/* + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d850a9b1..68eef7c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: - id: golangci-lint-repo-mod args: [ --config, .golangci.yaml, --, --fix ] - repo: https://github.com/norwoodj/helm-docs - rev: "v1.11.3" + rev: "v1.12.0" hooks: - id: helm-docs args: diff --git a/README.md b/README.md index 6b51e880..f81955f4 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,8 @@ Additionally check out the sparrow [configuration](#configuration) variants. ## Usage -Use `sparrow run` to execute the instance using the binary. A `sparrowName` (a valid DNS name) is required to be passed, else -the sparrow will not start: +Use `sparrow run` to execute the instance using the binary. A `sparrowName` (a valid DNS name) is required to be passed, +else the sparrow will not start: ```sh sparrow run --sparrowName sparrow.telekom.de @@ -143,6 +143,81 @@ Priority of configuration (high to low): 3. Defined configuration file 4. Default configuration file +Every value in the config file can be set through environment variables. + +You can set a token for the http loader: + +```bash +export SPARROW_LOADER_HTTP_TOKEN="Bearer xxxxxx" +``` + +Or for any other config attribute: + +```bash +export SPARROW_ANY_OTHER_OPTION="Some value" +``` + +Just write out the path to the attribute, delimited by `_`. + +#### Example configuration + +```yaml +# DNS sparrow is exposed on +name: sparrow.example.com +# Selects and configures a loader for continuosly fetching the configuration at runtime +loader: + # defines which loader to use. Options: "file | http" + type: http + # the interval in which sparrow tries to fetch a new configuration + interval: 30s + # config specific to the http loader + http: + # The url where the config is located + url: https://myconfig.example.com/config.yaml + # This token is passed in the Authorization header, when refreshing the config + token: xxxxxxx + # A timeout for the config refresh + timeout: 30s + retry: + # How long to wait in between retries + delay: 10s + # How many times to retry + count: 3 + + # config specific to the file loader + # The file loader is not intended for production use and does + # not refresh the config after reading it the first time + file: + # where to read the runtime config from + path: ./config.yaml + +# Configures the api +api: + # Which address to expose sparrows rest api on + address: :8080 + +# Configures the targetmanager +targetManager: + # time between checking for new targets + checkInterval: 1m + # how often the instance should register itself as a global target + registrationInterval: 1m + # the amount of time a target can be + # unhealthy before it is removed from the global target list + unhealthyThreshold: 3m + # Configuration options for the gitlab target manager + gitlab: + # The url of your gitlab host + baseUrl: https://gitlab.com + # Your gitlab api token + # you can also set this value through the + # SPARROW_TARGETMANAGER_GITLAB_TOKEN environment variable + token: glpat-xxxxxxxx + # the id of your gitlab project. This is where sparrow will register itself + # and grab the list of other sparrows from + projectId: 18923 +``` + #### Loader The loader component of the `sparrow` will load the [Runtime](#runtime) configuration dynamically. @@ -160,9 +235,12 @@ Available loader: ### Runtime -In addition to the technical startup configuration, the `sparrow` checks' configuration can be dynamically loaded from an HTTP endpoint during runtime. The `loader` is capable of dynamically loading and configuring checks. You can enable, disable, and configure checks as needed. +In addition to the technical startup configuration, the `sparrow` checks' configuration can be dynamically loaded from +an HTTP endpoint during runtime. The `loader` is capable of dynamically loading and configuring checks. You can enable, +disable, and configure checks as needed. -For detailed information on available loader configuration options, please refer to [this documentation](docs/sparrow_run.md). +For detailed information on available loader configuration options, please refer +to [this documentation](docs/sparrow_run.md). Example format of a runtime configuration: @@ -171,24 +249,27 @@ apiVersion: 0.0.1 kind: Config checks: health: - targets: [] + targets: [ ] ``` ### Target Manager The `sparrow` is able to manage the targets for the checks and register the `sparrow` as target on a (remote) backend. This is done via a `TargetManager` interface, which can be configured on startup. The available configuration options -are listed below and can be set in a startup YAML configuration file (per default `tmconfig.yaml` in the current -directory). - -| Type | Description | Default | -| ------------------------------------ | ------------------------------------------------------------------------------------ | -------------------- | -| `targetManager.checkInterval` | The interval in seconds to check for new targets. | `300s` | -| `targetManager.unhealthyThreshold` | The threshold in seconds to mark a target as unhealthy and remove it from the state. | `600s` | -| `targetManager.registrationInterval` | The interval in seconds to register the current sparrow at the targets backend. | `300s` | -| `targetManager.gitlab.token` | The token to authenticate against the gitlab instance. | `""` | -| `targetManager.gitlab.baseUrl` | The base URL of the gitlab instance. | `https://gitlab.com` | -| `targetManager.gitlab.projectId` | The project ID of the gitlab project to use as a remote state backend. | `""` | +are listed below and can be set in the startup YAML configuration file, as shown in +the [example configuration](#example-configuration). + +| Type | Description | Default | +|--------------------------------------|-------------------------------------------------------------------------------|----------------------| +| `targetManager.checkInterval` | The interval in seconds to check for new targets. | `300s` | +| `targetManager.unhealthyThreshold` | The threshold in seconds to mark a target as unhealthy and remove it from the + state. | `600s` | +| `targetManager.registrationInterval` | The interval in seconds to register the current sparrow at the targets + backend. | `300s` | +| `targetManager.gitlab.token` | The token to authenticate against the gitlab instance. | `""` | +| `targetManager.gitlab.baseUrl` | The base URL of the gitlab instance. | `https://gitlab.com` | +| `targetManager.gitlab.projectId` | The project ID of the gitlab project to use as a remote state + backend. | `""` | Currently, only one target manager exists: the Gitlab target manager. It uses a gitlab project as the remote state backend. The various `sparrow` instances will @@ -208,7 +289,8 @@ which is named after the DNS name of the `sparrow`. The state file contains the Available configuration options: - `checks.health.targets` (list of strings): List of targets to send health probe. Needs to be a valid url. Can be - another `sparrow` instance. Automatically used when target manager is activated otherwise use the health endpoint of the remote sparrow, e.g. `https://sparrow-dns.telekom.de/checks/health`. + another `sparrow` instance. Automatically used when target manager is activated otherwise use the health endpoint of + the remote sparrow, e.g. `https://sparrow-dns.telekom.de/checks/health`. Example configuration: @@ -232,11 +314,11 @@ Available configuration options: - `checks` - `latency` - - `interval` (integer): Interval in seconds to perform the latency check. - - `timeout` (integer): Timeout in seconds for the latency check. + - `interval` (duration): Interval to perform the latency check. + - `timeout` (duration): Timeout for the latency check. - `retry` - `count` (integer): Number of retries for the latency check. - - `delay` (integer): Delay in seconds between retries for the latency check. + - `delay` (duration): Delay between retries for the latency check. - `targets` (list of strings): List of targets to send latency probe. Needs to be a valid url. Can be another `sparrow` instance. Automatically used when the target manager is enabled otherwise use latency endpoint, e.g. `https://sparrow-dns.telekom.de/checks/latency`. @@ -247,11 +329,11 @@ Available configuration options: ```yaml checks: latency: - interval: 1 - timeout: 3 + interval: 1s + timeout: 3s retry: count: 3 - delay: 1 + delay: 1s targets: - https://example.com/ - https://google.com/ @@ -304,7 +386,7 @@ The application itself and all end-user facing content will be made available in The following channels are available for discussions, feedback, and support requests: | Type | Channel | -| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| | **Issues** | | ## How to Contribute diff --git a/chart/README.md b/chart/README.md index 538545ae..5788c1a8 100644 --- a/chart/README.md +++ b/chart/README.md @@ -57,6 +57,5 @@ A Helm chart to install Sparrow | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | | startupConfig | object | `{}` | startup configuration of the Sparrow see: https://github.com/caas-team/sparrow/blob/main/docs/sparrow_run.md | -| targetManagerConfig | object | `{}` | target manager configuration of the Sparrow (part of the startup) | | tolerations | list | `[]` | | diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 87f480ad..59df92a6 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -30,17 +30,13 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - {{- if or .Values.extraArgs .Values.startupConfig .Values.targetManagerConfig}} + {{- if or .Values.extraArgs .Values.startupConfig }} - args: {{- end }} {{- if .Values.startupConfig}} - --config - /startupconfig/.sparrow.yaml {{- end }} - {{- if .Values.targetManagerConfig}} - - --tmconfig - - /startupconfig/tmconfig.yaml - {{- end }} {{- if .Values.extraArgs }} {{- range $key, $value := .Values.extraArgs }} - --{{ $key }} diff --git a/chart/templates/secret.yaml b/chart/templates/secret.yaml index 64c8fa66..3cf660dd 100644 --- a/chart/templates/secret.yaml +++ b/chart/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if or .Values.startupConfig .Values.targetManagerConfig }} +{{- if .Values.startupConfig }} apiVersion: v1 kind: Secret type: Opaque @@ -10,7 +10,4 @@ data: {{- if .Values.startupConfig}} .sparrow.yaml: {{ toYaml .Values.startupConfig | b64enc }} {{- end }} - {{- if .Values.targetManagerConfig}} - tmconfig.yaml: {{ toYaml .Values.targetManagerConfig | b64enc }} - {{- end }} {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 5a06dd22..53610c42 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -10,7 +10,7 @@ image: # -- Overrides the image tag whose default is the chart appVersion. tag: "" -imagePullSecrets: [] +imagePullSecrets: [ ] nameOverride: "" fullnameOverride: "" @@ -20,13 +20,13 @@ serviceAccount: # -- Automatically mount a ServiceAccount's API credentials? automount: true # -- Annotations to add to the service account - annotations: {} + annotations: { } # -- The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" -podAnnotations: {} -podLabels: {} +podAnnotations: { } +podLabels: { } podSecurityContext: fsGroup: 1000 @@ -51,24 +51,24 @@ ingress: enabled: false className: "" annotations: - {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" + { } + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" hosts: - host: chart-example.local paths: - path: / pathType: ImplementationSpecific - tls: [] + tls: [ ] # - secretName: chart-example-tls # hosts: # - chart-example.local env: - {} - # HTTP_PROXY: - # HTTPS_PROXY: - # NO_PROXY: + { } +# HTTP_PROXY: +# HTTPS_PROXY: +# NO_PROXY: # -- define a network policy that will # open egress traffic to a proxy @@ -78,7 +78,7 @@ networkPolicies: # ip: 1.2.3.4 # port: 8080 -resources: {} +resources: { } # resources: # limits: # cpu: 500m @@ -87,11 +87,11 @@ resources: {} # cpu: 100m # memory: 128Mi -nodeSelector: {} +nodeSelector: { } -tolerations: [] +tolerations: [ ] -affinity: {} +affinity: { } # -- extra command line start parameters # see: https://github.com/caas-team/sparrow/blob/main/docs/sparrow_run.md @@ -101,21 +101,22 @@ extraArgs: # -- startup configuration of the Sparrow # see: https://github.com/caas-team/sparrow/blob/main/docs/sparrow_run.md -startupConfig: {} -# apiAddress: -# loaderFilePath: /runconfig/checks.yaml -# loaderHttpRetryCount: -# loaderHttpRetryDelay: -# loaderHttpTimeout: -# loaderHttpToken: -# loaderHttpUrl: -# loaderInterval: -# loaderType: http | file -# sparrowName: the-sparrow.com - - -# -- target manager configuration of the Sparrow (part of the startup) -targetManagerConfig: {} +startupConfig: { } +# name: the-sparrow.com +# api: +# address: +# loader: +# type: http | file +# interval: +# http: +# url: +# token: +# timeout: +# retryCount: +# retryDelay: +# file: +# path: /runconfig/checks.yaml +# targetManager: # checkInterval: 300s # unhealthyThreshold: 600s # registrationInterval: 300s @@ -124,6 +125,8 @@ targetManagerConfig: {} # baseUrl: https://gitlab.com # projectId: "" + + # -- runtime configuration of the Sparrow # see: https://github.com/caas-team/sparrow#runtime runtimeConfig: diff --git a/cmd/flag.go b/cmd/flag.go new file mode 100644 index 00000000..5dbb422b --- /dev/null +++ b/cmd/flag.go @@ -0,0 +1,99 @@ +package cmd + +import ( + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type Flag struct { + Config string + Cli string +} + +type StringFlag struct { + *Flag +} + +type IntFlag struct { + *Flag +} + +type DurationFlag struct { + *Flag +} + +type StringPFlag struct { + *Flag + sh string +} + +// Bind registers the flag with the command and binds it to the config +func (f *StringFlag) Bind(cmd *cobra.Command, value, usage string) { + cmd.PersistentFlags().String(f.Cli, value, usage) + if err := viper.BindPFlag(f.Config, cmd.PersistentFlags().Lookup(f.Cli)); err != nil { + panic(err) + } +} + +func (f *Flag) String() *StringFlag { + return &StringFlag{ + Flag: f, + } +} + +func (f *DurationFlag) Bind(cmd *cobra.Command, value time.Duration, usage string) { + cmd.PersistentFlags().Duration(f.Cli, value, usage) + if err := viper.BindPFlag(f.Config, cmd.PersistentFlags().Lookup(f.Cli)); err != nil { + panic(err) + } +} + +func (f *Flag) Duration() *DurationFlag { + return &DurationFlag{ + Flag: f, + } +} + +// Bind registers the flag with the command and binds it to the config +func (f *IntFlag) Bind(cmd *cobra.Command, value int, usage string) { + cmd.PersistentFlags().Int(f.Cli, value, usage) + if err := viper.BindPFlag(f.Config, cmd.PersistentFlags().Lookup(f.Cli)); err != nil { + panic(err) + } +} + +func (f *Flag) Int() *IntFlag { + return &IntFlag{ + Flag: f, + } +} + +// Bind registers the flag with the command and binds it to the config +func (f *StringPFlag) Bind(cmd *cobra.Command, value, usage string) { + cmd.PersistentFlags().StringP(f.Cli, f.sh, value, usage) + if err := viper.BindPFlag(f.Config, cmd.PersistentFlags().Lookup(f.Cli)); err != nil { + panic(err) + } +} + +func (f *Flag) StringP(shorthand string) *StringPFlag { + return &StringPFlag{ + Flag: f, + sh: shorthand, + } +} + +// NewFlag returns a flag builder +// It serves as a wrapper around cobra and viper, that allows creating and binding typed cli flags to config values +// +// Example: +// +// NewFlag("config", "c").String().Bind(cmd, "config.yaml", "config file") +func NewFlag(config, cli string) *Flag { + return &Flag{ + Config: config, + Cli: cli, + } +} diff --git a/cmd/root.go b/cmd/root.go index b1532023..0fcf85e9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,6 +21,7 @@ package cmd import ( "fmt" "os" + "strings" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -53,7 +54,7 @@ func Execute(version string) { cmd := BuildCmd(version) if err := cmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) os.Exit(1) } } @@ -73,12 +74,15 @@ func initConfig(cfgFile string) { home, err := os.UserHomeDir() cobra.CheckErr(err) - // Search config in home directory with name ".sparrow" (without extension) + // Search config in home directory with name ".sparrow" (without an extension) viper.AddConfigPath(home) viper.SetConfigType("yaml") viper.SetConfigName(".sparrow") } + viper.SetEnvPrefix("sparrow") + dotreplacer := strings.NewReplacer(".", "_") + viper.EnvKeyReplacer(dotreplacer) viper.AutomaticEnv() if err := viper.ReadInConfig(); err == nil { diff --git a/cmd/run.go b/cmd/run.go index 4f876ce9..5287ce38 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -20,7 +20,8 @@ package cmd import ( "context" - "os" + "fmt" + "time" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -31,96 +32,61 @@ import ( ) const ( - defaultLoaderHttpTimeout = 30 - defaultLoaderInterval = 300 + defaultLoaderHttpTimeout = 30 * time.Second + defaultLoaderInterval = 300 * time.Second defaultHttpRetryCount = 3 - defaultHttpRetryDelay = 1 + defaultHttpRetryDelay = 1 * time.Second ) // NewCmdRun creates a new run command func NewCmdRun() *cobra.Command { - flagMapping := config.RunFlagsNameMapping{ - ApiAddress: "apiAddress", - SparrowName: "sparrowName", - LoaderType: "loaderType", - LoaderInterval: "loaderInterval", - LoaderHttpUrl: "loaderHttpUrl", - LoaderHttpToken: "loaderHttpToken", - LoaderHttpTimeout: "loaderHttpTimeout", - LoaderHttpRetryCount: "loaderHttpRetryCount", - LoaderHttpRetryDelay: "loaderHttpRetryDelay", - LoaderFilePath: "loaderFilePath", - TargetManagerConfig: "tmconfig", - } - cmd := &cobra.Command{ Use: "run", Short: "Run sparrow", Long: `Sparrow will be started with the provided configuration`, - Run: run(&flagMapping), + RunE: run(), } - cmd.PersistentFlags().String(flagMapping.ApiAddress, ":8080", "api: The address the server is listening on") - cmd.PersistentFlags().String(flagMapping.SparrowName, "", "The DNS name of the sparrow") - cmd.PersistentFlags().StringP(flagMapping.LoaderType, "l", "http", - "defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader") - cmd.PersistentFlags().Int(flagMapping.LoaderInterval, defaultLoaderInterval, "defines the interval the loader reloads the configuration in seconds") - cmd.PersistentFlags().String(flagMapping.LoaderHttpUrl, "", "http loader: The url where to get the remote configuration") - cmd.PersistentFlags().String(flagMapping.LoaderHttpToken, "", "http loader: Bearer token to authenticate the http endpoint") - cmd.PersistentFlags().Int(flagMapping.LoaderHttpTimeout, defaultLoaderHttpTimeout, "http loader: The timeout for the http request in seconds") - cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryCount, defaultHttpRetryCount, "http loader: Amount of retries trying to load the configuration") - cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryDelay, defaultHttpRetryDelay, "http loader: The initial delay between retries in seconds") - cmd.PersistentFlags().String(flagMapping.LoaderFilePath, "config.yaml", "file loader: The path to the file to read the runtime config from") - cmd.PersistentFlags().String(flagMapping.TargetManagerConfig, "", "target manager: The path to the file to read the target manager config from") - - _ = viper.BindPFlag(flagMapping.ApiAddress, cmd.PersistentFlags().Lookup(flagMapping.ApiAddress)) - _ = viper.BindPFlag(flagMapping.SparrowName, cmd.PersistentFlags().Lookup(flagMapping.SparrowName)) - _ = viper.BindPFlag(flagMapping.LoaderType, cmd.PersistentFlags().Lookup(flagMapping.LoaderType)) - _ = viper.BindPFlag(flagMapping.LoaderInterval, cmd.PersistentFlags().Lookup(flagMapping.LoaderInterval)) - _ = viper.BindPFlag(flagMapping.LoaderHttpUrl, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpUrl)) - _ = viper.BindPFlag(flagMapping.LoaderHttpToken, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpToken)) - _ = viper.BindPFlag(flagMapping.LoaderHttpTimeout, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpTimeout)) - _ = viper.BindPFlag(flagMapping.LoaderHttpRetryCount, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryCount)) - _ = viper.BindPFlag(flagMapping.LoaderHttpRetryDelay, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryDelay)) - _ = viper.BindPFlag(flagMapping.LoaderFilePath, cmd.PersistentFlags().Lookup(flagMapping.LoaderFilePath)) - _ = viper.BindPFlag(flagMapping.TargetManagerConfig, cmd.PersistentFlags().Lookup(flagMapping.TargetManagerConfig)) + NewFlag("api.address", "apiAddress").String().Bind(cmd, ":8080", "api: The address the server is listening on") + NewFlag("name", "sparrowName").String().Bind(cmd, "", "The DNS name of the sparrow") + NewFlag("loader.type", "loaderType").StringP("l").Bind(cmd, "http", "Defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader") + NewFlag("loader.interval", "loaderInterval").Duration().Bind(cmd, defaultLoaderInterval, "defines the interval the loader reloads the configuration in seconds") + NewFlag("loader.http.url", "loaderHttpUrl").String().Bind(cmd, "", "http loader: The url where to get the remote configuration") + NewFlag("loader.http.token", "loaderHttpToken").String().Bind(cmd, "", "http loader: Bearer token to authenticate the http endpoint") + NewFlag("loader.http.timeout", "loaderHttpTimeout").Duration().Bind(cmd, defaultLoaderHttpTimeout, "http loader: The timeout for the http request in seconds") + NewFlag("loader.http.retry.count", "loaderHttpRetryCount").Int().Bind(cmd, defaultHttpRetryCount, "http loader: Amount of retries trying to load the configuration") + NewFlag("loader.http.retry.delay", "loaderHttpRetryDelay").Duration().Bind(cmd, defaultHttpRetryDelay, "http loader: The initial delay between retries in seconds") + NewFlag("loader.file.path", "loaderFilePath").String().Bind(cmd, "config.yaml", "file loader: The path to the file to read the runtime config from") return cmd } // run is the entry point to start the sparrow -func run(fm *config.RunFlagsNameMapping) func(cmd *cobra.Command, args []string) { - return func(cmd *cobra.Command, args []string) { +func run() func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { log := logger.NewLogger() ctx := logger.IntoContext(context.Background(), log) cfg := config.NewConfig() - cfg.SetTargetManagerConfig(config.NewTargetManagerConfig(viper.GetString(fm.TargetManagerConfig))) - cfg.SetApiAddress(viper.GetString(fm.ApiAddress)) - cfg.SetSparrowName(viper.GetString(fm.SparrowName)) - - cfg.SetLoaderType(viper.GetString(fm.LoaderType)) - cfg.SetLoaderInterval(viper.GetInt(fm.LoaderInterval)) - cfg.SetLoaderHttpUrl(viper.GetString(fm.LoaderHttpUrl)) - cfg.SetLoaderHttpToken(viper.GetString(fm.LoaderHttpToken)) - cfg.SetLoaderHttpTimeout(viper.GetInt(fm.LoaderHttpTimeout)) - cfg.SetLoaderHttpRetryCount(viper.GetInt(fm.LoaderHttpRetryCount)) - cfg.SetLoaderHttpRetryDelay(viper.GetInt(fm.LoaderHttpRetryDelay)) - cfg.SetLoaderFilePath(viper.GetString(fm.LoaderFilePath)) + err := viper.Unmarshal(cfg) + if err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } - if err := cfg.Validate(ctx, fm); err != nil { - log.Error("Error while validating the config", "error", err) - panic(err) + if err = cfg.Validate(ctx); err != nil { + return fmt.Errorf("error while validating the config: %w", err) } s := sparrow.New(cfg) log.Info("Running sparrow") - if err := s.Run(ctx); err != nil { - log.Error("Error while running sparrow", "error", err) + if err = s.Run(ctx); err != nil { + err = fmt.Errorf("error while running sparrow: %w", err) // by this time all shutdown routines should have been called // so we can exit here - os.Exit(1) + return err } + + return nil } } diff --git a/docs/sparrow_run.md b/docs/sparrow_run.md index 7e963df0..d9418a9a 100644 --- a/docs/sparrow_run.md +++ b/docs/sparrow_run.md @@ -13,18 +13,17 @@ sparrow run [flags] ### Options ``` - --apiAddress string api: The address the server is listening on (default ":8080") - -h, --help help for run - --loaderFilePath string file loader: The path to the file to read the runtime config from (default "config.yaml") - --loaderHttpRetryCount int http loader: Amount of retries trying to load the configuration (default 3) - --loaderHttpRetryDelay int http loader: The initial delay between retries in seconds (default 1) - --loaderHttpTimeout int http loader: The timeout for the http request in seconds (default 30) - --loaderHttpToken string http loader: Bearer token to authenticate the http endpoint - --loaderHttpUrl string http loader: The url where to get the remote configuration - --loaderInterval int defines the interval the loader reloads the configuration in seconds (default 300) - -l, --loaderType string defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader (default "http") - --sparrowName string The DNS name of the sparrow - --tmconfig string target manager: The path to the file to read the target manager config from + --apiAddress string api: The address the server is listening on (default ":8080") + -h, --help help for run + --loaderFilePath string file loader: The path to the file to read the runtime config from (default "config.yaml") + --loaderHttpRetryCount int http loader: Amount of retries trying to load the configuration (default 3) + --loaderHttpRetryDelay duration http loader: The initial delay between retries in seconds (default 1s) + --loaderHttpTimeout duration http loader: The timeout for the http request in seconds (default 30s) + --loaderHttpToken string http loader: Bearer token to authenticate the http endpoint + --loaderHttpUrl string http loader: The url where to get the remote configuration + --loaderInterval duration defines the interval the loader reloads the configuration in seconds (default 5m0s) + -l, --loaderType string Defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader (default "http") + --sparrowName string The DNS name of the sparrow ``` ### Options inherited from parent commands diff --git a/go.mod b/go.mod index 07a8f93f..42af17d3 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.21 require ( github.com/getkin/kin-openapi v0.120.0 github.com/go-chi/chi/v5 v5.0.10 - github.com/go-test/deep v1.0.8 github.com/jarcoal/httpmock v1.3.1 github.com/mitchellh/mapstructure v1.5.0 github.com/spf13/cobra v1.8.0 diff --git a/internal/helper/retry.go b/internal/helper/retry.go index d96d5610..5b44b358 100644 --- a/internal/helper/retry.go +++ b/internal/helper/retry.go @@ -28,8 +28,8 @@ import ( ) type RetryConfig struct { - Count int - Delay time.Duration + Count int `yaml:"count"` + Delay time.Duration `yaml:"delay"` } // Effector will be the function that is called by the Retry function diff --git a/pkg/checks/health.go b/pkg/checks/health.go index 75175d98..ee37a7c1 100644 --- a/pkg/checks/health.go +++ b/pkg/checks/health.go @@ -49,7 +49,7 @@ type Health struct { // HealthConfig contains the health check config type HealthConfig struct { - Targets []string `json:"targets,omitempty"` + Targets []string `json:"targets,omitempty" yaml:"targets,omitempty"` } // Data that will be stored in the database diff --git a/pkg/checks/latency.go b/pkg/checks/latency.go index 178debfa..0fa691d2 100644 --- a/pkg/checks/latency.go +++ b/pkg/checks/latency.go @@ -57,10 +57,10 @@ type Latency struct { } type LatencyConfig struct { - Targets []string - Interval time.Duration - Timeout time.Duration - Retry helper.RetryConfig + Targets []string `json:"targets" yaml:"targets"` + Interval time.Duration `json:"interval" yaml:"interval"` + Timeout time.Duration `json:"timeout" yaml:"timeout"` + Retry helper.RetryConfig `json:"retry" yaml:"retry"` } type LatencyResult struct { @@ -184,11 +184,11 @@ func newLatencyMetrics() latencyMetrics { } // GetMetricCollectors returns all metric collectors of check -func (h *Latency) GetMetricCollectors() []prometheus.Collector { +func (l *Latency) GetMetricCollectors() []prometheus.Collector { return []prometheus.Collector{ - h.metrics.latencyDuration, - h.metrics.latencyCount, - h.metrics.latencyHistogram, + l.metrics.latencyDuration, + l.metrics.latencyCount, + l.metrics.latencyHistogram, } } diff --git a/pkg/config/config.go b/pkg/config/config.go index b379bdcb..4311460d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,82 +19,58 @@ package config import ( - "os" "time" - "gopkg.in/yaml.v3" - "github.com/caas-team/sparrow/internal/helper" ) type GitlabTargetManagerConfig struct { - BaseURL string `yaml:"baseUrl"` - Token string `yaml:"token"` - ProjectID int `yaml:"projectId"` + BaseURL string `yaml:"baseUrl" mapstructure:"baseUrl"` + Token string `yaml:"token" mapstructure:"token"` + ProjectID int `yaml:"projectId" mapstructure:"projectId"` } type TargetManagerConfig struct { - CheckInterval time.Duration `yaml:"checkInterval"` - RegistrationInterval time.Duration `yaml:"registrationInterval"` - UnhealthyThreshold time.Duration `yaml:"unhealthyThreshold"` - Gitlab GitlabTargetManagerConfig `yaml:"gitlab"` + CheckInterval time.Duration `yaml:"checkInterval" mapstructure:"checkInterval"` + RegistrationInterval time.Duration `yaml:"registrationInterval" mapstructure:"registrationInterval"` + UnhealthyThreshold time.Duration `yaml:"unhealthyThreshold" mapstructure:"unhealthyThreshold"` + Gitlab GitlabTargetManagerConfig `yaml:"gitlab" mapstructure:"gitlab"` } type Config struct { // SparrowName is the DNS name of the sparrow - SparrowName string + SparrowName string `yaml:"name" mapstructure:"name"` // Checks is a map of configurations for the checks - Checks map[string]any - Loader LoaderConfig - Api ApiConfig - TargetManager TargetManagerConfig + Checks map[string]any `yaml:"checks" mapstructure:"checks"` + Loader LoaderConfig `yaml:"loader" mapstructure:"loader"` + Api ApiConfig `yaml:"api" mapstructure:"api"` + TargetManager TargetManagerConfig `yaml:"targetManager" mapstructure:"targetManager"` } // ApiConfig is the configuration for the data API type ApiConfig struct { - ListeningAddress string + ListeningAddress string `yaml:"address" mapstructure:"address"` } // LoaderConfig is the configuration for loader type LoaderConfig struct { - Type string - Interval time.Duration - http HttpLoaderConfig - file FileLoaderConfig + Type string `yaml:"type" mapstructure:"type"` + Interval time.Duration `yaml:"interval" mapstructure:"interval"` + Http HttpLoaderConfig `yaml:"http" mapstructure:"http"` + File FileLoaderConfig `yaml:"file" mapstructure:"file"` } // HttpLoaderConfig is the configuration // for the specific http loader type HttpLoaderConfig struct { - url string - token string - timeout time.Duration - retryCfg helper.RetryConfig + Url string `yaml:"url" mapstructure:"url"` + Token string `yaml:"token" mapstructure:"token"` + Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"` + RetryCfg helper.RetryConfig `yaml:"retry" mapstructure:"retry"` } type FileLoaderConfig struct { - path string -} - -// NewTargetManagerConfig creates a new TargetManagerConfig -// from the passed file -func NewTargetManagerConfig(path string) TargetManagerConfig { - if path == "" { - return TargetManagerConfig{} - } - - var res TargetManagerConfig - f, err := os.ReadFile(path) //#nosec G304 - if err != nil { - panic("failed to read config file " + err.Error()) - } - - err = yaml.Unmarshal(f, &res) - if err != nil { - panic("failed to parse config file: " + err.Error()) - } - - return res + Path string `yaml:"path" mapstructure:"path"` } // NewConfig creates a new Config @@ -104,62 +80,6 @@ func NewConfig() *Config { } } -func (c *Config) SetApiAddress(address string) { - c.Api.ListeningAddress = address -} - -// SetSparrowName sets the DNS name of the sparrow -func (c *Config) SetSparrowName(name string) { - c.SparrowName = name -} - -// SetLoaderType sets the loader type -func (c *Config) SetLoaderType(loaderType string) { - c.Loader.Type = loaderType -} - -func (c *Config) SetLoaderFilePath(loaderFilePath string) { - c.Loader.file.path = loaderFilePath -} - -// SetLoaderInterval sets the loader interval -// loaderInterval in seconds -func (c *Config) SetLoaderInterval(loaderInterval int) { - c.Loader.Interval = time.Duration(loaderInterval) * time.Second -} - -// SetLoaderHttpUrl sets the loader http url -func (c *Config) SetLoaderHttpUrl(url string) { - c.Loader.http.url = url -} - -// SetLoaderHttpToken sets the loader http token -func (c *Config) SetLoaderHttpToken(token string) { - c.Loader.http.token = token -} - -// SetLoaderHttpTimeout sets the loader http timeout -// timeout in seconds -func (c *Config) SetLoaderHttpTimeout(timeout int) { - c.Loader.http.timeout = time.Duration(timeout) * time.Second -} - -// SetLoaderHttpRetryCount sets the loader http retry count -func (c *Config) SetLoaderHttpRetryCount(retryCount int) { - c.Loader.http.retryCfg.Count = retryCount -} - -// SetLoaderHttpRetryDelay sets the loader http retry delay -// retryDelay in seconds -func (c *Config) SetLoaderHttpRetryDelay(retryDelay int) { - c.Loader.http.retryCfg.Delay = time.Duration(retryDelay) * time.Second -} - -// SetTargetManagerConfig sets the target manager config -func (c *Config) SetTargetManagerConfig(config TargetManagerConfig) { - c.TargetManager = config -} - // HasTargetManager returns true if the config has a target manager func (c *Config) HasTargetManager() bool { return c.TargetManager != TargetManagerConfig{} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go deleted file mode 100644 index 3a453611..00000000 --- a/pkg/config/config_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -import ( - "testing" - "time" - - "github.com/go-test/deep" -) - -func Test_NewTargetManagerConfig_Gitlab(t *testing.T) { - got := NewTargetManagerConfig("testdata/tmconfig.yaml") - want := TargetManagerConfig{ - CheckInterval: 300 * time.Second, - RegistrationInterval: 600 * time.Second, - UnhealthyThreshold: 900 * time.Second, - Gitlab: GitlabTargetManagerConfig{ - BaseURL: "https://gitlab.devops.telekom.de", - ProjectID: 666, - Token: "gitlab-token", - }, - } - - if diff := deep.Equal(got, want); diff != nil { - t.Error(diff) - } -} diff --git a/pkg/config/file.go b/pkg/config/file.go index 6538855b..ee7d1fda 100644 --- a/pkg/config/file.go +++ b/pkg/config/file.go @@ -36,7 +36,7 @@ type FileLoader struct { func NewFileLoader(cfg *Config, cCfgChecks chan<- map[string]any) *FileLoader { return &FileLoader{ - path: cfg.Loader.file.path, + path: cfg.Loader.File.Path, c: cCfgChecks, } } diff --git a/pkg/config/file_test.go b/pkg/config/file_test.go index bee4f547..a2c74bca 100644 --- a/pkg/config/file_test.go +++ b/pkg/config/file_test.go @@ -25,7 +25,7 @@ import ( ) func TestNewFileLoader(t *testing.T) { - l := NewFileLoader(&Config{Loader: LoaderConfig{file: FileLoaderConfig{path: "config.yaml"}}}, make(chan<- map[string]any, 1)) + l := NewFileLoader(&Config{Loader: LoaderConfig{File: FileLoaderConfig{Path: "config.yaml"}}}, make(chan<- map[string]any, 1)) if l.path != "config.yaml" { t.Errorf("Expected path to be config.yaml, got %s", l.path) diff --git a/pkg/config/flags.go b/pkg/config/flags.go deleted file mode 100644 index 2509c186..00000000 --- a/pkg/config/flags.go +++ /dev/null @@ -1,35 +0,0 @@ -// sparrow -// (C) 2023, Deutsche Telekom IT GmbH -// -// Deutsche Telekom IT GmbH and all other contributors / -// copyright owners license this file to you under the Apache -// License, Version 2.0 (the "License"); you may not use this -// file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package config - -type RunFlagsNameMapping struct { - ApiAddress string - SparrowName string - - LoaderType string - LoaderInterval string - LoaderHttpUrl string - LoaderHttpToken string - LoaderHttpTimeout string - LoaderHttpRetryCount string - LoaderHttpRetryDelay string - LoaderFilePath string - - TargetManagerConfig string -} diff --git a/pkg/config/http.go b/pkg/config/http.go index fc097f7f..7557fcc6 100644 --- a/pkg/config/http.go +++ b/pkg/config/http.go @@ -41,7 +41,7 @@ func NewHttpLoader(cfg *Config, cCfgChecks chan<- map[string]any) *HttpLoader { cfg: cfg, cCfgChecks: cCfgChecks, client: &http.Client{ - Timeout: cfg.Loader.http.timeout, + Timeout: cfg.Loader.Http.Timeout, }, } } @@ -62,7 +62,7 @@ func (gl *HttpLoader) Run(ctx context.Context) { var err error runtimeCfg, err = gl.GetRuntimeConfig(ctx) return err - }, gl.cfg.Loader.http.retryCfg) + }, gl.cfg.Loader.Http.RetryCfg) if err := getConfigRetry(ctx); err != nil { log.Error("Could not get remote runtime configuration", "error", err) @@ -82,15 +82,15 @@ func (gl *HttpLoader) Run(ctx context.Context) { // GetRuntimeConfig gets the remote runtime configuration func (gl *HttpLoader) GetRuntimeConfig(ctx context.Context) (*RuntimeConfig, error) { - log := logger.FromContext(ctx).With("url", gl.cfg.Loader.http.url) + log := logger.FromContext(ctx).With("url", gl.cfg.Loader.Http.Url) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, gl.cfg.Loader.http.url, http.NoBody) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, gl.cfg.Loader.Http.Url, http.NoBody) if err != nil { log.Error("Could not create http GET request", "error", err.Error()) return nil, err } - if gl.cfg.Loader.http.token != "" { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", gl.cfg.Loader.http.token)) + if gl.cfg.Loader.Http.Token != "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", gl.cfg.Loader.Http.Token)) } res, err := gl.client.Do(req) diff --git a/pkg/config/http_test.go b/pkg/config/http_test.go index 3192946f..d13bb35f 100644 --- a/pkg/config/http_test.go +++ b/pkg/config/http_test.go @@ -74,8 +74,8 @@ func TestHttpLoader_GetRuntimeConfig(t *testing.T) { Loader: LoaderConfig{ Type: "http", Interval: time.Second, - http: HttpLoaderConfig{ - token: "SECRET", + Http: HttpLoaderConfig{ + Token: "SECRET", }, }, }, @@ -127,8 +127,8 @@ func TestHttpLoader_GetRuntimeConfig(t *testing.T) { endpoint := "https://api.test.com/test" httpmock.RegisterResponder("GET", endpoint, func(req *http.Request) (*http.Response, error) { - if tt.cfg.Loader.http.token != "" { - require.Equal(t, req.Header.Get("Authorization"), fmt.Sprintf("Bearer %s", tt.cfg.Loader.http.token)) + if tt.cfg.Loader.Http.Token != "" { + require.Equal(t, req.Header.Get("Authorization"), fmt.Sprintf("Bearer %s", tt.cfg.Loader.Http.Token)) fmt.Println("TOKEN tested") } resp, _ := httpmock.NewStringResponder(tt.httpResponder.statusCode, tt.httpResponder.response)(req) @@ -145,10 +145,10 @@ func TestHttpLoader_GetRuntimeConfig(t *testing.T) { cfg: tt.cfg, cCfgChecks: make(chan<- map[string]any, 1), client: &http.Client{ - Timeout: tt.cfg.Loader.http.timeout, + Timeout: tt.cfg.Loader.Http.Timeout, }, } - gl.cfg.Loader.http.url = endpoint + gl.cfg.Loader.Http.Url = endpoint got, err := gl.GetRuntimeConfig(ctx) if (err != nil) != tt.wantErr { diff --git a/pkg/config/runtime_config.go b/pkg/config/runtime_config.go index 63e3ea65..fc1fdd00 100644 --- a/pkg/config/runtime_config.go +++ b/pkg/config/runtime_config.go @@ -19,5 +19,5 @@ package config type RuntimeConfig struct { - Checks map[string]any `json:"checks"` + Checks map[string]any `yaml:"checks"` } diff --git a/pkg/config/testdata/tmconfig.yaml b/pkg/config/testdata/tmconfig.yaml deleted file mode 100644 index 9f615528..00000000 --- a/pkg/config/testdata/tmconfig.yaml +++ /dev/null @@ -1,7 +0,0 @@ -checkInterval: 300s -registrationInterval: 600s -unhealthyThreshold: 900s -gitlab: - token: gitlab-token - baseUrl: https://gitlab.devops.telekom.de - projectId: 666 diff --git a/pkg/config/validate.go b/pkg/config/validate.go index e87b845e..f36450f6 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -28,7 +28,7 @@ import ( ) // Validate validates the config -func (c *Config) Validate(ctx context.Context, fm *RunFlagsNameMapping) error { +func (c *Config) Validate(ctx context.Context) error { ctx, cancel := logger.NewContextWithLogger(ctx, "configValidation") defer cancel() log := logger.FromContext(ctx) @@ -37,20 +37,18 @@ func (c *Config) Validate(ctx context.Context, fm *RunFlagsNameMapping) error { if !isDNSName(c.SparrowName) { ok = false - log.Error("The name of the sparrow must be DNS compliant", fm.SparrowName, c.SparrowName) + log.Error("The name of the sparrow must be DNS compliant") } switch c.Loader.Type { //nolint:gocritic case "http": - if _, err := url.ParseRequestURI(c.Loader.http.url); err != nil { + if _, err := url.ParseRequestURI(c.Loader.Http.Url); err != nil { ok = false - log.ErrorContext(ctx, "The loader http url is not a valid url", - fm.LoaderHttpUrl, c.Loader.http.url) + log.ErrorContext(ctx, "The loader http url is not a valid url") } - if c.Loader.http.retryCfg.Count < 0 || c.Loader.http.retryCfg.Count >= 5 { + if c.Loader.Http.RetryCfg.Count < 0 || c.Loader.Http.RetryCfg.Count >= 5 { ok = false - log.Error("The amount of loader http retries should be above 0 and below 6", - fm.LoaderHttpRetryCount, c.Loader.http.retryCfg.Count) + log.Error("The amount of loader http retries should be above 0 and below 6", "retryCount", c.Loader.Http.RetryCfg.Count) } } diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go index 451299bf..dc611138 100644 --- a/pkg/config/validate_test.go +++ b/pkg/config/validate_test.go @@ -44,10 +44,10 @@ func TestConfig_Validate(t *testing.T) { fields: fields{ Loader: LoaderConfig{ Type: "http", - http: HttpLoaderConfig{ - url: "https://test.de/config", - timeout: time.Second, - retryCfg: helper.RetryConfig{ + Http: HttpLoaderConfig{ + Url: "https://test.de/config", + Timeout: time.Second, + RetryCfg: helper.RetryConfig{ Count: 1, Delay: time.Second, }, @@ -62,10 +62,10 @@ func TestConfig_Validate(t *testing.T) { fields: fields{ Loader: LoaderConfig{ Type: "http", - http: HttpLoaderConfig{ - url: "", - timeout: time.Second, - retryCfg: helper.RetryConfig{ + Http: HttpLoaderConfig{ + Url: "", + Timeout: time.Second, + RetryCfg: helper.RetryConfig{ Count: 1, Delay: time.Second, }, @@ -80,10 +80,10 @@ func TestConfig_Validate(t *testing.T) { fields: fields{ Loader: LoaderConfig{ Type: "http", - http: HttpLoaderConfig{ - url: "this is not a valid url", - timeout: time.Second, - retryCfg: helper.RetryConfig{ + Http: HttpLoaderConfig{ + Url: "this is not a valid url", + Timeout: time.Second, + RetryCfg: helper.RetryConfig{ Count: 1, Delay: time.Second, }, @@ -98,10 +98,10 @@ func TestConfig_Validate(t *testing.T) { fields: fields{ Loader: LoaderConfig{ Type: "http", - http: HttpLoaderConfig{ - url: "test.de", - timeout: time.Minute, - retryCfg: helper.RetryConfig{ + Http: HttpLoaderConfig{ + Url: "test.de", + Timeout: time.Minute, + RetryCfg: helper.RetryConfig{ Count: 100000, Delay: time.Second, }, @@ -119,7 +119,7 @@ func TestConfig_Validate(t *testing.T) { SparrowName: "cool-dns-name.org", Loader: tt.fields.Loader, } - if err := c.Validate(ctx, &RunFlagsNameMapping{}); (err != nil) != tt.wantErr { + if err := c.Validate(ctx); (err != nil) != tt.wantErr { t.Errorf("Config.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/sparrow/run_test.go b/pkg/sparrow/run_test.go index f8f9b818..c724403c 100644 --- a/pkg/sparrow/run_test.go +++ b/pkg/sparrow/run_test.go @@ -206,7 +206,7 @@ func TestSparrow_Run(t *testing.T) { }, } - c.SetLoaderFilePath("../config/testdata/config.yaml") + c.Loader.File.Path = ("../config/testdata/config.yaml") // start sparrow s := New(c)