diff --git a/.gitignore b/.gitignore index 70eb07810..aa02c8470 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ pnpm-debug.log* # macOS-specific files .DS_Store +dicedb.yaml +dicedb.conf diff --git a/cmd/init_config.go b/cmd/init_config.go new file mode 100644 index 000000000..17e180adc --- /dev/null +++ b/cmd/init_config.go @@ -0,0 +1,26 @@ +// Copyright (c) 2022-present, DiceDB contributors +// All rights reserved. Licensed under the BSD 3-Clause License. See LICENSE file in the project root for full license information. + +package cmd + +import ( + "fmt" + + "github.com/dicedb/dice/config" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var initConfigCmd = &cobra.Command{ + Use: "init-config", + Short: "creates a config file at dicedb.yaml with default values", + Run: func(cmd *cobra.Command, args []string) { + config.Init(cmd.Flags()) + viper.WriteConfigAs("dicedb.yaml") + fmt.Println("config created at dicedb.yaml") + }, +} + +func init() { + rootCmd.AddCommand(initConfigCmd) +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 000000000..f532869c3 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,53 @@ +// Copyright (c) 2022-present, DiceDB contributors +// All rights reserved. Licensed under the BSD 3-Clause License. See LICENSE file in the project root for full license information. + +package cmd + +import ( + "fmt" + "os" + "reflect" + "strconv" + + "github.com/dicedb/dice/config" + "github.com/dicedb/dice/server" + "github.com/spf13/cobra" +) + +func init() { + c := config.DiceDBConfig{} + _type := reflect.TypeOf(c) + for i := 0; i < _type.NumField(); i++ { + field := _type.Field(i) + yamlTag := field.Tag.Get("mapstructure") + descriptionTag := field.Tag.Get("description") + defaultTag := field.Tag.Get("default") + + switch field.Type.Kind() { + case reflect.String: + rootCmd.PersistentFlags().String(yamlTag, defaultTag, descriptionTag) + case reflect.Int: + val, _ := strconv.Atoi(defaultTag) + rootCmd.PersistentFlags().Int(yamlTag, val, descriptionTag) + case reflect.Bool: + val, _ := strconv.ParseBool(defaultTag) + rootCmd.PersistentFlags().Bool(yamlTag, val, descriptionTag) + } + } +} + +var rootCmd = &cobra.Command{ + Use: "dicedb", + Short: "an in-memory database;", + Run: func(cmd *cobra.Command, args []string) { + config.Init(cmd.Flags()) + server.Start() + }, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/config/config.go b/config/config.go index 54d9622f4..b808808cf 100644 --- a/config/config.go +++ b/config/config.go @@ -12,6 +12,8 @@ import ( "time" "github.com/dicedb/dice/internal/server/utils" + "github.com/spf13/pflag" + "github.com/spf13/viper" ) const ( @@ -318,3 +320,34 @@ func MergeFlags(flags *Config) { } }) } + +type DiceDBConfig struct { + Host string `mapstructure:"host" description:"the host address to bind to" default:"0.0.0.0"` + Port int `mapstructure:"port" description:"the port to bind to" default:"7379"` + EnableHTTP bool `mapstructure:"enable-http" description:"enable http server" default:"false"` +} + +var GlobalDiceDBConfig *DiceDBConfig + +func Init(flags *pflag.FlagSet) { + viper.SetConfigName("dicedb") + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("/etc/dicedb") + + err := viper.ReadInConfig() + if _, ok := err.(viper.ConfigFileNotFoundError); !ok && err != nil { + panic(err) + } + + flags.VisitAll(func(flag *pflag.Flag) { + if flag.Name == "help" { + return + } + viper.Set(flag.Name, flag.Value.String()) + }) + + if err := viper.Unmarshal(&GlobalDiceDBConfig); err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod index dcf33b828..515bde104 100644 --- a/go.mod +++ b/go.mod @@ -11,20 +11,34 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.11.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -47,9 +61,11 @@ require ( github.com/ohler55/ojg v1.25.0 github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.33.0 + github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.8 - github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 golang.org/x/crypto v0.29.0 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f google.golang.org/protobuf v1.35.1 diff --git a/go.sum b/go.sum index 51567b272..ffc65ee77 100644 --- a/go.sum +++ b/go.sum @@ -20,7 +20,7 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs= github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -35,6 +35,10 @@ github.com/dicedb/dicedb-go v0.0.0-20241120175955-5eaa6c7e79bb h1:HVdPhxbTT7wLIN github.com/dicedb/dicedb-go v0.0.0-20241120175955-5eaa6c7e79bb/go.mod h1:DuggsMhSh810UH6hH4MXWLflPz+/ZgoFAhhsi53S9e0= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -56,6 +60,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -66,6 +74,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -74,11 +84,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/geohash v0.10.0 h1:9w1HchfDfdeLc+jFEf/04D27KP7E2QmpDu52wPbJWRE= github.com/mmcloughlin/geohash v0.10.0/go.mod h1:oNZxQo5yWJh0eMQEP/8hwQuVx9Z9tjwFUqcTB1SmG0c= github.com/ohler55/ojg v1.25.0 h1:sDwc4u4zex65Uz5Nm7O1QwDKTT+YRcpeZQTy1pffRkw= github.com/ohler55/ojg v1.25.0/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -90,21 +103,45 @@ github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= -github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= @@ -126,6 +163,8 @@ google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index b320bdc39..52864ce6a 100644 --- a/main.go +++ b/main.go @@ -3,279 +3,8 @@ package main -import ( - "context" - "errors" - "fmt" - "log/slog" - "net/http" - "os" - "os/signal" - "runtime" - "runtime/pprof" - "runtime/trace" - "sync" - "syscall" - "time" - - "github.com/dicedb/dice/internal/server/httpws" - - "github.com/dicedb/dice/internal/cli" - "github.com/dicedb/dice/internal/commandhandler" - "github.com/dicedb/dice/internal/logger" - "github.com/dicedb/dice/internal/server/abstractserver" - "github.com/dicedb/dice/internal/wal" - "github.com/dicedb/dice/internal/watchmanager" - - "github.com/dicedb/dice/config" - diceerrors "github.com/dicedb/dice/internal/errors" - "github.com/dicedb/dice/internal/iothread" - "github.com/dicedb/dice/internal/observability" - "github.com/dicedb/dice/internal/server/resp" - "github.com/dicedb/dice/internal/shard" - dstore "github.com/dicedb/dice/internal/store" -) - -const ( - WALEngineAOF = "aof" -) +import "github.com/dicedb/dice/cmd" func main() { - iid := observability.GetOrCreateInstanceID() - config.DiceConfig.InstanceID = iid - - // This is counter intuitive, but it's the first thing that should be done - // because this function parses the flags and prepares the config, - cli.Execute() - - slog.SetDefault(logger.New()) - go observability.Ping() - - ctx, cancel := context.WithCancel(context.Background()) - - // Handle SIGTERM and SIGINT - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) - - var ( - cmdWatchChan chan dstore.CmdWatchEvent - serverErrCh = make(chan error, 2) - cmdWatchSubscriptionChan = make(chan watchmanager.WatchSubscription) - wl wal.AbstractWAL - ) - - wl, _ = wal.NewNullWAL() - if config.DiceConfig.Persistence.Enabled { - if config.DiceConfig.Persistence.WALEngine == WALEngineAOF { - _wl, err := wal.NewAOFWAL(config.DiceConfig.WAL.LogDir) - if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) - sigs <- syscall.SIGKILL - return - } - wl = _wl - } else { - slog.Error("unsupported WAL engine", slog.String("engine", config.DiceConfig.Persistence.WALEngine)) - sigs <- syscall.SIGKILL - return - } - - if err := wl.Init(time.Now()); err != nil { - slog.Error("could not initialize WAL", slog.Any("error", err)) - } else { - go wal.InitBG(wl) - } - - slog.Debug("WAL initialization complete") - - if config.DiceConfig.Persistence.RestoreFromWAL { - slog.Info("restoring database from WAL") - wal.ReplayWAL(wl) - slog.Info("database restored from WAL") - } - } - - if config.DiceConfig.Performance.EnableWatch { - bufSize := config.DiceConfig.Performance.WatchChanBufSize - cmdWatchChan = make(chan dstore.CmdWatchEvent, bufSize) - } - - // Get the number of available CPU cores on the machine using runtime.NumCPU(). - // This determines the total number of logical processors that can be utilized - // for parallel execution. Setting the maximum number of CPUs to the available - // core count ensures the application can make full use of all available hardware. - var numShards int - numShards = runtime.NumCPU() - if config.DiceConfig.Performance.NumShards > 0 { - numShards = config.DiceConfig.Performance.NumShards - } - - // The runtime.GOMAXPROCS(numShards) call limits the number of operating system - // threads that can execute Go code simultaneously to the number of CPU cores. - // This enables Go to run more efficiently, maximizing CPU utilization and - // improving concurrency performance across multiple goroutines. - runtime.GOMAXPROCS(runtime.NumCPU()) - - // Initialize the ShardManager - shardManager := shard.NewShardManager(uint8(numShards), cmdWatchChan, serverErrCh) - - wg := sync.WaitGroup{} - - wg.Add(1) - go func() { - defer wg.Done() - shardManager.Run(ctx) - }() - - var serverWg sync.WaitGroup - - if config.DiceConfig.Performance.EnableProfiling { - stopProfiling, err := startProfiling() - if err != nil { - slog.Error("Profiling could not be started", slog.Any("error", err)) - sigs <- syscall.SIGKILL - } - defer stopProfiling() - } - ioThreadManager := iothread.NewManager(config.DiceConfig.Performance.MaxClients) - cmdHandlerManager := commandhandler.NewRegistry(config.DiceConfig.Performance.MaxClients, shardManager) - - respServer := resp.NewServer(shardManager, ioThreadManager, cmdHandlerManager, cmdWatchSubscriptionChan, cmdWatchChan, serverErrCh, wl) - serverWg.Add(1) - go runServer(ctx, &serverWg, respServer, serverErrCh) - - if config.DiceConfig.HTTP.Enabled { - httpServer := httpws.NewHTTPServer(shardManager, wl) - serverWg.Add(1) - go runServer(ctx, &serverWg, httpServer, serverErrCh) - } - - if config.DiceConfig.WebSocket.Enabled { - websocketServer := httpws.NewWebSocketServer(shardManager, config.DiceConfig.WebSocket.Port, wl) - serverWg.Add(1) - go runServer(ctx, &serverWg, websocketServer, serverErrCh) - } - - wg.Add(1) - go func() { - defer wg.Done() - <-sigs - cancel() - }() - - go func() { - serverWg.Wait() - close(serverErrCh) // Close the channel when both servers are done - }() - - for err := range serverErrCh { - if err != nil && errors.Is(err, diceerrors.ErrAborted) { - // if either the AsyncServer/RESPServer or the HTTPServer received an abort command, - // cancel the context, helping gracefully exiting all servers - cancel() - } - } - - close(sigs) - - if config.DiceConfig.Persistence.Enabled { - wal.ShutdownBG() - } - - cancel() - - wg.Wait() -} - -func runServer(ctx context.Context, wg *sync.WaitGroup, srv abstractserver.AbstractServer, errCh chan<- error) { - defer wg.Done() - if err := srv.Run(ctx); err != nil { - switch { - case errors.Is(err, context.Canceled): - slog.Debug(fmt.Sprintf("%T was canceled", srv)) - case errors.Is(err, diceerrors.ErrAborted): - slog.Debug(fmt.Sprintf("%T received abort command", srv)) - case errors.Is(err, http.ErrServerClosed): - slog.Debug(fmt.Sprintf("%T received abort command", srv)) - default: - slog.Error(fmt.Sprintf("%T error", srv), slog.Any("error", err)) - } - errCh <- err - } else { - slog.Debug("bye.") - } -} -func startProfiling() (func(), error) { - // Start CPU profiling - cpuFile, err := os.Create("cpu.prof") - if err != nil { - return nil, fmt.Errorf("could not create cpu.prof: %w", err) - } - - if err = pprof.StartCPUProfile(cpuFile); err != nil { - cpuFile.Close() - return nil, fmt.Errorf("could not start CPU profile: %w", err) - } - - // Start memory profiling - memFile, err := os.Create("mem.prof") - if err != nil { - pprof.StopCPUProfile() - cpuFile.Close() - return nil, fmt.Errorf("could not create mem.prof: %w", err) - } - - // Start block profiling - runtime.SetBlockProfileRate(1) - - // Start execution trace - traceFile, err := os.Create("trace.out") - if err != nil { - runtime.SetBlockProfileRate(0) - memFile.Close() - pprof.StopCPUProfile() - cpuFile.Close() - return nil, fmt.Errorf("could not create trace.out: %w", err) - } - - if err := trace.Start(traceFile); err != nil { - traceFile.Close() - runtime.SetBlockProfileRate(0) - memFile.Close() - pprof.StopCPUProfile() - cpuFile.Close() - return nil, fmt.Errorf("could not start trace: %w", err) - } - - // Return a cleanup function - return func() { - // Stop the CPU profiling and close cpuFile - pprof.StopCPUProfile() - cpuFile.Close() - - // Write heap profile - runtime.GC() - if err := pprof.WriteHeapProfile(memFile); err != nil { - slog.Warn("could not write memory profile", slog.Any("error", err)) - } - - memFile.Close() - - // Write block profile - blockFile, err := os.Create("block.prof") - if err != nil { - slog.Warn("could not create block profile", slog.Any("error", err)) - } else { - if err := pprof.Lookup("block").WriteTo(blockFile, 0); err != nil { - slog.Warn("could not write block profile", slog.Any("error", err)) - } - blockFile.Close() - } - - runtime.SetBlockProfileRate(0) - - // Stop trace and close traceFile - trace.Stop() - traceFile.Close() - }, nil + cmd.Execute() } diff --git a/server/main.go b/server/main.go new file mode 100644 index 000000000..971060f04 --- /dev/null +++ b/server/main.go @@ -0,0 +1,281 @@ +// Copyright (c) 2022-present, DiceDB contributors +// All rights reserved. Licensed under the BSD 3-Clause License. See LICENSE file in the project root for full license information. + +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "runtime" + "runtime/pprof" + "runtime/trace" + "sync" + "syscall" + "time" + + "github.com/dicedb/dice/internal/server/httpws" + + "github.com/dicedb/dice/internal/cli" + "github.com/dicedb/dice/internal/commandhandler" + "github.com/dicedb/dice/internal/logger" + "github.com/dicedb/dice/internal/server/abstractserver" + "github.com/dicedb/dice/internal/wal" + "github.com/dicedb/dice/internal/watchmanager" + + "github.com/dicedb/dice/config" + diceerrors "github.com/dicedb/dice/internal/errors" + "github.com/dicedb/dice/internal/iothread" + "github.com/dicedb/dice/internal/observability" + "github.com/dicedb/dice/internal/server/resp" + "github.com/dicedb/dice/internal/shard" + dstore "github.com/dicedb/dice/internal/store" +) + +const ( + WALEngineAOF = "aof" +) + +func Start() { + iid := observability.GetOrCreateInstanceID() + config.DiceConfig.InstanceID = iid + + // This is counter intuitive, but it's the first thing that should be done + // because this function parses the flags and prepares the config, + cli.Execute() + + slog.SetDefault(logger.New()) + go observability.Ping() + + ctx, cancel := context.WithCancel(context.Background()) + + // Handle SIGTERM and SIGINT + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) + + var ( + cmdWatchChan chan dstore.CmdWatchEvent + serverErrCh = make(chan error, 2) + cmdWatchSubscriptionChan = make(chan watchmanager.WatchSubscription) + wl wal.AbstractWAL + ) + + wl, _ = wal.NewNullWAL() + if config.DiceConfig.Persistence.Enabled { + if config.DiceConfig.Persistence.WALEngine == WALEngineAOF { + _wl, err := wal.NewAOFWAL(config.DiceConfig.WAL.LogDir) + if err != nil { + slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) + sigs <- syscall.SIGKILL + return + } + wl = _wl + } else { + slog.Error("unsupported WAL engine", slog.String("engine", config.DiceConfig.Persistence.WALEngine)) + sigs <- syscall.SIGKILL + return + } + + if err := wl.Init(time.Now()); err != nil { + slog.Error("could not initialize WAL", slog.Any("error", err)) + } else { + go wal.InitBG(wl) + } + + slog.Debug("WAL initialization complete") + + if config.DiceConfig.Persistence.RestoreFromWAL { + slog.Info("restoring database from WAL") + wal.ReplayWAL(wl) + slog.Info("database restored from WAL") + } + } + + if config.DiceConfig.Performance.EnableWatch { + bufSize := config.DiceConfig.Performance.WatchChanBufSize + cmdWatchChan = make(chan dstore.CmdWatchEvent, bufSize) + } + + // Get the number of available CPU cores on the machine using runtime.NumCPU(). + // This determines the total number of logical processors that can be utilized + // for parallel execution. Setting the maximum number of CPUs to the available + // core count ensures the application can make full use of all available hardware. + var numShards int + numShards = runtime.NumCPU() + if config.DiceConfig.Performance.NumShards > 0 { + numShards = config.DiceConfig.Performance.NumShards + } + + // The runtime.GOMAXPROCS(numShards) call limits the number of operating system + // threads that can execute Go code simultaneously to the number of CPU cores. + // This enables Go to run more efficiently, maximizing CPU utilization and + // improving concurrency performance across multiple goroutines. + runtime.GOMAXPROCS(runtime.NumCPU()) + + // Initialize the ShardManager + shardManager := shard.NewShardManager(uint8(numShards), cmdWatchChan, serverErrCh) + + wg := sync.WaitGroup{} + + wg.Add(1) + go func() { + defer wg.Done() + shardManager.Run(ctx) + }() + + var serverWg sync.WaitGroup + + if config.DiceConfig.Performance.EnableProfiling { + stopProfiling, err := startProfiling() + if err != nil { + slog.Error("Profiling could not be started", slog.Any("error", err)) + sigs <- syscall.SIGKILL + } + defer stopProfiling() + } + ioThreadManager := iothread.NewManager(config.DiceConfig.Performance.MaxClients) + cmdHandlerManager := commandhandler.NewRegistry(config.DiceConfig.Performance.MaxClients, shardManager) + + respServer := resp.NewServer(shardManager, ioThreadManager, cmdHandlerManager, cmdWatchSubscriptionChan, cmdWatchChan, serverErrCh, wl) + serverWg.Add(1) + go runServer(ctx, &serverWg, respServer, serverErrCh) + + if config.DiceConfig.HTTP.Enabled { + httpServer := httpws.NewHTTPServer(shardManager, wl) + serverWg.Add(1) + go runServer(ctx, &serverWg, httpServer, serverErrCh) + } + + if config.DiceConfig.WebSocket.Enabled { + websocketServer := httpws.NewWebSocketServer(shardManager, config.DiceConfig.WebSocket.Port, wl) + serverWg.Add(1) + go runServer(ctx, &serverWg, websocketServer, serverErrCh) + } + + wg.Add(1) + go func() { + defer wg.Done() + <-sigs + cancel() + }() + + go func() { + serverWg.Wait() + close(serverErrCh) // Close the channel when both servers are done + }() + + for err := range serverErrCh { + if err != nil && errors.Is(err, diceerrors.ErrAborted) { + // if either the AsyncServer/RESPServer or the HTTPServer received an abort command, + // cancel the context, helping gracefully exiting all servers + cancel() + } + } + + close(sigs) + + if config.DiceConfig.Persistence.Enabled { + wal.ShutdownBG() + } + + cancel() + + wg.Wait() +} + +func runServer(ctx context.Context, wg *sync.WaitGroup, srv abstractserver.AbstractServer, errCh chan<- error) { + defer wg.Done() + if err := srv.Run(ctx); err != nil { + switch { + case errors.Is(err, context.Canceled): + slog.Debug(fmt.Sprintf("%T was canceled", srv)) + case errors.Is(err, diceerrors.ErrAborted): + slog.Debug(fmt.Sprintf("%T received abort command", srv)) + case errors.Is(err, http.ErrServerClosed): + slog.Debug(fmt.Sprintf("%T received abort command", srv)) + default: + slog.Error(fmt.Sprintf("%T error", srv), slog.Any("error", err)) + } + errCh <- err + } else { + slog.Debug("bye.") + } +} +func startProfiling() (func(), error) { + // Start CPU profiling + cpuFile, err := os.Create("cpu.prof") + if err != nil { + return nil, fmt.Errorf("could not create cpu.prof: %w", err) + } + + if err = pprof.StartCPUProfile(cpuFile); err != nil { + cpuFile.Close() + return nil, fmt.Errorf("could not start CPU profile: %w", err) + } + + // Start memory profiling + memFile, err := os.Create("mem.prof") + if err != nil { + pprof.StopCPUProfile() + cpuFile.Close() + return nil, fmt.Errorf("could not create mem.prof: %w", err) + } + + // Start block profiling + runtime.SetBlockProfileRate(1) + + // Start execution trace + traceFile, err := os.Create("trace.out") + if err != nil { + runtime.SetBlockProfileRate(0) + memFile.Close() + pprof.StopCPUProfile() + cpuFile.Close() + return nil, fmt.Errorf("could not create trace.out: %w", err) + } + + if err := trace.Start(traceFile); err != nil { + traceFile.Close() + runtime.SetBlockProfileRate(0) + memFile.Close() + pprof.StopCPUProfile() + cpuFile.Close() + return nil, fmt.Errorf("could not start trace: %w", err) + } + + // Return a cleanup function + return func() { + // Stop the CPU profiling and close cpuFile + pprof.StopCPUProfile() + cpuFile.Close() + + // Write heap profile + runtime.GC() + if err := pprof.WriteHeapProfile(memFile); err != nil { + slog.Warn("could not write memory profile", slog.Any("error", err)) + } + + memFile.Close() + + // Write block profile + blockFile, err := os.Create("block.prof") + if err != nil { + slog.Warn("could not create block profile", slog.Any("error", err)) + } else { + if err := pprof.Lookup("block").WriteTo(blockFile, 0); err != nil { + slog.Warn("could not write block profile", slog.Any("error", err)) + } + blockFile.Close() + } + + runtime.SetBlockProfileRate(0) + + // Stop trace and close traceFile + trace.Stop() + traceFile.Close() + }, nil +}