From abcace90bb866412e018a74dc144f36027eba3a8 Mon Sep 17 00:00:00 2001 From: Filippov Alex Date: Sat, 23 Dec 2023 19:44:42 +0600 Subject: [PATCH] Technical debt (#252) * technical debt --- README.md | 101 ++++---------- cmd/server/commands/gate.go | 2 +- cmd/server/commands/reset.go | 12 +- cmd/server/commands/restore.go | 4 +- cmd/server/container/init.go | 1 + cmd/server/reset_container/config.go | 32 +++++ cmd/server/reset_container/container.go | 46 +++++++ cmd/server/reset_container/fx.go | 33 +++++ cmd/server/reset_container/logger.go | 25 ++++ cmd/server/reset_container/migrations.go | 29 +++++ conf/config.gate.json | 4 +- .../src/components/Infotip/src/Infotip.vue | 25 +++- static_source/admin/src/locales/en.ts | 2 + static_source/admin/src/locales/ru.ts | 2 + .../admin/src/views/Settings/index.vue | 39 ++++-- system/backup/backup.go | 6 +- system/gate/client/gate_client.go | 1 + system/gate/client/wsp/client.go | 4 +- system/gate/client/wsp/connection.go | 6 +- system/gate/client/wsp/pool.go | 4 +- system/gate/server/gate_server.go | 10 +- system/gate/server/wsp/config.go | 10 +- system/gate/server/wsp/connection.go | 123 ++++++++---------- system/gate/server/wsp/pool.go | 56 ++++---- system/gate/server/wsp/server.go | 36 ++--- system/gate/server/wsp/types.go | 31 +++++ system/initial/initial.go | 19 +-- system/initial/local_migrations/m_add_var1.go | 45 +++++++ .../initial/local_migrations/m_dashboard.go | 59 +++++---- .../initial/local_migrations/m_scheduler.go | 2 +- system/initial/local_migrations/migrations.go | 6 +- 31 files changed, 496 insertions(+), 279 deletions(-) create mode 100644 cmd/server/reset_container/config.go create mode 100644 cmd/server/reset_container/container.go create mode 100644 cmd/server/reset_container/fx.go create mode 100644 cmd/server/reset_container/logger.go create mode 100644 cmd/server/reset_container/migrations.go create mode 100644 system/gate/server/wsp/types.go create mode 100644 system/initial/local_migrations/m_add_var1.go diff --git a/README.md b/README.md index 2c77292da..a0dea84a1 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,7 @@ the component base. - [Supported system](#supported-system) - [Quick installation](#quick-installation) - [Postgresql](#database-postgresql) -- [Installation for development](#installation-for-development) - - [Server](#main-server-install) - [Docker](#docker) -- [Testing](#testing) - [Support](#support) - [Contributors](#contributors) - [See also](#see-also) @@ -49,32 +46,36 @@ the component base. ### Features -1. The ultimate smart thing solution - server, configurator, nodes, gateway, mobile application +1. The ultimate smart thing solution - server, configurator, gateway 2. Free and open source 3. Cross-platform Linux, MacOS, Windows ... 4. Convenient WEB-configurator for fine-tuning -5. Mobile application for equipment management +5. Powerful visual interface editor +5. Organization of remote access without white IP FREE 6. Role system for separation of access rights +7. Plugin system 7. Programs in javaScript, coffeeScript, typeScript -8. Notification system SMS, Email, Slack, Telegram -9. modbus, mqtt, [zigbee2mqtt](https://www.zigbee2mqtt.io/), rpc calling +8. Notification system SMS, Email, Slack, Telegram, Web push, html5 +9. Embedded MQTT server/client/bridge +9. MODBUS, [ZIGBEE2MQTT](https://www.zigbee2mqtt.io/), rpc calling, etc 10. Autonomous system. -11. Quick backup of all data, and recovery +11. Telegram bots +11. Automation scenarios +11. Quick backup/recovery 12. Have Docker images to enhance system security 13. Minimum consumption of resources. 14. Optimized for embedded devices like Raspberry Pi 15. 100% local home automation 16. Create and restore full backups of your whole configuration with ease 17. Management web interface integrated into Smart home -18. Alexa skills ### Demo access -outdated version, not supported:
-[dashboard](https://board.e154.ru) (https://board.e154.ru)
+dashboard:
+[dashboard](https://gate.e154.ru:8443) (https://gate.e154.ru:8443)
-outdated version, not supported:
-[swagger](https://sh.e154.ru/api/v1/swagger) (https://sh.e154.ru/api/v1/swagger) +server id:
+**cf4463fc-2f39-4271-a034-5c0c8087c56c** user: admin@e154.ru
pass: admin @@ -82,6 +83,9 @@ pass: admin user: user@e154.ru
pass: user +openapi v3 spec:
+[swagger](https://gate.e154.ru:8443/#/etc/swagger) (https://gate.e154.ru:8443/#/etc/swagger) + ### Supported system * macOS 386 10.6 @@ -101,69 +105,28 @@ Schematic smart home map smart-home map -### Quick installation - -#### Database postgresql - -System **Smart Home** works with **Postgresql database**. Create a database and database user with full rights to this -database. Connection parameters to the database must be specified in the configuration file. Updating the server version -may require updating the database. , migrations will start automatically, manual intervention is not required. - -```bash -sudo -u postgres psql -postgres=# create database mydb; -postgres=# create user myuser with encrypted password 'mypass'; -postgres=# grant all privileges on database mydb to myuser; -``` +### Database postgresql -### Installation for development +[Installing PostgreSQL 15](https://e154.github.io/smart-home/docs/install/postgresql/) -install brew +### Editing configuration files ```bash -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -``` - -Run these two commands in your terminal to add Homebrew to your PATH: - -```bash -(echo; echo 'eval "$(/opt/homebrew/bin/brew shellenv)"') >> /Users/delta54/.zprofile -eval "$(/opt/homebrew/bin/brew shellenv)" -``` - -install github.com/deepmap/oapi-codegen version v1.13.0 - -generate api stub -```bash -make server +cp conf/config.dev.json conf/config.json ``` -#### main server install +### Main server install -```bash -git clone https://github.com/e154/smart-home $GOPATH/src/github.com/e154/smart-home - -cd $GOPATH/src/github.com/e154/smart-home - -go mod vendor - -go build - -./smart-home -reset -./smart-home -``` - -editing configuration files +Download the latest server version from [releases](https://github.com/e154/smart-home/releases) +show help options ```bash -cp conf/config.dev.json conf/config.json -cp conf/dbconfig.dev.yml conf/dbconfig.yml +./server-linux-amd64 help ``` -run server - +run the server ```bash -./smart-home +./server-linux-amd64 ``` ### Docker @@ -174,18 +137,8 @@ cd smart-home docker-compose up ``` -connect to the database, create two smart-home databases, smart-home-gate - It's all -### Testing - -The system supports self-testing of internal components, and is started by the command - -```bash -go test -v ./tests/... -``` - ### Support Smart home Wiki: [e154.github.io/smart-home](https://e154.github.io/smart-home/) diff --git a/cmd/server/commands/gate.go b/cmd/server/commands/gate.go index 6490aa62b..735ecbdcf 100644 --- a/cmd/server/commands/gate.go +++ b/cmd/server/commands/gate.go @@ -34,7 +34,7 @@ import ( var ( gateCmd = &cobra.Command{ Use: "gate", - Short: "Proxy server", + Short: "Organization of remote access without white IP", Run: func(cmd *cobra.Command, args []string) { fmt.Printf(version.ShortVersionBanner, "") diff --git a/cmd/server/commands/reset.go b/cmd/server/commands/reset.go index a27cf5fd3..3d7804fa9 100644 --- a/cmd/server/commands/reset.go +++ b/cmd/server/commands/reset.go @@ -24,8 +24,8 @@ import ( . "github.com/e154/smart-home/cmd/server/container" . "github.com/e154/smart-home/common/app" - "github.com/e154/smart-home/system/initial" "github.com/e154/smart-home/system/logging" + "github.com/e154/smart-home/system/migrations" ) var ( @@ -35,10 +35,14 @@ var ( Run: func(cmd *cobra.Command, args []string) { app := BuildContainer(fx.Invoke(func( - logger *logging.Logging, - initialService *initial.Initial) { + _ *logging.Logging, + migrations *migrations.Migrations, + ) { + log.Info("full reset") - initialService.Reset() + _ = migrations.Purge() + + log.Info("complete") })) Start(app) }, diff --git a/cmd/server/commands/restore.go b/cmd/server/commands/restore.go index 4bdade11a..184f80996 100644 --- a/cmd/server/commands/restore.go +++ b/cmd/server/commands/restore.go @@ -45,7 +45,7 @@ var ( logger *logging.Logging, backup *backup.Backup) { - if err := backup.Restore(filename); err != nil { + if err := backup.RestoreFile(filename); err != nil { log.Error(err.Error()) } @@ -56,5 +56,5 @@ var ( ) func init() { - backupCmd.Flags().StringVarP(&filename, "filename", "f", "backup.zip", "backup file name") + restoreCmd.Flags().StringVarP(&filename, "filename", "f", "backup.zip", "backup file name") } diff --git a/cmd/server/container/init.go b/cmd/server/container/init.go index dd536796a..a50a768c8 100644 --- a/cmd/server/container/init.go +++ b/cmd/server/container/init.go @@ -52,5 +52,6 @@ func MigrationList(adaptors *adaptors.Adaptors, local_migrations.NewMigrationSpeedtest(adaptors), local_migrations.NewMigrationBackup(adaptors), local_migrations.NewMigrationGate(adaptors), + local_migrations.NewMigrationAddVar1(adaptors), } } diff --git a/cmd/server/reset_container/config.go b/cmd/server/reset_container/config.go new file mode 100644 index 000000000..3f43fe9df --- /dev/null +++ b/cmd/server/reset_container/config.go @@ -0,0 +1,32 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package container + +import ( + "path" + + "github.com/e154/smart-home/common/config" + "github.com/e154/smart-home/models" +) + +func ReadConfig() (conf *models.AppConfig) { + conf = &models.AppConfig{} + config.ReadConfig(path.Join("conf", "config.json"), "", conf) + return +} diff --git a/cmd/server/reset_container/container.go b/cmd/server/reset_container/container.go new file mode 100644 index 000000000..ac925d1ad --- /dev/null +++ b/cmd/server/reset_container/container.go @@ -0,0 +1,46 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package container + +import ( + "github.com/e154/smart-home/system/migrations" + "go.uber.org/fx" + + "github.com/e154/smart-home/system/bus" + "github.com/e154/smart-home/system/logging" +) + +// BuildContainer ... +func BuildContainer(opt fx.Option) (app *fx.App) { + + app = fx.New( + fx.Provide( + ReadConfig, + bus.NewBus, + NewLoggerConfig, + logging.NewLogger, + NewMigrationsConfig, + migrations.NewMigrations, + ), + fx.Logger(NewPrinter()), + opt, + ) + + return +} diff --git a/cmd/server/reset_container/fx.go b/cmd/server/reset_container/fx.go new file mode 100644 index 000000000..ccbefa6c9 --- /dev/null +++ b/cmd/server/reset_container/fx.go @@ -0,0 +1,33 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2016-2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package container + +// Printer ... +type Printer struct { +} + +// NewPrinter ... +func NewPrinter() *Printer { + return &Printer{} +} + +// Printf ... +func (p *Printer) Printf(msg string, fields ...interface{}) { + //fmt.Fprintln(os.Stderr, fmt.Sprintf(msg, fields...)) +} diff --git a/cmd/server/reset_container/logger.go b/cmd/server/reset_container/logger.go new file mode 100644 index 000000000..ff3810691 --- /dev/null +++ b/cmd/server/reset_container/logger.go @@ -0,0 +1,25 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package container + +import "github.com/e154/smart-home/system/logging" + +func NewLoggerConfig() *logging.Config { + return &logging.Config{} +} diff --git a/cmd/server/reset_container/migrations.go b/cmd/server/reset_container/migrations.go new file mode 100644 index 000000000..fe53a3573 --- /dev/null +++ b/cmd/server/reset_container/migrations.go @@ -0,0 +1,29 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2016-2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package container + +import "github.com/e154/smart-home/system/migrations" + +// NewMigrationsConfig ... +func NewMigrationsConfig() *migrations.Config { + return &migrations.Config{ + Source: "embed", + Dir: "migrations", + } +} diff --git a/conf/config.gate.json b/conf/config.gate.json index a5cd64066..0305c9211 100644 --- a/conf/config.gate.json +++ b/conf/config.gate.json @@ -5,7 +5,7 @@ "pprof": false, "domain": "localhost", "https": false, - "proxy_timeout": 1000, - "proxy_idle_timeout": 60000, + "proxy_timeout": 1, + "proxy_idle_timeout": 10, "proxy_secret_key": "" } diff --git a/static_source/admin/src/components/Infotip/src/Infotip.vue b/static_source/admin/src/components/Infotip/src/Infotip.vue index 01f59e62d..f50b9d895 100644 --- a/static_source/admin/src/components/Infotip/src/Infotip.vue +++ b/static_source/admin/src/components/Infotip/src/Infotip.vue @@ -11,6 +11,7 @@ const prefixCls = getPrefixCls('infotip') defineProps({ title: propTypes.string.def(''), + type: propTypes.string.def(''), schema: { type: Array as PropType>, required: true, @@ -28,7 +29,7 @@ const keyClick = (key: string) => { diff --git a/static_source/admin/src/locales/en.ts b/static_source/admin/src/locales/en.ts index fb473f87a..c6b13664e 100644 --- a/static_source/admin/src/locales/en.ts +++ b/static_source/admin/src/locales/en.ts @@ -619,6 +619,8 @@ export default { createBackupAt: 'Create Backup At (Cron)', maximumNumberOfBackups: 'Maximum Number Of Backups', info1: 'https://en.wikipedia.org/wiki/Cron', + info2: 'Organization of remote access without white IP. Free gate https://gate.e154.ru:8443', + info3: 'don\'t change it if you don\'t know what it\'s for', sendTheBackupInPartsMb: 'Send The Backup In Parts Mb', sendbackuptoTelegramBot: 'Send Backup To Telegram Bot', gate: 'Gate', diff --git a/static_source/admin/src/locales/ru.ts b/static_source/admin/src/locales/ru.ts index 88156fe89..f4fb72e9a 100644 --- a/static_source/admin/src/locales/ru.ts +++ b/static_source/admin/src/locales/ru.ts @@ -617,6 +617,8 @@ export default { createBackupAt: 'Когда создавать кипию (cron)', maximumNumberOfBackups: 'Максимальное количество резервных копий', info1: 'https://en.wikipedia.org/wiki/Cron', + info2: 'Организация удаленного доступа без белого IP. Тестовый шлюз https://gate.e154.ru:8443', + info3: 'не меняй, если не знаешь для чего это', sendTheBackupInPartsMb: 'Отправлять резервную копию по частям Mb', sendbackuptoTelegramBot: 'Отправить резервную копию боту Telegram', gate: 'Шлюз', diff --git a/static_source/admin/src/views/Settings/index.vue b/static_source/admin/src/views/Settings/index.vue index f94d6751e..d82a2c4fc 100644 --- a/static_source/admin/src/views/Settings/index.vue +++ b/static_source/admin/src/views/Settings/index.vue @@ -213,12 +213,12 @@ getSettings() - + - + @@ -226,12 +226,12 @@ getSettings() - + - + @@ -266,7 +266,7 @@ getSettings() - + @@ -279,13 +279,23 @@ getSettings() - + {{$t('settings.gate')}} + + @@ -300,13 +310,13 @@ getSettings() - + - + - + @@ -322,6 +332,17 @@ getSettings() {{$t('settings.hmacKey')}} + + diff --git a/system/backup/backup.go b/system/backup/backup.go index 386577055..f63f40bf9 100644 --- a/system/backup/backup.go +++ b/system/backup/backup.go @@ -92,7 +92,7 @@ func (b *Backup) Shutdown(ctx context.Context) (err error) { _ = b.eventBus.Unsubscribe("system/services/backup", b.eventHandler) if b.restoreImage != "" { - if err = b.restore(b.restoreImage); err != nil { + if err = b.RestoreFile(b.restoreImage); err != nil { log.Errorf("%+v", err) return } @@ -277,8 +277,8 @@ func (b *Backup) ApplyChanges() (err error) { return } -// restore ... -func (b *Backup) restore(name string) (err error) { +// RestoreFile ... +func (b *Backup) RestoreFile(name string) (err error) { log.Infof("restore backup file %s", name) var list []*m.Backup diff --git a/system/gate/client/gate_client.go b/system/gate/client/gate_client.go index 74f9652f7..06fc39e55 100644 --- a/system/gate/client/gate_client.go +++ b/system/gate/client/gate_client.go @@ -104,6 +104,7 @@ func (g *GateClient) initWspServer() { if !g.inProcess.CompareAndSwap(false, true) { return } + defer g.inProcess.Store(false) if g.proxy != nil { g.proxy.Shutdown() diff --git a/system/gate/client/wsp/client.go b/system/gate/client/wsp/client.go index ab88ccc04..67a896821 100644 --- a/system/gate/client/wsp/client.go +++ b/system/gate/client/wsp/client.go @@ -67,7 +67,7 @@ func NewClient(cfg *Config, api *api.Api, stream *stream.Stream, adaptors *adapt // Start the Proxy func (c *Client) Start(ctx context.Context) { - log.Info("Start") + //log.Info("Start") if !c.isStarted.CompareAndSwap(false, true) { return } @@ -80,7 +80,7 @@ func (c *Client) Start(ctx context.Context) { // Shutdown the Proxy func (c *Client) Shutdown() { - log.Info("Shutdown") + //log.Info("Shutdown") if !c.isStarted.CompareAndSwap(true, false) { return } diff --git a/system/gate/client/wsp/connection.go b/system/gate/client/wsp/connection.go index 81c9f9c8c..7640e0ec8 100644 --- a/system/gate/client/wsp/connection.go +++ b/system/gate/client/wsp/connection.go @@ -69,7 +69,7 @@ func NewConnection(pool *Pool, // Connect to the IsolatorServer using a HTTP websocket func (c *Connection) Connect(ctx context.Context) (err error) { - log.Infof("Connecting to %s", c.pool.target) + //log.Infof("Connecting to %s", c.pool.target) // Create a new TCP(/TLS) connection ( no use of net.http ) c.ws, _, err = c.pool.client.dialer.DialContext( @@ -83,6 +83,7 @@ func (c *Connection) Connect(ctx context.Context) (err error) { } log.Infof("Connected to %s", c.pool.target) + defer log.Info("Connection closed ...") // Send the greeting message with proxy id and wanted pool size. greeting := fmt.Sprintf( @@ -112,7 +113,7 @@ func (c *Connection) serve(ctx context.Context) { // Keep connection alive go func() { - timer := time.NewTicker(time.Second * 30) + timer := time.NewTicker(time.Second * 10) defer timer.Stop() for { select { @@ -274,6 +275,7 @@ func (c *Connection) error(msg string) (err error) { func (c *Connection) Close() { if c.ws != nil { + _ = c.ws.WriteMessage(websocket.CloseMessage, []byte{}) c.ws.Close() } } diff --git a/system/gate/client/wsp/pool.go b/system/gate/client/wsp/pool.go index 23cc07cbc..b1e251f07 100644 --- a/system/gate/client/wsp/pool.go +++ b/system/gate/client/wsp/pool.go @@ -73,7 +73,7 @@ func NewPool(client *Client, target string, // Start connect to the remote Server func (p *Pool) Start(ctx context.Context) { - log.Info("Start") + //log.Info("Start") p.connector(ctx) go func() { ticker := time.NewTicker(time.Second) @@ -92,7 +92,7 @@ func (p *Pool) Start(ctx context.Context) { // Shutdown close all connection in the pool func (p *Pool) Shutdown() { - log.Info("Shutdown") + //log.Info("Shutdown") close(p.done) p.connections.Range(func(key, value interface{}) bool { connection := value.(*Connection) diff --git a/system/gate/server/gate_server.go b/system/gate/server/gate_server.go index 70cb9eb07..e4a22c648 100644 --- a/system/gate/server/gate_server.go +++ b/system/gate/server/gate_server.go @@ -20,13 +20,15 @@ package server import ( "context" - "github.com/e154/smart-home/models" - "github.com/e154/smart-home/system/gate/server/wsp" + "time" + "go.uber.org/fx" "github.com/e154/smart-home/common/events" "github.com/e154/smart-home/common/logger" + "github.com/e154/smart-home/models" "github.com/e154/smart-home/system/bus" + "github.com/e154/smart-home/system/gate/server/wsp" ) var ( @@ -69,8 +71,8 @@ func NewGateServer(lc fx.Lifecycle, func (g *GateServer) Start(ctx context.Context) (err error) { config := &wsp.Config{ - Timeout: g.gateConfig.ProxyTimeout, - IdleTimeout: g.gateConfig.ProxyIdleTimeout, + Timeout: time.Duration(g.gateConfig.ProxyTimeout) * time.Second, + IdleTimeout: time.Duration(g.gateConfig.ProxyIdleTimeout) * time.Second, SecretKey: g.gateConfig.ProxySecretKey, } g.proxy = wsp.NewServer(config) diff --git a/system/gate/server/wsp/config.go b/system/gate/server/wsp/config.go index 4bf3220e8..6fe450194 100644 --- a/system/gate/server/wsp/config.go +++ b/system/gate/server/wsp/config.go @@ -22,14 +22,8 @@ import ( "time" ) -// Config configures an Server type Config struct { - Timeout int - IdleTimeout int + Timeout time.Duration + IdleTimeout time.Duration SecretKey string } - -// GetTimeout returns the time.Duration converted to millisecond -func (c Config) GetTimeout() time.Duration { - return time.Duration(c.Timeout) * time.Second -} diff --git a/system/gate/server/wsp/connection.go b/system/gate/server/wsp/connection.go index 866d44c12..41fb5cecc 100644 --- a/system/gate/server/wsp/connection.go +++ b/system/gate/server/wsp/connection.go @@ -55,6 +55,7 @@ type Connection struct { status ConnectionStatus idleSince time.Time lock sync.Mutex + queue chan Message } // NewConnection returns a new Connection. @@ -63,6 +64,7 @@ func NewConnection(pool *Pool, ws *websocket.Conn) *Connection { pool: pool, ws: ws, status: Idle, + queue: make(chan Message), } // Mark that this connection is ready to use for relay @@ -71,7 +73,27 @@ func NewConnection(pool *Pool, ws *websocket.Conn) *Connection { return c } +func (c *Connection) WritePump() { + var data []byte + var messageType int + var err error + for c.status != Closed { + messageType, data, err = c.ws.ReadMessage() + if messageType == -1 || err != nil { + c.status = Closed + close(c.queue) + return + } + msg := Message{ + Type: messageType, + Value: data, + } + c.queue <- msg + } +} + func (c *Connection) proxyWs(w http.ResponseWriter, r *http.Request) (err error) { + defer c.Release() // Only pass those headers to the upgrader. upgradeHeader := http.Header{} @@ -87,7 +109,9 @@ func (c *Connection) proxyWs(w http.ResponseWriter, r *http.Request) (err error) if accessToken == "" { accessToken = "NIL" } - _ = c.ws.WriteMessage(websocket.TextMessage, []byte("WS:" + accessToken)) + if err = c.ws.WriteMessage(websocket.TextMessage, []byte("WS:"+accessToken)); err != nil { + return + } upgrader.CheckOrigin = func(r *http.Request) bool { return true @@ -102,7 +126,7 @@ func (c *Connection) proxyWs(w http.ResponseWriter, r *http.Request) (err error) } defer connPub.Close() - errClient := make(chan error, 1) + //errClient := make(chan error, 1) errBackend := make(chan error, 1) replicateWebsocketConn := func(dst, src *websocket.Conn, errc chan error) { for { @@ -127,13 +151,26 @@ func (c *Connection) proxyWs(w http.ResponseWriter, r *http.Request) (err error) } } - go replicateWebsocketConn(connPub, c.ws, errClient) + go func() { + for c.status != Closed { + msg, ok := <-c.queue + if !ok { + return + } + err = connPub.WriteMessage(msg.Type, msg.Value) + if err != nil { + break + } + } + }() + + //go replicateWebsocketConn(connPub, c.ws, errClient) go replicateWebsocketConn(c.ws, connPub, errBackend) var message string select { - case err = <-errClient: - message = "websocketproxy: Error when copying from backend to client: %v" + //case err = <-errClient: + // message = "websocketproxy: Error when copying from backend to client: %v" case err = <-errBackend: message = "websocketproxy: Error when copying from client to backend: %v" } @@ -147,6 +184,7 @@ func (c *Connection) proxyWs(w http.ResponseWriter, r *http.Request) (err error) // Proxy a HTTP request through the Proxy over the websocket connection func (c *Connection) proxyRequest(w http.ResponseWriter, r *http.Request) (err error) { log.Infof("proxy request to %s", c.pool.id) + defer c.Release() // [1]: Serialize HTTP request jsonReq, err := json.Marshal(common.SerializeHTTPRequest(r)) @@ -179,39 +217,12 @@ func (c *Connection) proxyRequest(w http.ResponseWriter, r *http.Request) (err e return fmt.Errorf("unable to pipe request body (close) : %w", err) } - // [3]: Wait the HTTP response is ready - //responseChannel := make(chan (io.Reader)) - //c.nextResponse <- responseChannel - //responseReader, ok := <-responseChannel - //if responseReader == nil { - // if ok { - // // The value of ok is false, the channel is closed and empty. - // // See the Receiver operator in https://go.dev/ref/spec for more information. - // close(responseChannel) - // } - // return fmt.Errorf("unable to get http response reader : %w", err) - //} - // - //// [4]: Read the HTTP response from the peer - //// Get the serialized HTTP Response from the peer - //jsonResponse, err := io.ReadAll(responseReader) - //if err != nil { - // close(responseChannel) - // return fmt.Errorf("unable to read http response : %w", err) - //} - // - //// Notify the read() goroutine that we are done reading the response - //close(responseChannel) - - messageType, jsonResponse, err := c.ws.ReadMessage() - if messageType == -1 { + msg, ok := <-c.queue + if !ok { return } - if err != nil { - log.Error(err.Error()) - return - } + jsonResponse := msg.Value // Deserialize the HTTP Response httpResponse := new(common.HTTPResponse) @@ -227,40 +238,19 @@ func (c *Connection) proxyRequest(w http.ResponseWriter, r *http.Request) (err e } w.WriteHeader(httpResponse.StatusCode) - // [5]: Wait the HTTP response body is ready - // Get the HTTP Response body from the the peer - // To do so send a new channel to the read() goroutine - // to get the next message reader - //responseBodyChannel := make(chan (io.Reader)) - //c.nextResponse <- responseBodyChannel - //responseBodyReader, ok := <-responseBodyChannel - //if responseBodyReader == nil { - // if ok { - // // If more is false the channel is already closed - // close(responseChannel) - // } - // return fmt.Errorf("unable to get http response body reader : %w", err) - //} - - messageType, responseBody, err := c.ws.ReadMessage() - if messageType == -1 { + msg, ok = <-c.queue + if !ok { return } + responseBody := msg.Value + responseBodyReader := bytes.NewReader(responseBody) - // [6]: Read the HTTP response body from the peer - // Pipe the HTTP response body right from the remote Proxy to the client if _, err := io.Copy(w, responseBodyReader); err != nil { - //close(responseBodyChannel) return fmt.Errorf("unable to pipe response body : %w", err) } - // Notify read() that we are done reading the response body - //close(responseBodyChannel) - - c.Release() - return } @@ -269,11 +259,7 @@ func (c *Connection) Take() bool { c.lock.Lock() defer c.lock.Unlock() - if c.status == Closed { - return false - } - - if c.status == Busy { + if c.status == Closed || c.status == Busy { return false } @@ -309,15 +295,10 @@ func (c *Connection) close() { if c.status == Closed { return } + c.status = Closed log.Infof("Closing connection from %s", c.pool.id) - // This one will be executed *before* lock.Unlock() - defer func() { c.status = Closed }() - - // Unlock a possible read() wild message - //close(c.nextResponse) - // Close the underlying TCP connection c.ws.Close() } diff --git a/system/gate/server/wsp/pool.go b/system/gate/server/wsp/pool.go index 6666d4e05..88cd8d3f5 100644 --- a/system/gate/server/wsp/pool.go +++ b/system/gate/server/wsp/pool.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/gorilla/websocket" ) @@ -31,12 +32,11 @@ type Pool struct { id PoolID size int + idle chan *Connection - connections []*Connection - idle chan *Connection - - done bool - lock sync.RWMutex + done bool + lock sync.RWMutex + connections map[string]*Connection } // PoolID represents the identifier of the connected WebSocket client. @@ -44,10 +44,12 @@ type PoolID string // NewPool creates a new Pool func NewPool(server *Server, id PoolID) *Pool { - p := new(Pool) - p.server = server - p.id = id - p.idle = make(chan *Connection) + p := &Pool{ + server: server, + id: id, + idle: make(chan *Connection), + connections: make(map[string]*Connection), + } return p } @@ -63,7 +65,14 @@ func (p *Pool) Register(ws *websocket.Conn) { log.Infof("Registering new connection from %s", p.id) connection := NewConnection(p, ws) - p.connections = append(p.connections, connection) + id := uuid.NewString() + + go func() { + p.connections[id] = connection + connection.WritePump() + delete(p.connections, id) + }() + } // Offer offers an idle connection to the server. @@ -79,28 +88,26 @@ func (p *Pool) Offer(connection *Connection) { // This MUST be surrounded by pool.lock.Lock() func (p *Pool) Clean() { idle := 0 - var connections []*Connection - - for _, connection := range p.connections { + now := time.Now() + for id, connection := range p.connections { // We need to be sur we'll never close a BUSY or soon to be BUSY connection connection.lock.Lock() if connection.status == Idle { idle++ - if idle > p.size { + if idle > p.size+1 { // We have enough idle connections in the pool. // Terminate the connection if it is idle since more that IdleTimeout - if int(time.Now().Sub(connection.idleSince).Seconds())*1000 > p.server.Config.IdleTimeout { + if now.Sub(connection.idleSince).Seconds() > p.server.Config.IdleTimeout.Seconds() { connection.close() + delete(p.connections, id) } } } connection.lock.Unlock() if connection.status == Closed { - continue + connection.Close() } - connections = append(connections, connection) } - p.connections = connections } // IsEmpty clean the pool and return true if the pool is empty @@ -119,17 +126,10 @@ func (p *Pool) Shutdown() { p.done = true - for _, connection := range p.connections { + for id, connection := range p.connections { connection.Close() + delete(p.connections, id) } - p.Clean() -} - -// PoolSize is the number of connection in each state in the pool -type PoolSize struct { - Idle int - Busy int - Closed int } // Size return the number of connection in each state in the pool @@ -137,7 +137,7 @@ func (p *Pool) Size() (ps *PoolSize) { p.lock.Lock() defer p.lock.Unlock() - ps = new(PoolSize) + ps = &PoolSize{} for _, connection := range p.connections { if connection.status == Idle { ps.Idle++ diff --git a/system/gate/server/wsp/server.go b/system/gate/server/wsp/server.go index f49a9bfe1..5805bfd95 100644 --- a/system/gate/server/wsp/server.go +++ b/system/gate/server/wsp/server.go @@ -20,7 +20,6 @@ package wsp import ( "context" - "github.com/pkg/errors" "net/http" "net/url" "reflect" @@ -30,6 +29,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/pkg/errors" "github.com/e154/smart-home/common/logger" "github.com/e154/smart-home/system/gate/common" @@ -126,6 +126,7 @@ func (s *Server) clean() { idle := 0 busy := 0 + closed := 0 for _, pool := range s.pools { if pool.IsEmpty() { @@ -137,9 +138,10 @@ func (s *Server) clean() { ps := pool.Size() idle += ps.Idle busy += ps.Busy + closed = ps.Closed } - //log.Infof("%d pools, %d idle, %d busy", len(s.pools), idle, busy) + log.Infof("%d pools, %d idle, %d busy, %d closed", len(s.pools), idle, busy, closed) } // Dispatch connection from available pools to clients requests @@ -156,7 +158,7 @@ func (s *Server) dispatchConnections() { // A timeout is set for each dispatch request. ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, s.Config.GetTimeout()) + ctx, cancel := context.WithTimeout(ctx, s.Config.Timeout) defer cancel() L: @@ -247,7 +249,7 @@ func (s *Server) Ws(w http.ResponseWriter, r *http.Request) { } // [2]: Take an WebSocket connection available from pools for relaying received requests. - request := NewConnectionRequest(s.Config.GetTimeout(), PoolID(serverId)) + request := NewConnectionRequest(s.Config.Timeout, PoolID(serverId)) // "Dispatcher" is running in a separate thread from the server by `go s.dispatchConnections()`. // It waits to receive requests to dispatch connection from available pools to clients requests. // https://github.com/hgsgtk/wsp/blob/ea4902a8e11f820268e52a6245092728efeffd7f/server/server.go#L93 @@ -282,19 +284,6 @@ func (s *Server) Ws(w http.ResponseWriter, r *http.Request) { } func (s *Server) Request(w http.ResponseWriter, r *http.Request) { - // [1]: Receive requests to be proxied - // Parse destination URL - //dstURL := r.Header.Get("X-PROXY-DESTINATION") - //if dstURL == "" { - // common.ProxyErrorf(w, "Missing X-PROXY-DESTINATION header") - // return - //} - //URL, err := url.Parse(dstURL) - //if err != nil { - // common.ProxyErrorf(w, "Unable to parse X-PROXY-DESTINATION header") - // return - //} - //r.URL = URL if len(s.pools) == 0 { common.ProxyErrorf(w, "No proxy available") @@ -317,7 +306,7 @@ func (s *Server) Request(w http.ResponseWriter, r *http.Request) { log.Infof("[%s] %s", r.Method, r.URL.String()) // [2]: Take an WebSocket connection available from pools for relaying received requests. - request := NewConnectionRequest(s.Config.GetTimeout(), PoolID(serverId)) + request := NewConnectionRequest(s.Config.Timeout, PoolID(serverId)) // "Dispatcher" is running in a separate thread from the server by `go s.dispatchConnections()`. // It waits to receive requests to dispatch connection from available pools to clients requests. // https://github.com/hgsgtk/wsp/blob/ea4902a8e11f820268e52a6245092728efeffd7f/server/server.go#L93 @@ -388,18 +377,15 @@ func (s *Server) Register(w http.ResponseWriter, r *http.Request) { s.lock.Lock() defer s.lock.Unlock() - var pool *Pool - var ok bool - if pool, ok = s.pools[id]; !ok { - pool = NewPool(s, id) - s.pools[id] = pool + if _, ok := s.pools[id]; !ok { + s.pools[id] = NewPool(s, id) } // update pool size - pool.size = size + s.pools[id].size = size // Add the WebSocket connection to the pool - pool.Register(ws) + s.pools[id].Register(ws) } // Shutdown stop the Server diff --git a/system/gate/server/wsp/types.go b/system/gate/server/wsp/types.go new file mode 100644 index 000000000..61d5bbf41 --- /dev/null +++ b/system/gate/server/wsp/types.go @@ -0,0 +1,31 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package wsp + +type Message struct { + Type int + Value []byte +} + +// PoolSize is the number of connection in each state in the pool +type PoolSize struct { + Idle int + Busy int + Closed int +} diff --git a/system/initial/initial.go b/system/initial/initial.go index 38dbf9513..43bef3497 100644 --- a/system/initial/initial.go +++ b/system/initial/initial.go @@ -43,7 +43,6 @@ import ( localMigrations "github.com/e154/smart-home/system/initial/local_migrations" "github.com/e154/smart-home/system/logging_ws" "github.com/e154/smart-home/system/media" - "github.com/e154/smart-home/system/migrations" "github.com/e154/smart-home/system/scheduler" "github.com/e154/smart-home/system/scripts" "github.com/e154/smart-home/system/supervisor" @@ -56,7 +55,6 @@ var ( // Initial ... type Initial struct { - migrations *migrations.Migrations adaptors *Adaptors scriptService scripts.ScriptService accessList access_list.AccessListService @@ -73,7 +71,6 @@ type Initial struct { // NewInitial ... func NewInitial(lc fx.Lifecycle, - migrations *migrations.Migrations, adaptors *Adaptors, scriptService scripts.ScriptService, accessList access_list.AccessListService, @@ -90,7 +87,6 @@ func NewInitial(lc fx.Lifecycle, db *gorm.DB, eventBus bus.Bus) *Initial { initial := &Initial{ - migrations: migrations, adaptors: adaptors, scriptService: scriptService, accessList: accessList, @@ -115,16 +111,6 @@ func NewInitial(lc fx.Lifecycle, return initial } -// Reset ... -func (n *Initial) Reset() { - - log.Info("full reset") - - _ = n.migrations.Purge() - - log.Info("complete") -} - // InstallDemoData ... func (n *Initial) InstallDemoData() { @@ -153,8 +139,9 @@ func (n *Initial) checkForUpgrade() { if errors.Is(err, apperr.ErrNotFound) { v = m.Variable{ - Name: name, - Value: fmt.Sprintf("%d", 1), + Name: name, + Value: fmt.Sprintf("%d", 1), + System: true, } err = n.adaptors.Variable.Add(context.Background(), v) So(err, ShouldBeNil) diff --git a/system/initial/local_migrations/m_add_var1.go b/system/initial/local_migrations/m_add_var1.go new file mode 100644 index 000000000..9da984aa8 --- /dev/null +++ b/system/initial/local_migrations/m_add_var1.go @@ -0,0 +1,45 @@ +// This file is part of the Smart Home +// Program complex distribution https://github.com/e154/smart-home +// Copyright (C) 2023, Filippov Alex +// +// This library is free software: you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library. If not, see +// . + +package local_migrations + +import ( + "context" + + "github.com/e154/smart-home/adaptors" +) + +type MigrationAddVar1 struct { + adaptors *adaptors.Adaptors +} + +func NewMigrationAddVar1(adaptors *adaptors.Adaptors) *MigrationAddVar1 { + return &MigrationAddVar1{ + adaptors: adaptors, + } +} + +func (n *MigrationAddVar1) Up(ctx context.Context, adaptors *adaptors.Adaptors) error { + if adaptors != nil { + n.adaptors = adaptors + } + + AddVariableIfNotExist(n.adaptors, ctx, "restartComponentIfScriptChanged", "false") + AddVariableIfNotExist(n.adaptors, ctx, "sendTheBackupInPartsMb", "0") + return nil +} diff --git a/system/initial/local_migrations/m_dashboard.go b/system/initial/local_migrations/m_dashboard.go index 9d2425c81..cfb7ccd60 100644 --- a/system/initial/local_migrations/m_dashboard.go +++ b/system/initial/local_migrations/m_dashboard.go @@ -20,8 +20,11 @@ package local_migrations import ( "context" + "fmt" "github.com/e154/smart-home/adaptors" + m "github.com/e154/smart-home/models" + . "github.com/e154/smart-home/system/initial/assertions" ) type MigrationDashboard struct { @@ -34,25 +37,29 @@ func NewMigrationDashboard(adaptors *adaptors.Adaptors) *MigrationDashboard { } } -func (n *MigrationDashboard) addDashboard(ctx context.Context, name, src string) error { - - //req := &api.Dashboard{} - //_ = json.Unmarshal([]byte(src), req) - // - //board := dto.ImportDashboard(req) - // - //var err error - //if board.Id, err = n.adaptors.Dashboard.Import(ctx, board); err != nil { - // return err - //} - // - //err = n.adaptors.Variable.CreateOrUpdate(ctx, m.Variable{ - // Name: name, - // Value: fmt.Sprintf("%d", board.Id), - // System: true, - //}) +func (n *MigrationDashboard) addDashboard(ctx context.Context, name, _n, _d string) error { - return nil + if _, err := n.adaptors.Variable.GetByName(ctx, name); err == nil { + return nil + } + + board := &m.Dashboard{ + Name: _n, + Description: _d, + } + + var err error + if board.Id, err = n.adaptors.Dashboard.Add(ctx, board); err != nil { + return err + } + + err = n.adaptors.Variable.Update(ctx, m.Variable{ + Name: name, + Value: fmt.Sprintf("%d", board.Id), + System: true, + }) + + return err } func (n *MigrationDashboard) Up(ctx context.Context, adaptors *adaptors.Adaptors) error { @@ -60,11 +67,17 @@ func (n *MigrationDashboard) Up(ctx context.Context, adaptors *adaptors.Adaptors n.adaptors = adaptors } - //err := n.addDashboard(ctx, "devDashboard", devDashboardRaw) - //So(err, ShouldBeNil) - // - //err = n.addDashboard(ctx, "mainDashboard", mainDashboardRaw) - //So(err, ShouldBeNil) + err := n.addDashboard(ctx, "devDashboardLight", "develop (light theme)", "DEVELOP") + So(err, ShouldBeNil) + + err = n.addDashboard(ctx, "devDashboardDark", "develop (dark theme)", "DEVELOP") + So(err, ShouldBeNil) + + err = n.addDashboard(ctx, "mainDashboardLight", "main (light theme)", "MAIN") + So(err, ShouldBeNil) + + err = n.addDashboard(ctx, "mainDashboardDark", "main (dark theme)", "MAIN") + So(err, ShouldBeNil) return nil } diff --git a/system/initial/local_migrations/m_scheduler.go b/system/initial/local_migrations/m_scheduler.go index dd079a05c..22fa42d55 100644 --- a/system/initial/local_migrations/m_scheduler.go +++ b/system/initial/local_migrations/m_scheduler.go @@ -40,7 +40,7 @@ func (n *MigrationScheduler) Up(ctx context.Context, adaptors *adaptors.Adaptors n.adaptors = adaptors } - err := AddVariableIfNotExist(n.adaptors, ctx, "gate_client_id", "60") + err := AddVariableIfNotExist(n.adaptors, ctx, "clearMetricsDays", "60") So(err, ShouldBeNil) err = AddVariableIfNotExist(n.adaptors, ctx, "clearLogsDays", "60") So(err, ShouldBeNil) diff --git a/system/initial/local_migrations/migrations.go b/system/initial/local_migrations/migrations.go index 524934cce..52906c5e9 100644 --- a/system/initial/local_migrations/migrations.go +++ b/system/initial/local_migrations/migrations.go @@ -87,7 +87,11 @@ func (t *Migrations) Up(ctx context.Context, adaptors *adaptors.Adaptors, ver st return } - for _, migration := range t.list[position+1 : len(t.list)] { + if position > 0 { + position++ + } + + for _, migration := range t.list[position:len(t.list)] { if err = ctx.Err(); err != nil { log.Error(err.Error()) return