diff --git a/cmd/cli/from_2.13_to_2.14.go b/cmd/cli/from_2.13_to_2.14.go new file mode 100644 index 000000000..d4c739d90 --- /dev/null +++ b/cmd/cli/from_2.13_to_2.14.go @@ -0,0 +1,27 @@ +package main + +import "github.com/moira-alert/moira" + +func updateFrom213(logger moira.Logger, database moira.Database) error { + logger.Info().Msg("Update 2.13 -> 2.14 started") + + err := fillTeamNamesHash(logger, database) + if err != nil { + return err + } + + logger.Info().Msg("Update 2.13 -> 2.14 was finished") + return nil +} + +func downgradeTo213(logger moira.Logger, database moira.Database) error { + logger.Info().Msg("Downgrade 2.14 -> 2.13 started") + + err := removeTeamNamesHash(logger, database) + if err != nil { + return err + } + + logger.Info().Msg("Downgrade 2.14 -> 2.13 was finished") + return nil +} diff --git a/cmd/cli/main.go b/cmd/cli/main.go index e1b6bf899..2e1f91ec5 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -25,7 +25,15 @@ var ( GoVersion = "unknown" ) -var moiraValidVersions = []string{"2.3", "2.6", "2.7", "2.9", "2.11", "2.12"} +var moiraValidVersions = []string{ + "2.3", + "2.6", + "2.7", + "2.9", + "2.11", + "2.12", + "2.13", +} var ( configFileName = flag.String("config", "/etc/moira/cli.yml", "Path to configuration file") @@ -125,6 +133,13 @@ func main() { //nolint Error(err). Msg("Fail to update from version 2.12") } + case "2.13": + err := updateFrom213(logger, database) + if err != nil { + logger.Fatal(). + Error(err). + Msg("Fail to update from version 2.13") + } } } @@ -173,6 +188,13 @@ func main() { //nolint Error(err). Msg("Fail to update to version 2.12") } + case "2.13": + err := downgradeTo213(logger, database) + if err != nil { + logger.Fatal(). + Error(err). + Msg("Fail to update to version 2.13") + } } } diff --git a/cmd/cli/teams_names.go b/cmd/cli/teams_names.go new file mode 100644 index 000000000..06cfbcc28 --- /dev/null +++ b/cmd/cli/teams_names.go @@ -0,0 +1,264 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + goredis "github.com/go-redis/redis/v8" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/database/redis" +) + +const ( + teamsKey = "moira-teams" + teamsByNamesKey = "moira-teams-by-names" + delayBeforeRenamingTeams = time.Second * 10 +) + +var errTeamsCountAndUniqueNamesCountMismatch = errors.New( + "count of teams does not match count of unique names after transformation") + +type renameInfo struct { + oldTeamName string + newTeamName string + teamID string +} + +func (info *renameInfo) String() string { + return fmt.Sprintf("%s: '%s' -> '%s'", info.teamID, info.oldTeamName, info.newTeamName) +} + +// fillTeamNamesHash does the following +// 1. Get all teams from DB. +// 2. Group teams with same names. +// 3. For teams with same names change name (example: ["name", "name", "name"] -> ["name", "name1", "name2"]). +// 4. Update teams with name changed. +// 5. Save pairs teamName:team.ID to "moira-teams-by-names" redis hash. +func fillTeamNamesHash(logger moira.Logger, database moira.Database) error { + logger.Info().Msg("Start filling \"moira-teams-by-names\" hash") + + switch db := database.(type) { + case *redis.DbConnector: + logger.Info().Msg("collecting teams from redis node...") + + teamsMap, err := db.Client().HGetAll(db.Context(), teamsKey).Result() + if err != nil { + return fmt.Errorf("failed to fetch teams from redis node: %w", err) + } + + logger.Info(). + Int("total_teams_count", len(teamsMap)). + Msg("fetched teams") + + teamsByNameMap, err := groupTeamsByNames(logger, teamsMap) + if err != nil { + return fmt.Errorf("failed to group teams by names: %w", err) + } + + teamByUniqueName, renamingInfo := transformTeamsByNameMap(teamsByNameMap) + + if len(teamByUniqueName) != len(teamsMap) { + return errTeamsCountAndUniqueNamesCountMismatch + } + + builder := strings.Builder{} + for i, rename := range renamingInfo { + builder.WriteString(rename.String()) + if i != len(renamingInfo)-1 { + builder.WriteString(", ") + } + } + + logger.Info(). + String("rename_info", builder.String()). + Msg("Would rename teams") + + logger.Info(). + Int("teams_renamed_count", len(renamingInfo)). + String("delay", delayBeforeRenamingTeams.String()). + Msg("Teams will be renamed after delay") + + logger.Info().Msg("Press Ctrl+C to stop") + time.Sleep(delayBeforeRenamingTeams) + + logger.Info().Msg("Start renaming teams and filling \"moira-teams-by-names\" hash") + + client := db.Client() + ctx := db.Context() + + _, pipeErr := client.TxPipelined( + ctx, + func(pipe goredis.Pipeliner) error { + return updateTeamsInPipe(ctx, logger, pipe, teamByUniqueName) + }) + if pipeErr != nil { + return pipeErr + } + + logger.Info().Msg("\"moira-teams-by-names\" hash successfully filled") + + default: + return makeUnknownDBError(database) + } + return nil +} + +func groupTeamsByNames(logger moira.Logger, teamsMap map[string]string) (map[string][]teamWithID, error) { + teamsByNameMap := make(map[string][]teamWithID, len(teamsMap)) + + for teamID, marshaledTeam := range teamsMap { + team, err := unmarshalTeam(teamID, []byte(marshaledTeam)) + if err != nil { + return nil, err + } + + lowercaseTeamName := strings.ToLower(team.Name) + + teamWithNameList, exists := teamsByNameMap[lowercaseTeamName] + if exists { + teamWithNameList = append(teamWithNameList, team) + teamsByNameMap[lowercaseTeamName] = teamWithNameList + } else { + teamsByNameMap[lowercaseTeamName] = []teamWithID{team} + } + } + + logger.Info(). + Int("unique_team_names_count", len(teamsByNameMap)). + Msg("grouped teams with same names") + + return teamsByNameMap, nil +} + +func transformTeamsByNameMap(teamsByNameMap map[string][]teamWithID) (map[string]teamWithID, []renameInfo) { + teamByUniqueName := make(map[string]teamWithID, len(teamsByNameMap)) + renamingTeams := make([]renameInfo, 0) + + for _, teams := range teamsByNameMap { + for i, team := range teams { + iStr := strconv.FormatInt(int64(i), 10) + + oldTeamName := team.Name + if i > 0 { + team.Name += iStr + } + + for { + // sometimes we have the following situation in db (IDs and team names): + // moira-teams: { + // team1: "team name", + // team2: "team Name", + // team3: "Team name1" + // } + // so we can't just add 1 to one of [team1, team2] + lowercasedTeamName := strings.ToLower(team.Name) + + _, exists := teamByUniqueName[lowercasedTeamName] + if exists { + team.Name += "_" + iStr + } else { + teamByUniqueName[lowercasedTeamName] = team + if team.Name != oldTeamName { + renamingTeams = append(renamingTeams, renameInfo{ + teamID: team.ID, + oldTeamName: oldTeamName, + newTeamName: team.Name, + }) + } + break + } + } + } + } + + return teamByUniqueName, renamingTeams +} + +func updateTeamsInPipe(ctx context.Context, logger moira.Logger, pipe goredis.Pipeliner, teamsByUniqueName map[string]teamWithID) error { + for _, team := range teamsByUniqueName { + teamBytes, err := getTeamBytes(team) + if err != nil { + return err + } + + err = pipe.HSet(ctx, teamsKey, team.ID, teamBytes).Err() + if err != nil { + logger.Error(). + Error(err). + String("team_id", team.ID). + String("new_team_name", team.Name). + Msg("failed to update team name") + + return fmt.Errorf("failed to update team name: %w", err) + } + + err = pipe.HSet(ctx, teamsByNamesKey, strings.ToLower(team.Name), team.ID).Err() + if err != nil { + logger.Error(). + Error(err). + String("team_id", team.ID). + String("new_team_name", team.Name). + Msg("failed to add team name to redis hash") + + return fmt.Errorf("failed to add team name to redis hash: %w", err) + } + } + + return nil +} + +// removeTeamNamesHash remove "moira-teams-by-names" redis hash. +// Note that if fillTeamNamesHash have been called, then team names would not be changed back. +func removeTeamNamesHash(logger moira.Logger, database moira.Database) error { + logger.Info().Msg("Start removing \"moira-teams-by-names\" redis hash") + + switch db := database.(type) { + case *redis.DbConnector: + _, err := db.Client().Del(db.Context(), teamsByNamesKey).Result() + if err != nil { + return fmt.Errorf("failed to delete teamsByNameKey: %w", err) + } + + default: + return makeUnknownDBError(database) + } + + return nil +} + +type teamStorageElement struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type teamWithID struct { + teamStorageElement + ID string +} + +func unmarshalTeam(teamID string, teamBytes []byte) (teamWithID, error) { + var storedTeam teamStorageElement + err := json.Unmarshal(teamBytes, &storedTeam) + if err != nil { + return teamWithID{}, fmt.Errorf("failed to deserialize team: %w", err) + } + + return teamWithID{ + teamStorageElement: storedTeam, + ID: teamID, + }, nil +} + +func getTeamBytes(team teamWithID) ([]byte, error) { + bytes, err := json.Marshal(team.teamStorageElement) + if err != nil { + return nil, fmt.Errorf("failed to marshal team: %w", err) + } + + return bytes, nil +} diff --git a/cmd/cli/teams_names_test.go b/cmd/cli/teams_names_test.go new file mode 100644 index 000000000..e440ba498 --- /dev/null +++ b/cmd/cli/teams_names_test.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/moira-alert/moira/database/redis" + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + + . "github.com/smartystreets/goconvey/convey" +) + +var testTeams = []teamWithID{ + { + teamStorageElement: teamStorageElement{ + Name: "First team", + Description: "first desc", + }, + ID: "team1", + }, + { + teamStorageElement: teamStorageElement{ + Name: "Second team", + Description: "second desc", + }, + ID: "team2", + }, + { + teamStorageElement: teamStorageElement{ + Name: "Third team", + Description: "third desc", + }, + ID: "team3", + }, + { + teamStorageElement: teamStorageElement{ + Name: "Fourth team", + Description: "fourth desc", + }, + ID: "team4", + }, +} + +func Test_fillTeamNamesHash(t *testing.T) { + Convey("Test filling \"moira-teams-by-names\" redis hash", t, func() { + conf := getDefault() + logger, err := logging.ConfigureLog(conf.LogFile, conf.LogLevel, "test", conf.LogPrettyFormat) + if err != nil { + t.Fatal(err) + } + + db := redis.NewTestDatabase(logger) + db.Flush() + defer db.Flush() + + ctx := context.Background() + client := db.Client() + + Convey("with empty database", func() { + err = fillTeamNamesHash(logger, db) + So(err, ShouldBeNil) + + res, existErr := client.Exists(ctx, teamsByNamesKey).Result() + So(existErr, ShouldBeNil) + So(res, ShouldEqual, 0) + }) + + Convey("with teams which have unique names", func() { + defer db.Flush() + + var teamNames, actualTeamNames map[string]string + + teamNames = make(map[string]string, len(testTeams)) + + for _, team := range testTeams { + var teamBytes []byte + + teamBytes, err = getTeamBytes(team) + So(err, ShouldBeNil) + + err = client.HSet(ctx, teamsKey, team.ID, teamBytes).Err() + So(err, ShouldBeNil) + + teamNames[strings.ToLower(team.Name)] = team.ID + } + + err = fillTeamNamesHash(logger, db) + So(err, ShouldBeNil) + + actualTeamNames, err = client.HGetAll(ctx, teamsByNamesKey).Result() + So(err, ShouldBeNil) + So(actualTeamNames, ShouldResemble, teamNames) + }) + + Convey("with teams no unique names", func() { + defer db.Flush() + + testTeams[0].Name = "Team name" + testTeams[1].Name = "teaM name" + testTeams[2].Name = "Team name" + + for _, team := range testTeams { + var teamBytes []byte + + teamBytes, err = getTeamBytes(team) + So(err, ShouldBeNil) + + err = client.HSet(ctx, teamsKey, team.ID, teamBytes).Err() + So(err, ShouldBeNil) + } + + err = fillTeamNamesHash(logger, db) + So(err, ShouldBeNil) + + var actualTeamNames map[string]string + + actualTeamNames, err = client.HGetAll(ctx, teamsByNamesKey).Result() + So(err, ShouldBeNil) + So(actualTeamNames, ShouldHaveLength, len(testTeams)) + + expectedLowercasedTeamNames := []string{"team name", "team name1", "team name2", strings.ToLower(testTeams[3].Name)} + for _, name := range expectedLowercasedTeamNames { + _, ok := actualTeamNames[name] + So(ok, ShouldBeTrue) + } + + for i, team := range testTeams { + Convey(fmt.Sprintf("for team %v fields ok", i), func() { + var marshaledTeam string + + marshaledTeam, err = client.HGet(ctx, teamsKey, team.ID).Result() + So(err, ShouldBeNil) + + var actualTeam teamWithID + + actualTeam, err = unmarshalTeam(team.ID, []byte(marshaledTeam)) + So(err, ShouldBeNil) + So(actualTeam.ID, ShouldEqual, team.ID) + So(actualTeam.Description, ShouldEqual, team.Description) + + if i < 3 { + So(actualTeam.Name, ShouldBeIn, []string{team.Name, team.Name + "1", team.Name + "2"}) + } else { + So(actualTeam.Name, ShouldEqual, team.Name) + } + }) + } + }) + + Convey("with teams has no unique names and adding one number does not help", func() { + testTeams[0].Name = "Team name" + testTeams[1].Name = "teaM name" + testTeams[2].Name = "Team name1" + + for _, team := range testTeams { + var teamBytes []byte + + teamBytes, err = getTeamBytes(team) + So(err, ShouldBeNil) + + err = client.HSet(ctx, teamsKey, team.ID, teamBytes).Err() + So(err, ShouldBeNil) + } + + err = fillTeamNamesHash(logger, db) + So(err, ShouldBeNil) + + var actualTeamNames map[string]string + + actualTeamNames, err = client.HGetAll(ctx, teamsByNamesKey).Result() + So(err, ShouldBeNil) + So(actualTeamNames, ShouldHaveLength, len(testTeams)) + + // depends on order of map iteration + expectedLowercasedTeamNames := []string{"team name", "team name1", "team name1_0", "team name1_1", strings.ToLower(testTeams[3].Name)} + for name := range actualTeamNames { + So(name, ShouldBeIn, expectedLowercasedTeamNames) + } + + for i, team := range testTeams { + Convey(fmt.Sprintf("for team %v fields ok", i), func() { + var marshaledTeam string + + marshaledTeam, err = client.HGet(ctx, teamsKey, team.ID).Result() + So(err, ShouldBeNil) + + var actualTeam teamWithID + + actualTeam, err = unmarshalTeam(team.ID, []byte(marshaledTeam)) + So(err, ShouldBeNil) + So(actualTeam.ID, ShouldEqual, team.ID) + So(actualTeam.Description, ShouldEqual, team.Description) + + if i < 3 { + So(actualTeam.Name, ShouldBeIn, []string{team.Name, team.Name + "1", team.Name + "1_1", team.Name + "_0"}) + } else { + So(actualTeam.Name, ShouldEqual, team.Name) + } + }) + } + }) + }) +} + +func Test_removeTeamNamesHash(t *testing.T) { + Convey("Test removing \"moira-teams-by-names\" hash", t, func() { + conf := getDefault() + logger, err := logging.ConfigureLog(conf.LogFile, conf.LogLevel, "test", conf.LogPrettyFormat) + if err != nil { + t.Fatal(err) + } + + db := redis.NewTestDatabase(logger) + db.Flush() + defer db.Flush() + + ctx := context.Background() + client := db.Client() + + Convey("with empty database", func() { + err = removeTeamNamesHash(logger, db) + So(err, ShouldBeNil) + + res, existErr := client.Exists(ctx, teamsByNamesKey).Result() + So(existErr, ShouldBeNil) + So(res, ShouldEqual, 0) + }) + + Convey("with filled teams and teams by names hashes", func() { + defer db.Flush() + + for _, team := range testTeams { + var teamBytes []byte + + teamBytes, err = getTeamBytes(team) + So(err, ShouldBeNil) + + err = client.HSet(ctx, teamsKey, team.ID, teamBytes).Err() + So(err, ShouldBeNil) + + err = client.HSet(ctx, teamsByNamesKey, strings.ToLower(team.Name), team.ID).Err() + So(err, ShouldBeNil) + } + + err = removeTeamNamesHash(logger, db) + So(err, ShouldBeNil) + + res, existErr := client.Exists(ctx, teamsByNamesKey).Result() + So(existErr, ShouldBeNil) + So(res, ShouldEqual, 0) + }) + }) +}