Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(dbm-services): 优化模拟执行解析表的返回 #9053 #9054

Open
wants to merge 1 commit into
base: v1.5.0
Choose a base branch
from
+55 −21
Open

refactor(dbm-services): 优化模拟执行解析表的返回 #9053 #9054

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions dbm-services/common/db-resource/internal/model/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,6 @@ func init() {
logger.Info("tb_rp_operation_info columns %v", TbRpOperationInfoColumns)
}

// func migration() {
// err := DB.Self.AutoMigrate(&TbRpDailySnapShot{})
// if err != nil {
// logger.Error("auto migrate failed %v", err)
// }
// }

func createSysDb() {
user := config.AppConfig.Db.UserName
pwd := config.AppConfig.Db.PassWord
Expand Down
2 changes: 2 additions & 0 deletions dbm-services/mysql/db-simulation/app/service/kubernets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,8 @@ func (k *DbPodSets) getLoadSchemaSQLCmd(bkpath, file string) (cmd string) {
// 从中控dump的schema文件,默认是添加了tc_admin=0,需要删除
// 因为模拟执行是需要将中控进行sql转发
commands = append(commands, fmt.Sprintf("sed -i '/50720 SET tc_admin=0/d' %s", file))
// del definer
commands = append(commands, fmt.Sprintf("sed -i 's/\\sDEFINER=`[^`]*`@`[^`]*`//g' %s", file))
commands = append(commands, fmt.Sprintf("mysql -uroot -p%s --default-character-set=%s -vvv < %s", k.BaseInfo.RootPwd,
k.BaseInfo.Charset, file))
return strings.Join(commands, " && ")
Expand Down
22 changes: 18 additions & 4 deletions dbm-services/mysql/db-simulation/app/syntax/parse_relation_db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,18 +90,19 @@ func (t *TmysqlParse) doParseInchan(alreadExecutedSqlfileCh chan string,
wg.Add(1)
c <- struct{}{}
go func(fileName string) {
defer wg.Done()
cdbs, dbs, commands, dumpAllDbs, err := t.analyzeRelationDbs(fileName, mysqlVersion)
logger.Info("createDbs:%v,dbs:%v,dumpAllDbs:%v,err:%v", cdbs, dbs, dumpAllDbs, err)
if err != nil {
logger.Error("analyzeRelationDbs failed %s", err.Error())
errChan <- err
wg.Done()
return
}
// 如果有dumpall 则直接返回退出,不在继续分析
if dumpAllDbs {
dumpAll = true
<-c
wg.Done()
stopChan <- struct{}{}
}
t.mu.Lock()
Expand All @@ -110,6 +111,7 @@ func (t *TmysqlParse) doParseInchan(alreadExecutedSqlfileCh chan string,
allCommands = append(allCommands, commands...)
t.mu.Unlock()
<-c
wg.Done()
}(sqlfile)
}

Expand Down Expand Up @@ -170,12 +172,24 @@ func (t *TmysqlParse) analyzeRelationDbs(inputfileName, mysqlVersion string) (
return nil, nil, nil, false, fmt.Errorf("%s", res.ErrorMsg)
}
if lo.IsNotEmpty(res.Command) {
allCommandType = append(allCommandType, res.Command)
if res.Command == SQLTypeCreateTable {
var c CreateTableResult
if err = json.Unmarshal(line, &c); err != nil {
logger.Error("json unmasrshal line:%s failed %s", string(line), err.Error())
return nil, nil, nil, false, err
}
// 需要排除create table like
if !c.IsCreateTableLike && !c.IsCreateTableSelect {
allCommandType = append(allCommandType, res.Command)
}
} else {
allCommandType = append(allCommandType, res.Command)
}
}
if slices.Contains([]string{SQLTypeCreateProcedure, SQLTypeCreateFunction, SQLTypeCreateView, SQLTypeCreateTrigger,
SQLTypeInsertSelect, SQLTypeRelaceSelect},
res.Command) {
return nil, nil, nil, true, nil
return nil, nil, allCommandType, true, nil
}
if lo.IsEmpty(res.DbName) {
continue
Expand Down Expand Up @@ -207,7 +221,7 @@ func (tf *TmysqlParseFile) ParseSpecialTbls(mysqlVersion string) (relationTbls [
for k, v := range m {
relationTbls = append(relationTbls, RelationTbl{
DbName: k,
Tbls: v,
Tbls: lo.Uniq(v),
})
}
return relationTbls, nil
Expand Down
6 changes: 4 additions & 2 deletions dbm-services/mysql/db-simulation/app/syntax/tmysqlpase_schema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ const (
// SQLTypeCreateView is create view sql
SQLTypeCreateView = "create_view"

// SQLTypeInsert is insert sql
SQLTypeInsert = "insert"
// SQLTypeReplace is replace sql
SQLTypeReplace = "replace"
// SQLTypeAlterTable is alter table sql
SQLTypeAlterTable = "alter_table"
// SQLTypeDelete is delete sql
Expand Down Expand Up @@ -276,14 +280,12 @@ type CreateTrigger struct {
type CreateFunction struct {
ParseBase
Definer UserHost `json:"definer,omitempty"`
// TODO
}

// CreateEvent tmysqlparse create event result
type CreateEvent struct {
ParseBase
Definer UserHost `json:"definer,omitempty"`
// TODO
}

// CreateIndex tmysqlparse create index result
Expand Down
39 changes: 31 additions & 8 deletions dbm-services/mysql/db-simulation/handler/syntax_check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,14 +257,29 @@ func (s SyntaxHandler) ParseSQLFileRelationDb(r *gin.Context) {
s.SendResponse(r, err, nil)
return
}
defer p.DelTempDir()
// 如果所有的命令都是alter table, dump指定库表
logger.Debug("debug: %v,%d", allCommands, len(allCommands))
if isAllOperateTable(allCommands) || isAllCreateTable(allCommands) {
if isAllOperateTable(allCommands) {
relationTbls, err := p.ParseSpecialTbls("")
if err != nil {
s.SendResponse(r, err, nil)
return
}
tblCount := 0
for _, tbl := range relationTbls {
tblCount += len(lo.Uniq(tbl.Tbls))
}
// sql语句的变更表数量大于5000,防止mysqldump 拼接参数过长导致执行失败
if tblCount > 5000 || len(relationTbls) > 100 {
s.SendResponse(r, nil, gin.H{
"create_dbs": createDbs,
"dbs": dbs,
"dump_all": dumpall,
"timestamp": time.Now().Unix(),
})
return
}
s.SendResponse(r, nil, gin.H{
"create_dbs": createDbs,
"dbs": dbs,
Expand All @@ -285,13 +300,21 @@ func (s SyntaxHandler) ParseSQLFileRelationDb(r *gin.Context) {
}

func isAllOperateTable(allCommands []string) bool {
return lo.Every([]string{syntax.SQLTypeAlterTable, syntax.SQLTypeUseDb,
syntax.SQLTypeCreateIndex, syntax.SQLTypeDropTable}, allCommands)
if len(allCommands) == 0 {
return false
}
return lo.Every([]string{
syntax.SQLTypeAlterTable, syntax.SQLTypeUseDb,
syntax.SQLTypeCreateIndex, syntax.SQLTypeDropTable,
syntax.SQLTypeInsert, syntax.SQLTypeUpdate,
syntax.SQLTypeDelete, syntax.SQLTypeCreateTable,
syntax.SQLTypeReplace,
}, allCommands)
}

func isAllCreateTable(allCommands []string) bool {
return lo.Every([]string{syntax.SQLTypeCreateTable, syntax.SQLTypeUseDb}, allCommands)
}
// func isAllCreateTable(allCommands []string) bool {
// return lo.Every([]string{, syntax.SQLTypeUseDb}, allCommands)
// }

// ParseSQLRelationDb 语法检查入参SQL string
func (s *SyntaxHandler) ParseSQLRelationDb(r *gin.Context) {
Expand Down Expand Up @@ -333,8 +356,8 @@ func (s *SyntaxHandler) ParseSQLRelationDb(r *gin.Context) {
return
}
// 如果所有的命令都是alter table, dump指定库表
logger.Info("make debug: %v,%d", allCommands, len(allCommands))
if isAllOperateTable(allCommands) || isAllCreateTable(allCommands) {
logger.Info("all command types: %v,%d", allCommands, len(allCommands))
if isAllOperateTable(allCommands) {
relationTbls, err := p.ParseSpecialTbls("")
if err != nil {
s.SendResponse(r, err, nil)
Expand Down