refactor(dbm-services): 优化模拟执行解析表的返回 #9053

dbm-services/common/db-resource/internal/model/model.go

@@ -63,13 +63,6 @@ func init() {
 	logger.Info("tb_rp_operation_info columns %v", TbRpOperationInfoColumns)
 }
 
-// func migration() {
-// 	err := DB.Self.AutoMigrate(&TbRpDailySnapShot{})
-// 	if err != nil {
-// 		logger.Error("auto migrate failed %v", err)
-// 	}
-// }
-
 func createSysDb() {
 	user := config.AppConfig.Db.UserName
 	pwd := config.AppConfig.Db.PassWord

dbm-services/mysql/db-simulation/app/service/kubernets.go

@@ -495,6 +495,8 @@ func (k *DbPodSets) getLoadSchemaSQLCmd(bkpath, file string) (cmd string) {
 	// 从中控dump的schema文件,默认是添加了tc_admin=0,需要删除
 	// 因为模拟执行是需要将中控进行sql转发
 	commands = append(commands, fmt.Sprintf("sed -i '/50720 SET tc_admin=0/d' %s", file))
+	// del definer
+	commands = append(commands, fmt.Sprintf("sed -i 's/\\sDEFINER=`[^`]*`@`[^`]*`//g'  %s", file))
 	commands = append(commands, fmt.Sprintf("mysql -uroot -p%s --default-character-set=%s -vvv < %s", k.BaseInfo.RootPwd,
 		k.BaseInfo.Charset, file))
 	return strings.Join(commands, " && ")

dbm-services/mysql/db-simulation/app/syntax/parse_relation_db.go

@@ -90,18 +90,19 @@ func (t *TmysqlParse) doParseInchan(alreadExecutedSqlfileCh chan string,
 		wg.Add(1)
 		c <- struct{}{}
 		go func(fileName string) {
-			defer wg.Done()
 			cdbs, dbs, commands, dumpAllDbs, err := t.analyzeRelationDbs(fileName, mysqlVersion)
 			logger.Info("createDbs:%v,dbs:%v,dumpAllDbs:%v,err:%v", cdbs, dbs, dumpAllDbs, err)
 			if err != nil {
 				logger.Error("analyzeRelationDbs failed %s", err.Error())
 				errChan <- err
+				wg.Done()
 				return
 			}
 			// 如果有dumpall 则直接返回退出,不在继续分析
 			if dumpAllDbs {
 				dumpAll = true
 				<-c
+				wg.Done()
 				stopChan <- struct{}{}
 			}
 			t.mu.Lock()
@@ -110,6 +111,7 @@ func (t *TmysqlParse) doParseInchan(alreadExecutedSqlfileCh chan string,
 			allCommands = append(allCommands, commands...)
 			t.mu.Unlock()
 			<-c
+			wg.Done()
 		}(sqlfile)
 	}
 
@@ -170,12 +172,24 @@ func (t *TmysqlParse) analyzeRelationDbs(inputfileName, mysqlVersion string) (
 			return nil, nil, nil, false, fmt.Errorf("%s", res.ErrorMsg)
 		}
 		if lo.IsNotEmpty(res.Command) {
-			allCommandType = append(allCommandType, res.Command)
+			if res.Command == SQLTypeCreateTable {
+				var c CreateTableResult
+				if err = json.Unmarshal(line, &c); err != nil {
+					logger.Error("json unmasrshal line:%s failed %s", string(line), err.Error())
+					return nil, nil, nil, false, err
+				}
+				// 需要排除create table like
+				if !c.IsCreateTableLike && !c.IsCreateTableSelect {
+					allCommandType = append(allCommandType, res.Command)
+				}
+			} else {
+				allCommandType = append(allCommandType, res.Command)
+			}
 		}
 		if slices.Contains([]string{SQLTypeCreateProcedure, SQLTypeCreateFunction, SQLTypeCreateView, SQLTypeCreateTrigger,
 			SQLTypeInsertSelect, SQLTypeRelaceSelect},
 			res.Command) {
-			return nil, nil, nil, true, nil
+			return nil, nil, allCommandType, true, nil
 		}
 		if lo.IsEmpty(res.DbName) {
 			continue
@@ -207,7 +221,7 @@ func (tf *TmysqlParseFile) ParseSpecialTbls(mysqlVersion string) (relationTbls [
 	for k, v := range m {
 		relationTbls = append(relationTbls, RelationTbl{
 			DbName: k,
-			Tbls:   v,
+			Tbls:   lo.Uniq(v),
 		})
 	}
 	return relationTbls, nil

dbm-services/mysql/db-simulation/app/syntax/tmysqlpase_schema.go

@@ -39,6 +39,10 @@ const (
 	// SQLTypeCreateView is create view sql
 	SQLTypeCreateView = "create_view"
 
+	// SQLTypeInsert is insert sql
+	SQLTypeInsert = "insert"
+	// SQLTypeReplace is replace sql
+	SQLTypeReplace = "replace"
 	// SQLTypeAlterTable is alter table sql
 	SQLTypeAlterTable = "alter_table"
 	// SQLTypeDelete is delete sql
@@ -276,14 +280,12 @@ type CreateTrigger struct {
 type CreateFunction struct {
 	ParseBase
 	Definer UserHost `json:"definer,omitempty"`
-	// TODO
 }
 
 // CreateEvent  tmysqlparse create event result
 type CreateEvent struct {
 	ParseBase
 	Definer UserHost `json:"definer,omitempty"`
-	// TODO
 }
 
 // CreateIndex  tmysqlparse create index result

dbm-services/mysql/db-simulation/handler/syntax_check.go

@@ -257,14 +257,29 @@ func (s SyntaxHandler) ParseSQLFileRelationDb(r *gin.Context) {
 		s.SendResponse(r, err, nil)
 		return
 	}
+	defer p.DelTempDir()
 	// 如果所有的命令都是alter table, dump指定库表
 	logger.Debug("debug: %v,%d", allCommands, len(allCommands))
-	if isAllOperateTable(allCommands) || isAllCreateTable(allCommands) {
+	if isAllOperateTable(allCommands) {
 		relationTbls, err := p.ParseSpecialTbls("")
 		if err != nil {
 			s.SendResponse(r, err, nil)
 			return
 		}
+		tblCount := 0
+		for _, tbl := range relationTbls {
+			tblCount += len(lo.Uniq(tbl.Tbls))
+		}
+		// sql语句的变更表数量大于5000,防止mysqldump 拼接参数过长导致执行失败
+		if tblCount > 5000 || len(relationTbls) > 100 {
+			s.SendResponse(r, nil, gin.H{
+				"create_dbs": createDbs,
+				"dbs":        dbs,
+				"dump_all":   dumpall,
+				"timestamp":  time.Now().Unix(),
+			})
+			return
+		}
 		s.SendResponse(r, nil, gin.H{
 			"create_dbs":             createDbs,
 			"dbs":                    dbs,
@@ -285,12 +300,20 @@ func (s SyntaxHandler) ParseSQLFileRelationDb(r *gin.Context) {
 }
 
 func isAllOperateTable(allCommands []string) bool {
-	return lo.Every([]string{syntax.SQLTypeAlterTable, syntax.SQLTypeUseDb,
-		syntax.SQLTypeCreateIndex, syntax.SQLTypeDropTable}, allCommands)
-}
-
-func isAllCreateTable(allCommands []string) bool {
-	return lo.Every([]string{syntax.SQLTypeCreateTable, syntax.SQLTypeUseDb}, allCommands)
+	if len(allCommands) == 0 {
+		return false
+	}
+	// 不允许只用use db
+	if len(allCommands) == 1 && allCommands[0] == syntax.SQLTypeUseDb {
+		return false
+	}
+	return lo.Every([]string{
+		syntax.SQLTypeAlterTable, syntax.SQLTypeUseDb,
+		syntax.SQLTypeCreateIndex, syntax.SQLTypeDropTable,
+		syntax.SQLTypeInsert, syntax.SQLTypeUpdate,
+		syntax.SQLTypeDelete, syntax.SQLTypeCreateTable,
+		syntax.SQLTypeReplace,
+	}, allCommands)
 }
 
 // ParseSQLRelationDb  语法检查入参SQL string
@@ -333,8 +356,8 @@ func (s *SyntaxHandler) ParseSQLRelationDb(r *gin.Context) {
 		return
 	}
 	// 如果所有的命令都是alter table, dump指定库表
-	logger.Info("make debug: %v,%d", allCommands, len(allCommands))
-	if isAllOperateTable(allCommands) || isAllCreateTable(allCommands) {
+	logger.Info("all command types: %v,%d", allCommands, len(allCommands))
+	if isAllOperateTable(allCommands) {
 		relationTbls, err := p.ParseSpecialTbls("")
 		if err != nil {
 			s.SendResponse(r, err, nil)

helm-charts/bk-dbm/charts/db-simulation/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 0.0.1-alpha.91
+appVersion: 0.0.1-alpha.92
 description: A Helm chart for Kubernetes
 name: db-simulation
 type: application