enhance: Add repair legacy-collection-remnant command (#270)

See also milvus-io/milvus#33608

Command removes collection with no field to fix the bug in previously
mentioned issue

Signed-off-by: Congqi Xia <[email protected]>

states/etcd/commands.go

@@ -108,21 +108,35 @@ func RawCommands(cli clientv3.KV) []*cobra.Command {
 		Use:   "get",
 		Short: "equivalent to etcd get(withPrefix) command to fetch raw kv values from backup file",
 		Run: func(cmd *cobra.Command, args []string) {
+			withValue, err := cmd.Flags().GetBool("withValue")
+			if err != nil {
+				fmt.Println(err.Error())
+				return
+			}
 			for _, arg := range args {
 				fmt.Println("list with", arg)
-				resp, err := cli.Get(context.Background(), arg, clientv3.WithPrefix())
+				var resp *clientv3.GetResponse
+				var err error
+				if withValue {
+					resp, err = cli.Get(context.Background(), arg, clientv3.WithPrefix())
+				} else {
+					resp, err = cli.Get(context.Background(), arg, clientv3.WithPrefix(), clientv3.WithKeysOnly())
+				}
 				if err != nil {
 					fmt.Println(err.Error())
 					continue
 				}
 				for _, kv := range resp.Kvs {
 					fmt.Printf("key: %s\n", string(kv.Key))
-					fmt.Printf("Value: %s\n", string(kv.Value))
+					if withValue {
+						fmt.Printf("Value: %s\n", string(kv.Value))
+					}
 				}
 			}
 		},
 	}
 
+	cmd.Flags().Bool("withValue", false, "print values")
 	return []*cobra.Command{cmd}
 }
 

states/etcd/repair/collection_legacy.go

@@ -0,0 +1,51 @@
+package repair
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/milvus-io/birdwatcher/framework"
+	"github.com/milvus-io/birdwatcher/models"
+	"github.com/milvus-io/birdwatcher/states/etcd/common"
+	etcdversion "github.com/milvus-io/birdwatcher/states/etcd/version"
+)
+
+type CollectionLegacyDroppedParams struct {
+	framework.ParamBase `use:"repair legacy-collection-remnant"`
+	CollectionID        int64 `name:""`
+	Run                 bool  `name:"run" default:"false" desc:"whether to remove legacy collection meta, default set to \"false\" to dry run"`
+}
+
+func (c *ComponentRepair) CollectionLegacyDroppedCommand(ctx context.Context, p *CollectionLegacyDroppedParams) error {
+	collections, err := common.ListCollectionsVersion(ctx, c.client, c.basePath, etcdversion.GetVersion(), func(coll *models.Collection) bool {
+		fmt.Println(coll.Key())
+		return coll.DBID == 0 && len(coll.Schema.Fields) == 0 && (p.CollectionID == 0 || p.CollectionID == coll.ID)
+	})
+
+	if err != nil {
+		return err
+	}
+
+	var removed int
+	for _, collection := range collections {
+		fmt.Printf("collection [%d]%s is suspect of legacy collection remnant\n", collection.ID, collection.Schema.Name)
+		if p.Run {
+			key := collection.Key()
+			fmt.Printf("start to remove remnant meta for %s, key:%s\n", collection.Schema.Name, key)
+			_, err := c.client.Delete(ctx, collection.Key())
+			if err != nil {
+				fmt.Printf("failed to remove %s, error: %s\n", key, err.Error())
+				continue
+			}
+			fmt.Println("Removal done!")
+			removed++
+		}
+	}
+	if len(collections) == 0 {
+		fmt.Println("no suspect found")
+	} else if removed > 0 {
+		fmt.Println("Remnant meta removed, please restart rootcoord/mixtcord to check")
+	}
+
+	return nil
+}

states/etcd/repair/component.go

@@ -0,0 +1,20 @@
+package repair
+
+import (
+	"github.com/milvus-io/birdwatcher/configs"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type ComponentRepair struct {
+	client   clientv3.KV
+	config   *configs.Config
+	basePath string
+}
+
+func NewComponent(cli clientv3.KV, config *configs.Config, basePath string) *ComponentRepair {
+	return &ComponentRepair{
+		client:   cli,
+		config:   config,
+		basePath: basePath,
+	}
+}

states/instance.go

@@ -10,6 +10,7 @@ import (
 	"github.com/milvus-io/birdwatcher/states/etcd"
 	"github.com/milvus-io/birdwatcher/states/etcd/audit"
 	"github.com/milvus-io/birdwatcher/states/etcd/remove"
+	"github.com/milvus-io/birdwatcher/states/etcd/repair"
 	"github.com/milvus-io/birdwatcher/states/etcd/show"
 	"github.com/spf13/cobra"
 	clientv3 "go.etcd.io/etcd/client/v3"
@@ -20,6 +21,7 @@ type InstanceState struct {
 	cmdState
 	*show.ComponentShow
 	*remove.ComponentRemove
+	*repair.ComponentRepair
 	instanceName string
 	client       clientv3.KV
 	auditFile    *os.File
@@ -149,6 +151,7 @@ func getInstanceState(cli clientv3.KV, instanceName, metaPath string, etcdState
 		},
 		ComponentShow:   show.NewComponent(cli, config, basePath),
 		ComponentRemove: remove.NewComponent(cli, config, basePath),
+		ComponentRepair: repair.NewComponent(cli, config, basePath),
 		instanceName:    instanceName,
 		client:          kv,
 		auditFile:       file,