-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add remove segment-orphan command (#163)
Signed-off-by: Congqi Xia <[email protected]>
- Loading branch information
Showing
35 changed files
with
788 additions
and
800 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package framework | ||
|
|
||
| // CmdParam is the interface definition for command parameter. | ||
| type CmdParam interface { | ||
| ParseArgs(args []string) error | ||
| Desc() (string, string) | ||
| } | ||
|
|
||
| // ParamBase implmenet CmdParam when empty args parser. | ||
| type ParamBase struct{} | ||
|
|
||
| func (pb ParamBase) ParseArgs(args []string) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (pb ParamBase) Desc() (string, string) { | ||
| return "", "" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package remove | ||
|
|
||
| import ( | ||
| "github.com/milvus-io/birdwatcher/configs" | ||
| clientv3 "go.etcd.io/etcd/client/v3" | ||
| ) | ||
|
|
||
| type ComponentRemove struct { | ||
| client clientv3.KV | ||
| config *configs.Config | ||
| basePath string | ||
| } | ||
|
|
||
| func NewComponent(cli clientv3.KV, config *configs.Config, basePath string) *ComponentRemove { | ||
| return &ComponentRemove{ | ||
| client: cli, | ||
| config: config, | ||
| basePath: basePath, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package remove | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/milvus-io/birdwatcher/framework" | ||
| "github.com/milvus-io/birdwatcher/models" | ||
| "github.com/milvus-io/birdwatcher/states/etcd/common" | ||
| etcdversion "github.com/milvus-io/birdwatcher/states/etcd/version" | ||
| "github.com/samber/lo" | ||
| ) | ||
|
|
||
| type SegmentOrphan struct { | ||
| framework.ParamBase `use:"remove segment-orphan" desc:"remove orphan segments that collection meta already gone"` | ||
| CollectionID int64 `name:"collection" default:"0" desc:"collection id to filter with"` | ||
| Run bool `name:"run" default:"false" desc:"flag to control actually run or dry"` | ||
| } | ||
|
|
||
| // SegmentOrphanCommand returns command to remove | ||
| func (c *ComponentRemove) SegmentOrphanCommand(ctx context.Context, p *SegmentOrphan) error { | ||
| segments, err := common.ListSegmentsVersion(ctx, c.client, c.basePath, etcdversion.GetVersion(), func(segment *models.Segment) bool { | ||
| return (p.CollectionID == 0 || segment.CollectionID == p.CollectionID) | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| groups := lo.GroupBy(segments, func(segment *models.Segment) int64 { | ||
| return segment.CollectionID | ||
| }) | ||
|
|
||
| for collectionID, segments := range groups { | ||
| _, err := common.GetCollectionByIDVersion(ctx, c.client, c.basePath, etcdversion.GetVersion(), collectionID) | ||
| if errors.Is(err, common.ErrCollectionNotFound) { | ||
| // print segments | ||
| fmt.Printf("Collection %d missing, orphan segments: %v\n", collectionID, lo.Map(segments, func(segment *models.Segment, idx int) int64 { | ||
| return segment.ID | ||
| })) | ||
|
|
||
| if p.Run { | ||
| for _, segment := range segments { | ||
| err := common.RemoveSegmentByID(ctx, c.client, c.basePath, segment.CollectionID, segment.PartitionID, segment.ID) | ||
| if err != nil { | ||
| fmt.Printf("failed to remove segment %d, err: %s\n", segment.ID, err.Error()) | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.