space cleanup: force option (part one)

* lmeta-not-found * flip "too early to remove anything" * clone misplaced Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Nov 20, 2024 · 91fe18e · 91fe18e
1 parent 7c6f318
commit 91fe18e
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 14 deletions.
diff --git a/api/xaction.go b/api/xaction.go
@@ -19,8 +19,7 @@ import (
 	"github.com/NVIDIA/aistore/xact"
 )
 
-// Start xaction
-func StartXaction(bp BaseParams, args *xact.ArgsMsg, extra string) (xid string, err error) {
+func StartXaction(bp BaseParams, args *xact.ArgsMsg, extra string /* e.g. blob-downloader objname */) (xid string, err error) {
 	if !xact.Table[args.Kind].Startable {
 		return "", fmt.Errorf("xaction %q is not startable", args.Kind)
 	}
@@ -43,7 +42,7 @@ func StartXaction(bp BaseParams, args *xact.ArgsMsg, extra string) (xid string,
 	return
 }
 
-// Abort ("stop") xactions
+// a.k.a. stop
 func AbortXaction(bp BaseParams, args *xact.ArgsMsg) (err error) {
 	msg := apc.ActMsg{Action: apc.ActXactStop, Value: args}
 	bp.Method = http.MethodPut

diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go
@@ -495,7 +495,12 @@ var (
 			indent1 + "\t see also: 'ais bucket props show' and 'ais bucket props set')",
 	}
 
-	forceFlag = cli.BoolFlag{Name: "force,f", Usage: "force an action"}
+	forceFlag    = cli.BoolFlag{Name: "force,f", Usage: "force an action"}
+	forceClnFlag = cli.BoolFlag{
+		Name: forceFlag.Name,
+		Usage: "disregard interrupted rebalance and possibly other conditions preventing full cleanup\n" +
+			indent1 + "\t(tip: check 'ais config cluster lru.dont_evict_time' as well)",
+	}
 
 	// units enum { unitsIEC, unitsSI, unitsRaw }
 	unitsFlag = cli.StringFlag{

diff --git a/cmd/cli/cli/storage_hdlr.go b/cmd/cli/cli/storage_hdlr.go
@@ -111,6 +111,7 @@ var (
 
 var (
 	cleanupFlags = []cli.Flag{
+		forceClnFlag,
 		waitFlag,
 		waitJobXactFinishedFlag,
 	}
@@ -210,7 +211,7 @@ func showStorageHandler(c *cli.Context) (err error) {
 }
 
 //
-// cleanup
+// cleanup space: remove deleted, misplaced
 //
 
 func cleanupStorageHandler(c *cli.Context) (err error) {
@@ -227,10 +228,13 @@ func cleanupStorageHandler(c *cli.Context) (err error) {
 			return
 		}
 	}
-	xargs := xact.ArgsMsg{Kind: apc.ActStoreCleanup, Bck: bck}
+
+	force := flagIsSet(c, forceFlag)
+	xargs := xact.ArgsMsg{Kind: apc.ActStoreCleanup, Bck: bck, Force: force}
 	if id, err = api.StartXaction(apiBP, &xargs, ""); err != nil {
 		return
 	}
+
 	xargs.ID = id
 	if !flagIsSet(c, waitFlag) && !flagIsSet(c, waitJobXactFinishedFlag) {
 		if id != "" {

diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.23.2
 
 require (
-	github.com/NVIDIA/aistore v1.3.26-0.20241119185518-0eb3279624bd
+	github.com/NVIDIA/aistore v1.3.26-0.20241119190643-7c6f3181bf9f
 	github.com/fatih/color v1.18.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.21.0

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/NVIDIA/aistore v1.3.26-0.20241119185518-0eb3279624bd h1:lM4UacWc6nltBIZBIRF7pFNvB66Ts68IExzu0DahrWE=
-github.com/NVIDIA/aistore v1.3.26-0.20241119185518-0eb3279624bd/go.mod h1:978dp5ldMS/UILDQ/z29dcvU01ZyyaQXUGtT/OD6d4w=
+github.com/NVIDIA/aistore v1.3.26-0.20241119190643-7c6f3181bf9f h1:+Cl8s4DwoXEX9XKqXZBvDt+Fhbv4zGcu8TFm9j9vujc=
+github.com/NVIDIA/aistore v1.3.26-0.20241119190643-7c6f3181bf9f/go.mod h1:I3l9tS47EY4ODo7mQQGsFFrIjHa6mwnNRF4t1LXKCvA=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

diff --git a/fs/fs.go b/fs/fs.go
@@ -281,7 +281,8 @@ func (mi *Mountpath) MakePathCT(bck *cmn.Bck, contentType string) string {
 }
 
 func (mi *Mountpath) MakePathFQN(bck *cmn.Bck, contentType, objName string) string {
-	debug.Assert(contentType != "" && objName != "")
+	debug.Assert(contentType != "")
+	debug.Assert(objName != "")
 	buf := mi.makePathBuf(bck, contentType, 1+len(objName))
 	buf = append(buf, filepath.Separator)
 	buf = append(buf, objName...)

diff --git a/space/cleanup.go b/space/cleanup.go
@@ -191,6 +191,7 @@ func (p *clnP) rmMisplaced() bool {
 		l = xreg.GetResilverMarked()
 	)
 	if g.Xact == nil && l.Xact == nil && !g.Interrupted && !g.Restarted && !l.Interrupted {
+		// TODO -- FIXME: force
 		return true
 	}
 
@@ -437,17 +438,17 @@ func (j *clnJ) visitObj(fqn string, lom *core.LOM) {
 			return
 		}
 		// too early to remove anything
-		if atimefs+int64(j.config.LRU.DontEvictTime) < j.now {
+		if atimefs+int64(j.config.LRU.DontEvictTime) > j.now {
 			return
 		}
-		if cmn.IsErrLmetaCorrupted(err) {
+		if cmn.IsErrLmetaCorrupted(errLoad) {
 			if err := lom.RemoveMain(); err != nil {
 				nlog.Errorf("%s: failed to rm MD-corrupted %s: %v (nested: %v)", j, lom, errLoad, err)
 				j.ini.Xaction.AddErr(err)
 			} else {
 				nlog.Errorf("%s: removed MD-corrupted %s: %v", j, lom, errLoad)
 			}
-		} else if cmn.IsErrLmetaNotFound(err) {
+		} else if cmn.IsErrLmetaNotFound(errLoad) {
 			if err := lom.RemoveMain(); err != nil {
 				nlog.Errorf("%s: failed to rm no-MD %s: %v (nested: %v)", j, lom, errLoad, err)
 				j.ini.Xaction.AddErr(err)
@@ -479,6 +480,8 @@ func (j *clnJ) visitObj(fqn string, lom *core.LOM) {
 			j.misplaced.ec = append(j.misplaced.ec, core.NewCTFromLOM(lom, fs.ObjectType))
 		}
 	} else {
+		// TODO -- FIXME: instead, do not free in the visitObj() caller
+		lom = lom.CloneMD(lom.FQN)
 		j.misplaced.loms = append(j.misplaced.loms, lom)
 	}
 }
@@ -602,7 +605,7 @@ func (j *clnJ) rmLeftovers() (size int64, err error) {
 				fqn     = mlom.FQN
 				removed bool
 			)
-			lom := core.AllocLOM(mlom.ObjName) // yes placed
+			lom := core.AllocLOM(mlom.ObjName)
 			switch {
 			case lom.InitBck(&j.bck) != nil:
 				removed = os.Remove(fqn) == nil