list-objects entries; CLI "cached"

* refactor-out redundant LsoEnt accessors * CLI: "cached" column in a list-objects output vs explicitly specified props * check for prop duplications; refactor Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Jan 17, 2025 · 278c735 · 278c735
1 parent 5867ddf
commit 278c735
Show file tree

Hide file tree

Showing 23 changed files with 71 additions and 48 deletions.
diff --git a/ais/plstcx.go b/ais/plstcx.go
@@ -127,11 +127,11 @@ func (c *lstcx) do() (string, error) {
 	c.tcomsg.ToBck = c.bckTo.Clone()
 	lr, cnt := &c.tcomsg.ListRange, len(lst.Entries)
 	lr.ObjNames = make([]string, 0, cnt)
-	for _, e := range lst.Entries {
-		if e.IsDir() { // NOTE: always skip virtual dir (apc.EntryIsDir)
+	for _, en := range lst.Entries {
+		if en.IsAnyFlagSet(apc.EntryIsDir) { // always skip virtual dirs
 			continue
 		}
-		lr.ObjNames = append(lr.ObjNames, e.Name)
+		lr.ObjNames = append(lr.ObjNames, en.Name)
 	}
 
 	// 5. multi-obj action: transform/copy 1st page
@@ -191,11 +191,11 @@ func (c *lstcx) _page() (int, error) {
 	lr := &c.tcomsg.ListRange
 	clear(lr.ObjNames)
 	lr.ObjNames = lr.ObjNames[:0]
-	for _, e := range lst.Entries {
-		if e.IsDir() { // NOTE: always skip virtual dir (apc.EntryIsDir)
+	for _, en := range lst.Entries {
+		if en.IsAnyFlagSet(apc.EntryIsDir) { // always skip virtual dirs
 			continue
 		}
-		lr.ObjNames = append(lr.ObjNames, e.Name)
+		lr.ObjNames = append(lr.ObjNames, en.Name)
 	}
 	c.altmsg.Name = c.xid
 	c.altmsg.Value = &c.tcomsg

diff --git a/ais/proxy.go b/ais/proxy.go
@@ -3457,7 +3457,7 @@ func dedupLso(entries cmn.LsoEntries, maxSize int, noDirs bool) []*cmn.LsoEnt {
 			continue
 		}
 
-		debug.Assert(!(noDirs && en.IsDir())) // expecting backends for filter out accordingly
+		debug.Assert(!(noDirs && en.IsAnyFlagSet(apc.EntryIsDir))) // expecting backends for filter out accordingly
 
 		entries[j] = en
 		j++

diff --git a/ais/test/archive_test.go b/ais/test/archive_test.go
@@ -419,7 +419,7 @@ func testArch(t *testing.T, bck *meta.Bck) {
 				mime    = "application/x-" + test.ext[1:]
 			)
 			for _, en := range lst.Entries {
-				if !en.IsInsideArch() {
+				if !en.IsAnyFlagSet(apc.EntryInArch) {
 					objName = en.Name
 					continue
 				}

diff --git a/api/apc/actmsg.go b/api/apc/actmsg.go
@@ -1,6 +1,6 @@
 // Package apc: API constant and control messages
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package apc
 

diff --git a/api/ls.go b/api/ls.go
@@ -1,6 +1,6 @@
 // Package api provides native Go-based API/SDK over HTTP(S).
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package api
 

diff --git a/bench/microbenchmarks/apitests/listobj_test.go b/bench/microbenchmarks/apitests/listobj_test.go
@@ -1,6 +1,6 @@
 // Package integration contains AIS integration tests.
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package apitests_test
 

diff --git a/cmd/cli/cli/get.go b/cmd/cli/cli/get.go
@@ -312,7 +312,7 @@ func getMultiObj(c *cli.Context, bck cmn.Bck, outFile string, lsarch, extract bo
 			continue
 		}
 
-		if en.IsInsideArch() {
+		if en.IsAnyFlagSet(apc.EntryInArch) {
 			if origPrefix != msg.Prefix {
 				if !strings.HasPrefix(en.Name, origPrefix) {
 					// skip
@@ -324,7 +324,7 @@ func getMultiObj(c *cli.Context, bck cmn.Bck, outFile string, lsarch, extract bo
 				}
 			}
 			for _, shardEntry := range lst.Entries {
-				if shardEntry.IsListedArch() && strings.HasPrefix(en.Name, shardEntry.Name+"/") {
+				if shardEntry.IsAnyFlagSet(apc.EntryIsArchive) && strings.HasPrefix(en.Name, shardEntry.Name+"/") {
 					shardName = shardEntry.Name
 					break
 				}

diff --git a/cmd/cli/cli/ls.go b/cmd/cli/cli/ls.go
@@ -272,7 +272,7 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
 		addCachedCol bool
 	)
 	if bck.IsRemote() {
-		addCachedCol = true
+		addCachedCol = true           // preliminary; may change below
 		msg.SetFlag(apc.LsBckPresent) // default
 	}
 	if flagIsSet(c, verChangedFlag) {
@@ -292,7 +292,8 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
 			briefPause(1)
 		}
 		msg.SetFlag(apc.LsObjCached)
-		addCachedCol = false // redundant
+		// addCachedCol: correction #1
+		addCachedCol = false
 	}
 
 	// NOTE: `--all` combines two separate meanings:
@@ -326,6 +327,17 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
 	if propsStr != "" {
 		debug.Assert(apc.LsPropsSepa == ",", "',' is documented in 'objPropsFlag' usage and elsewhere")
 		props = splitCsv(propsStr) // split apc.LsPropsSepa
+
+		for i := range props {
+			for j := range props {
+				if i == j {
+					continue
+				}
+				if props[i] == props[j] {
+					return fmt.Errorf("'%s %s' contains duplication: %q", flprn(objPropsFlag), propsStr, props[i])
+				}
+			}
+		}
 	}
 
 	// add _implied_ props into control lsmsg
@@ -352,6 +364,13 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
 		default:
 			msg.AddProps(apc.GetPropsMinimal...)
 		}
+	case propsStr == apc.GetPropsName:
+		msg.SetFlag(apc.LsNameOnly)
+		msg.Props = apc.GetPropsName
+	case len(props) == 2 &&
+		((props[0] == apc.GetPropsName || props[1] == apc.GetPropsName) && (props[0] == apc.GetPropsSize || props[1] == apc.GetPropsSize)):
+		msg.SetFlag(apc.LsNameSize)
+		msg.AddProps([]string{apc.GetPropsName, apc.GetPropsSize}...)
 	default:
 		if cos.StringInSlice(allPropsFlag.GetName(), props) {
 			msg.AddProps(apc.GetPropsAll...)
@@ -361,9 +380,14 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
 		}
 	}
 
-	if flagIsSet(c, allObjsOrBcksFlag) {
-		// Show status. Object name can then be displayed multiple times
-		// (due to mirroring, EC). The status helps to tell an object from its replica(s).
+	// addCachedCol: correction #2
+	if addCachedCol && (msg.IsFlagSet(apc.LsNameOnly) || msg.IsFlagSet(apc.LsNameSize)) {
+		addCachedCol = false
+	}
+
+	// when props are _not_ explicitly specified
+	// but (somewhat ambiguous) flag `--all` is
+	if len(props) == 0 && flagIsSet(c, allObjsOrBcksFlag) {
 		msg.AddProps(apc.GetPropsStatus)
 	}
 	propsStr = msg.Props // show these and _only_ these props

diff --git a/cmd/cli/cli/scrub.go b/cmd/cli/cli/scrub.go
@@ -283,7 +283,7 @@ func (ctx *scrCtx) ls(bck cmn.Bck) (*scrBp, error) {
 		}
 		// one page
 		for _, en := range lst.Entries {
-			if en.IsDir() || cos.IsLastB(en.Name, filepath.Separator) {
+			if en.IsAnyFlagSet(apc.EntryIsDir) || cos.IsLastB(en.Name, filepath.Separator) {
 				continue
 			}
 			scr.upd(ctx, en)

diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.23.4
 
 require (
-	github.com/NVIDIA/aistore v1.3.26-0.20250115193147-939d7a64d226
+	github.com/NVIDIA/aistore v1.3.26-0.20250117165906-a8d69066bbde
 	github.com/fatih/color v1.18.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.21.0

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/NVIDIA/aistore v1.3.26-0.20250115193147-939d7a64d226 h1:lOtbuxNtpQIx26uh1JYINTu+SsCvpNrZrit3hbW12vk=
-github.com/NVIDIA/aistore v1.3.26-0.20250115193147-939d7a64d226/go.mod h1:CqUKZjhqSTocBlK0XMSYRPGMsoNOFKLwHIkyhOto6uc=
+github.com/NVIDIA/aistore v1.3.26-0.20250117165906-a8d69066bbde h1:ShH3TpJzgZ2P/eTdvnO497s4/nzwlZrnzDqFPA5j/Ww=
+github.com/NVIDIA/aistore v1.3.26-0.20250117165906-a8d69066bbde/go.mod h1:CqUKZjhqSTocBlK0XMSYRPGMsoNOFKLwHIkyhOto6uc=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

diff --git a/cmd/cli/teb/lso.go b/cmd/cli/teb/lso.go
@@ -111,7 +111,7 @@ func fmtLsObjStatus(en *cmn.LsoEnt) string {
 }
 
 func fmtLsObjIsCached(en *cmn.LsoEnt) string {
-	if en.IsDir() {
+	if en.IsAnyFlagSet(apc.EntryIsDir) {
 		return ""
 	}
 	return FmtBool(en.IsPresent())

diff --git a/cmd/cli/test/list_bucket.in b/cmd/cli/test/list_bucket.in
@@ -3,7 +3,7 @@ ais ls ais://$BUCKET_1
 ais ls ais://$BUCKET_1/
 ais ls ais://$BUCKET_1
 ais ls ais://$BUCKET_1 --props=name,size,version
-ais ls ais://$BUCKET_1 --props=name,name,size,size,version
+ais ls ais://$BUCKET_1 --props=name,size,version
 ais ls ais://$BUCKET_1 --props=all
 ais ls ais://$BUCKET_1/
 

diff --git a/cmd/ishard/go.mod b/cmd/ishard/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard
 go 1.23.4
 
 require (
-	github.com/NVIDIA/aistore v1.3.26-0.20250115193147-939d7a64d226
+	github.com/NVIDIA/aistore v1.3.26-0.20250117165906-a8d69066bbde
 	github.com/json-iterator/go v1.1.12
 	github.com/vbauerster/mpb/v4 v4.12.2
 )

diff --git a/cmd/ishard/go.sum b/cmd/ishard/go.sum
@@ -1,6 +1,6 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
-github.com/NVIDIA/aistore v1.3.26-0.20250115193147-939d7a64d226 h1:lOtbuxNtpQIx26uh1JYINTu+SsCvpNrZrit3hbW12vk=
-github.com/NVIDIA/aistore v1.3.26-0.20250115193147-939d7a64d226/go.mod h1:CqUKZjhqSTocBlK0XMSYRPGMsoNOFKLwHIkyhOto6uc=
+github.com/NVIDIA/aistore v1.3.26-0.20250117165906-a8d69066bbde h1:ShH3TpJzgZ2P/eTdvnO497s4/nzwlZrnzDqFPA5j/Ww=
+github.com/NVIDIA/aistore v1.3.26-0.20250117165906-a8d69066bbde/go.mod h1:CqUKZjhqSTocBlK0XMSYRPGMsoNOFKLwHIkyhOto6uc=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=

diff --git a/cmd/ishard/ishard/ishard_test.go b/cmd/ishard/ishard/ishard_test.go
@@ -172,7 +172,7 @@ func TestIshardShardSize(t *testing.T) {
 
 				for _, en := range lst.Entries {
 					// Only counts objects inside archive
-					if !en.IsInsideArch() {
+					if !en.IsAnyFlagSet(apc.EntryInArch) {
 						continue
 					}
 					ext := filepath.Ext(en.Name)

diff --git a/cmn/objlist_utils.go b/cmn/objlist_utils.go
@@ -28,29 +28,28 @@ func (entries LsoEntries) cmp(i, j int) bool {
 // LsoEnt //
 ////////////
 
-// The terms "cached" and "present" are interchangeable:
-// "object is cached" and "is present" is actually the same thing
+// flags:
+// - see above "LsoEntry Flags" enum
+// - keeping IsPresent for client convenience, with Set/IsAnyFlag covering for the rest
+// - terms "cached", "present" and "in-cluster" - are interchangeable
 func (be *LsoEnt) IsPresent() bool { return be.Flags&apc.EntryIsCached != 0 }
-func (be *LsoEnt) SetPresent()     { be.Flags |= apc.EntryIsCached }
 
 func (be *LsoEnt) SetFlag(fl uint16)           { be.Flags |= fl }
 func (be *LsoEnt) IsAnyFlagSet(fl uint16) bool { return be.Flags&fl != 0 }
 
-func (be *LsoEnt) IsStatusOK() bool   { return be.Status() == 0 }
-func (be *LsoEnt) Status() uint16     { return be.Flags & apc.EntryStatusMask }
-func (be *LsoEnt) IsDir() bool        { return be.Flags&apc.EntryIsDir != 0 }
-func (be *LsoEnt) IsInsideArch() bool { return be.Flags&apc.EntryInArch != 0 }
-func (be *LsoEnt) IsListedArch() bool { return be.Flags&apc.EntryIsArchive != 0 }
-func (be *LsoEnt) String() string     { return "{" + be.Name + "}" }
+// location _status_
+func (be *LsoEnt) IsStatusOK() bool { return be.Status() == 0 }
+func (be *LsoEnt) Status() uint16   { return be.Flags & apc.EntryStatusMask }
 
+// sorting
 func (be *LsoEnt) less(oe *LsoEnt) bool {
-	if be.IsDir() {
-		if oe.IsDir() {
+	if be.IsAnyFlagSet(apc.EntryIsDir) {
+		if oe.IsAnyFlagSet(apc.EntryIsDir) {
 			return be.Name < oe.Name
 		}
 		return true
 	}
-	if oe.IsDir() {
+	if oe.IsAnyFlagSet(apc.EntryIsDir) {
 		return false
 	}
 	if be.Name == oe.Name {

diff --git a/xact/api.go b/xact/api.go
@@ -1,6 +1,6 @@
 // Package xact provides core functionality for the AIStore eXtended Actions (xactions).
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package xact
 

diff --git a/xact/xs/lrit.go b/xact/xs/lrit.go
@@ -1,7 +1,7 @@
 // Package xs is a collection of eXtended actions (xactions), including multi-object
 // operations, list-objects, (cluster) rebalance and (target) resilver, ETL, and more.
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package xs
 
@@ -295,7 +295,7 @@ func (r *lrit) _prefix(wi lrwi, smap *meta.Smap) error {
 			if !be.IsStatusOK() {
 				continue
 			}
-			if be.IsDir() { // NOTE: always skip virtual dirs (apc.EntryIsDir)
+			if be.IsAnyFlagSet(apc.EntryIsDir) { // always skip virtual dirs
 				continue
 			}
 			if r.done() {

diff --git a/xact/xs/lso.go b/xact/xs/lso.go
@@ -1,7 +1,7 @@
 // Package xs is a collection of eXtended actions (xactions), including multi-object
 // operations, list-objects, (cluster) rebalance and (target) resilver, ETL, and more.
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package xs
 

diff --git a/xact/xs/nextpage.go b/xact/xs/nextpage.go
@@ -116,7 +116,7 @@ func (npg *npgCtx) filterAddLmeta(lst *cmn.LsoRes) error {
 		i    int
 	)
 	for _, en := range lst.Entries {
-		if en.IsDir() {
+		if en.IsAnyFlagSet(apc.EntryIsDir) {
 			// collecting virtual dir-s when apc.LsNoRecursion is on - skipping here
 			continue
 		}
@@ -141,7 +141,7 @@ func (npg *npgCtx) filterAddLmeta(lst *cmn.LsoRes) error {
 		}
 
 		npg.wi.setWanted(en, lom)
-		en.SetPresent()
+		en.SetFlag(apc.EntryIsCached) // formerly, SetPresent
 
 		if post != nil {
 			post(lom)

diff --git a/xact/xs/nsumm.go b/xact/xs/nsumm.go
@@ -1,7 +1,7 @@
 // Package xs is a collection of eXtended actions (xactions), including multi-object
 // operations, list-objects, (cluster) rebalance and (target) resilver, ETL, and more.
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package xs
 

diff --git a/xact/xs/prefetch.go b/xact/xs/prefetch.go
@@ -1,7 +1,7 @@
 // Package xs is a collection of eXtended actions (xactions), including multi-object
 // operations, list-objects, (cluster) rebalance and (target) resilver, ETL, and more.
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package xs