core: BID and object flags (metadata)

* amend core/lombid; remove meta/bid; clarify and refactor

Signed-off-by: Alex Aizman <[email protected]>

core/lombid.go

@@ -1,19 +1,25 @@
 // Package core provides core metadata and in-cluster API
 /*
- * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package core
 
 import (
+	"math"
+
 	"github.com/NVIDIA/aistore/cmn/debug"
-	"github.com/NVIDIA/aistore/core/meta"
 )
 
 const (
+	AisBID = uint64(1 << 63)
+
 	bitshift = 52
-	flagsBID = uint64(0xfff<<bitshift) & ^meta.AisBID
+	flagmask = math.MaxUint64 >> bitshift       // 0xfff
+	flagsBID = (flagmask << bitshift) & ^AisBID // 0x7ff0000000000000
 )
 
+type lomBID uint64
+
 // lomBID is a 64-bit field in the LOM - specifically, `lmeta` structure.
 // As the name implies, lomBID is a union of two values:
 // - bucket ID
@@ -26,22 +32,31 @@ const (
 // * next 11 bits: bit flags
 // * remaining (64 - 12) = bitshift bits contain the bucket's serial number.
 
-type lomBID meta.BID
+func NewBID(serial uint64, isAis bool) uint64 {
+	// not adding runtime check given the time reasonably
+	// required to create (1 << 52) buckets
+	debug.Assert(serial&(flagmask<<bitshift) == 0)
+
+	if isAis {
+		return serial | AisBID
+	}
+	return serial
+}
 
 func (lid lomBID) bid() uint64   { return uint64(lid) & ^flagsBID }
 func (lid lomBID) flags() uint16 { return uint16((uint64(lid) & flagsBID) >> bitshift) }
 
 func (lid lomBID) setbid(bid uint64) lomBID {
-	debug.Assert(bid&flagsBID == 0)
+	debug.Assert(bid&flagsBID == 0, bid)
 	return lomBID((uint64(lid) & flagsBID) | bid)
 }
 
 func (lid lomBID) setflags(fl uint16) lomBID {
-	debug.Assert(fl <= 0x7ff)
+	debug.Assert(fl <= uint16(flagsBID>>bitshift), fl)
 	return lomBID(uint64(lid) | (uint64(fl) << bitshift))
 }
 
 func (lid lomBID) clrflags(fl uint16) lomBID {
-	debug.Assert(fl <= 0x7ff)
+	debug.Assert(fl <= uint16(flagsBID>>bitshift))
 	return lomBID(uint64(lid) & ^(uint64(fl) << bitshift))
 }

core/lombid_internal_test.go

@@ -7,7 +7,6 @@ package core
 import (
 	"testing"
 
-	"github.com/NVIDIA/aistore/core/meta"
 	"github.com/NVIDIA/aistore/tools/tassert"
 )
 
@@ -16,18 +15,23 @@ func TestLomBid(t *testing.T) {
 		bid1, bid2 uint64
 		flags      uint16
 	}{
-		{1, 1 | meta.AisBID, 1},
-		{2 | meta.AisBID, 1<<bitshift - 1, 1},
-		{1<<bitshift - 1, 2 | meta.AisBID, 1},
-		{1234567 | meta.AisBID, 1, 0x7ff},
-		{1<<bitshift - 1 | meta.AisBID, 1234567 | meta.AisBID, 0x7f5},
+		{1, 1 | AisBID, 1},
+		{2 | AisBID, 1<<bitshift - 1, 1},
+		{1<<bitshift - 1, 2 | AisBID, 1},
+		{1234567 | AisBID, 1, 0x7ff},
+		{1<<bitshift - 1 | AisBID, 1234567 | AisBID, 0x7f5},
 	}
 
 	for _, test := range tests {
 		var lid lomBID
 		lid = lid.setbid(test.bid1)
 		lid = lid.setflags(test.flags)
-		tassert.Errorf(t, uint64(lid)&meta.AisBID == test.bid1&meta.AisBID, "expected high bit to match")
+		tassert.Errorf(t, uint64(lid)&AisBID == test.bid1&AisBID, "expected high bit to match")
+		tassert.Errorf(t, lid.flags() == test.flags, "expected %x, got %x", test.flags, lid.flags())
+		tassert.Errorf(t, lid.bid() == test.bid1, "expected %x, got %x", test.bid1, lid.bid())
+
+		lid = lid.setbid(test.bid1)
+		tassert.Errorf(t, uint64(lid)&AisBID == test.bid1&AisBID, "expected high bit to match")
 		tassert.Errorf(t, lid.flags() == test.flags, "expected %x, got %x", test.flags, lid.flags())
 		tassert.Errorf(t, lid.bid() == test.bid1, "expected %x, got %x", test.bid1, lid.bid())
 
@@ -37,7 +41,7 @@ func TestLomBid(t *testing.T) {
 
 		lid = lid.setbid(test.bid2)
 		lid = lid.setflags(test.flags)
-		tassert.Errorf(t, uint64(lid)&meta.AisBID == test.bid2&meta.AisBID, "expected high bit to match")
+		tassert.Errorf(t, uint64(lid)&AisBID == test.bid2&AisBID, "expected high bit to match")
 		tassert.Errorf(t, lid.flags() == test.flags, "expected %x, got %x", test.flags, lid.flags())
 		tassert.Errorf(t, lid.bid() == test.bid2, "expected %x, got %x", test.bid2, lid.bid())
 

transport/obj_test.go

@@ -124,7 +124,7 @@ func Example_headers() {
 				break
 			}
 
-			tlog.Logf("Bck:%s ObjName:%s SID:%s Opaque:%v ObjAttrs:{%s} (%d)\n",
+			fmt.Printf("Bck:%s ObjName:%s SID:%s Opaque:%v ObjAttrs:{%s} (%d)\n",
 				hdr.Bck.String(), hdr.ObjName, hdr.SID, hdr.Opaque, hdr.ObjAttrs.String(), hlen)
 			off += hlen + int(hdr.ObjAttrs.Size)
 		}
@@ -203,7 +203,7 @@ func Example_obj() {
 		if int64(len(object)) != hdr.ObjAttrs.Size {
 			panic(fmt.Sprintf("size %d != %d", len(object), hdr.ObjAttrs.Size))
 		}
-		tlog.Logf("%s...\n", string(object[:16]))
+		fmt.Printf("%s...\n", string(object[:16]))
 		return nil
 	}
 	ts := httptest.NewServer(objmux)
@@ -260,7 +260,7 @@ func printNetworkStats() {
 	for trname, eps := range netstats {
 		for uid, stats := range eps { // RxStats by session ID
 			xx, sessID := transport.UID2SessID(uid)
-			tlog.Logf("recv$ %s[%d:%d]: offset=%d, num=%d\n",
+			fmt.Printf("recv$ %s[%d:%d]: offset=%d, num=%d\n",
 				trname, xx, sessID, stats.Offset.Load(), stats.Num.Load())
 		}
 	}
@@ -272,18 +272,18 @@ func compareNetworkStats(netstats1 map[string]transport.RxStats) {
 		eps1, ok := netstats1[trname]
 		for uid, stats2 := range eps2 { // RxStats by session ID
 			xx, sessID := transport.UID2SessID(uid)
-			tlog.Logf("recv$ %s[%d:%d]: offset=%d, num=%d\n", trname, xx, sessID,
+			fmt.Printf("recv$ %s[%d:%d]: offset=%d, num=%d\n", trname, xx, sessID,
 				stats2.Offset.Load(), stats2.Num.Load())
 			if ok {
 				stats1, ok := eps1[sessID]
 				if ok {
-					tlog.Logf("send$ %s[%d]: offset=%d, num=%d\n",
+					fmt.Printf("send$ %s[%d]: offset=%d, num=%d\n",
 						trname, sessID, stats1.Offset.Load(), stats1.Num.Load())
 				} else {
-					tlog.Logf("send$ %s[%d]: -- not present --\n", trname, sessID)
+					fmt.Printf("send$ %s[%d]: -- not present --\n", trname, sessID)
 				}
 			} else {
-				tlog.Logf("send$ %s[%d]: -- not present --\n", trname, sessID)
+				fmt.Printf("send$ %s[%d]: -- not present --\n", trname, sessID)
 			}
 		}
 	}
@@ -518,7 +518,7 @@ func TestCompressedOne(t *testing.T) {
 
 	slab.Free(buf)
 
-	tlog.Logf("send$ %s: offset=%d, num=%d(%d/%d), compression-ratio=%.2f\n",
+	fmt.Printf("send$ %s: offset=%d, num=%d(%d/%d), compression-ratio=%.2f\n",
 		stream, stats.Offset.Load(), stats.Num.Load(), num, numhdr, stats.CompressionRatio())
 
 	printNetworkStats()
@@ -559,7 +559,7 @@ func TestDryRun(t *testing.T) {
 			size += hdr.ObjAttrs.Size
 			if size-prevsize >= cos.GiB*100 {
 				prevsize = size
-				tlog.Logf("[dry]: %d GiB\n", size/cos.GiB)
+				fmt.Printf("[dry]: %d GiB\n", size/cos.GiB)
 			}
 		}
 	}
@@ -703,7 +703,7 @@ func streamWriteUntil(t *testing.T, ii int, wg *sync.WaitGroup, ts *httptest.Ser
 	if netstats == nil {
 		reason, termErr := stream.TermInfo()
 		tassert.Errorf(t, reason != "", "expecting reason for termination")
-		tlog.Logf("send$ %s[%d]: offset=%d, num=%d(%d), term(%q, %v)\n",
+		fmt.Printf("send$ %s[%d]: offset=%d, num=%d(%d), term(%q, %v)\n",
 			trname, sessID, stats.Offset.Load(), stats.Num.Load(), num, reason, termErr)
 	} else {
 		lock.Lock()