Enable gosec (milvus-io#8202)

Signed-off-by: Congqi Xia <[email protected]>

.golangci.yml

@@ -24,6 +24,7 @@ linters:
     - golint
     - ineffassign
     - gosimple
+    - gosec
 
 issues:
   exclude-use-default: false
@@ -33,6 +34,19 @@ issues:
     - should be of the form
     - should not use dot imports
     - which can be annoying to use
+    # Binds to all network interfaces
+    - G102
+    # Errors unhandled
+    - G104
+    # file/folder Permission
+    - G301
+    - G302
+    # Potential file inclusion via variable
+    - G304
+    # Deferring unsafe method like *os.File Close 
+    - G307
+    # Use of weak random number generator math/rand
+    - G404
 service:
   golangci-lint-version: 1.27.0 # use the fixed version to not introduce new linters unexpectedly
 

internal/datanode/flow_graph_insert_buffer_node.go

@@ -396,6 +396,7 @@ func (ibNode *insertBufferNode) updateSegStatesInReplica(insertMsgs []*msgstream
 	return
 }
 
+/* #nosec G103 */
 // bufferInsertMsg put InsertMsg into buffer
 // 	1.1 fetch related schema from replica
 // 	1.2 Get buffer data and put data into each field buffer

internal/indexcoord/meta_table.go

@@ -229,6 +229,8 @@ func (mt *metaTable) MarkIndexAsDeleted(indexID UniqueID) error {
 	for _, meta := range mt.indexBuildID2Meta {
 		if meta.indexMeta.Req.IndexID == indexID && !meta.indexMeta.MarkDeleted {
 			meta.indexMeta.MarkDeleted = true
+			// marshal inside
+			/* #nosec G601 */
 			if err := mt.saveIndexMeta(&meta); err != nil {
 				log.Debug("IndexCoord metaTable MarkIndexAsDeleted saveIndexMeta failed", zap.Error(err))
 				fn := func() error {

internal/metrics/metrics.go

@@ -595,7 +595,7 @@ var (
 
 //RegisterDataCoord register DataCoord metrics
 func RegisterDataCoord() {
-	prometheus.Register(DataCoordDataNodeList)
+	prometheus.MustRegister(DataCoordDataNodeList)
 }
 
 var (
@@ -620,8 +620,8 @@ var (
 
 //RegisterDataNode register DataNode metrics
 func RegisterDataNode() {
-	prometheus.Register(DataNodeFlushSegmentsCounter)
-	prometheus.Register(DataNodeWatchDmChannelsCounter)
+	prometheus.MustRegister(DataNodeFlushSegmentsCounter)
+	prometheus.MustRegister(DataNodeWatchDmChannelsCounter)
 }
 
 //RegisterIndexCoord register IndexCoord metrics

internal/proxy/plan_parser.go

@@ -393,6 +393,8 @@ func (context *ParserContext) handleArrayExpr(node *ant_ast.Node, dataType schem
 	}
 	var arr []*planpb.GenericValue
 	for _, element := range arrayNode.Nodes {
+		// use value inside
+		// #nosec G601
 		val, err := context.handleLeafValue(&element, dataType)
 		if err != nil {
 			return nil, err

internal/proxy/task.go

@@ -906,6 +906,7 @@ func (it *insertTask) _assignSegmentID(stream msgstream.MsgStream, pack *msgstre
 	threshold := Params.PulsarMaxMessageSize / factor
 	log.Debug("Proxy", zap.Int("threshold of message size: ", threshold))
 	// not accurate
+	/* #nosec G103 */
 	getFixedSizeOfInsertMsg := func(msg *msgstream.InsertMsg) int {
 		size := 0
 
@@ -974,6 +975,7 @@ func (it *insertTask) _assignSegmentID(stream msgstream.MsgStream, pack *msgstre
 			curMsg.Timestamps = append(curMsg.Timestamps, ts)
 			curMsg.RowIDs = append(curMsg.RowIDs, rowID)
 			curMsg.RowData = append(curMsg.RowData, row)
+			/* #nosec G103 */
 			curMsgSize += 4 + 8 + int(unsafe.Sizeof(row.Value))
 			curMsgSize += len(row.Value)
 

internal/querynode/query_collection.go

@@ -706,6 +706,7 @@ func translateHits(schema *typeutil.SchemaHelper, fieldIDs []int64, rawHits [][]
 				for _, row := range hit.RowData {
 					dataBlob := row[blobOffset : blobOffset+blobLen]
 					//ref https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+					/* #nosec G103 */
 					ptr := unsafe.Pointer(&dataBlob[0])
 					farray := (*[1 << 28]float32)(ptr)
 					colData = append(colData, farray[:dim:dim]...)

internal/storage/binlog_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+/* #nosec G103 */
 func TestInsertBinlog(t *testing.T) {
 	w := NewInsertBinlogWriter(schemapb.DataType_Int64, 10, 20, 30, 40)
 
@@ -280,6 +281,7 @@ func TestInsertBinlog(t *testing.T) {
 	assert.Equal(t, ed2.EndTimestamp, Timestamp(400))
 }
 
+/* #nosec G103 */
 func TestDeleteBinlog(t *testing.T) {
 	w := NewDeleteBinlogWriter(schemapb.DataType_Int64, 50)
 
@@ -536,6 +538,7 @@ func TestDeleteBinlog(t *testing.T) {
 	assert.Equal(t, ed2.EndTimestamp, Timestamp(400))
 }
 
+/* #nosec G103 */
 func TestDDLBinlog1(t *testing.T) {
 	w := NewDDLBinlogWriter(schemapb.DataType_Int64, 50)
 
@@ -792,6 +795,7 @@ func TestDDLBinlog1(t *testing.T) {
 	assert.Equal(t, ed2.EndTimestamp, Timestamp(400))
 }
 
+/* #nosec G103 */
 func TestDDLBinlog2(t *testing.T) {
 	w := NewDDLBinlogWriter(schemapb.DataType_Int64, 50)
 

internal/storage/event_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+/* #nosec G103 */
 func checkEventHeader(
 	t *testing.T,
 	buf []byte,
@@ -44,6 +45,7 @@ func checkEventHeader(
 	assert.Equal(t, nPos, length)
 }
 
+/* #nosec G103 */
 func TestDescriptorEvent(t *testing.T) {
 	desc := newDescriptorEvent()
 
@@ -167,6 +169,7 @@ func TestDescriptorEvent(t *testing.T) {
 	}
 }
 
+/* #nosec G103 */
 func TestInsertEvent(t *testing.T) {
 	insertT := func(t *testing.T,
 		dt schemapb.DataType,
@@ -414,6 +417,7 @@ func TestInsertEvent(t *testing.T) {
 	})
 }
 
+/* #nosec G103 */
 func TestDeleteEvent(t *testing.T) {
 	deleteT := func(t *testing.T,
 		dt schemapb.DataType,
@@ -661,6 +665,7 @@ func TestDeleteEvent(t *testing.T) {
 	})
 }
 
+/* #nosec G103 */
 func TestCreateCollectionEvent(t *testing.T) {
 	t.Run("create_event", func(t *testing.T) {
 		w, err := newCreateCollectionEventWriter(schemapb.DataType_Float)
@@ -782,6 +787,7 @@ func TestCreateCollectionEvent(t *testing.T) {
 	})
 }
 
+/* #nosec G103 */
 func TestDropCollectionEvent(t *testing.T) {
 	t.Run("drop_event", func(t *testing.T) {
 		w, err := newDropCollectionEventWriter(schemapb.DataType_Float)
@@ -903,6 +909,7 @@ func TestDropCollectionEvent(t *testing.T) {
 	})
 }
 
+/* #nosec G103 */
 func TestCreatePartitionEvent(t *testing.T) {
 	t.Run("create_event", func(t *testing.T) {
 		w, err := newCreatePartitionEventWriter(schemapb.DataType_Float)
@@ -1024,6 +1031,7 @@ func TestCreatePartitionEvent(t *testing.T) {
 	})
 }
 
+/* #nosec G103 */
 func TestDropPartitionEvent(t *testing.T) {
 	t.Run("drop_event", func(t *testing.T) {
 		w, err := newDropPartitionEventWriter(schemapb.DataType_Float)

internal/storage/local_chunk_manager.go

@@ -53,7 +53,7 @@ func (lcm *LocalChunkManager) Write(key string, content []byte) error {
 			return err
 		}
 	}
-	err := ioutil.WriteFile(filePath, content, 0644)
+	err := ioutil.WriteFile(filePath, content, 0600)
 	if err != nil {
 		return err
 	}

internal/storage/print_binlog_test.go

@@ -13,6 +13,7 @@ package storage
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -60,7 +61,7 @@ func TestPrintBinlogFilesInt64(t *testing.T) {
 	buf, err := w.GetBuffer()
 	assert.Nil(t, err)
 
-	fd, err := os.Create("/tmp/binlog_int64.db")
+	fd, err := ioutil.TempFile("", "binlog_int64.db")
 	assert.Nil(t, err)
 	num, err := fd.Write(buf)
 	assert.Nil(t, err)

internal/storage/unsafe.go

@@ -13,31 +13,37 @@ package storage
 
 import "unsafe"
 
+/* #nosec G103 */
 func UnsafeReadInt8(buf []byte, idx int) int8 {
 	ptr := unsafe.Pointer(&(buf[idx]))
 	return *((*int8)(ptr))
 }
 
+/* #nosec G103 */
 func UnsafeReadInt16(buf []byte, idx int) int16 {
 	ptr := unsafe.Pointer(&(buf[idx]))
 	return *((*int16)(ptr))
 }
 
+/* #nosec G103 */
 func UnsafeReadInt32(buf []byte, idx int) int32 {
 	ptr := unsafe.Pointer(&(buf[idx]))
 	return *((*int32)(ptr))
 }
 
+/* #nosec G103 */
 func UnsafeReadInt64(buf []byte, idx int) int64 {
 	ptr := unsafe.Pointer(&(buf[idx]))
 	return *((*int64)(ptr))
 }
 
+/* #nosec G103 */
 func UnsafeReadFloat32(buf []byte, idx int) float32 {
 	ptr := unsafe.Pointer(&(buf[idx]))
 	return *((*float32)(ptr))
 }
 
+/* #nosec G103 */
 func UnsafeReadFloat64(buf []byte, idx int) float64 {
 	ptr := unsafe.Pointer(&(buf[idx]))
 	return *((*float64)(ptr))

internal/tso/tso.go

@@ -117,6 +117,8 @@ func (t *timestampOracle) InitTimestamp() error {
 	current := &atomicObject{
 		physical: next,
 	}
+	// atomic unsafe pointer
+	/* #nosec G103 */
 	atomic.StorePointer(&t.TSO, unsafe.Pointer(current))
 
 	return nil
@@ -144,6 +146,8 @@ func (t *timestampOracle) ResetUserTimestamp(tso uint64) error {
 	update := &atomicObject{
 		physical: next,
 	}
+	// atomic unsafe pointer
+	/* #nosec G103 */
 	atomic.CompareAndSwapPointer(&t.TSO, unsafe.Pointer(prev), unsafe.Pointer(update))
 	return nil
 }
@@ -196,7 +200,8 @@ func (t *timestampOracle) UpdateTimestamp() error {
 		physical: next,
 		logical:  0,
 	}
-
+	// atomic unsafe pointer
+	/* #nosec G103 */
 	atomic.StorePointer(&t.TSO, unsafe.Pointer(current))
 
 	return nil
@@ -207,5 +212,7 @@ func (t *timestampOracle) ResetTimestamp() {
 	zero := &atomicObject{
 		physical: time.Now(),
 	}
+	// atomic unsafe pointer
+	/* #nosec G103 */
 	atomic.StorePointer(&t.TSO, unsafe.Pointer(zero))
 }

internal/util/mqclient/pulsar_consumer.go

@@ -12,7 +12,6 @@
 package mqclient
 
 import (
-	"reflect"
 	"unsafe"
 
 	"github.com/apache/pulsar-client-go/pulsar"
@@ -88,8 +87,6 @@ func (pc *pulsarConsumer) Close() {
 // ONLY used in Chan() function
 // DON'T use elsewhere
 func patchEarliestMessageID(mid *pulsar.MessageID) {
-	v := reflect.ValueOf(mid)
-	v = v.Elem()
 	// cannot use field.SetInt(), since partitionIdx is not exported
 
 	// this reflect+ unsafe solution is disable by go vet
@@ -101,6 +98,7 @@ func patchEarliestMessageID(mid *pulsar.MessageID) {
 	//*(*int32)(unsafe.Pointer(v.InterfaceData()[1] + 20)) = 0
 
 	// use direct unsafe conversion
+	/* #nosec G103 */
 	r := (*iface)(unsafe.Pointer(mid))
 	id := (*messageID)(r.Data)
 	id.partitionIdx = 0

internal/util/typeutil/hash.go

@@ -27,6 +27,8 @@ func Hash32Bytes(b []byte) (uint32, error) {
 }
 
 func Hash32Uint64(v uint64) (uint32, error) {
+	// need unsafe package to get element byte size
+	/* #nosec G103 */
 	b := make([]byte, unsafe.Sizeof(v))
 	binary.LittleEndian.PutUint64(b, v)
 	return Hash32Bytes(b)

internal/util/typeutil/hash_test.go

@@ -37,6 +37,7 @@ func TestHash32_Uint64(t *testing.T) {
 	t.Log(h1)
 	assert.Equal(t, h, h1)
 
+	/* #nosec G103 */
 	b := make([]byte, unsafe.Sizeof(u))
 	b[0] = 0x12
 	h2, err := Hash32Bytes(b)