enhance: enhance the accuracy of memory usage (#28554) (#28559)

before this, Milvus use container/system's memory info to get the memory
usage, which could be inaccurate.

we allocates the memory by private anon mmap,
then rss - shared would be the accurate memory usage

resolve https://github.com/milvus-io/milvus/issues/28553
pr: #28554

---------

Signed-off-by: yah01 <yah2er0ne@outlook.com>

.golangci.yml

@@ -155,3 +155,6 @@ issues:
   max-issues-per-linter: 0
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
+
+service:
+  golangci-lint-version: 1.55.2 # use the fixed version to not introduce new linters unexpectedly

Makefile

@@ -35,7 +35,7 @@ ifdef USE_DYNAMIC_SIMD
 	use_dynamic_simd = ${USE_DYNAMIC_SIMD}
 endif
 # golangci-lint
-GOLANGCI_LINT_VERSION := 1.53.1
+GOLANGCI_LINT_VERSION := 1.55.2
 GOLANGCI_LINT_OUTPUT := $(shell $(INSTALL_PATH)/golangci-lint --version 2>/dev/null)
 INSTALL_GOLANGCI_LINT := $(findstring $(GOLANGCI_LINT_VERSION), $(GOLANGCI_LINT_OUTPUT))
 # mockery
@@ -80,7 +80,7 @@ get-build-deps:
 getdeps:
 	@mkdir -p $(INSTALL_PATH)
 	@if [ -z "$(INSTALL_GOLANGCI_LINT)" ]; then \
-		echo "Installing golangci-lint into ./bin/" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(INSTALL_PATH) v1.53.1 ; \
+		echo "Installing golangci-lint into ./bin/" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(INSTALL_PATH) v1.55.2 ; \
 	else \
 		echo "golangci-lint v@$(GOLANGCI_LINT_VERSION) already installed"; \
 	fi

internal/core/src/mmap/Column.h

@@ -33,8 +33,6 @@
 
 namespace milvus {
 
-static int mmap_flags = MAP_SHARED;
-
 class ColumnBase {
  public:
     // memory mode ctor
@@ -56,7 +54,7 @@ class ColumnBase {
         data_ = static_cast<char*>(mmap(nullptr,
                                         cap_size_ + padding_,
                                         PROT_READ | PROT_WRITE,
-                                        mmap_flags | MAP_ANON,
+                                        MAP_PRIVATE | MAP_ANON,
                                         -1,
                                         0));
         AssertInfo(
@@ -77,7 +75,7 @@ class ColumnBase {
         data_ = static_cast<char*>(mmap(nullptr,
                                         cap_size_ + padding_,
                                         PROT_READ,
-                                        mmap_flags,
+                                        MAP_SHARED,
                                         file.Descriptor(),
                                         0));
         AssertInfo(data_ != MAP_FAILED,
@@ -100,7 +98,7 @@ class ColumnBase {
         data_ = static_cast<char*>(mmap(nullptr,
                                         cap_size_ + padding_,
                                         PROT_READ,
-                                        mmap_flags,
+                                        MAP_SHARED,
                                         file.Descriptor(),
                                         0));
         AssertInfo(data_ != MAP_FAILED,
@@ -191,7 +189,7 @@ class ColumnBase {
         auto data = static_cast<char*>(mmap(nullptr,
                                             new_size + padding_,
                                             PROT_READ | PROT_WRITE,
-                                            mmap_flags | MAP_ANON,
+                                            MAP_PRIVATE | MAP_ANON,
                                             -1,
                                             0));
 

internal/kv/tikv/txn_tikv.go

@@ -772,8 +772,7 @@ func convertEmptyStringToByte(value string) ([]byte, error) {
 	if len(value) == 0 {
 		return EmptyValueByte, nil
 	} else if value == EmptyValueString {
-		return nil, fmt.Errorf("Value for key is reserved by EmptyValue: %s", EmptyValueString)
-	} else {
-		return []byte(value), nil
+		return nil, fmt.Errorf("value for key is reserved by EmptyValue: %s", EmptyValueString)
 	}
+	return []byte(value), nil
 }

internal/parser/planparserv2/utils.go

@@ -432,9 +432,8 @@ func HandleCompare(op int, left, right *ExprWithType) (*planpb.Expr, error) {
 		return handleCompareRightValue(op, right, valueExpr)
 	} else if valueExpr := right.expr.GetValueExpr(); valueExpr != nil {
 		return handleCompareRightValue(cmpOp, left, valueExpr)
-	} else {
-		return handleCompare(cmpOp, left, right)
 	}
+	return handleCompare(cmpOp, left, right)
 }
 
 func isEmptyExpression(s string) bool {

internal/util/importutil/binlog_adapter.go

@@ -316,10 +316,9 @@ func (p *BinlogAdapter) readDeltalogs(segmentHolder *SegmentFilesHolder) (map[in
 		}
 		log.Info("Binlog adapter: count of deleted entities", zap.Int("deletedCount", len(deletedIDDict)))
 		return nil, deletedIDDict, nil
-	} else {
-		log.Warn("Binlog adapter: unsupported primary key type", zap.Int("type", int(primaryKey.GetDataType())))
-		return nil, nil, fmt.Errorf("unsupported primary key type %d, primary key should be int64 or varchar", primaryKey.GetDataType())
 	}
+	log.Warn("Binlog adapter: unsupported primary key type", zap.Int("type", int(primaryKey.GetDataType())))
+	return nil, nil, fmt.Errorf("unsupported primary key type %d, primary key should be int64 or varchar", primaryKey.GetDataType())
 }
 
 // decodeDeleteLogs decodes string array(read from delta log) to storage.DeleteLog array
@@ -495,10 +494,9 @@ func (p *BinlogAdapter) readPrimaryKeys(logPath string) ([]int64, []string, erro
 		}
 		log.Info("Binlog adapter: succeed to read varchar primary key binlog", zap.Int("len", len(idList)))
 		return nil, idList, nil
-	} else {
-		log.Warn("Binlog adapter: unsupported primary key type", zap.Int("type", int(primaryKey.GetDataType())))
-		return nil, nil, fmt.Errorf("unsupported primary key type %d, primary key should be int64 or varchar", primaryKey.GetDataType())
 	}
+	log.Warn("Binlog adapter: unsupported primary key type", zap.Int("type", int(primaryKey.GetDataType())))
+	return nil, nil, fmt.Errorf("unsupported primary key type %d, primary key should be int64 or varchar", primaryKey.GetDataType())
 }
 
 // getShardingListByPrimaryInt64 method generates a shard id list by primary key(int64) list and deleted list.

pkg/util/hardware/hardware_info.go

@@ -106,35 +106,6 @@ func GetMemoryCount() uint64 {
 	return stats.Total
 }
 
-// GetUsedMemoryCount returns the memory usage in bytes.
-func GetUsedMemoryCount() uint64 {
-	icOnce.Do(func() {
-		ic, icErr = inContainer()
-	})
-	if icErr != nil {
-		log.Error(icErr.Error())
-		return 0
-	}
-	if ic {
-		// in container, calculate by `cgroups`
-		used, err := getContainerMemUsed()
-		if err != nil {
-			log.Warn("failed to get container memory used", zap.Error(err))
-			return 0
-		}
-		return used
-	}
-	// not in container, calculate by `gopsutil`
-	stats, err := mem.VirtualMemory()
-	if err != nil {
-		log.Warn("failed to get memory usage count",
-			zap.Error(err))
-		return 0
-	}
-
-	return stats.Used
-}
-
 // GetFreeMemoryCount returns the free memory in bytes.
 func GetFreeMemoryCount() uint64 {
 	return GetMemoryCount() - GetUsedMemoryCount()

pkg/util/hardware/mem_info.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !darwin && !openbsd && !freebsd
+// +build !darwin,!openbsd,!freebsd
+
+package hardware
+
+import (
+	"os"
+
+	"github.com/shirou/gopsutil/v3/process"
+	"go.uber.org/zap"
+
+	"github.com/milvus-io/milvus/pkg/log"
+)
+
+var proc *process.Process
+
+func init() {
+	var err error
+	proc, err = process.NewProcess(int32(os.Getpid()))
+	if err != nil {
+		panic(err)
+	}
+}
+
+// GetUsedMemoryCount returns the memory usage in bytes.
+func GetUsedMemoryCount() uint64 {
+	memInfo, err := proc.MemoryInfoEx()
+	if err != nil {
+		log.Warn("failed to get memory info", zap.Error(err))
+		return 0
+	}
+
+	// sub the shared memory to filter out the file-backed map memory usage
+	return memInfo.RSS - memInfo.Shared
+}

pkg/util/hardware/mem_info_darwin.go

@@ -0,0 +1,54 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build darwin || openbsd || freebsd
+// +build darwin openbsd freebsd
+
+package hardware
+
+import (
+	"github.com/shirou/gopsutil/v3/mem"
+	"go.uber.org/zap"
+
+	"github.com/milvus-io/milvus/pkg/log"
+)
+
+// GetUsedMemoryCount returns the memory usage in bytes.
+func GetUsedMemoryCount() uint64 {
+	icOnce.Do(func() {
+		ic, icErr = inContainer()
+	})
+	if icErr != nil {
+		log.Error(icErr.Error())
+		return 0
+	}
+	if ic {
+		// in container, calculate by `cgroups`
+		used, err := getContainerMemUsed()
+		if err != nil {
+			log.Warn("failed to get container memory used", zap.Error(err))
+			return 0
+		}
+		return used
+	}
+	// not in container, calculate by `gopsutil`
+	stats, err := mem.VirtualMemory()
+	if err != nil {
+		log.Warn("failed to get memory usage count",
+			zap.Error(err))
+		return 0
+	}
+
+	return stats.Used
+}