Refine Rocksdb option (#24394) (#24503)

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>

internal/core/conanfile.py

@@ -50,6 +50,7 @@ class MilvusConan(ConanFile):
     generators = ("cmake", "cmake_find_package")
     default_options = {
         "rocksdb:shared": True,
+        "rocksdb:with_zstd": True,
         "arrow:parquet": True,
         "arrow:compute": True,
         "arrow:with_zstd": True,

internal/core/thirdparty/rocksdb/rocksdb-win.pc.in

@@ -5,5 +5,5 @@ Description: Rocksdb
 Version: @ROCKSDB_VERSION@
 
 Libs: -L${libdir} -lrocksdb
-Libs.private: -lz -lbz2                                                                    │
+Libs.private: -lz -lbz2 
 Cflags: -I${includedir}

internal/core/thirdparty/rocksdb/rocksdb.pc.in

@@ -4,6 +4,6 @@ Name: Rocksdb
 Description: Rocksdb
 Version: @ROCKSDB_VERSION@
 
-Libs: -L${libdir} -lrocksdb
+Libs: -L${libdir} -lrocksdb 
 Libs.private: -lz -lbz2
 Cflags: -I${includedir}

internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go

@@ -159,9 +159,16 @@ func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error)
 	log.Debug("Start rocksmq ", zap.Int("max proc", maxProcs),
 		zap.Int("parallism", parallelism), zap.Uint64("lru cache", rocksDBLRUCacheCapacity))
 	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
+	bbto.SetBlockSize(64 << 10)
 	bbto.SetBlockCache(gorocksdb.NewLRUCache(rocksDBLRUCacheCapacity))
+
 	optsKV := gorocksdb.NewDefaultOptions()
+	// L0:No Compression
+	// L1,L2: ZSTD
+	optsKV.SetNumLevels(3)
+	optsKV.SetCompressionPerLevel([]gorocksdb.CompressionType{0, 7, 7})
 	optsKV.SetBlockBasedTableFactory(bbto)
+	optsKV.SetTargetFileSizeMultiplier(2)
 	optsKV.SetCreateIfMissing(true)
 	// by default there are only 1 thread for flush compaction, which may block each other.
 	// increase to a reasonable thread numbers
@@ -179,7 +186,10 @@ func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error)
 	// finish rocks mq store initialization, rocks mq store has to set the prefix extractor
 	optsStore := gorocksdb.NewDefaultOptions()
 	// share block cache with kv
+	optsKV.SetNumLevels(3)
+	optsStore.SetCompressionPerLevel([]gorocksdb.CompressionType{0, 7, 7})
 	optsStore.SetBlockBasedTableFactory(bbto)
+	optsStore.SetTargetFileSizeMultiplier(2)
 	optsStore.SetCreateIfMissing(true)
 	// by default there are only 1 thread for flush compaction, which may block each other.
 	// increase to a reasonable thread numbers