print azure sdk log (#28241)

Signed-off-by: PowderLi <min.li@zilliz.com>

internal/core/src/storage/AzureChunkManager.cpp

@@ -16,6 +16,7 @@
 
 #include <iostream>
 #include <sstream>
+#include "common/Consts.h"
 #include "common/EasyAssert.h"
 #include "log/Log.h"
 #include "storage/AzureChunkManager.h"
@@ -26,9 +27,37 @@ namespace storage {
 std::atomic<size_t> AzureChunkManager::init_count_(0);
 std::mutex AzureChunkManager::client_mutex_;
 
+std::string const ErrorText = "ERROR";
+std::string const WarningText = "WARN ";
+std::string const InformationalText = "INFO ";
+std::string const VerboseText = "DEBUG";
+std::string const UnknownText = "?????";
+
+inline std::string const&
+LogLevelToConsoleString(Azure::Core::Diagnostics::Logger::Level logLevel) {
+    switch (logLevel) {
+        case Azure::Core::Diagnostics::Logger::Level::Error:
+            return ErrorText;
+
+        case Azure::Core::Diagnostics::Logger::Level::Warning:
+            return WarningText;
+
+        case Azure::Core::Diagnostics::Logger::Level::Informational:
+            return InformationalText;
+
+        case Azure::Core::Diagnostics::Logger::Level::Verbose:
+            return VerboseText;
+
+        default:
+            return UnknownText;
+    }
+}
+
 void
-AzureLogger(std::string const& level, std::string const& msg) {
-    LOG_SEGCORE_INFO_ << "[AZURE LOG] " << msg;
+AzureLogger(Azure::Core::Diagnostics::Logger::Level level,
+            std::string const& msg) {
+    LOG_SEGCORE_INFO_ << "[AZURE LOG] [" << LogLevelToConsoleString(level)
+                      << "] " << msg;
 }
 
 AzureChunkManager::AzureChunkManager(const StorageConfig& storage_config)
@@ -37,13 +66,16 @@ AzureChunkManager::AzureChunkManager(const StorageConfig& storage_config)
     std::scoped_lock lock{client_mutex_};
     const size_t initCount = init_count_++;
     if (initCount == 0) {
-        //        azure::AzureBlobChunkManager::InitLog(storage_config.log_level, AzureLogger);
+        azure::AzureBlobChunkManager::InitLog(storage_config.log_level,
+                                              AzureLogger);
     }
     client_ = std::make_shared<azure::AzureBlobChunkManager>(
         storage_config.access_key_id,
         storage_config.access_key_value,
         storage_config.address,
-        storage_config.requestTimeoutMs,
+        storage_config.requestTimeoutMs == 0
+            ? DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS
+            : storage_config.requestTimeoutMs,
         storage_config.useIAM);
 }
 

internal/core/src/storage/AzureChunkManager.h

@@ -41,6 +41,10 @@ ThrowAzureError(const std::string& func,
     throw SegcoreError(S3Error, oss.str());
 }
 
+void
+AzureLogger(Azure::Core::Diagnostics::Logger::Level level,
+            std::string const& msg);
+
 /**
  * @brief This AzureChunkManager is responsible for read and write file in blob.
    */

internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp

@@ -16,7 +16,6 @@
 
 #include <iostream>
 #include <sstream>
-#include <azure/core/diagnostics/logger.hpp>
 #include <azure/identity/workload_identity_credential.hpp>
 #include "AzureBlobChunkManager.h"
 
@@ -52,11 +51,10 @@ GetConnectionString(const std::string& access_key_id,
 void
 AzureBlobChunkManager::InitLog(
     std::string level_str,
-    std::function<void(std::string const& level, std::string const& message)>
-        listener) {
+    std::function<void(Azure::Core::Diagnostics::Logger::Level level,
+                       std::string const& message)> listener) {
     // SetListener accepts std::function<>, which can be either lambda or a function pointer.
-    Azure::Core::Diagnostics::Logger::SetListener(
-        [&](auto lvl, auto msg) { listener("info", msg); });
+    Azure::Core::Diagnostics::Logger::SetListener(listener);
     Azure::Core::Diagnostics::Logger::Level level =
         Azure::Core::Diagnostics::Logger::Level::Verbose;
     if (level_str == "fatal" || level_str == "error") {

internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.h

@@ -16,11 +16,12 @@
 
 #pragma once
 
-#include <azure/storage/blobs.hpp>
 #include <iostream>
 #include <stdlib.h>
 #include <string>
 #include <vector>
+#include <azure/core/diagnostics/logger.hpp>
+#include <azure/storage/blobs.hpp>
 #include <azure/storage/common/storage_exception.hpp>
 
 namespace azure {
@@ -31,7 +32,7 @@ class AzureBlobChunkManager {
  public:
     static void
     InitLog(std::string level_string,
-            std::function<void(std::string const& level,
+            std::function<void(Azure::Core::Diagnostics::Logger::Level level,
                                std::string const& message)> listener);
     explicit AzureBlobChunkManager(const std::string& access_key_id,
                                    const std::string& access_key_value,

internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt

@@ -3,5 +3,5 @@
 
 project(azure-blob-test)
 
-add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp)
-target_link_libraries(azure-blob-test PRIVATE blob-chunk-manager Azure::azure-identity Azure::azure-storage-blobs)
+add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp)
+target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs)

internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp

@@ -3,15 +3,17 @@
 using namespace azure;
 
 void
-print(std::string const& level, std::string const& message) {
-    std::cout << "level: " << level << ", message: " << message << std::endl;
+print(Azure::Core::Diagnostics::Logger::Level level,
+      std::string const& message) {
+    std::cout << "level: "
+              << ", message: " << message << std::endl;
 }
 
 int
 main() {
     const char* containerName = "default";
     const char* blobName = "sample-blob";
-    //    AzureBlobChunkManager::InitLog("info", print);
+    AzureBlobChunkManager::InitLog("info", print);
     AzureBlobChunkManager chunkManager = AzureBlobChunkManager("", "", "");
     std::vector<std::string> buckets = chunkManager.ListBuckets();
     std::cout << "list buckets." << std::endl;

internal/core/unittest/test_azure_chunk_manager.cpp

@@ -25,7 +25,9 @@ StorageConfig
 get_default_storage_config() {
     auto endpoint = "core.windows.net";
     auto accessKey = "devstoreaccount1";
-    auto accessValue = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+    auto accessValue =
+        "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
+        "K1SZFPTOtr/KBHBeksoGMGw==";
     auto rootPath = "files";
     auto useSSL = false;
     auto useIam = false;
@@ -66,12 +68,18 @@ class AzureChunkManagerTest : public testing::Test {
     StorageConfig configs_;
 };
 
+TEST_F(AzureChunkManagerTest, AzureLogger) {
+    AzureLogger(Azure::Core::Diagnostics::Logger::Level::Error, "");
+    AzureLogger(Azure::Core::Diagnostics::Logger::Level::Warning, "");
+    AzureLogger(Azure::Core::Diagnostics::Logger::Level::Informational, "");
+    AzureLogger(Azure::Core::Diagnostics::Logger::Level::Verbose, "");
+}
+
 TEST_F(AzureChunkManagerTest, BasicFunctions) {
     EXPECT_TRUE(chunk_manager_->GetName() == "AzureChunkManager");
     EXPECT_TRUE(chunk_manager_ptr_->GetName() == "AzureChunkManager");
     EXPECT_TRUE(chunk_manager_->GetRootPath() == "files");
 
-
     string path = "test";
     uint8_t readdata[20] = {0};
     try {
@@ -132,9 +140,9 @@ TEST_F(AzureChunkManagerTest, WritePositive) {
     string testBucketName = configs_.bucket_name;
     EXPECT_EQ(chunk_manager_->GetBucketName(), testBucketName);
 
-     if (!chunk_manager_->BucketExists(testBucketName)) {
-         chunk_manager_->CreateBucket(testBucketName);
-     }
+    if (!chunk_manager_->BucketExists(testBucketName)) {
+        chunk_manager_->CreateBucket(testBucketName);
+    }
     auto has_bucket = chunk_manager_->BucketExists(testBucketName);
     uint8_t data[5] = {0x17, 0x32, 0x45, 0x34, 0x23};
     string path = "1";

scripts/azure_build.sh

@@ -1,4 +1,6 @@
-while getopts "p:s:h" arg; do
+#!/usr/bin/env bash
+BUILD_UNITTEST="OFF"
+while getopts "p:s:t:h" arg; do
   case $arg in
   p)
     INSTALL_PREFIX=$OPTARG
@@ -6,15 +8,19 @@ while getopts "p:s:h" arg; do
   s)
     SOURCE_DIR=$OPTARG
     ;;
+  t)
+    BUILD_UNITTEST=$OPTARG
+    ;;
   h) # help
     echo "
 parameter:
 -p: install prefix
 -s: source directory
+-t: ON means build with unittest
 -h: help
 
 usage:
-./azure_build.sh -p \${INSTALL_PREFIX} -s \${SOURCE_DIR} [-h]
+./azure_build.sh -p \${INSTALL_PREFIX} -s \${SOURCE_DIR} [-t \${BUILD_UNITTEST}] [-h]
 "
     exit 0
     ;;
@@ -30,7 +36,7 @@ if [[ ${ARCHITECTURE} == "aarch64" ]]; then
   export VCPKG_FORCE_SYSTEM_BINARIES="arm"
 fi
 
-AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=on \
+AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
 ${SOURCE_DIR}"
 echo ${AZURE_CMAKE_CMD}

scripts/core_build.sh

@@ -184,7 +184,7 @@ if [ -z "$BUILD_WITHOUT_AZURE" ]; then
     mkdir -p ${AZURE_BUILD_DIR}
   fi
   pushd ${AZURE_BUILD_DIR}
-  env bash ${ROOT_DIR}/scripts/azure_build.sh -p ${INSTALL_PREFIX} -s ${ROOT_DIR}/internal/core/src/storage/azure-blob-storage
+  env bash ${ROOT_DIR}/scripts/azure_build.sh -p ${INSTALL_PREFIX} -s ${ROOT_DIR}/internal/core/src/storage/azure-blob-storage -t ${BUILD_UNITTEST}
   if [ ! -e libblob-chunk-manager* ]; then
     cat vcpkg-bootstrap.log
     exit 1