milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use json benchmark format log in proxy 

Signed-off-by: shengjh <1572099106@qq.com>
pull/4973/head^2
shengjh 2020-09-29 10:40:43 +08:00 committed by yefu.chen
parent 26860c1041
commit 14d81f19cd
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
proxy/src/message_client/ClientV2.cpp
View File

@ -8,6 +8,7 @@
#include <numeric>
#include <algorithm>
#include <unistd.h>
#include "nlohmann/json.hpp"
#include "log/Log.h"
namespace milvus::message_client {
@ -222,6 +223,14 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
// auto r = getcwd(buff, 128);
auto path = std::string("/tmp");
std::ofstream file(path + "/proxy2pulsar.benchmark", std::fstream::app);
nlohmann::json json;
json["InsertTime"] = milvus::CommonUtil::TimeToString(start);
json["DurationInMilliseconds"] = std::chrono::duration_cast<std::chrono::milliseconds>(time_cost).count();
json["SizeInMB"] = size_inserted / 1024.0 / 1024.0;
json["ThroughputInMB"] = double(size_inserted) / std::chrono::duration_cast<std::chrono::milliseconds>(time_cost).count() * 1000 / 1024.0 / 1024;
json["NumRecords"] = num_inserted;
file << json.dump() << std::endl;
/*
file << "[" << milvus::CommonUtil::TimeToString(start) << "]"
<< " Insert " << num_inserted << " records, "
<< "size:" << size_inserted / 1024.0 / 1024.0 << "M, "
@ -230,6 +239,7 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
<< double(size_inserted) / std::chrono::duration_cast<std::chrono::milliseconds>(time_cost).count() * 1000 / 1024.0
/ 1024
<< "M/s" << std::endl;
*/
time_cost = stdclock::duration(0);
num_inserted = 0;
size_inserted = 0;

24
proxy/src/server/delivery/request/InsertReq.cpp
View File

@ -25,6 +25,7 @@
#include <vector>
#include <unistd.h>
#include "utils/CommonUtil.h"
#include "nlohmann/json.hpp"
#ifdef ENABLE_CPU_PROFILING
#include <gperftools/profiler.h>
@ -95,14 +96,21 @@ InsertReq::OnExecute() {
ready_log_records += inserted_count;
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() / 1000.0;
if (duration > interval) {
log << "[" << milvus::CommonUtil::TimeToString(start) << "] "
<< "Insert "
<< inserted_count << " records, "
<< "size: " << inserted_size / MB << "MB, "
<< "cost: " << duration << "s, "
<< "throughput: "
<< double(inserted_size) / duration / MB
<< "M/s\n";
nlohmann::json json;
json["InsertTime"] = milvus::CommonUtil::TimeToString(start);
json["DurationInMilliseconds"] = duration * 1000;
json["SizeInMB"] = inserted_size / MB;
json["ThroughputInMB"] = double(inserted_size) / duration / MB;
json["NumRecords"] = inserted_count;
file << json.dump() << std::endl;
// log << "[" << milvus::CommonUtil::TimeToString(start) << "] "
// << "Insert "
// << inserted_count << " records, "
// << "size: " << inserted_size / MB << "MB, "
// << "cost: " << duration << "s, "
// << "throughput: "
// << double(inserted_size) / duration / MB
// << "M/s\n";
auto new_flag = ready_log_records / per_log_records;
if (new_flag != log_flag) {
log_flag = new_flag;