milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add segment/channel/task/slow query render (#37561) 

issue: #36621

Signed-off-by: jaime <yun.zhang@zilliz.com>
pull/37510/head
jaime 2024-11-12 17:44:29 +08:00 committed by GitHub
parent 2742e9573f
commit 1e8ea4a7e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
44 changed files with 2699 additions and 715 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/datacoord/compaction_task_meta.go
View File

@ -63,7 +63,7 @@ func newCompactionTaskMeta(ctx context.Context, catalog metastore.DataCoordCatal
ctx: ctx,
catalog: catalog,
compactionTasks: make(map[int64]map[int64]*datapb.CompactionTask, 0),
taskStats: expirable.NewLRU[UniqueID, *metricsinfo.CompactionTask](1024, nil, time.Minute*60),
taskStats: expirable.NewLRU[UniqueID, *metricsinfo.CompactionTask](32, nil, time.Minute*15),
}
if err := csm.reloadFromKV(); err != nil {
return nil, err
@ -178,10 +178,6 @@ func (csm *compactionTaskMeta) DropCompactionTask(task *datapb.CompactionTask) e
func (csm *compactionTaskMeta) TaskStatsJSON() string {
tasks := csm.taskStats.Values()
if len(tasks) == 0 {
return ""
}
ret, err := json.Marshal(tasks)
if err != nil {
return ""

2
internal/datacoord/compaction_task_meta_test.go
View File

@ -111,7 +111,7 @@ func (suite *CompactionTaskMetaSuite) TestTaskStatsJSON() {
// testing return empty string
actualJSON := suite.meta.TaskStatsJSON()
suite.Equal("", actualJSON)
suite.Equal("[]", actualJSON)
err := suite.meta.SaveCompactionTask(task1)
suite.NoError(err)

5
internal/datacoord/import_meta.go
View File

@ -52,7 +52,7 @@ type importTasks struct {
func newImportTasks() *importTasks {
return &importTasks{
tasks: make(map[int64]ImportTask),
taskStats: expirable.NewLRU[UniqueID, ImportTask](4096, nil, time.Minute*60),
taskStats: expirable.NewLRU[UniqueID, ImportTask](64, nil, time.Minute*30),
}
}
@ -301,9 +301,6 @@ func (m *importMeta) RemoveTask(taskID int64) error {
func (m *importMeta) TaskStatsJSON() string {
tasks := m.tasks.listTaskStats()
if len(tasks) == 0 {
return ""
}
ret, err := json.Marshal(tasks)
if err != nil {

2
internal/datacoord/import_meta_test.go
View File

@ -251,7 +251,7 @@ func TestTaskStatsJSON(t *testing.T) {
assert.NoError(t, err)
statsJSON := im.TaskStatsJSON()
assert.Equal(t, "", statsJSON)
assert.Equal(t, "[]", statsJSON)
task1 := &importTask{
ImportTaskV2: &datapb.ImportTaskV2{

4
internal/datacoord/import_task.go
View File

@ -185,7 +185,7 @@ func (p *preImportTask) MarshalJSON() ([]byte, error) {
NodeID: p.GetNodeID(),
State: p.GetState().String(),
Reason: p.GetReason(),
TaskType: "PreImportTask",
TaskType: p.GetType().String(),
CreatedTime: p.GetCreatedTime(),
CompleteTime: p.GetCompleteTime(),
}
@ -231,7 +231,7 @@ func (t *importTask) MarshalJSON() ([]byte, error) {
NodeID: t.GetNodeID(),
State: t.GetState().String(),
Reason: t.GetReason(),
TaskType: "ImportTask",
TaskType: t.GetType().String(),
CreatedTime: t.GetCreatedTime(),
CompleteTime: t.GetCompleteTime(),
}

6
internal/datacoord/index_meta.go
View File

@ -102,7 +102,7 @@ func newSegmentIndexBuildInfo() *segmentBuildInfo {
// build ID -> segment index
buildID2SegmentIndex: make(map[UniqueID]*model.SegmentIndex),
// build ID -> task stats
taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](1024, nil, time.Minute*60),
taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](64, nil, time.Minute*30),
}
}
@ -1075,10 +1075,6 @@ func (m *indexMeta) HasIndex(collectionID int64) bool {
func (m *indexMeta) TaskStatsJSON() string {
tasks := m.segmentBuildInfo.GetTaskStats()
if len(tasks) == 0 {
return ""
}
ret, err := json.Marshal(tasks)
if err != nil {
return ""

2
internal/datacoord/index_meta_test.go
View File

@ -1543,7 +1543,7 @@ func TestBuildIndexTaskStatsJSON(t *testing.T) {
}
actualJSON := im.TaskStatsJSON()
assert.Equal(t, "", actualJSON)
assert.Equal(t, "[]", actualJSON)
im.segmentBuildInfo.Add(si1)
im.segmentBuildInfo.Add(si2)

2
internal/datacoord/job_manager_test.go
View File

@ -104,7 +104,7 @@ func (s *jobManagerSuite) TestJobManager_triggerStatsTaskLoop() {
allocator: alloc,
tasks: make(map[int64]Task),
meta: mt,
taskStats: expirable.NewLRU[UniqueID, Task](1024, nil, time.Minute*5),
taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*5),
},
allocator: alloc,
}

6
internal/datacoord/metrics_info_test.go
View File

@ -325,7 +325,7 @@ func TestGetSyncTaskMetrics(t *testing.T) {
mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)})
svr.cluster = mockCluster
expectedJSON := ""
expectedJSON := "null"
actualJSON, err := svr.getSyncTaskJSON(ctx, req)
assert.NoError(t, err)
assert.Equal(t, expectedJSON, actualJSON)
@ -449,7 +449,7 @@ func TestGetSegmentsJSON(t *testing.T) {
mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)})
svr.cluster = mockCluster
expectedJSON := ""
expectedJSON := "null"
actualJSON, err := svr.getSegmentsJSON(ctx, req)
assert.NoError(t, err)
assert.Equal(t, expectedJSON, actualJSON)
@ -591,7 +591,7 @@ func TestGetChannelsJSON(t *testing.T) {
svr.cluster = mockCluster
svr.meta = &meta{channelCPs: newChannelCps()}
expectedJSON := ""
expectedJSON := "null"
actualJSON, err := svr.getChannelsJSON(ctx, req)
assert.NoError(t, err)
assert.Equal(t, expectedJSON, actualJSON)

2
internal/datacoord/task_scheduler.go
View File

@ -91,7 +91,7 @@ func newTaskScheduler(
handler: handler,
indexEngineVersionManager: indexEngineVersionManager,
allocator: allocator,
taskStats: expirable.NewLRU[UniqueID, Task](1024, nil, time.Minute*5),
taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*15),
}
ts.reloadFromMeta()
return ts

21
internal/distributed/proxy/httpserver/handler.go
View File

@ -27,6 +27,9 @@ func (h *Handlers) RegisterRoutesTo(router gin.IRouter) {
router.GET("/health", wrapHandler(h.handleGetHealth))
router.POST("/dummy", wrapHandler(h.handleDummy))
router.GET("/databases", wrapHandler(h.handleListDatabases))
router.GET("/database", wrapHandler(h.handleDescribeDatabases))
router.POST("/collection", wrapHandler(h.handleCreateCollection))
router.DELETE("/collection", wrapHandler(h.handleDropCollection))
router.GET("/collection/existence", wrapHandler(h.handleHasCollection))
@ -96,6 +99,24 @@ func (h *Handlers) handleDummy(c *gin.Context) (interface{}, error) {
return h.proxy.Dummy(c, &req)
}
func (h *Handlers) handleListDatabases(c *gin.Context) (interface{}, error) {
req := milvuspb.ListDatabasesRequest{}
err := shouldBind(c, &req)
if err != nil {
return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err)
}
return h.proxy.ListDatabases(c, &req)
}
func (h *Handlers) handleDescribeDatabases(c *gin.Context) (interface{}, error) {
req := milvuspb.DescribeDatabaseRequest{}
err := shouldBind(c, &req)
if err != nil {
return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err)
}
return h.proxy.DescribeDatabase(c, &req)
}
func (h *Handlers) handleCreateCollection(c *gin.Context) (interface{}, error) {
wrappedReq := WrappedCreateCollectionRequest{}
err := shouldBind(c, &wrappedReq)

2
internal/flushcommon/syncmgr/sync_manager.go
View File

@ -70,7 +70,7 @@ func NewSyncManager(chunkManager storage.ChunkManager) SyncManager {
keyLockDispatcher: dispatcher,
chunkManager: chunkManager,
tasks: typeutil.NewConcurrentMap[string, Task](),
taskStats: expirable.NewLRU[string, Task](512, nil, time.Minute*15),
taskStats: expirable.NewLRU[string, Task](16, nil, time.Minute*15),
}
// setup config update watcher
params.Watch(params.DataNodeCfg.MaxParallelSyncMgrTasks.Key, config.NewHandler("datanode.syncmgr.poolsize", syncMgr.resizeHandler))

2
internal/http/router.go
View File

@ -75,6 +75,8 @@ const (
ClusterDependenciesPath = "/_cluster/dependencies"
// HookConfigsPath is the path to get hook configurations.
HookConfigsPath = "/_hook/configs"
// SlowQueryPath is the path to get slow queries metrics
SlowQueryPath = "/_cluster/slow_query"
// QCDistPath is the path to get QueryCoord distribution.
QCDistPath = "/_qc/dist"

158
internal/http/webui/channels.html
View File

@ -1,158 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Channels</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h2>
Channel Checkpoints
</h2>
<table id="channelCP" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Checkpoint Ts</th>
<th scope="col">Checkpoint Offset</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>channel1</td>
<td>11</td>
<td>2022-11-11 12:00:00</td>
<td>{ledgerID:1, entryID:1, batchIdx:0}</td>
<td>datanode1</td>
</tr>
<tr>
<td>channel2</td>
<td>22</td>
<td>2022-11-11 12:00:00</td>
<td>{ledgerID:2, entryID:2, batchIdx:0}</td>
<td>datanode1</td>
</tr>
</tbody>
</table>
<h2>
Watched Channels On Datanode
</h2>
<table id="watchedChDataNode" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Consume Rate/s</th>
<th scope="col">Latency</th>
<th scope="col">TimeTickLag</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>channel1</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watching</td>
<td>datanode1</td>
</tr>
<tr>
<td>channel1</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watched</td>
<td>datanode2</td>
</tr>
</tbody>
</table>
<h2>
Watched Channels On QueryNode
</h2>
<table id="watchedChQueryNode" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Consume Rate/s</th>
<!-- consume latency-->
<th scope="col">Latency</th>
<th scope="col">TimeTickLag</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>channel1</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watching</td>
<td>querynode1</td>
</tr>
<tr>
<td>channel2</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watching</td>
<td>querynode2</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
// load cluster information data
document.addEventListener("DOMContentLoaded", function() {
fetch(MILVUS_URI + "/")
.then(text => {
//TODO add channel render
throw new Error("Unimplemented API")
})
.catch(error => {
handleError(error);
});
});
</script>
</body>
</html>

159
internal/http/webui/collections.html
View File

@ -23,110 +23,52 @@
</div>
<div class="col-md-8">
<h2>
Database List
Database
</h2>
<table id="database" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Database ID</th>
<th scope="col">Name</th>
<th scope="col">Create Time</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>db1</td>
<td>2022-11-11 12:00:00</td>
</tr>
<tr>
<td>2</td>
<td>db2</td>
<td>2022-11-11 12:00:00</td>
</tr>
</tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<p id="db_totalCount"></p>
<p id="dbPaginationControls"></p>
</div>
<h2>
Collection List
Collection
</h2>
<table id="collectionMeta" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Collection ID</th>
<th scope="col">Collection Name</th>
<th scope="col">Partition Key Count</th>
<th scope="col">Partition Count</th>
<th scope="col">Channel Count</th>
<th scope="col">Segment Count</th>
<th scope="col">Binlog Count</th>
<th scope="col">Size</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>db1.coll1-fake</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>1</td>
</tr>
<tr>
<td>2</td>
<td>db1.coll2-fake</td>
<td>2</td>
<td>2</td>
<td>2</td>
<td>2</td>
<td>2</td>
<td>2</td>
</tr>
</tbody>
</table>
<!-- Navigation Tabs -->
<ul class="nav nav-tabs" id="componentTabs" role="tablist">
<li class="nav-item">
<a class="nav-link active" id="coll-base-stats-tab" data-toggle="tab" href="#coll-base-stats" role="tab" aria-controls="coll-base-stats" aria-selected="true">Base </a>
</li>
<li class="nav-item">
<a class="nav-link" id="coll-request-tab" data-toggle="tab" href="#coll-request-metrics" role="tab" aria-controls="coll-request-metrics" aria-selected="false">Requests</a>
</li>
<h2>
Collection Metrics
</h2>
<table id="collectionMetrics" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Collection Name</th>
<th scope="col">isQueryable</th>
<th scope="col">isWritable</th>
<th scope="col">Query Ops/s</th>
<th scope="col">Search Ops/s</th>
<th scope="col">Insert Throughput(MB/s)</th>
<th scope="col">Delete Throughput(MB/s)</th>
</tr>
</thead>
<tbody>
<tr>
<td>db1.coll-fake</td>
<td>true</td>
<td>true</td>
<td>20</td>
<td>20</td>
<td>1</td>
<td>0.5</td>
</tr>
<tr>
<td>db1.col2-fake</td>
<td>true</td>
<td>true</td>
<td>20</td>
<td>20</td>
<td>1</td>
<td>0.5</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-2">
<!-- Search Input -->
<li class="nav-item ml-auto">
<input type="text" class="form-control" placeholder="Search database..." id="databaseSearch" oninput="searchCollections()">
</li>
</ul>
<!-- Tab Content -->
<div class="tab-content" id="componentTabsContent">
<div class="tab-pane fade show active" id="coll-base-stats" role="tabpanel" aria-labelledby="coll-base-stats-tab">
<table id="collectionList" class="table table-hover"></table>
</div>
<div class="tab-pane fade" id="coll-request-metrics" role="tabpanel" aria-labelledby="coll-request-tab">
<table id="collectionRequests" class="table table-hover mt-3"></table>
</div>
</div>
<div style="display: flex; justify-content: space-between;">
<p id="collection_totalCount"></p>
<p id="collectionPaginationControls"></p>
</div>
</div>
<div class="col-md-2"></div>
</div>
<div id="footer"></div>
</div>
@ -137,12 +79,33 @@
$('#footer').load("footer.html");
});
fetchData(MILVUS_URI + "/", clientInfos)
function searchCollections() {
const searchTerm = document.getElementById('databaseSearch').value;
let dbName = 'default';
if (searchTerm !== '') {
dbName = searchTerm;
}
fetchCollections(dbName);
}
searchCollections()
// TODO - Implement the following functions and support search with db name
// fetchData(MILVUS_URI + "/_collection/metrics", collectionRequest)
// .then(data => {
// collectionRequestsData = data;
// renderCollectionRequests(startPage, paginationSize);
// })
// .catch(error => {
// handleError(error);
// });
fetchData(MILVUS_URI + "/databases", databases)
.then(data => {
//TODO add collection render
databaseData = data;
renderDatabases(startPage, paginationSize)
})
.catch(error => {
handleError(new Error("Unimplemented API"));
handleError(error);
});
</script>
</body>

11
internal/http/webui/configs.html
View File

@ -24,7 +24,12 @@
<h2>
Milvus Configurations
</h2>
<div>
<input type="text" id="searchInput" placeholder="Search..." oninput="searchConfigs()" class="form-control mb-3" />
</div>
<table id="mConfig" class="table table-hover"></table>
<div id="paginationControls"></div>
<h2>
Hook Configurations
@ -41,6 +46,12 @@
$('#footer').load("footer.html");
});
function searchConfigs() {
const searchTerm = document.getElementById('searchInput').value;
currentPage = 0; // Reset to the first page on new search
renderConfigs(configData, searchTerm);
}
// load cluster configurations
fetchData(MILVUS_URI + '/_cluster/configs', mconfigs)
.then(data => {

90
internal/http/webui/data_component.html Normal file
View File

@ -0,0 +1,90 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Data Component</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h2>Data Channels</h2>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Watch State</th>
<th scope="col">Node ID</th>
<th scope="col">Latest Time Tick</th>
<th scope="col">Start Watch Ts</th>
<th scope="col">Checkpoint Ts</th>
</tr>
</thead>
<tbody id="dataChannelsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<p id="dchannel_totalCount"></p>
<div id="dchannelPaginationControls"></div>
</div>
<!-- Notifications Container -->
<div id="notificationsDChannels"></div>
<h2>Data Segments</h2>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Segment ID</th>
<th scope="col">Collection ID</th>
<th scope="col">Partition ID</th>
<th scope="col">Channel</th>
<th scope="col">Num of Rows</th>
<th scope="col">State</th>
<th scope="col">Level</th>
</tr>
</thead>
<tbody id="dataSegmentsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<p id="dsegment_totalCount"></p>
<div id="dsegmentPaginationControls"></div>
</div>
<!-- Notifications Container -->
<div id="notificationsDSegments"></div>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
fetchAndRenderDataChannels(startPage, paginationSize);
fetchAndRenderDataSegments(dataSegmentsStartPage, dataSegmentsPaginationSize);
</script>
</body>
</html>

10
internal/http/webui/header.html
View File

@ -11,17 +11,23 @@
<a class="nav-link" href="collections.html">Collections</a>
</li>
<li class="nav-item">
<a class="nav-link" href="segments.html">Segments</a>
<a class="nav-link" href="query_component.html">Query</a>
</li>
<li class="nav-item">
<a class="nav-link" href="channels.html">Channels</a>
<a class="nav-link" href="data_component.html">Data</a>
</li>
<li class="nav-item">
<a class="nav-link" href="tasks.html">Tasks</a>
</li>
<li class="nav-item">
<a class="nav-link " href="slow_requests.html">Slow Requests</a>
</li>
<li class="nav-item">
<a class="nav-link " href="configs.html">Configurations</a>
</li>
<li class="nav-item">
<a class="nav-link " href="tools.html">Tools</a>
</li>
</ul>
</div>
</nav>

18
internal/http/webui/index.html
View File

@ -39,6 +39,9 @@
<li class="nav-item">
<a class="nav-link" id="runtime-metrics-tab" data-toggle="tab" href="#runtime-metrics" role="tab" aria-controls="runtime-metrics" aria-selected="false">Runtime Metrics</a>
</li>
<li class="nav-item">
<a class="nav-link" id="request-tab" data-toggle="tab" href="#request-metrics" role="tab" aria-controls="request-metrics" aria-selected="false">Requests</a>
</li>
</ul>
<!-- Tab Content -->
@ -52,7 +55,12 @@
<div class="tab-pane fade" id="runtime-metrics" role="tabpanel" aria-labelledby="runtime-metrics-tab">
<table id="nodeMetrics" class="table table-hover mt-3"></table>
</div>
</div>
<!-- Node Request Table -->
<div class="tab-pane fade" id="request-metrics" role="tabpanel" aria-labelledby="request-tab">
<table id="nodeRequests" class="table table-hover mt-3"></table>
</div>
</div>
<h2>
Connected Clients
@ -86,6 +94,14 @@
handleError(error);
});
// fetchData(MILVUS_URI + "/_node/requests", nodeRequests)
// .then(data => {
// renderNodeRequests(data)
// })
// .catch(error => {
// handleError(error);
// });
fetchData(MILVUS_URI + "/_cluster/clients", clientInfos)
.then(data => {
renderClientsInfo(data);

63
internal/http/webui/query_component.html Normal file
View File

@ -0,0 +1,63 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Query Component</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h3>Segments</h3>
<table id="querySegmentsTable" class="table table-bordered"></table>
<div style="display: flex; justify-content: space-between;">
<div id="querySegmentsTotalCount"></div>
<div id="querySegmentsPagination"></div>
</div>
<h3>Channels</h3>
<table id="queryChannelsTable" class="table table-bordered"></table>
<div style="display: flex; justify-content: space-between;">
<div id="queryChannelsTotalCount"></div>
<div id="queryChannelsPagination"></div>
</div>
<h3>Replicas</h3>
<table id="replicasTable" class="table table-bordered"></table>
<h3>Resource Groups</h3>
<table id="resourceGroupTable" class="table table-bordered"></table>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
fetchAndRenderQuerySegmentsAndChannels();
fetchAndRenderReplicas();
fetchAndRenderResourceGroup()
</script>
</body>
</html>

86
internal/http/webui/query_target.html Normal file
View File

@ -0,0 +1,86 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Query Component</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h3>Target Segments</h3>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Segment ID</th>
<th scope="col">Collection ID</th>
<th scope="col">Partition ID</th>
<th scope="col">Channel</th>
<th scope="col">Num of Rows</th>
<th scope="col">State</th>
<th scope="col">Target Scope</th>
</tr>
</thead>
<tbody id="dataSegmentsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<div id="segment_totalCount"></div>
<div id="segmentPaginationControls"></div>
</div>
<h3>Target Channels</h3>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Node ID</th>
<th scope="col">Version</th>
<th scope="col">Unflushed Segments</th>
<th scope="col">Flushed Segments</th>
<th scope="col">Dropped Segments</th>
<th scope="col">Target Scope</th>
</tr>
</thead>
<tbody id="dataChannelsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<div id="channel_totalCount"></div>
<div id="channelPaginationControls"></div>
</div>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
let targetTableStartPage = 0
let pageSize = 10
fetchAndRenderTargets(targetTableStartPage, pageSize)
</script>
</body>
</html>

196
internal/http/webui/segments.html
View File

@ -1,196 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Segments</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h2>
Loading Segments
</h2>
<!-- TODO: -->
<table id="loadingSegments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">isIndexed</th>
<th scope="col">segmentSize</th>
<th scope="col">queryNode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode1</td>
</tr>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode2</td>
</tr>
</tbody>
</table>
<h2>
Releasing Segments
</h2>
<!-- TODO: -->
<table id="releasingSegments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">isIndexed</th>
<th scope="col">segmentSize</th>
<th scope="col">queryNode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode1</td>
</tr>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode2</td>
</tr>
</tbody>
</table>
<h2>
Loaded Segments
</h2>
<!-- TODO: -->
<table id="loadedSegments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">isIndexed</th>
<th scope="col">segmentSize</th>
<th scope="col">queryNode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode1</td>
</tr>
<tr>
<td>111</td>
<td>coll1</td>
<td>11</td>
<td>6</td>
<td>faked-querynode2</td>
</tr>
</tbody>
</table>
<h2>
All Segments
</h2>
<!-- TODO: data from data component -->
<table id="segments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">state</th>
<th scope="col">rowCount</th>
<th scope="col">binlog count</th>
<th scope="col">binlogs size</th>
<th scope="col">statslogs size</th>
<th scope="col">dmlChannel</th>
<th scope="col">level</th>
<th scope="col">datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>Flushing</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>faked-channel-1</td>
<td>L0</td>
<td>faked-datanode1</td>
</tr>
<tr>
<td>22222</td>
<td>coll1</td>
<td>Flushing</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>faked-channel-2</td>
<td>L0</td>
<td>faked-datanode2</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
document.addEventListener("DOMContentLoaded", function() {
fetchData(MILVUS_URI + "/", sysmetrics)
.then(data => {
//TODO add segment render
})
.catch(error => {
handleError(new Error("Unimplemented API"));
});
});
</script>
</body>
</html>

58
internal/http/webui/slow_requests.html Normal file
View File

@ -0,0 +1,58 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Slow Requests</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<div class="alert alert-warning alert-dismissible fade show" role="alert">
<strong>Notice:</strong> Slow request in the last 5 minutes.
<!-- <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>-->
</div>
<table id="slowQueries" class="table table-bordered table-hover table-auto">
</table>
<p id="slowQueriesTotalCount"></p>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
document.addEventListener("DOMContentLoaded", function() {
fetchData(MILVUS_URI + "/_cluster/slow_query", slowQueries)
.then(data => {
renderSlowQueries(data);
})
.catch(error => {
handleError(error);
});
});
</script>
</body>
</html>

6
internal/http/webui/static/css/style.css
View File

@ -31,4 +31,10 @@
height: 1px;
background-color: #000;
margin: 40px 0;
}
.table td.fit,
.table th.fit {
white-space: nowrap;
width: 1%;
}

35
internal/http/webui/static/js/common.js
View File

@ -27,18 +27,29 @@ toggleDebugMode();
const handleError = (error) => {
console.error('Error fetching data:', error);
const errorMessage = encodeURIComponent(error.message || 'Unknown error');
window.location.href = `5xx.html?error=${errorMessage}`;
// const errorMessage = encodeURIComponent(error.message || 'Unknown error');
// window.location.href = `5xx.html?error=${errorMessage}`;
};
const fetchData = (url, localData) => {
const fetchData = (url, localData, kvParams) => {
if (DEBUG_MODE) {
return new Promise((resolve) => {
resolve(JSON.parse(localData));
});
} else if (kvParams && kvParams.length !== 0) {
return fetch(url, {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json',
},
mode: 'no-cors',
body: JSON.stringify(kvParams)
}).then(response => response.json())
} else {
return fetch(url)
.then(response => response.json())
return fetch(url).then(response => {
return response.json();
})
}
};
@ -51,4 +62,18 @@ function getQueryParams() {
params[decodeURIComponent(key)] = decodeURIComponent(value || '');
});
return params;
}
function formatTimestamp(timestamp) {
const date = new Date(timestamp); // Convert timestamp to a Date object
// Format the date components
const year = date.getFullYear();
const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are zero-indexed
const day = ('0' + date.getDate()).slice(-2);
const hours = ('0' + date.getHours()).slice(-2);
const minutes = ('0' + date.getMinutes()).slice(-2);
const seconds = ('0' + date.getSeconds()).slice(-2);
// Return formatted date string
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`
}

571
internal/http/webui/static/js/mockdata.js
View File

@ -1,4 +1,4 @@
var sysmetrics = `{
const sysmetrics = `{
"nodes_info": [
{
"identifier": 1,
@ -341,7 +341,40 @@ var sysmetrics = `{
]
}`
var clientInfos = `[
const nodeRequests = `
[
{
"node_name": "querynode1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
},
{
"node_name": "datanode1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
},
{
"node_name": "indexnode1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
},
{
"node_name": "proxy1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
}
]
`
const clientInfos = `[
{
"sdk_type": "python",
"sdk_version": "1.0.0",
@ -364,7 +397,7 @@ var clientInfos = `[
}
]`
var dependencies = `
const dependencies = `
{
"metastore": {
"health_status": true,
@ -386,7 +419,7 @@ var dependencies = `
}
`
var mconfigs = `
const mconfigs = `
{
"MILVUS_GIT_BUILD_TAGS": "v2.2-testing-20240702-811-g38211f2b81-dev",
"MILVUS_GIT_COMMIT": "38211f2b81",
@ -412,7 +445,223 @@ var mconfigs = `
}
`;
var qcTargets = `
const collections =`
{
"status": {
"error_code": "Success",
"reason": ""
},
"collection_names": [
"collection1",
"collection2",
"collection3",
"collection4",
"collection5",
"collection6",
"collection7",
"collection8",
"collection9",
"collection10"
],
"collection_ids": [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10
],
"created_timestamps": [
1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400
],
"created_utc_timestamps": [
1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400
],
"inMemory_percentages": [
100, 90, 80, 70, 60, 50, 40, 30, 20, 10
],
"query_service_available": [
true, false, false, false, false, false, false, false, false, false
]
}
`
const collectionRequest = `
[
{
"collection_name": "collection1",
"search_QPS": 10,
"query_QPS": 5,
"write_throughput": 20,
"delete_QPS": 2
},
{
"collection_name": "collection2",
"search_QPS": 15,
"query_QPS": 7,
"write_throughput": 25,
"delete_QPS": 3
},
{
"collection_name": "collection3",
"search_QPS": 20,
"query_QPS": 10,
"write_throughput": 30,
"delete_QPS": 4
},
{
"collection_name": "collection4",
"search_QPS": 25,
"query_QPS": 12,
"write_throughput": 35,
"delete_QPS": 5
},
{
"collection_name": "collection5",
"search_QPS": 30,
"query_QPS": 15,
"write_throughput": 40,
"delete_QPS": 6
},
{
"collection_name": "collection6",
"search_QPS": 35,
"query_QPS": 17,
"write_throughput": 45,
"delete_QPS": 7
},
{
"collection_name": "collection7",
"search_QPS": 40,
"query_QPS": 20,
"write_throughput": 50,
"delete_QPS": 8
},
{
"collection_name": "collection8",
"search_QPS": 45,
"query_QPS": 22,
"write_throughput": 55,
"delete_QPS": 9
},
{
"collection_name": "collection9",
"search_QPS": 50,
"query_QPS": 25,
"write_throughput": 60,
"delete_QPS": 10
},
{
"collection_name": "collection10",
"search_QPS": 55,
"query_QPS": 27,
"write_throughput": 65,
"delete_QPS": 11
}
]
`
const describeCollectionResp = `
{
"status": {
"error_code": 0,
"reason": "Success"
},
"schema": {
"name": "example_collection",
"description": "This is an example collection schema",
"fields": [
{
"name": "field1",
"data_type": "INT64",
"is_primary_key": true,
"auto_id": false
},
{
"name": "field2",
"data_type": "FLOAT",
"is_primary_key": false,
"auto_id": false
}
]
},
"collectionID": 12345,
"virtual_channel_names": ["vchan1", "vchan2"],
"physical_channel_names": ["pchan1", "pchan2"],
"created_timestamp": 1633036800,
"created_utc_timestamp": 1633036800,
"shards_num": 2,
"aliases": ["alias1", "alias2"],
"start_positions": [
{
"key": "start_key",
"data": "start_data"
}
],
"consistency_level": 0,
"collection_name": "example_collection",
"properties": [
{
"key": "property_key",
"value": "property_value"
}
],
"db_name": "example_db",
"num_partitions": 1,
"db_id": 1
}
`
const databases = `
{
"status": {
"error_code": "Success",
"reason": ""
},
"db_names": [
"database_1",
"database_2",
"database_3",
"database_4",
"database_5",
"database_6",
"database_7",
"database_8",
"database_9",
"database_10"
],
"created_timestamp": [
1633036800,
1633123200,
1633209600,
1633296000,
1633382400,
1633468800,
1633555200,
1633641600,
1633728000,
1633814400
],
"db_ids": [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10
]
}
`
const describeDatabaseResp = `
{
"status": {
"error_code": 0,
"reason": "Success"
},
"db_name": "example_db",
"dbID": 1,
"created_timestamp": 1633036800,
"properties": [
{
"key": "property_key",
"value": "property_value"
}
]
}
`
const qcCurrentTargets = `
[
{
"collection_id": 1,
@ -442,7 +691,7 @@ var qcTargets = `
],
"resource_group": "rg1",
"loaded_insert_row_count": 1000,
"mem_size": 2048,
"mem_size": 2048
}
],
"dm_channels": [
@ -451,10 +700,18 @@ var qcTargets = `
"version": 1,
"collection_id": 1,
"channel_name": "channel1",
"unflushed_segment_ids": [1],
"flushed_segment_ids": [2],
"dropped_segment_ids": [3],
"level_zero_segment_ids": [4],
"unflushed_segment_ids": [
1
],
"flushed_segment_ids": [
2
],
"dropped_segment_ids": [
3
],
"level_zero_segment_ids": [
4
],
"partition_stats_versions": {
"1": 1
}
@ -464,7 +721,44 @@ var qcTargets = `
]
`
var qcDist =`
const qcNextTargets = `
[
{
"collection_id": 1,
"segments": [
{
"segment_id": 2,
"collection_id": 1,
"partition_id": 1,
"channel": "channel2",
"num_of_rows": 1000,
"state": "Sealed",
"is_importing": false,
"compacted": false,
"level": "L0",
"is_sorted": true,
"node_id": 1,
"is_invisible": false,
"loaded_timestamp": 1633072800,
"index": [
{
"field_id": 1,
"index_id": 1,
"build_id": 1,
"index_size": 1024,
"is_loaded": true
}
],
"resource_group": "rg1",
"loaded_insert_row_count": 1000,
"mem_size": 2048
}
]
}
]
`;
const qcDist = `
{
"segments": [
{
@ -493,6 +787,9 @@ var qcDist =`
"resource_group": "rg1",
"loaded_insert_row_count": 1000,
"mem_size": 2048,
"flushed_rows": 1000,
"sync_buffer_rows": 0,
"syncing_rows": 0
}
],
"dm_channels": [
@ -507,18 +804,24 @@ var qcDist =`
"level_zero_segment_ids": [4],
"partition_stats_versions": {
"1": 1
}
},
"watch_state": "Healthy",
"start_watch_ts": 1633072800
}
],
"leader_views": [
{
"node_id": 1,
"leader_id": 1,
"collection_id": 1,
"channel_name": "channel1",
"segments": [
"node_id": 1,
"channel": "channel1",
"version": 1,
"sealed_segments": [
{
"segment_id": 1,
"collection_id": 1,
"partition_id": 1,
"channel": "channel1",
"num_of_rows": 1000,
"state": "Sealed",
"is_importing": false,
@ -540,52 +843,63 @@ var qcDist =`
"resource_group": "rg1",
"loaded_insert_row_count": 1000,
"mem_size": 2048,
"flushed_rows": 1000,
"sync_buffer_rows": 0,
"syncing_rows": 0
}
]
],
"growing_segments": [],
"target_version": 1,
"num_of_growing_rows": 0,
"unserviceable_error": ""
}
]
}
`
var qcReplica = `
const qcReplica = `
[
{
"ID": 1,
"CollectionID": 1,
"RWNodes": [1, 2],
"ResourceGroup": "rg1",
"RONodes": [3],
"ChannelToRWNodes": {
"rw_nodes": [1, 2],
"resource_group": "rg1",
"ro_nodes": [3],
"channel_to_rw_nodes": {
"channel1": [1, 2]
}
},
{
"ID": 2,
"CollectionID": 2,
"RWNodes": [4, 5],
"ResourceGroup": "rg2",
"RONodes": [6],
"ChannelToRWNodes": {
"rw_nodes": [4, 5],
"resource_group": "rg2",
"ro_nodes": [6],
"channel_to_rw_nodes": {
"channel2": [4, 5]
}
}
]
`
`;
var qcResourceGroup = `
const qcResourceGroup = `
[
{
"Name": "rg1",
"Nodes": [1, 2]
"name": "rg1",
"nodes": [1, 2],
"cfg": {
"requests":{},
"limits":{"node_num":1000000}
}
},
{
"Name": "rg2",
"Nodes": [3, 4]
"name": "rg2",
"nodes": [3, 4]
}
]
`
`;
var qcTasks = `
const qcTasks = `
[
{
"task_name": "balance_checker-ChannelTask[1]-ch1",
@ -598,7 +912,7 @@ var qcTasks = `
"type:Grow node id : 1 channel name:channel_1"
],
"step": 1,
"reason": "some reason"
"reason": ""
},
{
"task_name": "index_checker-SegmentTask[2]-54321",
@ -611,7 +925,7 @@ var qcTasks = `
"type:Grow node id: 2 segment id:123 scope:DataScope_Streaming"
],
"step": 2,
"reason": "another reason"
"reason": ""
},
{
"task_name": "leader_checker-LeaderSegmentTask[3]-1",
@ -627,9 +941,9 @@ var qcTasks = `
"reason": "yet another reason"
}
]
`
`;
var qn_segments = `
const qnSegments = `
[
{
"segment_id": 1,
@ -656,7 +970,7 @@ var qn_segments = `
],
"resource_group": "rg1",
"loaded_insert_row_count": 1000,
"mem_size": 2048,
"mem_size": 2048
},
{
"segment_id": 2,
@ -683,12 +997,12 @@ var qn_segments = `
],
"resource_group": "rg2",
"loaded_insert_row_count": 2000,
"mem_size": 4096,
"mem_size": 4096
}
]
`
`;
var qn_channels = `
const qnChannels = `
[
{
"name": "channel1",
@ -696,7 +1010,7 @@ var qn_channels = `
"assign_state": "assigned",
"latest_time_tick": "2023-10-01 12:00:00",
"node_id": 1,
"collection_id": 1,
"collection_id": 1
},
{
"name": "channel2",
@ -704,26 +1018,39 @@ var qn_channels = `
"assign_state": "assigned",
"latest_time_tick": "2023-10-01 12:05:00",
"node_id": 2,
"collection_id": 2,
"collection_id": 2
}
]
`
`;
var dc_dist = `
const dc_dist = `
{
"segments": [
{
"segment_id": 1,
"collection_id": 100,
"partition_id": 10,
"collection_id": 1,
"partition_id": 1,
"channel": "channel1",
"num_of_rows": 1000,
"state": "flushed",
"state": "Growing",
"is_importing": false,
"compacted": false,
"level": "L1",
"is_sorted": true,
"node_id": 1
},
{
"segment_id": 3,
"collection_id": 2,
"partition_id": 2,
"channel": "channel2",
"num_of_rows": 2000,
"state": "Growing",
"is_importing": true,
"compacted": true,
"level": "L2",
"is_sorted": false,
"node_id": 2
}
],
"dm_channels": [
@ -737,12 +1064,23 @@ var dc_dist = `
"dropped_segment_ids": [7, 8, 9],
"watch_state": "success",
"start_watch_ts": 123456789
}
},
{
"node_id": 1,
"version": 1,
"collection_id": 100,
"channel_name": "channel3",
"unflushed_segment_ids": [1, 2, 3],
"flushed_segment_ids": [4, 5, 6],
"dropped_segment_ids": [7, 8, 9],
"watch_state": "to_watch",
"start_watch_ts": 123456789
}
]
}
`
`;
var dc_build_index_task = `
const dc_build_index_task = `
[
{
"index_id": 1,
@ -767,7 +1105,7 @@ var dc_build_index_task = `
}
]`
var dc_compaction_task = `
const dc_compaction_task = `
[
{
"plan_id": 1,
@ -795,7 +1133,7 @@ var dc_compaction_task = `
}
]`
var dn_sync_task = `
const dn_sync_task = `
[
{
"segment_id": 1,
@ -822,7 +1160,7 @@ var dn_sync_task = `
]
`
var dc_import_task = `
const dc_import_task = `
[
{
"job_id": 1,
@ -840,9 +1178,9 @@ var dc_import_task = `
"task_id": 6,
"collection_id": 7,
"node_id": 8,
"state": "ImportTaskStateCompleted",
"state": "Completed",
"reason": "",
"task_type": "Completed",
"task_type": "ImportTask",
"created_time": "2023-10-01T00:00:00Z",
"complete_time": "2023-10-01T01:00:00Z"
},
@ -860,7 +1198,7 @@ var dc_import_task = `
]
`
var dn_segments = `
const dn_segments = `
[
{
"segment_id": 1,
@ -868,7 +1206,7 @@ var dn_segments = `
"partition_id": 1,
"channel": "channel1",
"num_of_rows": 1000,
"state": "active",
"state": "Growing",
"is_importing": false,
"compacted": false,
"level": "L1",
@ -884,7 +1222,7 @@ var dn_segments = `
"partition_id": 2,
"channel": "channel2",
"num_of_rows": 2000,
"state": "inactive",
"state": "Sealed",
"is_importing": true,
"compacted": true,
"level": "L2",
@ -897,7 +1235,7 @@ var dn_segments = `
]
`
var dn_channels = `
const dn_channels = `
[
{
"name": "channel1",
@ -918,4 +1256,115 @@ var dn_channels = `
"check_point_ts": "2023-10-01 12:05:00"
}
]
`
`
const slowQueries = `[
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.1s",
"user": "test_user",
"query_params": {
"search_params": [
{
"dsl": ["dsl1"],
"search_params": ["param2=value2"],
"nq": [10]
}
],
"output_fields": "field1,field2"
},
"type": "Search",
"trace_id": "729b10a6a7f32ddd7ab5c16dd30f60dc",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.2s",
"user": "test_user",
"query_params": {
"expr": "expr1",
"output_fields": "field1,field2"
},
"type": "Query",
"trace_id": "232955b7f33b135708d34c3c761b57e7",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.3s",
"user": "test_user",
"query_params": {
"search_params": [
{
"dsl": ["dsl2"],
"search_params": ["param3=value3"],
"nq": [20]
}
],
"output_fields": "field3,field4"
},
"type": "HybridSearch",
"trace_id": "3a4b5c6d7e8f9a0b1c2d3e4f5g6h7i8j",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.4s",
"user": "test_user",
"query_params": {
"expr": "expr2",
"output_fields": "field5,field6"
},
"type": "Query",
"trace_id": "4b5c6d7e8f9a0b1c2d3e4f5g6h7i8j9k",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.5s",
"user": "test_user",
"query_params": {
"search_params": [
{
"dsl": ["dsl3"],
"search_params": ["param4=value4"],
"nq": [30]
}
],
"output_fields": "field7,field8"
},
"type": "Search",
"trace_id": "5c6d7e8f9a0b1c2d3e4f5g6h7i8j9k0l",
"time": "2024-11-05 08:14:05"
}
]`;

1350
internal/http/webui/static/js/render.js
View File

File diff suppressed because it is too large Load Diff

162
internal/http/webui/tasks.html
View File

@ -25,123 +25,29 @@
<h2>
QueryCoord Tasks
</h2>
<table id="channelCP" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Source</th>
<th scope="col">Actions</th>
<th scope="col">Type</th>
<th scope="col">State</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>indexChecker</td>
<td>querynode1</td>
<td>querynode2</td>
<td>queued</td>
</tr>
<tr>
<td>2</td>
<td>loadChannel</td>
<td>querynode1</td>
<td>querynode2</td>
<td>queued</td>
</tr>
</tbody>
</table>
<table id="qcTasks" class="table table-hover"></table>
<h2>
Compaction Tasks
</h2>
<table id="compactionTasks" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Plan</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode1</td>
</tr>
<tr>
<td>2</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode2</td>
</tr>
</tbody>
</table>
<h2>
Flush Tasks
</h2>
<table id="flushTasks" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Segment ID</th>
<th scope="col">Insertion Size(MB)</th>
<th scope="col">Deletion Size(MB)</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>1111</td>
<td>10</td>
<td>5</td>
<td>running</td>
<td>datanode1</td>
</tr>
<tr>
<td>2</td>
<td>2222</td>
<td>5</td>
<td>5</td>
<td>running</td>
<td>datanode2</td>
</tr>
</tbody>
</table>
<table id="compactionTasks" class="table table-hover"></table>
<h2>
Index Build Tasks
</h2>
<table id="indexBuildTasks" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Plan</th>
<th scope="col">State</th>
<th scope="col">Indexnode</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode1</td>
</tr>
<tr>
<td>2</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode2</td>
</tr>
</tbody>
<table id="buildIndexTasks" class="table table-hover"></table>
<h2>
Import Tasks
</h2>
<table id="importTasks" class="table table-hover"></table>
<h2>
Sync Task
</h2>
<table id="syncTasks" class="table table-hover">
</table>
</div>
<div class="col-md-2">
</div>
@ -156,13 +62,47 @@
});
document.addEventListener("DOMContentLoaded", function() {
fetchData(MILVUS_URI + "/", sysmetrics)
fetchData(MILVUS_URI + "/_qc/tasks", qcTasks)
.then(data => {
//TODO add tasks render
renderQCTasks(data)
})
.catch(error => {
handleError(new Error("Unimplemented API"));
handleError(error);
});
fetchData(MILVUS_URI + "/_dc/tasks/compaction", dc_compaction_task)
.then(data => {
renderCompactionTasks(data)
})
.catch(error => {
handleError(error);
});
fetchData(MILVUS_URI + "/_dc/tasks/build_index", dc_build_index_task)
.then(data => {
renderBuildIndexTasks(data)
})
.catch(error => {
handleError(error);
});
fetchData(MILVUS_URI + "/_dc/tasks/import", dc_import_task)
.then(data => {
renderImportTasks(data)
})
.catch(error => {
handleError(error);
});
fetchData(MILVUS_URI + "/_dn/tasks/sync", dn_sync_task)
.then(data => {
renderSyncTasks(data)
})
.catch(error => {
handleError(error);
});
});
</script>
</body>

52
internal/http/webui/tools.html Normal file
View File

@ -0,0 +1,52 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>tools</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<div class="row" style="height: 100px;"></div>
<!-- Centered Links Section -->
<div class="row text-center mb-3">
<div class="col">
<a href="#link1" class="btn btn-link" style="font-size: 1.5em;">Pprof</a>
</div>
</div>
<div class="row text-center mb-3">
<div class="col">
<a href="#link2" class="btn btn-link" style="font-size: 1.5em;">Memory Data Visualization</a>
</div>
</div>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
</script>
</body>
</html>

14
internal/proxy/http_req_impl.go
View File

@ -118,6 +118,20 @@ func getDependencies(c *gin.Context) {
c.Data(http.StatusOK, contentType, ret)
}
func getSlowQuery(node *Proxy) gin.HandlerFunc {
return func(c *gin.Context) {
slowQueries := node.slowQueries.Values()
ret, err := json.Marshal(slowQueries)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
mhttp.HTTPReturnMessage: err.Error(),
})
return
}
c.Data(http.StatusOK, contentType, ret)
}
}
// buildReqParams fetch all parameters from query parameter of URL, add them into a map data structure.
// put key and value from query parameter into map, concatenate values with separator if values size is greater than 1
func buildReqParams(c *gin.Context, metricsType string) map[string]interface{} {

33
internal/proxy/impl.go
View File

@ -30,6 +30,7 @@ import (
"github.com/samber/lo"
"github.com/tidwall/gjson"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
@ -3020,6 +3021,14 @@ func (node *Proxy) search(ctx context.Context, request *milvuspb.SearchRequest,
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.SearchLabel,
).Inc()
user, _ := GetCurUserFromContext(ctx)
traceID := ""
if sp != nil {
traceID = sp.SpanContext().TraceID().String()
}
if node.slowQueries != nil {
node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithSearchRequest(request, user, span, traceID))
}
}
}()
@ -3230,6 +3239,14 @@ func (node *Proxy) hybridSearch(ctx context.Context, request *milvuspb.HybridSea
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.HybridSearchLabel,
).Inc()
user, _ := GetCurUserFromContext(ctx)
traceID := ""
if sp != nil {
traceID = sp.SpanContext().TraceID().String()
}
if node.slowQueries != nil {
node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithSearchRequest(newSearchReq, user, span, traceID))
}
}
}()
@ -3471,7 +3488,7 @@ func (node *Proxy) Flush(ctx context.Context, request *milvuspb.FlushRequest) (*
}
// Query get the records by primary keys.
func (node *Proxy) query(ctx context.Context, qt *queryTask) (*milvuspb.QueryResults, error) {
func (node *Proxy) query(ctx context.Context, qt *queryTask, sp trace.Span) (*milvuspb.QueryResults, error) {
request := qt.request
method := "Query"
@ -3513,6 +3530,15 @@ func (node *Proxy) query(ctx context.Context, qt *queryTask) (*milvuspb.QueryRes
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.QueryLabel,
).Inc()
user, _ := GetCurUserFromContext(ctx)
traceID := ""
if sp != nil {
traceID = sp.SpanContext().TraceID().String()
}
if node.slowQueries != nil {
node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithQueryRequest(request, user, span, traceID))
}
}
}()
@ -3628,7 +3654,7 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (*
request.GetCollectionName(),
).Inc()
res, err := node.query(ctx, qt)
res, err := node.query(ctx, qt, sp)
if err != nil || !merr.Ok(res.Status) {
return res, err
}
@ -6484,6 +6510,9 @@ func (node *Proxy) RegisterRestRouter(router gin.IRouter) {
// Hook request that executed by proxy
router.GET(http.HookConfigsPath, getConfigs(paramtable.GetHookParams().GetAll()))
// Slow query request that executed by proxy
router.GET(http.SlowQueryPath, getSlowQuery(node))
// QueryCoord requests that are forwarded from proxy
router.GET(http.QCTargetPath, getQueryComponentMetrics(node, metricsinfo.QueryTarget))
router.GET(http.QCDistPath, getQueryComponentMetrics(node, metricsinfo.QueryDist))

4
internal/proxy/proxy.go
View File

@ -26,6 +26,7 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/hashicorp/golang-lru/v2/expirable"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/atomic"
"go.uber.org/zap"
@ -130,6 +131,8 @@ type Proxy struct {
// delete rate limiter
enableComplexDeleteLimit bool
slowQueries *expirable.LRU[Timestamp, *metricsinfo.SlowQuery]
}
// NewProxy returns a Proxy struct.
@ -152,6 +155,7 @@ func NewProxy(ctx context.Context, factory dependency.Factory) (*Proxy, error) {
lbPolicy: lbPolicy,
resourceManager: resourceManager,
replicateStreamManager: replicateStreamManager,
slowQueries: expirable.NewLRU[Timestamp, *metricsinfo.SlowQuery](20, nil, time.Minute*15),
}
node.UpdateStateCode(commonpb.StateCode_Abnormal)
expr.Register("proxy", node)

7
internal/proxy/task_search.go
View File

@ -9,6 +9,7 @@ import (
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@ -734,7 +735,7 @@ func (t *searchTask) PostExecute(ctx context.Context) error {
t.fillInFieldInfo()
if t.requery {
err = t.Requery()
err = t.Requery(sp)
if err != nil {
log.Warn("failed to requery", zap.Error(err))
return err
@ -819,7 +820,7 @@ func (t *searchTask) estimateResultSize(nq int64, topK int64) (int64, error) {
//return int64(sizePerRecord) * nq * topK, nil
}
func (t *searchTask) Requery() error {
func (t *searchTask) Requery(span trace.Span) error {
queryReq := &milvuspb.QueryRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Retrieve,
@ -864,7 +865,7 @@ func (t *searchTask) Requery() error {
fastSkip: true,
reQuery: true,
}
queryResult, err := t.node.(*Proxy).query(t.ctx, qt)
queryResult, err := t.node.(*Proxy).query(t.ctx, qt, span)
if err != nil {
return err
}

6
internal/proxy/task_search_test.go
View File

@ -2744,7 +2744,7 @@ func TestSearchTask_Requery(t *testing.T) {
node: node,
}
err := qt.Requery()
err := qt.Requery(nil)
assert.NoError(t, err)
assert.Len(t, qt.result.Results.FieldsData, 2)
for _, field := range qt.result.Results.FieldsData {
@ -2773,7 +2773,7 @@ func TestSearchTask_Requery(t *testing.T) {
node: node,
}
err := qt.Requery()
err := qt.Requery(nil)
t.Logf("err = %s", err)
assert.Error(t, err)
})
@ -2807,7 +2807,7 @@ func TestSearchTask_Requery(t *testing.T) {
node: node,
}
err := qt.Requery()
err := qt.Requery(nil)
t.Logf("err = %s", err)
assert.Error(t, err)
})

4
internal/querycoordv2/meta/dist_manager.go
View File

@ -48,10 +48,6 @@ func (dm *DistributionManager) GetDistributionJSON() string {
channels := dm.GetChannelDist()
leaderView := dm.GetLeaderView()
if len(segments) == 0 && len(channels) == 0 && len(leaderView) == 0 {
return ""
}
dist := &metricsinfo.QueryCoordDist{
Segments: segments,
DMChannels: channels,

9
internal/querycoordv2/server.go
View File

@ -200,11 +200,16 @@ func (s *Server) registerMetricsRequest() {
}
QueryDistAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) {
return s.targetMgr.GetTargetJSON(meta.CurrentTarget), nil
return s.dist.GetDistributionJSON(), nil
}
QueryTargetAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) {
return s.dist.GetDistributionJSON(), nil
scope := meta.CurrentTarget
v := jsonReq.Get(metricsinfo.MetricRequestParamTargetScopeKey)
if v.Exists() {
scope = meta.TargetScope(v.Int())
}
return s.targetMgr.GetTargetJSON(scope), nil
}
QueryReplicasAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) {

6
internal/querycoordv2/task/scheduler.go
View File

@ -202,7 +202,7 @@ func NewScheduler(ctx context.Context,
channelTasks: make(map[replicaChannelIndex]Task),
processQueue: newTaskQueue(),
waitQueue: newTaskQueue(),
taskStats: expirable.NewLRU[UniqueID, Task](512, nil, time.Minute*30),
taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*15),
}
}
@ -574,10 +574,6 @@ func (scheduler *taskScheduler) GetSegmentTaskNum() int {
// GetTasksJSON returns the JSON string of all tasks.
// the task stats object is thread safe and can be accessed without lock
func (scheduler *taskScheduler) GetTasksJSON() string {
if scheduler.taskStats.Len() == 0 {
return ""
}
tasks := scheduler.taskStats.Values()
ret, err := json.Marshal(tasks)
if err != nil {

5
pkg/go.mod
View File

@ -26,6 +26,7 @@ require (
github.com/shirou/gopsutil/v3 v3.22.9
github.com/sirupsen/logrus v1.9.0
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.3.0
github.com/streamnative/pulsarctl v0.5.0
github.com/stretchr/testify v1.9.0
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
@ -55,7 +56,7 @@ require (
google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.1
gopkg.in/yaml.v2 v2.4.0
k8s.io/apimachinery v0.28.6
)
@ -169,7 +170,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

1
pkg/go.sum
View File

@ -655,6 +655,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=

2
pkg/util/metricsinfo/metric_request.go
View File

@ -88,6 +88,8 @@ const (
// MetricRequestParamVerboseKey as a request parameter decide to whether return verbose value
MetricRequestParamVerboseKey = "verbose"
MetricRequestParamTargetScopeKey = "target_scope"
)
type MetricsRequestAction func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error)

48
pkg/util/metricsinfo/metrics_info.go
View File

@ -70,6 +70,34 @@ const (
MilvusUsedGoVersion = "MILVUS_USED_GO_VERSION"
)
type SearchParams struct {
DSL []string `json:"dsl,omitempty"`
SearchParams []string `json:"search_params,omitempty"`
NQ []int64 `json:"nq,omitempty"`
}
type QueryParams struct {
SearchParams []*SearchParams `json:"search_params,omitempty"`
Expr string `json:"expr,omitempty"`
OutputFields string `json:"output_fields,omitempty"`
}
type SlowQuery struct {
Time string `json:"time,omitempty"`
Role string `json:"role,omitempty"`
Database string `json:"database,omitempty"`
Collection string `json:"collection,omitempty"`
Partitions string `json:"partitions,omitempty"`
ConsistencyLevel string `json:"consistency_level,omitempty"`
UseDefaultConsistency bool `json:"use_default_consistency,omitempty"`
GuaranteeTimestamp uint64 `json:"guarantee_timestamp,omitempty"`
Duration string `json:"duration,omitempty"`
User string `json:"user,omitempty"`
QueryParams *QueryParams `json:"query_params,omitempty"`
Type string `json:"type,omitempty"`
TraceID string `json:"trace_id,omitempty"`
}
type DmChannel struct {
NodeID int64 `json:"node_id,omitempty"`
Version int64 `json:"version,omitempty"`
@ -109,7 +137,6 @@ type Segment struct {
FlushedRows int64 `json:"flushed_rows,omitempty"`
SyncBufferRows int64 `json:"sync_buffer_rows,omitempty"`
SyncingRows int64 `json:"syncing_rows,omitempty"`
// TODO add checkpoints
}
type SegmentIndex struct {
@ -127,15 +154,16 @@ type QueryCoordTarget struct {
}
type LeaderView struct {
LeaderID int64 `json:"leader_id"`
CollectionID int64 `json:"collection_id"`
Channel string `json:"channel"`
Version int64 `json:"version"`
SealedSegments []*Segment `json:"sealed_segments"`
GrowingSegments []*Segment `json:"growing_segments"`
TargetVersion int64 `json:"target_version"`
NumOfGrowingRows int64 `json:"num_of_growing_rows"`
UnServiceableError string `json:"unserviceable_error"`
LeaderID int64 `json:"leader_id,omitempty"`
CollectionID int64 `json:"collection_id,omitempty"`
NodeID int64 `json:"node_id,omitempty"`
Channel string `json:"channel,omitempty"`
Version int64 `json:"version,omitempty"`
SealedSegments []*Segment `json:"sealed_segments,omitempty"`
GrowingSegments []*Segment `json:"growing_segments,omitempty"`
TargetVersion int64 `json:"target_version,omitempty"`
NumOfGrowingRows int64 `json:"num_of_growing_rows,omitempty"`
UnServiceableError string `json:"unserviceable_error,omitempty"`
}
type QueryCoordDist struct {

88
pkg/util/metricsinfo/utils.go
View File

@ -14,10 +14,15 @@ package metricsinfo
import (
"encoding/json"
"os"
"strings"
"time"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
// FillDeployMetricsWithEnv fill deploy metrics with env.
@ -34,10 +39,6 @@ func MarshalGetMetricsValues[T any](metrics []T, err error) (string, error) {
return "", err
}
if len(metrics) == 0 {
return "", nil
}
bs, err := json.Marshal(metrics)
if err != nil {
log.Warn("marshal metrics value failed", zap.Any("metrics", metrics), zap.String("err", err.Error()))
@ -45,3 +46,82 @@ func MarshalGetMetricsValues[T any](metrics []T, err error) (string, error) {
}
return string(bs), nil
}
func getSearchParamString(params []*commonpb.KeyValuePair) string {
searchParams := ""
for _, kv := range params {
searchParams += kv.Key + "=" + kv.Value + ","
}
if len(searchParams) > 0 {
searchParams = searchParams[:len(searchParams)-1]
}
return searchParams
}
func NewSlowQueryWithQueryRequest(request *milvuspb.QueryRequest, user string, cost time.Duration, traceID string) *SlowQuery {
queryParams := &QueryParams{
Expr: request.GetExpr(),
OutputFields: strings.Join(request.GetOutputFields(), ","),
}
return &SlowQuery{
Role: typeutil.ProxyRole,
Database: request.GetDbName(),
Collection: request.GetCollectionName(),
Partitions: strings.Join(request.GetPartitionNames(), ","),
ConsistencyLevel: request.GetConsistencyLevel().String(),
UseDefaultConsistency: request.GetUseDefaultConsistency(),
GuaranteeTimestamp: request.GetGuaranteeTimestamp(),
Duration: cost.String(),
User: user,
QueryParams: queryParams,
Type: "Query",
TraceID: traceID,
Time: time.Now().Format(time.DateTime),
}
}
func NewSlowQueryWithSearchRequest(request *milvuspb.SearchRequest, user string, cost time.Duration, traceID string) *SlowQuery {
searchParams := getSearchParamString(request.GetSearchParams())
var subReqs []*SearchParams
for _, req := range request.GetSubReqs() {
subReqs = append(subReqs, &SearchParams{
DSL: []string{req.GetDsl()},
SearchParams: []string{getSearchParamString(req.GetSearchParams())},
NQ: []int64{req.GetNq()},
})
}
searchType := "HybridSearch"
if len(request.GetSubReqs()) == 0 {
subReqs = append(subReqs, &SearchParams{
DSL: []string{request.GetDsl()},
SearchParams: []string{searchParams},
NQ: []int64{request.GetNq()},
})
searchType = "Search"
}
queryParams := &QueryParams{
SearchParams: subReqs,
Expr: request.GetDsl(),
OutputFields: strings.Join(request.GetOutputFields(), ","),
}
return &SlowQuery{
Role: typeutil.ProxyRole,
Database: request.GetDbName(),
Collection: request.GetCollectionName(),
Partitions: strings.Join(request.GetPartitionNames(), ","),
ConsistencyLevel: request.GetConsistencyLevel().String(),
UseDefaultConsistency: request.GetUseDefaultConsistency(),
GuaranteeTimestamp: request.GetGuaranteeTimestamp(),
Duration: cost.String(),
User: user,
QueryParams: queryParams,
Type: searchType,
TraceID: traceID,
Time: time.Now().Format(time.DateTime),
}
}

75
pkg/util/metricsinfo/utils_test.go
View File

@ -13,8 +13,12 @@ package metricsinfo
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
func TestFillDeployMetricsWithEnv(t *testing.T) {
@ -43,3 +47,74 @@ func TestFillDeployMetricsWithEnv(t *testing.T) {
assert.Equal(t, goVersion, m.UsedGoVersion)
assert.Equal(t, buildTime, m.BuildTime)
}
func TestNewSlowQueryWithSearchRequest(t *testing.T) {
request := &milvuspb.SearchRequest{
DbName: "test_db",
CollectionName: "test_collection",
PartitionNames: []string{"partition1", "partition2"},
ConsistencyLevel: commonpb.ConsistencyLevel_Bounded,
UseDefaultConsistency: true,
GuaranteeTimestamp: 123456789,
SearchParams: []*commonpb.KeyValuePair{{Key: "param1", Value: "value1"}},
SubReqs: []*milvuspb.SubSearchRequest{{Dsl: "dsl1", SearchParams: []*commonpb.KeyValuePair{{Key: "param2", Value: "value2"}}, Nq: 10}},
Dsl: "dsl2",
Nq: 20,
OutputFields: []string{"field1", "field2"},
}
user := "test_user"
cost := time.Duration(100) * time.Millisecond
slowQuery := NewSlowQueryWithSearchRequest(request, user, cost, "")
assert.NotNil(t, slowQuery)
assert.Equal(t, "proxy", slowQuery.Role)
assert.Equal(t, "test_db", slowQuery.Database)
assert.Equal(t, "test_collection", slowQuery.Collection)
assert.Equal(t, "partition1,partition2", slowQuery.Partitions)
assert.Equal(t, "Bounded", slowQuery.ConsistencyLevel)
assert.True(t, slowQuery.UseDefaultConsistency)
assert.Equal(t, uint64(123456789), slowQuery.GuaranteeTimestamp)
assert.Equal(t, "100ms", slowQuery.Duration)
assert.Equal(t, user, slowQuery.User)
assert.Equal(t, "HybridSearch", slowQuery.Type)
assert.NotNil(t, slowQuery.QueryParams)
assert.Equal(t, "dsl2", slowQuery.QueryParams.Expr)
assert.Equal(t, "field1,field2", slowQuery.QueryParams.OutputFields)
assert.Len(t, slowQuery.QueryParams.SearchParams, 1)
assert.Equal(t, []string{"dsl1"}, slowQuery.QueryParams.SearchParams[0].DSL)
assert.Equal(t, []string{"param2=value2"}, slowQuery.QueryParams.SearchParams[0].SearchParams)
assert.Equal(t, []int64{10}, slowQuery.QueryParams.SearchParams[0].NQ)
}
func TestNewSlowQueryWithQueryRequest(t *testing.T) {
request := &milvuspb.QueryRequest{
DbName: "test_db",
CollectionName: "test_collection",
PartitionNames: []string{"partition1", "partition2"},
ConsistencyLevel: commonpb.ConsistencyLevel_Bounded,
UseDefaultConsistency: true,
GuaranteeTimestamp: 123456789,
Expr: "expr1",
OutputFields: []string{"field1", "field2"},
}
user := "test_user"
cost := time.Duration(100) * time.Millisecond
slowQuery := NewSlowQueryWithQueryRequest(request, user, cost, "")
assert.NotNil(t, slowQuery)
assert.Equal(t, "proxy", slowQuery.Role)
assert.Equal(t, "test_db", slowQuery.Database)
assert.Equal(t, "test_collection", slowQuery.Collection)
assert.Equal(t, "partition1,partition2", slowQuery.Partitions)
assert.Equal(t, "Bounded", slowQuery.ConsistencyLevel)
assert.True(t, slowQuery.UseDefaultConsistency)
assert.Equal(t, uint64(123456789), slowQuery.GuaranteeTimestamp)
assert.Equal(t, "100ms", slowQuery.Duration)
assert.Equal(t, user, slowQuery.User)
assert.Equal(t, "Query", slowQuery.Type)
assert.NotNil(t, slowQuery.QueryParams)
assert.Equal(t, "expr1", slowQuery.QueryParams.Expr)
assert.Equal(t, "field1,field2", slowQuery.QueryParams.OutputFields)
}

25
tests/integration/minicluster_v2.go
View File

@ -464,14 +464,23 @@ func (cluster *MiniClusterV2) Stop() error {
if cluster.clientConn != nil {
cluster.clientConn.Close()
}
cluster.RootCoord.Stop()
log.Info("mini cluster rootCoord stopped")
cluster.DataCoord.Stop()
log.Info("mini cluster dataCoord stopped")
cluster.QueryCoord.Stop()
log.Info("mini cluster queryCoord stopped")
cluster.Proxy.Stop()
log.Info("mini cluster proxy stopped")
if cluster.RootCoord != nil {
cluster.RootCoord.Stop()
log.Info("mini cluster rootCoord stopped")
}
if cluster.DataCoord != nil {
cluster.DataCoord.Stop()
log.Info("mini cluster dataCoord stopped")
}
if cluster.QueryCoord != nil {
cluster.QueryCoord.Stop()
log.Info("mini cluster queryCoord stopped")
}
if cluster.Proxy != nil {
cluster.Proxy.Stop()
log.Info("mini cluster proxy stopped")
}
cluster.StopAllDataNodes()
cluster.StopAllStreamingNodes()