milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: Check response size before add to counter (#35778) 

Related to #35767

prometheus counter cannot add negative value
when response is not written(say timeout/network broken) panicking may
happen if not check

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/35738/head
congqixia 2024-08-29 00:05:00 +08:00 committed by GitHub
parent 1cd9123a6c
commit c5c9ef9876
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/distributed/proxy/service.go
View File

@ -192,9 +192,14 @@ func (s *Server) startHTTPServer(errChan chan error) {
metrics.RestfulReqLatency.WithLabelValues( metrics.RestfulReqLatency.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10), path, strconv.FormatInt(paramtable.GetNodeID(), 10), path,
).Observe(float64(latency.Milliseconds())) ).Observe(float64(latency.Milliseconds()))
metrics.RestfulSendBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10), path, // see https://github.com/milvus-io/milvus/issues/35767, counter cannot add negative value
).Add(float64(c.Writer.Size())) // when response is not written(say timeout/network broken), panicking may happen if not check
if size := c.Writer.Size(); size > 0 {
metrics.RestfulSendBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10), path,
).Add(float64(c.Writer.Size()))
}
}) })
ginHandler.Use(accesslog.AccessLogMiddleware) ginHandler.Use(accesslog.AccessLogMiddleware)