Exponential backoff if any hinted-handoff fails
parent
4eba2c1725
commit
878f776403
|
@ -119,17 +119,26 @@ func (s *Service) WriteShard(shardID, ownerID uint64, points []models.Point) err
|
||||||
|
|
||||||
func (s *Service) retryWrites() {
|
func (s *Service) retryWrites() {
|
||||||
defer s.wg.Done()
|
defer s.wg.Done()
|
||||||
ticker := time.NewTicker(time.Duration(s.cfg.RetryInterval))
|
currInterval := time.Duration(s.cfg.RetryInterval)
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
for {
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-s.closing:
|
case <-s.closing:
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-time.After(currInterval):
|
||||||
s.statMap.Add(processReq, 1)
|
s.statMap.Add(processReq, 1)
|
||||||
if err := s.HintedHandoff.Process(); err != nil && err != io.EOF {
|
if err := s.HintedHandoff.Process(); err != nil && err != io.EOF {
|
||||||
s.statMap.Add(processReqFail, 1)
|
s.statMap.Add(processReqFail, 1)
|
||||||
s.Logger.Printf("retried write failed: %v", err)
|
s.Logger.Printf("retried write failed: %v", err)
|
||||||
|
|
||||||
|
currInterval = currInterval * 2
|
||||||
|
if currInterval > time.Duration(s.cfg.RetryMaxInterval) {
|
||||||
|
currInterval = time.Duration(s.cfg.RetryMaxInterval)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Success! Return to configured interval.
|
||||||
|
currInterval = time.Duration(s.cfg.RetryInterval)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue