Exponential backoff if any hinted-handoff fails

pull/4284/head
Philip O'Toole 2015-09-30 21:27:13 -07:00
parent 4eba2c1725
commit 878f776403
1 changed files with 12 additions and 3 deletions

View File

@ -119,17 +119,26 @@ func (s *Service) WriteShard(shardID, ownerID uint64, points []models.Point) err
func (s *Service) retryWrites() { func (s *Service) retryWrites() {
defer s.wg.Done() defer s.wg.Done()
ticker := time.NewTicker(time.Duration(s.cfg.RetryInterval)) currInterval := time.Duration(s.cfg.RetryInterval)
defer ticker.Stop()
for { for {
select { select {
case <-s.closing: case <-s.closing:
return return
case <-ticker.C: case <-time.After(currInterval):
s.statMap.Add(processReq, 1) s.statMap.Add(processReq, 1)
if err := s.HintedHandoff.Process(); err != nil && err != io.EOF { if err := s.HintedHandoff.Process(); err != nil && err != io.EOF {
s.statMap.Add(processReqFail, 1) s.statMap.Add(processReqFail, 1)
s.Logger.Printf("retried write failed: %v", err) s.Logger.Printf("retried write failed: %v", err)
currInterval = currInterval * 2
if currInterval > time.Duration(s.cfg.RetryMaxInterval) {
currInterval = time.Duration(s.cfg.RetryMaxInterval)
}
} else {
// Success! Return to configured interval.
currInterval = time.Duration(s.cfg.RetryInterval)
} }
} }
} }