Retry algorithm
Retry with exponential back-off
Update on July 24, 2016: I just discovered Polly, which does this and more a lot better.
I think this is an useful class so I'm just going to leave it here. (I'm annoyed by the duplication between Retry
and RetryAsync
but I haven't been able to remove it.)
public interface RetryPolicy { T Retry<T>(Func<T> func); void Retry(Action action); Task<T> RetryAsync<T>(Func<Task<T>> func); Task RetryAsync(Func<Task> action); } public class RetryPolicyWithExponentialDelay : RetryPolicy { // ReSharper disable once InconsistentNaming public Func<double> GetRandom = () => RND.NextDouble(); // ReSharper disable once InconsistentNaming public Action<int> Sleep = timeout => Thread.Sleep(timeout); public RetryPolicyWithExponentialDelay(int maxCount, TimeSpan initialDelay, TimeSpan maxDelay) { this.maxCount = maxCount; this.initialDelay = initialDelay; this.maxDelay = maxDelay; } public T Retry<T>(Func<T> func) { var count = 0; var delay = initialDelay; while (true) { try { return func(); } catch { count++; if (count >= maxCount) throw; SleepUpTo(delay); delay = IncreaseDelay(delay); } } } public void Retry(Action action) { Retry(() => { action(); return 0; }); } public async Task<T> RetryAsync<T>(Func<Task<T>> func) { var count = 0; var delay = initialDelay; while (true) { try { return await func(); } catch { count++; if (count >= maxCount) throw; SleepUpTo(delay); delay = IncreaseDelay(delay); } } } public async Task RetryAsync(Func<Task> action) { await RetryAsync(async () => { await action(); return 0; }); } // private static readonly Random RND = new Random(); private readonly int maxCount; private readonly TimeSpan initialDelay; private readonly TimeSpan maxDelay; private void SleepUpTo(TimeSpan delay) { var actualDelay = (int) Math.Truncate(GetRandom() * delay.TotalMilliseconds); Sleep(actualDelay); } private TimeSpan IncreaseDelay(TimeSpan delay) { delay = delay.Add(delay); if (delay > maxDelay) delay = maxDelay; return delay; } }
I'm also adding the tests here:
[TestClass] public class RetryPolicyWithExponentialDelayTests { private const int RESULT = 100; private RetryPolicyWithExponentialDelay sut; private int called; private int sleepTime; [TestInitialize] public void SetUp() { sut = new RetryPolicyWithExponentialDelay(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)) { GetRandom = () => 1.0, Sleep = timeout => sleepTime += timeout, }; called = 0; sleepTime = 0; } [TestClass] public class Sync : RetryPolicyWithExponentialDelayTests { private Action action; private Func<int> func; [TestInitialize] public void InnerSetup() { action = null; func = () => { action(); return RESULT; }; } [TestClass] public class SyncFunc : Sync { [TestMethod] public void NoErrors() { action = GetActionWithErrors(0); var result = sut.Retry(func); Assert.AreEqual(1, called, "Function was not called"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(0, sleepTime, "Should not have slept"); } [TestMethod] public void OneError() { action = GetActionWithErrors(1); var result = sut.Retry(func); Assert.AreEqual(2, called, "The call was not retried"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public void TwoErrors() { action = GetActionWithErrors(2); var result = sut.Retry(func); Assert.AreEqual(3, called, "The call was not retried twice"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public void FourErrors() { action = GetActionWithErrors(4); var result = sut.Retry(func); Assert.AreEqual(5, called, "The call was not retried four times"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } [TestMethod] public void TooManyErrors() { action = GetActionWithErrors(10); try { sut.Retry(func); Assert.Fail("The call did not throw"); } catch { Assert.AreEqual(5, called, "The call was not tried five times"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } } } [TestClass] public class SyncAction : Sync { [TestMethod] public void NoErrors() { action = GetActionWithErrors(0); sut.Retry(action); Assert.AreEqual(1, called, "Function was not called"); Assert.AreEqual(0, sleepTime, "Should not have slept"); } [TestMethod] public void OneError() { action = GetActionWithErrors(1); sut.Retry(action); Assert.AreEqual(2, called, "The call was not retried"); Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public void TwoErrors() { action = GetActionWithErrors(2); sut.Retry(action); Assert.AreEqual(3, called, "The call was not retried twice"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public void FourErrors() { action = GetActionWithErrors(4); sut.Retry(action); Assert.AreEqual(5, called, "The call was not retried four times"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } [TestMethod] public void TooManyErrors() { action = GetActionWithErrors(10); try { sut.Retry(action); Assert.Fail("The call did not throw"); } catch { Assert.AreEqual(5, called, "The call was not tried five times"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } } } } [TestClass] public class Async : RetryPolicyWithExponentialDelayTests { private Action action; private Func<Task<int>> func; [TestInitialize] public void InnerSetup() { action = null; func = async () => { await Task.Run(action); return RESULT; }; } [TestClass] public class AsyncFunc : Async { [TestMethod] public async Task NoErrorsAsync() { action = GetActionWithErrors(0); var result = await sut.RetryAsync(func); Assert.AreEqual(1, called, "Function was not called"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(0, sleepTime, "Should not have slept"); } [TestMethod] public async Task OneErrorAsync() { action = GetActionWithErrors(1); var result = await sut.RetryAsync(func); Assert.AreEqual(2, called, "The call was not retried"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public async Task TwoErrorsAsync() { action = GetActionWithErrors(2); var result = await sut.RetryAsync(func); Assert.AreEqual(3, called, "The call was not retried twice"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public async Task FourErrorsAsync() { action = GetActionWithErrors(4); var result = await sut.RetryAsync(func); Assert.AreEqual(5, called, "The call was not retried four times"); Assert.AreEqual(RESULT, result, "Invalid result"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } [TestMethod] public async Task TooManyErrorsAsync() { action = GetActionWithErrors(10); try { await sut.RetryAsync(func); Assert.Fail("The call did not throw"); } catch { Assert.AreEqual(5, called, "The call was not tried five times"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } } } [TestClass] public class AsyncAction : Async { [TestMethod] public async Task NoErrorsAsync() { action = GetActionWithErrors(0); await sut.RetryAsync(() => Task.Run(action)); Assert.AreEqual(1, called, "Function was not called"); Assert.AreEqual(0, sleepTime, "Should not have slept"); } [TestMethod] public async Task OneErrorAsync() { action = GetActionWithErrors(1); await sut.RetryAsync(() => Task.Run(action)); Assert.AreEqual(2, called, "The call was not retried"); Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public async Task TwoErrorsAsync() { action = GetActionWithErrors(2); await sut.RetryAsync(() => Task.Run(action)); Assert.AreEqual(3, called, "The call was not retried twice"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount"); } [TestMethod] public async Task FourErrorsAsync() { action = GetActionWithErrors(4); await sut.RetryAsync(() => Task.Run(action)); Assert.AreEqual(5, called, "The call was not retried four times"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } [TestMethod] public async Task TooManyErrorsAsync() { action = GetActionWithErrors(10); try { await sut.RetryAsync(() => Task.Run(action)); Assert.Fail("The call did not throw"); } catch { Assert.AreEqual(5, called, "The call was not tried five times"); Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)"); } } } } // private Action GetActionWithErrors(int errorCount) { return () => { called++; if (called <= errorCount) throw new Exception(); }; } }
Comments