Tuesday, October 20, 2015

Retry algorithm

Retry with exponential back-off

Update on July 24, 2016: I just discovered Polly, which does this and more a lot better.

I think this is an useful class so I'm just going to leave it here. (I'm annoyed by the duplication between Retry and RetryAsync but I haven't been able to remove it.)

  public interface RetryPolicy
  {
    T Retry<T>(Func<T> func);
    void Retry(Action action);

    Task<T> RetryAsync<T>(Func<Task<T>> func);
    Task RetryAsync(Func<Task> action);
  }

  public class RetryPolicyWithExponentialDelay : RetryPolicy
  {
    // ReSharper disable once InconsistentNaming
    public Func<double> GetRandom = () => RND.NextDouble();

    // ReSharper disable once InconsistentNaming
    public Action<int> Sleep = timeout => Thread.Sleep(timeout);

    public RetryPolicyWithExponentialDelay(int maxCount, TimeSpan initialDelay, TimeSpan maxDelay)
    {
      this.maxCount = maxCount;
      this.initialDelay = initialDelay;
      this.maxDelay = maxDelay;
    }

    public T Retry<T>(Func<T> func)
    {
      var count = 0;
      var delay = initialDelay;

      while (true)
      {
        try
        {
          return func();
        }
        catch
        {
          count++;
          if (count >= maxCount)
            throw;

          SleepUpTo(delay);
          delay = IncreaseDelay(delay);
        }
      }
    }

    public void Retry(Action action)
    {
      Retry(() =>
      {
        action();
        return 0;
      });
    }

    public async Task<T> RetryAsync<T>(Func<Task<T>> func)
    {
      var count = 0;
      var delay = initialDelay;

      while (true)
      {
        try
        {
          return await func();
        }
        catch
        {
          count++;
          if (count >= maxCount)
            throw;

          SleepUpTo(delay);
          delay = IncreaseDelay(delay);
        }
      }
    }

    public async Task RetryAsync(Func<Task> action)
    {
      await RetryAsync(async () =>
      {
        await action();
        return 0;
      });
    }

    //

    private static readonly Random RND = new Random();

    private readonly int maxCount;
    private readonly TimeSpan initialDelay;
    private readonly TimeSpan maxDelay;

    private void SleepUpTo(TimeSpan delay)
    {
      var actualDelay = (int) Math.Truncate(GetRandom() * delay.TotalMilliseconds);
      Sleep(actualDelay);
    }

    private TimeSpan IncreaseDelay(TimeSpan delay)
    {
      delay = delay.Add(delay);
      if (delay > maxDelay)
        delay = maxDelay;

      return delay;
    }
  }

I'm also adding the tests here:

  [TestClass]
  public class RetryPolicyWithExponentialDelayTests
  {
    private const int RESULT = 100;

    private RetryPolicyWithExponentialDelay sut;

    private int called;
    private int sleepTime;

    [TestInitialize]
    public void SetUp()
    {
      sut = new RetryPolicyWithExponentialDelay(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5))
      {
        GetRandom = () => 1.0,
        Sleep = timeout => sleepTime += timeout,
      };

      called = 0;
      sleepTime = 0;
    }

    [TestClass]
    public class Sync : RetryPolicyWithExponentialDelayTests
    {
      private Action action;
      private Func<int> func;

      [TestInitialize]
      public void InnerSetup()
      {
        action = null;
        func = () =>
        {
          action();
          return RESULT;
        };
      }

      [TestClass]
      public class SyncFunc : Sync
      {
        [TestMethod]
        public void NoErrors()
        {
          action = GetActionWithErrors(0);

          var result = sut.Retry(func);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public void OneError()
        {
          action = GetActionWithErrors(1);

          var result = sut.Retry(func);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void TwoErrors()
        {
          action = GetActionWithErrors(2);

          var result = sut.Retry(func);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void FourErrors()
        {
          action = GetActionWithErrors(4);

          var result = sut.Retry(func);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public void TooManyErrors()
        {
          action = GetActionWithErrors(10);

          try
          {
            sut.Retry(func);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }

      [TestClass]
      public class SyncAction : Sync
      {
        [TestMethod]
        public void NoErrors()
        {
          action = GetActionWithErrors(0);

          sut.Retry(action);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public void OneError()
        {
          action = GetActionWithErrors(1);

          sut.Retry(action);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void TwoErrors()
        {
          action = GetActionWithErrors(2);

          sut.Retry(action);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void FourErrors()
        {
          action = GetActionWithErrors(4);

          sut.Retry(action);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public void TooManyErrors()
        {
          action = GetActionWithErrors(10);

          try
          {
            sut.Retry(action);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }
    }

    [TestClass]
    public class Async : RetryPolicyWithExponentialDelayTests
    {
      private Action action;
      private Func<Task<int>> func;

      [TestInitialize]
      public void InnerSetup()
      {
        action = null;
        func = async () =>
        {
          await Task.Run(action);
          return RESULT;
        };
      }

      [TestClass]
      public class AsyncFunc : Async
      {
        [TestMethod]
        public async Task NoErrorsAsync()
        {
          action = GetActionWithErrors(0);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public async Task OneErrorAsync()
        {
          action = GetActionWithErrors(1);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task TwoErrorsAsync()
        {
          action = GetActionWithErrors(2);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task FourErrorsAsync()
        {
          action = GetActionWithErrors(4);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public async Task TooManyErrorsAsync()
        {
          action = GetActionWithErrors(10);

          try
          {
            await sut.RetryAsync(func);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }

      [TestClass]
      public class AsyncAction : Async
      {
        [TestMethod]
        public async Task NoErrorsAsync()
        {
          action = GetActionWithErrors(0);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public async Task OneErrorAsync()
        {
          action = GetActionWithErrors(1);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task TwoErrorsAsync()
        {
          action = GetActionWithErrors(2);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task FourErrorsAsync()
        {
          action = GetActionWithErrors(4);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public async Task TooManyErrorsAsync()
        {
          action = GetActionWithErrors(10);

          try
          {
            await sut.RetryAsync(() => Task.Run(action));
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }
    }

    //

    private Action GetActionWithErrors(int errorCount)
    {
      return () =>
      {
        called++;
        if (called <= errorCount)
          throw new Exception();
      };
    }
  }

No comments: