Retry algorithm

Retry with exponential back-off

Update on July 24, 2016: I just discovered Polly, which does this and more a lot better.

I think this is an useful class so I'm just going to leave it here. (I'm annoyed by the duplication between Retry and RetryAsync but I haven't been able to remove it.)

  public interface RetryPolicy
  {
    T Retry<T>(Func<T> func);
    void Retry(Action action);

    Task<T> RetryAsync<T>(Func<Task<T>> func);
    Task RetryAsync(Func<Task> action);
  }

  public class RetryPolicyWithExponentialDelay : RetryPolicy
  {
    // ReSharper disable once InconsistentNaming
    public Func<double> GetRandom = () => RND.NextDouble();

    // ReSharper disable once InconsistentNaming
    public Action<int> Sleep = timeout => Thread.Sleep(timeout);

    public RetryPolicyWithExponentialDelay(int maxCount, TimeSpan initialDelay, TimeSpan maxDelay)
    {
      this.maxCount = maxCount;
      this.initialDelay = initialDelay;
      this.maxDelay = maxDelay;
    }

    public T Retry<T>(Func<T> func)
    {
      var count = 0;
      var delay = initialDelay;

      while (true)
      {
        try
        {
          return func();
        }
        catch
        {
          count++;
          if (count >= maxCount)
            throw;

          SleepUpTo(delay);
          delay = IncreaseDelay(delay);
        }
      }
    }

    public void Retry(Action action)
    {
      Retry(() =>
      {
        action();
        return 0;
      });
    }

    public async Task<T> RetryAsync<T>(Func<Task<T>> func)
    {
      var count = 0;
      var delay = initialDelay;

      while (true)
      {
        try
        {
          return await func();
        }
        catch
        {
          count++;
          if (count >= maxCount)
            throw;

          SleepUpTo(delay);
          delay = IncreaseDelay(delay);
        }
      }
    }

    public async Task RetryAsync(Func<Task> action)
    {
      await RetryAsync(async () =>
      {
        await action();
        return 0;
      });
    }

    //

    private static readonly Random RND = new Random();

    private readonly int maxCount;
    private readonly TimeSpan initialDelay;
    private readonly TimeSpan maxDelay;

    private void SleepUpTo(TimeSpan delay)
    {
      var actualDelay = (int) Math.Truncate(GetRandom() * delay.TotalMilliseconds);
      Sleep(actualDelay);
    }

    private TimeSpan IncreaseDelay(TimeSpan delay)
    {
      delay = delay.Add(delay);
      if (delay > maxDelay)
        delay = maxDelay;

      return delay;
    }
  }

I'm also adding the tests here:

  [TestClass]
  public class RetryPolicyWithExponentialDelayTests
  {
    private const int RESULT = 100;

    private RetryPolicyWithExponentialDelay sut;

    private int called;
    private int sleepTime;

    [TestInitialize]
    public void SetUp()
    {
      sut = new RetryPolicyWithExponentialDelay(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5))
      {
        GetRandom = () => 1.0,
        Sleep = timeout => sleepTime += timeout,
      };

      called = 0;
      sleepTime = 0;
    }

    [TestClass]
    public class Sync : RetryPolicyWithExponentialDelayTests
    {
      private Action action;
      private Func<int> func;

      [TestInitialize]
      public void InnerSetup()
      {
        action = null;
        func = () =>
        {
          action();
          return RESULT;
        };
      }

      [TestClass]
      public class SyncFunc : Sync
      {
        [TestMethod]
        public void NoErrors()
        {
          action = GetActionWithErrors(0);

          var result = sut.Retry(func);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public void OneError()
        {
          action = GetActionWithErrors(1);

          var result = sut.Retry(func);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void TwoErrors()
        {
          action = GetActionWithErrors(2);

          var result = sut.Retry(func);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void FourErrors()
        {
          action = GetActionWithErrors(4);

          var result = sut.Retry(func);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public void TooManyErrors()
        {
          action = GetActionWithErrors(10);

          try
          {
            sut.Retry(func);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }

      [TestClass]
      public class SyncAction : Sync
      {
        [TestMethod]
        public void NoErrors()
        {
          action = GetActionWithErrors(0);

          sut.Retry(action);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public void OneError()
        {
          action = GetActionWithErrors(1);

          sut.Retry(action);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void TwoErrors()
        {
          action = GetActionWithErrors(2);

          sut.Retry(action);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void FourErrors()
        {
          action = GetActionWithErrors(4);

          sut.Retry(action);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public void TooManyErrors()
        {
          action = GetActionWithErrors(10);

          try
          {
            sut.Retry(action);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }
    }

    [TestClass]
    public class Async : RetryPolicyWithExponentialDelayTests
    {
      private Action action;
      private Func<Task<int>> func;

      [TestInitialize]
      public void InnerSetup()
      {
        action = null;
        func = async () =>
        {
          await Task.Run(action);
          return RESULT;
        };
      }

      [TestClass]
      public class AsyncFunc : Async
      {
        [TestMethod]
        public async Task NoErrorsAsync()
        {
          action = GetActionWithErrors(0);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public async Task OneErrorAsync()
        {
          action = GetActionWithErrors(1);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task TwoErrorsAsync()
        {
          action = GetActionWithErrors(2);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task FourErrorsAsync()
        {
          action = GetActionWithErrors(4);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public async Task TooManyErrorsAsync()
        {
          action = GetActionWithErrors(10);

          try
          {
            await sut.RetryAsync(func);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }

      [TestClass]
      public class AsyncAction : Async
      {
        [TestMethod]
        public async Task NoErrorsAsync()
        {
          action = GetActionWithErrors(0);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public async Task OneErrorAsync()
        {
          action = GetActionWithErrors(1);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task TwoErrorsAsync()
        {
          action = GetActionWithErrors(2);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task FourErrorsAsync()
        {
          action = GetActionWithErrors(4);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public async Task TooManyErrorsAsync()
        {
          action = GetActionWithErrors(10);

          try
          {
            await sut.RetryAsync(() => Task.Run(action));
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }
    }

    //

    private Action GetActionWithErrors(int errorCount)
    {
      return () =>
      {
        called++;
        if (called <= errorCount)
          throw new Exception();
      };
    }
  }

Comments

Post a Comment

Popular posts from this blog

Posting dynamic Master / Detail forms with Knockout

Image
I have the following dynamic form in my GitHub Inventory project : I created the form with KnockoutJS , using some ideas from this article . The detail view looks like this: <table id="acquisition_items" class="jtable"> <thead> <tr> <th>Product Name</th> <th>Quantity</th> <th>Price</th> <th>Value</th> <th><button data-bind="click: addItem">[+]</button></th> </tr> </thead> <tbody data-bind="foreach: items"> <tr> <td> <input data-bind="value: ProductName" type="text" value="" /> </td> <td> <input data-bind="value: Quantity" type="text" value="" /> </td> <td> ...
Read more

Comparing Excel files, take two

Image
I already wrote this project once, on GitHub , but I can't say that I like the result. I wrote that code without TDD, mostly to see if I can still work in the "normal" fashion. I can, but I had a bug that took me a while to even discover and then was difficult to write tests for. I also hate the temporal dependency in that design (if you don't call the SortBy method first, the ExcludeRecords method will return incorrect results). Anyway; here's attempt number two, writing that code using TDD. Structure I start by creating my usual structure for projects of this type - a library for the logic, a console application for the main program and a test project for tests. I also use the Enable NuGet Package Restore option (right-click on the solution node) to add the three NuGet files that will allow someone to automatically download all the packages I'm referencing without wasting space in the source code repository. (Right now I've only added the Moq pa...
Read more

POST-ing to an ASP.NET MVC form with an anti-forgery token

I've had some issues trying to write an acceptance test that was registering a new user by POST-ing the required information to a MVC site and I got back these messages: The required anti-forgery cookie "__RequestVerificationToken" is not present. The required anti-forgery form field "__RequestVerificationToken" is not present. Validation of the provided anti-forgery token failed. The cookie "_RequestVerificationToken" and the form field "_RequestVerificationToken" were swapped. Since it took me a bit of fiddling with the code before I managed to make it work, I thought I'd share the solution. The site is an ASP.NET MVC version 5 and I am trying to register a new user (POST-ing to the /Account/Register URL). The main issue you will encounter is having to extract two anti-forgery tokens, one from the cookies and one from the form, and then sending both of them in the appropriate places (cookies vs form field). I have used LinqPad 5 w...
Read more