Retry algorithm

Retry with exponential back-off

Update on July 24, 2016: I just discovered Polly, which does this and more a lot better.

I think this is an useful class so I'm just going to leave it here. (I'm annoyed by the duplication between Retry and RetryAsync but I haven't been able to remove it.)

  public interface RetryPolicy
  {
    T Retry<T>(Func<T> func);
    void Retry(Action action);

    Task<T> RetryAsync<T>(Func<Task<T>> func);
    Task RetryAsync(Func<Task> action);
  }

  public class RetryPolicyWithExponentialDelay : RetryPolicy
  {
    // ReSharper disable once InconsistentNaming
    public Func<double> GetRandom = () => RND.NextDouble();

    // ReSharper disable once InconsistentNaming
    public Action<int> Sleep = timeout => Thread.Sleep(timeout);

    public RetryPolicyWithExponentialDelay(int maxCount, TimeSpan initialDelay, TimeSpan maxDelay)
    {
      this.maxCount = maxCount;
      this.initialDelay = initialDelay;
      this.maxDelay = maxDelay;
    }

    public T Retry<T>(Func<T> func)
    {
      var count = 0;
      var delay = initialDelay;

      while (true)
      {
        try
        {
          return func();
        }
        catch
        {
          count++;
          if (count >= maxCount)
            throw;

          SleepUpTo(delay);
          delay = IncreaseDelay(delay);
        }
      }
    }

    public void Retry(Action action)
    {
      Retry(() =>
      {
        action();
        return 0;
      });
    }

    public async Task<T> RetryAsync<T>(Func<Task<T>> func)
    {
      var count = 0;
      var delay = initialDelay;

      while (true)
      {
        try
        {
          return await func();
        }
        catch
        {
          count++;
          if (count >= maxCount)
            throw;

          SleepUpTo(delay);
          delay = IncreaseDelay(delay);
        }
      }
    }

    public async Task RetryAsync(Func<Task> action)
    {
      await RetryAsync(async () =>
      {
        await action();
        return 0;
      });
    }

    //

    private static readonly Random RND = new Random();

    private readonly int maxCount;
    private readonly TimeSpan initialDelay;
    private readonly TimeSpan maxDelay;

    private void SleepUpTo(TimeSpan delay)
    {
      var actualDelay = (int) Math.Truncate(GetRandom() * delay.TotalMilliseconds);
      Sleep(actualDelay);
    }

    private TimeSpan IncreaseDelay(TimeSpan delay)
    {
      delay = delay.Add(delay);
      if (delay > maxDelay)
        delay = maxDelay;

      return delay;
    }
  }

I'm also adding the tests here:

  [TestClass]
  public class RetryPolicyWithExponentialDelayTests
  {
    private const int RESULT = 100;

    private RetryPolicyWithExponentialDelay sut;

    private int called;
    private int sleepTime;

    [TestInitialize]
    public void SetUp()
    {
      sut = new RetryPolicyWithExponentialDelay(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5))
      {
        GetRandom = () => 1.0,
        Sleep = timeout => sleepTime += timeout,
      };

      called = 0;
      sleepTime = 0;
    }

    [TestClass]
    public class Sync : RetryPolicyWithExponentialDelayTests
    {
      private Action action;
      private Func<int> func;

      [TestInitialize]
      public void InnerSetup()
      {
        action = null;
        func = () =>
        {
          action();
          return RESULT;
        };
      }

      [TestClass]
      public class SyncFunc : Sync
      {
        [TestMethod]
        public void NoErrors()
        {
          action = GetActionWithErrors(0);

          var result = sut.Retry(func);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public void OneError()
        {
          action = GetActionWithErrors(1);

          var result = sut.Retry(func);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void TwoErrors()
        {
          action = GetActionWithErrors(2);

          var result = sut.Retry(func);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void FourErrors()
        {
          action = GetActionWithErrors(4);

          var result = sut.Retry(func);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public void TooManyErrors()
        {
          action = GetActionWithErrors(10);

          try
          {
            sut.Retry(func);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }

      [TestClass]
      public class SyncAction : Sync
      {
        [TestMethod]
        public void NoErrors()
        {
          action = GetActionWithErrors(0);

          sut.Retry(action);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public void OneError()
        {
          action = GetActionWithErrors(1);

          sut.Retry(action);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void TwoErrors()
        {
          action = GetActionWithErrors(2);

          sut.Retry(action);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public void FourErrors()
        {
          action = GetActionWithErrors(4);

          sut.Retry(action);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public void TooManyErrors()
        {
          action = GetActionWithErrors(10);

          try
          {
            sut.Retry(action);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }
    }

    [TestClass]
    public class Async : RetryPolicyWithExponentialDelayTests
    {
      private Action action;
      private Func<Task<int>> func;

      [TestInitialize]
      public void InnerSetup()
      {
        action = null;
        func = async () =>
        {
          await Task.Run(action);
          return RESULT;
        };
      }

      [TestClass]
      public class AsyncFunc : Async
      {
        [TestMethod]
        public async Task NoErrorsAsync()
        {
          action = GetActionWithErrors(0);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public async Task OneErrorAsync()
        {
          action = GetActionWithErrors(1);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task TwoErrorsAsync()
        {
          action = GetActionWithErrors(2);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task FourErrorsAsync()
        {
          action = GetActionWithErrors(4);

          var result = await sut.RetryAsync(func);

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(RESULT, result, "Invalid result");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public async Task TooManyErrorsAsync()
        {
          action = GetActionWithErrors(10);

          try
          {
            await sut.RetryAsync(func);
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }

      [TestClass]
      public class AsyncAction : Async
      {
        [TestMethod]
        public async Task NoErrorsAsync()
        {
          action = GetActionWithErrors(0);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(1, called, "Function was not called");
          Assert.AreEqual(0, sleepTime, "Should not have slept");
        }

        [TestMethod]
        public async Task OneErrorAsync()
        {
          action = GetActionWithErrors(1);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(2, called, "The call was not retried");
          Assert.AreEqual(TimeSpan.FromSeconds(1).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task TwoErrorsAsync()
        {
          action = GetActionWithErrors(2);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(3, called, "The call was not retried twice");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount");
        }

        [TestMethod]
        public async Task FourErrorsAsync()
        {
          action = GetActionWithErrors(4);

          await sut.RetryAsync(() => Task.Run(action));

          Assert.AreEqual(5, called, "The call was not retried four times");
          Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
        }

        [TestMethod]
        public async Task TooManyErrorsAsync()
        {
          action = GetActionWithErrors(10);

          try
          {
            await sut.RetryAsync(() => Task.Run(action));
            Assert.Fail("The call did not throw");
          }
          catch
          {
            Assert.AreEqual(5, called, "The call was not tried five times");
            Assert.AreEqual(TimeSpan.FromSeconds(1 + 2 + 4 + 5).TotalMilliseconds, sleepTime, 1, "Did not sleep the correct amount (limited by max delay)");
          }
        }
      }
    }

    //

    private Action GetActionWithErrors(int errorCount)
    {
      return () =>
      {
        called++;
        if (called <= errorCount)
          throw new Exception();
      };
    }
  }

Comments

Post a Comment

Popular posts from this blog

Posting dynamic Master / Detail forms with Knockout

Image
I have the following dynamic form in my GitHub Inventory project : I created the form with KnockoutJS , using some ideas from this article . The detail view looks like this: <table id="acquisition_items" class="jtable"> <thead> <tr> <th>Product Name</th> <th>Quantity</th> <th>Price</th> <th>Value</th> <th><button data-bind="click: addItem">[+]</button></th> </tr> </thead> <tbody data-bind="foreach: items"> <tr> <td> <input data-bind="value: ProductName" type="text" value="" /> </td> <td> <input data-bind="value: Quantity" type="text" value="" /> </td> <td>
Read more

Comparing Excel files, take two

Image
I already wrote this project once, on GitHub , but I can't say that I like the result. I wrote that code without TDD, mostly to see if I can still work in the "normal" fashion. I can, but I had a bug that took me a while to even discover and then was difficult to write tests for. I also hate the temporal dependency in that design (if you don't call the SortBy method first, the ExcludeRecords method will return incorrect results). Anyway; here's attempt number two, writing that code using TDD. Structure I start by creating my usual structure for projects of this type - a library for the logic, a console application for the main program and a test project for tests. I also use the Enable NuGet Package Restore option (right-click on the solution node) to add the three NuGet files that will allow someone to automatically download all the packages I'm referencing without wasting space in the source code repository. (Right now I've only added the Moq pa
Read more

EF Code First: seeding with foreign keys

This is just a quick tip, more for future reference for myself than anything else. When using Entity Framework Code First, one of the things you might want to do while developing the application is to seed your database with some test data. This can be something as easy as: protected override void Seed(InventoryDB context) { context.Products.AddOrUpdate( p => p.Name, new Product { Name = "Hammer", SalePrice = 11.99m, }, new Product { Name = "Nail Pack x100", SalePrice = 0.05m, }, new Product { Name = "Saw", SalePrice = 19.99m, }, new Product { Name = "Toolkit", SalePrice = 39.99m, } ); //... } (I am using the AddOrUpdate method so the code detects whether those products already exist and doesn't add them again if they do.) The Product class is quite simple and – most importantly – doesn't depend on any other: public class Product { public int Id { g
Read more