Added a backfill in 7-day windows, and fetches now keep their data on a failure

Backfilling a long window (an early --since, or a long time offline)
hit the API's rate limit, which could still be exceeded after waiting
once, and then everything fetched so far was lost.

Before each Find a Tender source's worker starts, a start point more
than 7 days ago is now backfilled in 7-day windows bounded on both
sides, oldest first, each saved before the next is fetched. A window
that fails is saved as far as it got and fetched again from its start
after the source's interval. The worker then carries on from the end
of the last window.

A 429 is now waited out up to sources.<source>.rate_limit_retries
times (default 5) rather than once; fts-once --ignore-rate-limit uses
0. FetchAll returns what it fetched before an error along with it, and
the workers save it, so a failed run no longer loses its data.
findatender's FetchAll and FetchOnce take RequestOptions, which gains
Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.

internal/config/README.md

@@ -72,7 +72,9 @@ supported.
 | `sources.<source>.enabled` | bool | `false` | For every source |
 | `sources.<source>.interval` | duration | `10m` | For every source; e.g. `30s`, `1h`. Must be greater than 0; turn a source off with `enabled` |
 | `sources.find_a_tender.api_root` | string | `https://www.find-tender.service.gov.uk/api/1.0` | URL of the Find a Tender API. Empty: the source fails to start if enabled, and `fetcher fts-once` fails |
+| `sources.find_a_tender.rate_limit_retries` | int | `5` | How many times a request refused for the API's rate limit is waited out (for its `Retry-After`) and retried. 0 fails at the first; can't be negative. `fetcher fts-once --ignore-rate-limit` uses 0 |
 | `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`). Empty: the source fails to start if enabled |
+| `sources.fts_mock.rate_limit_retries` | int | `5` | As for `find_a_tender` |
 | `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown |
 
 The current sources are `find_a_tender`, `fts_mock` and `sleeper`. See

internal/config/conf_sections_test.go

@@ -56,6 +56,14 @@ func TestDefaults(t *testing.T) {
 	if conf.Sources.FTSMock.APIRoot != "" {
 		t.Errorf("FTSMock.APIRoot = %q; want empty", conf.Sources.FTSMock.APIRoot)
 	}
+	if conf.Sources.FindATender.RateLimitRetries != 5 ||
+		conf.Sources.FTSMock.RateLimitRetries != 5 {
+		t.Errorf(
+			"RateLimitRetries = %d, %d; want 5, 5",
+			conf.Sources.FindATender.RateLimitRetries,
+			conf.Sources.FTSMock.RateLimitRetries,
+		)
+	}
 	for name, src := range map[string]JSCSourcesSource{
 		"find_a_tender": conf.Sources.FindATender.JSCSourcesSource,
 		"fts_mock":      conf.Sources.FTSMock.JSCSourcesSource,
@@ -236,6 +244,20 @@ func TestConcreteSourceSections(t *testing.T) {
 				}
 				wantMap["api_root"] = "http://localhost:8080"
 				wantFlat[tt.prefix+".api_root"] = "http://localhost:8080"
+
+				// The Find a Tender sources' own validate must still check
+				// interval, as JSCSourcesSource's does.
+				if err := s.SetValue("rate_limit_retries", "3"); err != nil {
+					t.Fatalf("SetValue(rate_limit_retries): %v", err)
+				}
+				if err := s.SetValue("rate_limit_retries", "-1"); err == nil {
+					t.Error("SetValue(rate_limit_retries, -1) succeeded")
+				}
+				if err := s.SetValue("interval", "0s"); err == nil {
+					t.Error("SetValue(interval, 0s) succeeded")
+				}
+				wantMap["rate_limit_retries"] = 3
+				wantFlat[tt.prefix+".rate_limit_retries"] = 3
 			} else if _, err := s.TypeOf("api_root"); err == nil {
 				t.Error("TypeOf(api_root) succeeded on a source without one")
 			}
@@ -295,8 +317,8 @@ func TestSourcesSection(t *testing.T) {
 	if flat["sources.fts_mock.api_root"] != "http://mock" {
 		t.Errorf("Flatten missing sources.fts_mock.api_root: %v", flat)
 	}
-	if len(flat) != 10 {
-		t.Errorf("Flatten has %d keys; want 10: %v", len(flat), flat)
+	if len(flat) != 12 {
+		t.Errorf("Flatten has %d keys; want 12: %v", len(flat), flat)
 	}
 
 	if err := c.SetValue("skip_rules_file", "rules.json"); err != nil ||

internal/config/conf_source_findatender.go

@@ -7,10 +7,13 @@ const JSCKSourceFindATender string = "find_a_tender"
 
 // JSCSourceFindATender configures fetching from the Find a Tender
 // OCDS API at APIRoot. Empty: the source fails to start if it is enabled,
-// and `fetcher fts-once` fails.
+// and `fetcher fts-once` fails. RateLimitRetries is how many times a request
+// refused for the API's rate limit is waited out and retried; 0 fails at the
+// first, and it can't be negative.
 type JSCSourceFindATender struct {
 	JSCSourcesSource
-	APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"`
+	APIRoot          string `cfg:"api_root"           default:"https://www.find-tender.service.gov.uk/api/1.0"`
+	RateLimitRetries int    `cfg:"rate_limit_retries" default:"5"`
 }
 
 // ---------- Section methods (see Section) ----------
@@ -37,3 +40,12 @@ func (c JSCSourceFindATender) ValueOf(key string) (any, error) {
 func (c *JSCSourceFindATender) SetValue(key string, value any) error {
 	return setValue(c, key, value)
 }
+
+// ---------- Section specific methods ----------
+
+// validate checks values before they are stored (see validator), as
+// JSCSourcesSource's does, which this one hides. rate_limit_retries can't be
+// negative.
+func (c *JSCSourceFindATender) validate(key string, value any) error {
+	return validateRateLimitRetries(&c.JSCSourcesSource, key, value)
+}

internal/config/conf_source_ftsmock.go

@@ -7,10 +7,12 @@ const JSCKSourceFTSMock string = "fts_mock"
 
 // JSCSourceFTSMock configures fetching from a Find a Tender mock
 // server (cmd/mock-fts) at APIRoot, which has no default. Empty: the source
-// fails to start if it is enabled.
+// fails to start if it is enabled. RateLimitRetries is as for
+// JSCSourceFindATender.
 type JSCSourceFTSMock struct {
 	JSCSourcesSource
-	APIRoot string `cfg:"api_root"`
+	APIRoot          string `cfg:"api_root"`
+	RateLimitRetries int    `cfg:"rate_limit_retries" default:"5"`
 }
 
 // ---------- Section methods (see Section) ----------
@@ -37,3 +39,12 @@ func (c JSCSourceFTSMock) ValueOf(key string) (any, error) {
 func (c *JSCSourceFTSMock) SetValue(key string, value any) error {
 	return setValue(c, key, value)
 }
+
+// ---------- Section specific methods ----------
+
+// validate checks values before they are stored (see validator), as
+// JSCSourcesSource's does, which this one hides. rate_limit_retries can't be
+// negative.
+func (c *JSCSourceFTSMock) validate(key string, value any) error {
+	return validateRateLimitRetries(&c.JSCSourcesSource, key, value)
+}

internal/config/conf_sources.go

@@ -97,3 +97,17 @@ func (c *JSCSourcesSource) validate(key string, value any) error {
 	}
 	return nil
 }
+
+// validateRateLimitRetries is the validate of a Find a Tender source's
+// section: source's checks, then rate_limit_retries, which can't be
+// negative.
+func validateRateLimitRetries(
+	source *JSCSourcesSource, key string, value any,
+) error {
+	if key == "rate_limit_retries" && value.(int) < 0 {
+		return errors.New(
+			"config: rate_limit_retries can't be negative; 0 fails at the first rate limit",
+		)
+	}
+	return source.validate(key, value)
+}