Added a backfill in 7-day windows, and fetches now keep their data on a failure

Backfilling a long window (an early --since, or a long time offline)
hit the API's rate limit, which could still be exceeded after waiting
once, and then everything fetched so far was lost.

Before each Find a Tender source's worker starts, a start point more
than 7 days ago is now backfilled in 7-day windows bounded on both
sides, oldest first, each saved before the next is fetched. A window
that fails is saved as far as it got and fetched again from its start
after the source's interval. The worker then carries on from the end
of the last window.

A 429 is now waited out up to sources.<source>.rate_limit_retries
times (default 5) rather than once; fts-once --ignore-rate-limit uses
0. FetchAll returns what it fetched before an error along with it, and
the workers save it, so a failed run no longer loses its data.
findatender's FetchAll and FetchOnce take RequestOptions, which gains
Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.

internal/fetcher/worker_ftsmock.go

@@ -16,7 +16,7 @@ import (
 const ftsMockSource = "FTSMock"
 
 // runFTSMock polls the mock Find a Tender server every configured interval
-// until ctx is cancelled, passing what each run fetches to save.
+// until ctx is cancelled, as runFindATender polls Find a Tender.
 func runFTSMock(
 	ctx context.Context,
 	logger *slog.Logger,
@@ -28,31 +28,44 @@ func runFTSMock(
 		return errors.New("fetcher: sources.fts_mock.api_root is not set")
 	}
 
+	fetch := func(
+		ctx context.Context, since, until time.Time,
+	) ([]*types.Opportunity, error) {
+		return fetchFTSMock(ctx, logger, conf, since, until)
+	}
+
+	since = backfill(
+		ctx,
+		logger,
+		ftsMockSource,
+		conf.Interval,
+		fetch,
+		save,
+		since,
+	)
+
 	worker := util.Worker{
 		Logger:   tagged(logger),
 		Name:     ftsMockSource,
 		Interval: conf.Interval,
 		Task: func(ctx context.Context, since time.Time) error {
-			opportunities, err := fetchFTSMock(ctx, logger, conf, since)
-			if err != nil {
-				return err
-			}
-			return save(ctx, logger, opportunities)
+			opportunities, err := fetch(ctx, since, time.Time{})
+			return saveFetched(ctx, logger, save, opportunities, err)
 		},
 	}
 	return worker.Run(ctx, since)
 }
 
-// fetchFTSMock fetches everything updated since since from the mock Find a
-// Tender server. The mock serves the real API's format, so the converter
-// marks the opportunities as Find a Tender's; their Source is set to
-// ftsMockSource here instead. Finding nothing returns ErrNoNewReleases, so
-// the worker still moves its checkpoint on.
+// fetchFTSMock fetches everything updated from since up to until (zero: no
+// bound) from the mock Find a Tender server, as fetchFindATender does. The
+// mock serves the real API's format, so the converter marks the
+// opportunities as Find a Tender's; their Source is set to ftsMockSource here
+// instead.
 func fetchFTSMock(
 	ctx context.Context,
 	logger *slog.Logger,
 	conf *config.JSCSourceFTSMock,
-	since time.Time,
+	since, until time.Time,
 ) ([]*types.Opportunity, error) {
 	opportunities, err := util.AwaitTask(
 		ctx,
@@ -61,15 +74,16 @@ func fetchFTSMock(
 				conf.APIRoot, findatender.WithLogger(logger),
 			)
 			defer findATender.Shutdown()
-			return findATender.With(ctx).FetchAll(since, true)
+			return findATender.With(ctx).FetchAll(findatender.RequestOptions{
+				Since:            since,
+				Until:            until,
+				RateLimitRetries: conf.RateLimitRetries,
+			})
 		},
 	)
 	if errors.Is(err, findatender.ErrNoReleases) {
 		return nil, ErrNoNewReleases
 	}
-	if err != nil {
-		return nil, err
-	}
 
 	for _, opportunity := range opportunities {
 		opportunity.Source = ftsMockSource
@@ -77,5 +91,5 @@ func fetchFTSMock(
 
 	tagged(logger).Info("fetched opportunities", "count", len(opportunities))
 
-	return opportunities, nil
+	return opportunities, err
 }