Added a backfill in 7-day windows, and fetches now keep their data on a failure

Backfilling a long window (an early --since, or a long time offline)
hit the API's rate limit, which could still be exceeded after waiting
once, and then everything fetched so far was lost.

Before each Find a Tender source's worker starts, a start point more
than 7 days ago is now backfilled in 7-day windows bounded on both
sides, oldest first, each saved before the next is fetched. A window
that fails is saved as far as it got and fetched again from its start
after the source's interval. The worker then carries on from the end
of the last window.

A 429 is now waited out up to sources.<source>.rate_limit_retries
times (default 5) rather than once; fts-once --ignore-rate-limit uses
0. FetchAll returns what it fetched before an error along with it, and
the workers save it, so a failed run no longer loses its data.
findatender's FetchAll and FetchOnce take RequestOptions, which gains
Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.

internal/findatender/api.go

@@ -20,6 +20,10 @@ import (
 	"git.avsdev.uk/Go/job-search/internal/logging"
 )
 
+// apiTimeLayout is the layout of updatedFrom and updatedTo: wall-clock time
+// with no offset, which the API reads as UK time.
+const apiTimeLayout = "2006-01-02T15:04:05"
+
 // Errors returned when fetching from the API.
 var (
 	// ErrNoNext means a release package has no next page link.
@@ -27,7 +31,8 @@ var (
 	// ErrNoReleases means the API returned no releases.
 	ErrNoReleases = errors.New("findatender: no releases")
 	// ErrRateLimitExceeded means the API refused a request with 429 Too Many
-	// Requests, and waiting for it was not allowed or did not help.
+	// Requests, and waiting for it was not allowed or did not help (see
+	// RequestOptions.RateLimitRetries).
 	ErrRateLimitExceeded = errors.New("findatender: rate limit has been exceeded")
 	// ErrFetchCancelled means the fetch's context was cancelled while waiting
 	// out a rate limit.
@@ -40,11 +45,15 @@ var (
 type RequestOptions struct {
 	// Since asks for releases updated from this time. Zero means 24 hours ago.
 	Since time.Time
+	// Until asks for releases updated up to this time. Zero means no bound
+	// (the API's: now).
+	Until time.Time
 	// Limit is the number of releases per page. Zero means 100.
 	Limit int
-	// AllowRateLimit waits out a 429 response and retries once, instead of
-	// failing with ErrRateLimitExceeded.
-	AllowRateLimit bool
+	// RateLimitRetries is how many times a request refused with 429 Too Many
+	// Requests is retried, each after waiting as the response's Retry-After
+	// asks, before failing with ErrRateLimitExceeded. 0 fails at the first.
+	RateLimitRetries int
 }
 
 // ---------- Utility methods for handling API irregularities ----------
@@ -409,54 +418,47 @@ func logTrace(logger *slog.Logger, res *resty.Response) {
 	}
 }
 
-// fetchURI performs a fetch against the API. If the API returns with a rate
-// limit reached response and allowRateLimit is true, then wait and retry like a
-// good client. If allowRateLimit is false, return to the caller informing them
-// of the rate limit exceeded error. If, after waiting, the API still thinks we
-// have exceeded the rate limit, bail out anyway.
+// fetchURI performs a fetch against the API. A response of 429 Too Many
+// Requests is waited out for as long as its Retry-After asks and the request
+// retried, up to rateLimitRetries times, like a good client; one more fails
+// with ErrRateLimitExceeded.
 func (ft *FindATender) fetchURI(
-	uri string, allowRateLimit bool,
+	uri string, rateLimitRetries int,
 ) (*resty.Response, error) {
-	res, err := fetchURI(ft, uri)
-	if err != nil {
-		return nil, err
-	}
-
 	ctx := ft.ctx
 	if ctx == nil {
 		ctx = context.Background()
 	}
 
-	if res.StatusCode() == 429 {
+	res, err := fetchURI(ft, uri)
+	for retry := 1; err == nil && res.StatusCode() == 429; retry++ {
 		closeBody(res)
 
-		if !allowRateLimit {
+		if retry > rateLimitRetries {
 			return nil, ErrRateLimitExceeded
 		}
 
 		retryAfterText := res.Header().Get("Retry-After")
-		retryAfterSeconds, err := strconv.ParseInt(retryAfterText, 10, 32)
-		if err != nil {
-			return nil, fmt.Errorf("unknown retry-after value: %w", err)
+		retryAfterSeconds, parseErr := strconv.ParseInt(retryAfterText, 10, 32)
+		if parseErr != nil {
+			return nil, fmt.Errorf("unknown retry-after value: %w", parseErr)
 		}
 
 		retryAfter := time.Duration(retryAfterSeconds) * time.Second
 		ft.logger().Warn(
 			"api rate limit hit, waiting as asked",
 			"retry_after", retryAfter,
+			"retry", retry,
+			"retries", rateLimitRetries,
 		)
 		if rateSleep(ctx, retryAfter) != nil {
 			return nil, ErrFetchCancelled
 		}
 
 		res, err = fetchURI(ft, uri)
-		if err != nil {
-			return nil, err
-		}
-		if res.StatusCode() == 429 {
-			closeBody(res)
-			return nil, ErrRateLimitExceeded
-		}
+	}
+	if err != nil {
+		return nil, err
 	}
 
 	if res.StatusCode() != 200 {
@@ -482,23 +484,26 @@ func (ft *FindATender) fetchWithOpts(
 	uri := fmt.Sprintf(
 		"%s/ocdsReleasePackages?updatedFrom=%s&limit=%d",
 		ft.apiRoot,
-		opts.Since.Format("2006-01-02T15:04:05"),
+		opts.Since.Format(apiTimeLayout),
 		opts.Limit,
 	)
+	if !opts.Until.IsZero() {
+		uri += "&updatedTo=" + opts.Until.Format(apiTimeLayout)
+	}
 
-	return ft.fetchURI(uri, opts.AllowRateLimit)
+	return ft.fetchURI(uri, opts.RateLimitRetries)
 }
 
 // fetchNext fetches the page pkg's "Next" link points to. It returns
 // ErrNoNext if there is no "Next" link; the caller can decide whether asking
 // for it was an error.
 func (ft *FindATender) fetchNext(
-	releasePackage *ocds.ReleasePackage, allowRateLimit bool,
+	releasePackage *ocds.ReleasePackage, rateLimitRetries int,
 ) (*resty.Response, error) {
 	if releasePackage.Links == nil || releasePackage.Links.Next == "" {
 		return nil, ErrNoNext
 	}
-	return ft.fetchURI(releasePackage.Links.Next, allowRateLimit)
+	return ft.fetchURI(releasePackage.Links.Next, rateLimitRetries)
 }
 
 // fetchOnce makes one call on the API. It does not follow any "Next" links.
@@ -521,7 +526,9 @@ func (ft *FindATender) fetchOnce(opts RequestOptions) ([]ocds.Release, error) {
 }
 
 // fetchAll makes a call on the API. If there are any "Next" links, it calls
-// them in succession until the end has been reached.
+// them in succession until the end has been reached. If a page fails, the
+// releases of the pages before it are returned with the error, so the caller
+// can keep them.
 func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
 	var releases []ocds.Release
 
@@ -551,17 +558,17 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) {
 			seen[releasePackage.Links.Next] = true
 		}
 
-		res, err := ft.fetchNext(releasePackage, opts.AllowRateLimit)
+		res, err := ft.fetchNext(releasePackage, opts.RateLimitRetries)
 		if errors.Is(err, ErrNoNext) {
 			break
 		}
 		if err != nil {
-			return nil, err
+			return releases, err
 		}
 
 		releasePackage, err = safeUnpackReleases(ft.logger(), res)
 		if err != nil {
-			return nil, err
+			return releases, err
 		}
 		releases = append(releases, releasePackage.Releases...)
 	}