bwilliams
c09c30ae71
Backfilling a long window (an early --since, or a long time offline) hit the API's rate limit, which could still be exceeded after waiting once, and then everything fetched so far was lost. Before each Find a Tender source's worker starts, a start point more than 7 days ago is now backfilled in 7-day windows bounded on both sides, oldest first, each saved before the next is fetched. A window that fails is saved as far as it got and fetched again from its start after the source's interval. The worker then carries on from the end of the last window. A 429 is now waited out up to sources.<source>.rate_limit_retries times (default 5) rather than once; fts-once --ignore-rate-limit uses 0. FetchAll returns what it fetched before an error along with it, and the workers save it, so a failed run no longer loses its data. findatender's FetchAll and FetchOnce take RequestOptions, which gains Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
91 lines
2.7 KiB
Go
91 lines
2.7 KiB
Go
package fetcher
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
"git.avsdev.uk/Go/job-search/internal/util"
|
|
)
|
|
|
|
// backfillChunk is the longest window a backfill fetches at once (the user's
|
|
// choice: fixed, not a setting).
|
|
const backfillChunk = 7 * 24 * time.Hour
|
|
|
|
// fetchWindowFunc fetches the opportunities a source updated from since up
|
|
// to until; a zero until means no bound. If it fails partway, it returns
|
|
// what it fetched before the error along with it.
|
|
type fetchWindowFunc func(
|
|
ctx context.Context, since, until time.Time,
|
|
) ([]*types.Opportunity, error)
|
|
|
|
// backfill catches a source up from since, before its worker starts, so a
|
|
// long window (an early --since, or a long time offline) isn't fetched in
|
|
// one go. It fetches backfillChunk windows, bounded on both sides, oldest
|
|
// first, saving each before fetching the next, until less than a window is
|
|
// left before now. It returns where the worker carries on from: since when
|
|
// it is zero (the client picks) or within backfillChunk of now.
|
|
//
|
|
// A window that fails is saved as far as it got, and fetched again from its
|
|
// start after interval, until it succeeds or ctx is cancelled. Cancelling
|
|
// ctx returns the start of the window it stopped in.
|
|
//
|
|
// logger is given untagged, as to the workers (see Fetcher.Run); source is
|
|
// the source's name, for the backfill's own lines.
|
|
func backfill(
|
|
ctx context.Context,
|
|
logger *slog.Logger,
|
|
source string,
|
|
interval time.Duration,
|
|
fetch fetchWindowFunc,
|
|
save saveFunc,
|
|
since time.Time,
|
|
) time.Time {
|
|
backfillLogger := tagged(logger).With("source", source)
|
|
|
|
for !since.IsZero() && time.Since(since) > backfillChunk {
|
|
until := since.Add(backfillChunk)
|
|
backfillLogger.Info("backfilling", "since", since, "until", until)
|
|
|
|
opportunities, err := fetch(ctx, since, until)
|
|
err = saveFetched(ctx, logger, save, opportunities, err)
|
|
switch {
|
|
case err == nil || errors.Is(err, util.ErrWorkerNoWork):
|
|
since = until
|
|
case ctx.Err() != nil:
|
|
return since
|
|
default:
|
|
backfillLogger.Error(
|
|
"backfill window failed, retrying",
|
|
"since", since,
|
|
"until", until,
|
|
"error", err,
|
|
"retry_in", interval,
|
|
)
|
|
if sleep(ctx, false, interval) != nil {
|
|
return since
|
|
}
|
|
}
|
|
}
|
|
|
|
return since
|
|
}
|
|
|
|
// saveFetched saves what a fetch returned, even when it failed partway, so
|
|
// nothing fetched is lost. It returns fetchErr, joined with save's error if
|
|
// that failed too.
|
|
func saveFetched(
|
|
ctx context.Context,
|
|
logger *slog.Logger,
|
|
save saveFunc,
|
|
opportunities []*types.Opportunity,
|
|
fetchErr error,
|
|
) error {
|
|
if len(opportunities) == 0 {
|
|
return fetchErr
|
|
}
|
|
return errors.Join(fetchErr, save(ctx, logger, opportunities))
|
|
}
|