package fetcher import ( "context" "errors" "log/slog" "time" "git.avsdev.uk/Go/job-search/internal/types" "git.avsdev.uk/Go/job-search/internal/util" ) // backfillChunk is the longest window a backfill fetches at once (the user's // choice: fixed, not a setting). const backfillChunk = 7 * 24 * time.Hour // fetchWindowFunc fetches the opportunities a source updated from since up // to until; a zero until means no bound. If it fails partway, it returns // what it fetched before the error along with it. type fetchWindowFunc func( ctx context.Context, since, until time.Time, ) ([]*types.Opportunity, error) // backfill catches a source up from since, before its worker starts, so a // long window (an early --since, or a long time offline) isn't fetched in // one go. It fetches backfillChunk windows, bounded on both sides, oldest // first, saving each before fetching the next, until less than a window is // left before now. It returns where the worker carries on from: since when // it is zero (the client picks) or within backfillChunk of now. // // A window that fails is saved as far as it got, and fetched again from its // start after interval, until it succeeds or ctx is cancelled. Cancelling // ctx returns the start of the window it stopped in. // // logger is given untagged, as to the workers (see Fetcher.Run); source is // the source's name, for the backfill's own lines. func backfill( ctx context.Context, logger *slog.Logger, source string, interval time.Duration, fetch fetchWindowFunc, save saveFunc, since time.Time, ) time.Time { backfillLogger := tagged(logger).With("source", source) for !since.IsZero() && time.Since(since) > backfillChunk { until := since.Add(backfillChunk) backfillLogger.Info("backfilling", "since", since, "until", until) opportunities, err := fetch(ctx, since, until) err = saveFetched(ctx, logger, save, opportunities, err) switch { case err == nil || errors.Is(err, util.ErrWorkerNoWork): since = until case ctx.Err() != nil: return since default: backfillLogger.Error( "backfill window failed, retrying", "since", since, "until", until, "error", err, "retry_in", interval, ) if sleep(ctx, false, interval) != nil { return since } } } return since } // saveFetched saves what a fetch returned, even when it failed partway, so // nothing fetched is lost. It returns fetchErr, joined with save's error if // that failed too. func saveFetched( ctx context.Context, logger *slog.Logger, save saveFunc, opportunities []*types.Opportunity, fetchErr error, ) error { if len(opportunities) == 0 { return fetchErr } return errors.Join(fetchErr, save(ctx, logger, opportunities)) }