bwilliams
c09c30ae71
Backfilling a long window (an early --since, or a long time offline) hit the API's rate limit, which could still be exceeded after waiting once, and then everything fetched so far was lost. Before each Find a Tender source's worker starts, a start point more than 7 days ago is now backfilled in 7-day windows bounded on both sides, oldest first, each saved before the next is fetched. A window that fails is saved as far as it got and fetched again from its start after the source's interval. The worker then carries on from the end of the last window. A 429 is now waited out up to sources.<source>.rate_limit_retries times (default 5) rather than once; fts-once --ignore-rate-limit uses 0. FetchAll returns what it fetched before an error along with it, and the workers save it, so a failed run no longer loses its data. findatender's FetchAll and FetchOnce take RequestOptions, which gains Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
93 lines
2.5 KiB
Go
93 lines
2.5 KiB
Go
package fetcher
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/config"
|
|
"git.avsdev.uk/Go/job-search/internal/findatender"
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
"git.avsdev.uk/Go/job-search/internal/util"
|
|
)
|
|
|
|
// findATenderSource is the Source of the opportunities Find a Tender's
|
|
// converter makes, and so of the ones the Find a Tender source fetches.
|
|
const findATenderSource = "FindATender"
|
|
|
|
// runFindATender polls Find a Tender every configured interval until ctx
|
|
// is cancelled, passing what each run fetches to save, even when the run
|
|
// fails partway. A long window is backfilled first (see backfill).
|
|
func runFindATender(
|
|
ctx context.Context,
|
|
logger *slog.Logger,
|
|
conf *config.JSCSourceFindATender,
|
|
save saveFunc,
|
|
since time.Time,
|
|
) error {
|
|
if conf.APIRoot == "" {
|
|
return errors.New("fetcher: sources.find_a_tender.api_root is not set")
|
|
}
|
|
|
|
fetch := func(
|
|
ctx context.Context, since, until time.Time,
|
|
) ([]*types.Opportunity, error) {
|
|
return fetchFindATender(ctx, logger, conf, since, until)
|
|
}
|
|
|
|
since = backfill(
|
|
ctx,
|
|
logger,
|
|
findATenderSource,
|
|
conf.Interval,
|
|
fetch,
|
|
save,
|
|
since,
|
|
)
|
|
|
|
worker := util.Worker{
|
|
Logger: tagged(logger),
|
|
Name: findATenderSource,
|
|
Interval: conf.Interval,
|
|
Task: func(ctx context.Context, since time.Time) error {
|
|
opportunities, err := fetch(ctx, since, time.Time{})
|
|
return saveFetched(ctx, logger, save, opportunities, err)
|
|
},
|
|
}
|
|
return worker.Run(ctx, since)
|
|
}
|
|
|
|
// fetchFindATender fetches everything updated from since up to until (zero:
|
|
// no bound) from Find a Tender. If it fails partway, what it fetched before
|
|
// the error is returned along with it. Finding nothing returns
|
|
// ErrNoNewReleases, so the worker still moves its checkpoint on.
|
|
func fetchFindATender(
|
|
ctx context.Context,
|
|
logger *slog.Logger,
|
|
conf *config.JSCSourceFindATender,
|
|
since, until time.Time,
|
|
) ([]*types.Opportunity, error) {
|
|
opportunities, err := util.AwaitTask(
|
|
ctx,
|
|
func() ([]*types.Opportunity, error) {
|
|
findATender := findatender.New(
|
|
conf.APIRoot, findatender.WithLogger(logger),
|
|
)
|
|
defer findATender.Shutdown()
|
|
return findATender.With(ctx).FetchAll(findatender.RequestOptions{
|
|
Since: since,
|
|
Until: until,
|
|
RateLimitRetries: conf.RateLimitRetries,
|
|
})
|
|
},
|
|
)
|
|
if errors.Is(err, findatender.ErrNoReleases) {
|
|
return nil, ErrNoNewReleases
|
|
}
|
|
|
|
tagged(logger).Info("fetched opportunities", "count", len(opportunities))
|
|
|
|
return opportunities, err
|
|
}
|