package fetcher import ( "context" "errors" "log/slog" "time" "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/findatender" "git.avsdev.uk/Go/job-search/internal/types" "git.avsdev.uk/Go/job-search/internal/util" ) // findATenderSource is the Source of the opportunities Find a Tender's // converter makes, and so of the ones the Find a Tender source fetches. const findATenderSource = "FindATender" // runFindATender polls Find a Tender every configured interval until ctx // is cancelled, passing what each run fetches to save, even when the run // fails partway. A long window is backfilled first (see backfill). func runFindATender( ctx context.Context, logger *slog.Logger, conf *config.JSCSourceFindATender, save saveFunc, since time.Time, ) error { if conf.APIRoot == "" { return errors.New("fetcher: sources.find_a_tender.api_root is not set") } fetch := func( ctx context.Context, since, until time.Time, ) ([]*types.Opportunity, error) { return fetchFindATender(ctx, logger, conf, since, until) } since = backfill( ctx, logger, findATenderSource, conf.Interval, fetch, save, since, ) worker := util.Worker{ Logger: tagged(logger), Name: findATenderSource, Interval: conf.Interval, Task: func(ctx context.Context, since time.Time) error { opportunities, err := fetch(ctx, since, time.Time{}) return saveFetched(ctx, logger, save, opportunities, err) }, } return worker.Run(ctx, since) } // fetchFindATender fetches everything updated from since up to until (zero: // no bound) from Find a Tender. If it fails partway, what it fetched before // the error is returned along with it. Finding nothing returns // ErrNoNewReleases, so the worker still moves its checkpoint on. func fetchFindATender( ctx context.Context, logger *slog.Logger, conf *config.JSCSourceFindATender, since, until time.Time, ) ([]*types.Opportunity, error) { opportunities, err := util.AwaitTask( ctx, func() ([]*types.Opportunity, error) { findATender := findatender.New( conf.APIRoot, findatender.WithLogger(logger), ) defer findATender.Shutdown() return findATender.With(ctx).FetchAll(findatender.RequestOptions{ Since: since, Until: until, RateLimitRetries: conf.RateLimitRetries, }) }, ) if errors.Is(err, findatender.ErrNoReleases) { return nil, ErrNoNewReleases } tagged(logger).Info( "fetched opportunities", "source", findATenderSource, "count", len(opportunities), ) return opportunities, err }