bwilliams
284a589627
Without --since, fetcher service and job-search serve start each source from the newest release stored for it (the new store.LatestReleaseDate, by Source), so a restart carries on where it left off instead of fetching the last day again; with nothing stored, the client's default still applies. --since overrides it for every source. The Find a Tender source's name is now a constant, as the Source its opportunities are stored with. The README's guide to adding a source is brought up to date with the current worker, start and config type names.
77 lines
2.0 KiB
Go
77 lines
2.0 KiB
Go
package fetcher
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/config"
|
|
"git.avsdev.uk/Go/job-search/internal/findatender"
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
"git.avsdev.uk/Go/job-search/internal/util"
|
|
)
|
|
|
|
// findATenderSource is the Source of the opportunities Find a Tender's
|
|
// converter makes, and so of the ones the Find a Tender source fetches.
|
|
const findATenderSource = "FindATender"
|
|
|
|
// runFindATender polls Find a Tender every configured interval until ctx
|
|
// is cancelled, passing what each run fetches to save.
|
|
func runFindATender(
|
|
ctx context.Context,
|
|
logger *slog.Logger,
|
|
conf *config.JSCSourceFindATender,
|
|
save saveFunc,
|
|
since time.Time,
|
|
) error {
|
|
if conf.APIRoot == "" {
|
|
return errors.New("fetcher: sources.find_a_tender.api_root is not set")
|
|
}
|
|
|
|
worker := util.Worker{
|
|
Logger: tagged(logger),
|
|
Name: findATenderSource,
|
|
Interval: conf.Interval,
|
|
Task: func(ctx context.Context, since time.Time) error {
|
|
opportunities, err := fetchFindATender(ctx, logger, conf, since)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return save(ctx, logger, opportunities)
|
|
},
|
|
}
|
|
return worker.Run(ctx, since)
|
|
}
|
|
|
|
// fetchFindATender fetches everything updated since since from Find a
|
|
// Tender. Finding nothing returns ErrNoNewReleases, so the worker still moves
|
|
// its checkpoint on.
|
|
func fetchFindATender(
|
|
ctx context.Context,
|
|
logger *slog.Logger,
|
|
conf *config.JSCSourceFindATender,
|
|
since time.Time,
|
|
) ([]*types.Opportunity, error) {
|
|
opportunities, err := util.AwaitTask(
|
|
ctx,
|
|
func() ([]*types.Opportunity, error) {
|
|
findATender := findatender.New(
|
|
conf.APIRoot, findatender.WithLogger(logger),
|
|
)
|
|
defer findATender.Shutdown()
|
|
return findATender.With(ctx).FetchAll(since, true)
|
|
},
|
|
)
|
|
if errors.Is(err, findatender.ErrNoReleases) {
|
|
return nil, ErrNoNewReleases
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tagged(logger).Info("fetched opportunities", "count", len(opportunities))
|
|
|
|
return opportunities, nil
|
|
}
|