Made the fetcher a Fetcher type with per-source config, and ran it in job-search serve
fetcher.Run(ctx, conf, since) becomes New(&conf.Sources, opts...) and Run(ctx, since). Each worker now takes only its own source's config, and Run starts each one through a closure that binds it. Cancelling ctx stops the workers through the Fetcher's background context, and Run always waits for every worker before returning their joined errors. The findatender client no longer holds the configuration: New takes only the API root, ReloadConfig is gone, and request tracing follows logging.IsDebugEnabled(). The FindAOpportunity worker is now FindATender. job-search serve runs the fetcher alongside the HTTP server. A source that fails stops the server, and the server stopping stops the sources; with no sources enabled the server keeps running. fetcher service and fetcher fts-once use the new APIs.
This commit is contained in:
1 parent
f0c171c8a8
commit
18db462d89
12 files changed
+269
-214
No files matched your search
@@ -12,43 +12,43 @@ import (
|
||||
"git.avsdev.uk/Go/job-search/internal/util"
|
||||
)
|
||||
|
||||
// runFindAOpportunity polls Find a Tender every configured interval until ctx
|
||||
// runFindATender polls Find a Tender every configured interval until ctx
|
||||
// is cancelled.
|
||||
func runFindAOpportunity(
|
||||
func runFindATender(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
conf *config.JSCSourceFindATender,
|
||||
since time.Time,
|
||||
) error {
|
||||
if conf.Sources.FindATender.APIRoot == "" {
|
||||
if conf.APIRoot == "" {
|
||||
return errors.New("fetcher: sources.find_a_tender.api_root is not set")
|
||||
}
|
||||
|
||||
worker := util.Worker{
|
||||
Logger: tagged(logger),
|
||||
Name: "FindAOpportunity",
|
||||
Interval: conf.Sources.FindATender.Interval,
|
||||
Name: "FindATender",
|
||||
Interval: conf.Interval,
|
||||
Task: func(ctx context.Context, since time.Time) error {
|
||||
return fetchFindAOpportunity(ctx, logger, conf, since)
|
||||
return fetchFindATender(ctx, logger, conf, since)
|
||||
},
|
||||
}
|
||||
return worker.Run(ctx, since)
|
||||
}
|
||||
|
||||
// fetchFindAOpportunity fetches everything updated since since from Find a
|
||||
// fetchFindATender fetches everything updated since since from Find a
|
||||
// Tender. Finding nothing returns ErrNoNewReleases, so the worker still moves
|
||||
// its checkpoint on.
|
||||
func fetchFindAOpportunity(
|
||||
func fetchFindATender(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JobSearchConfig,
|
||||
conf *config.JSCSourceFindATender,
|
||||
since time.Time,
|
||||
) error {
|
||||
opportunities, err := util.AwaitTask(
|
||||
ctx,
|
||||
func() ([]*types.Opportunity, error) {
|
||||
findATender := findatender.New(
|
||||
conf, conf.Sources.FindATender.APIRoot, findatender.WithLogger(logger),
|
||||
conf.APIRoot, findatender.WithLogger(logger),
|
||||
)
|
||||
defer findATender.Shutdown()
|
||||
return findATender.With(ctx).FetchAll(since, true)
|
||||
|
||||
Reference in new issue
Block a user