Made each source start from its newest stored release, with --since as an override

Without --since, fetcher service and job-search serve start each source
from the newest release stored for it (the new store.LatestReleaseDate,
by Source), so a restart carries on where it left off instead of fetching
the last day again; with nothing stored, the client's default still
applies. --since overrides it for every source. The Find a Tender source's
name is now a constant, as the Source its opportunities are stored with.

The README's guide to adding a source is brought up to date with the
current worker, start and config type names.
This commit is contained in:
bwilliams committed 2026-10-16 11:37:09 +01:00
1 parent 3c4fde8dfd
commit 284a589627
10 files changed
+256 -22

No files matched your search

+49 -8
View File
@@ -99,6 +99,11 @@ func New(conf *config.JSCSources, opts ...Option) *Fetcher {
// logged as they happen and returned joined; nil means a clean shutdown. Run
// cancels the Fetcher's background context as it returns, so it can only be
// called once.
//
// Every worker starts from since. A zero since starts each source from the
// newest release stored for it (see WithStore and startFrom); with no store
// or nothing stored, the source's client picks (Find a Tender's: the last
// day).
func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
defer f.backgroundCancel()
@@ -114,14 +119,22 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
errs []error
)
// start runs one source's worker until f.backgroundCtx is cancelled. Each
// worker is given logger untagged: it tags its own lines with tagged, and
// passes logger as it is to any client it creates.
start := func(name string, run func(ctx context.Context) error) {
// start runs one source's worker from where it starts (see startFrom)
// until f.backgroundCtx is cancelled. name is also the Source its
// opportunities are stored with. Each worker is given logger untagged: it
// tags its own lines with tagged, and passes logger as it is to any client
// it creates.
start := func(
name string, run func(ctx context.Context, since time.Time) error,
) {
wg.Add(1)
go func() {
defer wg.Done()
if err := run(f.backgroundCtx); err != nil {
sourceSince, err := f.startFrom(f.backgroundCtx, logger, name, since)
if err == nil {
err = run(f.backgroundCtx, sourceSince)
}
if err != nil {
tagged(logger).Error(
"source stopped, shutting down",
"source", name,
@@ -136,17 +149,17 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
}
if f.conf.FindATender.Enabled {
start("FindATender", func(ctx context.Context) error {
start(findATenderSource, func(ctx context.Context, since time.Time) error {
return runFindATender(ctx, logger, &f.conf.FindATender, f.save, since)
})
}
if f.conf.FTSMock.Enabled {
start("FTSMock", func(ctx context.Context) error {
start(ftsMockSource, func(ctx context.Context, since time.Time) error {
return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
})
}
if f.conf.Sleeper.Enabled {
start("Sleeper", func(ctx context.Context) error {
start("Sleeper", func(ctx context.Context, since time.Time) error {
return runSleeper(ctx, logger, &f.conf.Sleeper, since)
})
}
@@ -158,6 +171,34 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
return errors.Join(errs...)
}
// startFrom returns the time source's worker starts from: since if it is
// set, as an override. Otherwise it is the date of the newest release stored
// for source, so a restart carries on where the last run left off, or the
// zero time with no store or nothing stored yet, which leaves the start to
// the source's client. Fetching from the newest release's date fetches it
// again; the store skips it.
func (f *Fetcher) startFrom(
ctx context.Context, logger *slog.Logger, source string, since time.Time,
) (time.Time, error) {
if !since.IsZero() || f.store == nil {
return since, nil
}
latest, err := f.store.LatestReleaseDate(ctx, source)
if err != nil {
return time.Time{}, err
}
if !latest.IsZero() {
tagged(logger).Info(
"starting from the newest stored release",
"source", source,
"since", latest,
)
}
return latest, nil
}
// save saves opportunities to the store given WithStore, and logs what
// changed. Without a store it does nothing. A failure fails the worker's run,
// so its checkpoint isn't moved on and the next run fetches them again.