Made each source start from its newest stored release, with --since as an override
Without --since, fetcher service and job-search serve start each source from the newest release stored for it (the new store.LatestReleaseDate, by Source), so a restart carries on where it left off instead of fetching the last day again; with nothing stored, the client's default still applies. --since overrides it for every source. The Find a Tender source's name is now a constant, as the Source its opportunities are stored with. The README's guide to adding a source is brought up to date with the current worker, start and config type names.
This commit is contained in:
1 parent
3c4fde8dfd
commit
284a589627
10 files changed
+256
-22
No files matched your search
@@ -99,6 +99,11 @@ func New(conf *config.JSCSources, opts ...Option) *Fetcher {
|
||||
// logged as they happen and returned joined; nil means a clean shutdown. Run
|
||||
// cancels the Fetcher's background context as it returns, so it can only be
|
||||
// called once.
|
||||
//
|
||||
// Every worker starts from since. A zero since starts each source from the
|
||||
// newest release stored for it (see WithStore and startFrom); with no store
|
||||
// or nothing stored, the source's client picks (Find a Tender's: the last
|
||||
// day).
|
||||
func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
|
||||
defer f.backgroundCancel()
|
||||
|
||||
@@ -114,14 +119,22 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
|
||||
errs []error
|
||||
)
|
||||
|
||||
// start runs one source's worker until f.backgroundCtx is cancelled. Each
|
||||
// worker is given logger untagged: it tags its own lines with tagged, and
|
||||
// passes logger as it is to any client it creates.
|
||||
start := func(name string, run func(ctx context.Context) error) {
|
||||
// start runs one source's worker from where it starts (see startFrom)
|
||||
// until f.backgroundCtx is cancelled. name is also the Source its
|
||||
// opportunities are stored with. Each worker is given logger untagged: it
|
||||
// tags its own lines with tagged, and passes logger as it is to any client
|
||||
// it creates.
|
||||
start := func(
|
||||
name string, run func(ctx context.Context, since time.Time) error,
|
||||
) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := run(f.backgroundCtx); err != nil {
|
||||
sourceSince, err := f.startFrom(f.backgroundCtx, logger, name, since)
|
||||
if err == nil {
|
||||
err = run(f.backgroundCtx, sourceSince)
|
||||
}
|
||||
if err != nil {
|
||||
tagged(logger).Error(
|
||||
"source stopped, shutting down",
|
||||
"source", name,
|
||||
@@ -136,17 +149,17 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
|
||||
}
|
||||
|
||||
if f.conf.FindATender.Enabled {
|
||||
start("FindATender", func(ctx context.Context) error {
|
||||
start(findATenderSource, func(ctx context.Context, since time.Time) error {
|
||||
return runFindATender(ctx, logger, &f.conf.FindATender, f.save, since)
|
||||
})
|
||||
}
|
||||
if f.conf.FTSMock.Enabled {
|
||||
start("FTSMock", func(ctx context.Context) error {
|
||||
start(ftsMockSource, func(ctx context.Context, since time.Time) error {
|
||||
return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
|
||||
})
|
||||
}
|
||||
if f.conf.Sleeper.Enabled {
|
||||
start("Sleeper", func(ctx context.Context) error {
|
||||
start("Sleeper", func(ctx context.Context, since time.Time) error {
|
||||
return runSleeper(ctx, logger, &f.conf.Sleeper, since)
|
||||
})
|
||||
}
|
||||
@@ -158,6 +171,34 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// startFrom returns the time source's worker starts from: since if it is
|
||||
// set, as an override. Otherwise it is the date of the newest release stored
|
||||
// for source, so a restart carries on where the last run left off, or the
|
||||
// zero time with no store or nothing stored yet, which leaves the start to
|
||||
// the source's client. Fetching from the newest release's date fetches it
|
||||
// again; the store skips it.
|
||||
func (f *Fetcher) startFrom(
|
||||
ctx context.Context, logger *slog.Logger, source string, since time.Time,
|
||||
) (time.Time, error) {
|
||||
if !since.IsZero() || f.store == nil {
|
||||
return since, nil
|
||||
}
|
||||
|
||||
latest, err := f.store.LatestReleaseDate(ctx, source)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if !latest.IsZero() {
|
||||
tagged(logger).Info(
|
||||
"starting from the newest stored release",
|
||||
"source", source,
|
||||
"since", latest,
|
||||
)
|
||||
}
|
||||
|
||||
return latest, nil
|
||||
}
|
||||
|
||||
// save saves opportunities to the store given WithStore, and logs what
|
||||
// changed. Without a store it does nothing. A failure fails the worker's run,
|
||||
// so its checkpoint isn't moved on and the next run fetches them again.
|
||||
|
||||
Reference in new issue
Block a user