Made the fetcher save what it fetches, with fetcher.WithStore
With WithStore, the Find a Tender and mock workers save each run's opportunities with SaveOpportunities and log what changed; a failed save fails the run, so the checkpoint stays put and the next run retries. Without a store nothing is persisted. The mock's opportunities are saved with Source "FTSMock" rather than the converter's "FindATender". fetcher service and job-search serve open the store at database.file (cli.OpenConfiguredStore), or warn and run without one. A missing file isn't created, and a database needing migration says to run job-search migrate. Fixed upsertOpportunity, which never wrote the source column, so every saved opportunity had an empty Source.
This commit is contained in:
1 parent
eba6af680f
commit
b3d20c1db0
10 files changed
+407
-38
No files matched your search
@@ -13,11 +13,12 @@ import (
|
||||
)
|
||||
|
||||
// runFindATender polls Find a Tender every configured interval until ctx
|
||||
// is cancelled.
|
||||
// is cancelled, passing what each run fetches to save.
|
||||
func runFindATender(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JSCSourceFindATender,
|
||||
save saveFunc,
|
||||
since time.Time,
|
||||
) error {
|
||||
if conf.APIRoot == "" {
|
||||
@@ -29,7 +30,11 @@ func runFindATender(
|
||||
Name: "FindATender",
|
||||
Interval: conf.Interval,
|
||||
Task: func(ctx context.Context, since time.Time) error {
|
||||
return fetchFindATender(ctx, logger, conf, since)
|
||||
opportunities, err := fetchFindATender(ctx, logger, conf, since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return save(ctx, logger, opportunities)
|
||||
},
|
||||
}
|
||||
return worker.Run(ctx, since)
|
||||
@@ -43,7 +48,7 @@ func fetchFindATender(
|
||||
logger *slog.Logger,
|
||||
conf *config.JSCSourceFindATender,
|
||||
since time.Time,
|
||||
) error {
|
||||
) ([]*types.Opportunity, error) {
|
||||
opportunities, err := util.AwaitTask(
|
||||
ctx,
|
||||
func() ([]*types.Opportunity, error) {
|
||||
@@ -55,13 +60,13 @@ func fetchFindATender(
|
||||
},
|
||||
)
|
||||
if errors.Is(err, findatender.ErrNoReleases) {
|
||||
return ErrNoNewReleases
|
||||
return nil, ErrNoNewReleases
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tagged(logger).Info("fetched opportunities", "count", len(opportunities))
|
||||
|
||||
return nil
|
||||
return opportunities, nil
|
||||
}
|
||||
Reference in new issue
Block a user