Made the fetcher save what it fetches, with fetcher.WithStore

With WithStore, the Find a Tender and mock workers save each run's
opportunities with SaveOpportunities and log what changed; a failed save
fails the run, so the checkpoint stays put and the next run retries.
Without a store nothing is persisted. The mock's opportunities are saved
with Source "FTSMock" rather than the converter's "FindATender".

fetcher service and job-search serve open the store at database.file
(cli.OpenConfiguredStore), or warn and run without one. A missing file
isn't created, and a database needing migration says to run job-search
migrate.

Fixed upsertOpportunity, which never wrote the source column, so every
saved opportunity had an empty Source.
This commit is contained in:
bwilliams committed 2026-10-16 11:26:16 +01:00
1 parent eba6af680f
commit b3d20c1db0
10 files changed
+407 -38

No files matched your search

+47 -2
View File
@@ -9,6 +9,8 @@ import (
"time"
"git.avsdev.uk/Go/job-search/internal/config"
"git.avsdev.uk/Go/job-search/internal/store"
"git.avsdev.uk/Go/job-search/internal/types"
"git.avsdev.uk/Go/job-search/internal/util"
)
@@ -27,6 +29,21 @@ func WithLogger(logger *slog.Logger) Option {
}
}
// WithStore makes the workers save the opportunities they fetch to
// opportunityStore. Without it, nothing is persisted.
func WithStore(opportunityStore *store.Store) Option {
return func(f *Fetcher) {
f.store = opportunityStore
}
}
// saveFunc saves the opportunities a worker fetched; see Fetcher.save.
type saveFunc func(
ctx context.Context,
logger *slog.Logger,
opportunities []*types.Opportunity,
) error
// logger returns the logger Run logs to: the given one, or slog.Default().
// It isn't tagged here, as it is also given to the clients the workers
// create, which add their own component; see tagged.
@@ -48,6 +65,7 @@ func tagged(logger *slog.Logger) *slog.Logger {
type Fetcher struct {
conf *config.JSCSources
log *slog.Logger
store *store.Store // nil: nothing is persisted
started time.Time
// backgroundCtx is the context the workers run under. backgroundCancel
@@ -119,12 +137,12 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
if f.conf.FindATender.Enabled {
start("FindATender", func(ctx context.Context) error {
return runFindATender(ctx, logger, &f.conf.FindATender, since)
return runFindATender(ctx, logger, &f.conf.FindATender, f.save, since)
})
}
if f.conf.FTSMock.Enabled {
start("FTSMock", func(ctx context.Context) error {
return runFTSMock(ctx, logger, &f.conf.FTSMock, since)
return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
})
}
if f.conf.Sleeper.Enabled {
@@ -139,3 +157,30 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
return errors.Join(errs...)
}
// save saves opportunities to the store given WithStore, and logs what
// changed. Without a store it does nothing. A failure fails the worker's run,
// so its checkpoint isn't moved on and the next run fetches them again.
func (f *Fetcher) save(
ctx context.Context,
logger *slog.Logger,
opportunities []*types.Opportunity,
) error {
if f.store == nil {
return nil
}
result, err := f.store.SaveOpportunities(ctx, opportunities)
if err != nil {
return fmt.Errorf("saving opportunities: %w", err)
}
tagged(logger).Info(
"saved opportunities",
"new_opportunities", result.NewOpportunities,
"updated_opportunities", result.UpdatedOpportunities,
"new_releases", result.NewReleases,
)
return nil
}