Made the fetcher save what it fetches, with fetcher.WithStore
With WithStore, the Find a Tender and mock workers save each run's opportunities with SaveOpportunities and log what changed; a failed save fails the run, so the checkpoint stays put and the next run retries. Without a store nothing is persisted. The mock's opportunities are saved with Source "FTSMock" rather than the converter's "FindATender". fetcher service and job-search serve open the store at database.file (cli.OpenConfiguredStore), or warn and run without one. A missing file isn't created, and a database needing migration says to run job-search migrate. Fixed upsertOpportunity, which never wrote the source column, so every saved opportunity had an empty Source.
This commit is contained in:
1 parent
eba6af680f
commit
b3d20c1db0
10 files changed
+407
-38
No files matched your search
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/config"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
"git.avsdev.uk/Go/job-search/internal/util"
|
||||
)
|
||||
|
||||
@@ -27,6 +29,21 @@ func WithLogger(logger *slog.Logger) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithStore makes the workers save the opportunities they fetch to
|
||||
// opportunityStore. Without it, nothing is persisted.
|
||||
func WithStore(opportunityStore *store.Store) Option {
|
||||
return func(f *Fetcher) {
|
||||
f.store = opportunityStore
|
||||
}
|
||||
}
|
||||
|
||||
// saveFunc saves the opportunities a worker fetched; see Fetcher.save.
|
||||
type saveFunc func(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
opportunities []*types.Opportunity,
|
||||
) error
|
||||
|
||||
// logger returns the logger Run logs to: the given one, or slog.Default().
|
||||
// It isn't tagged here, as it is also given to the clients the workers
|
||||
// create, which add their own component; see tagged.
|
||||
@@ -48,6 +65,7 @@ func tagged(logger *slog.Logger) *slog.Logger {
|
||||
type Fetcher struct {
|
||||
conf *config.JSCSources
|
||||
log *slog.Logger
|
||||
store *store.Store // nil: nothing is persisted
|
||||
started time.Time
|
||||
|
||||
// backgroundCtx is the context the workers run under. backgroundCancel
|
||||
@@ -119,12 +137,12 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
|
||||
|
||||
if f.conf.FindATender.Enabled {
|
||||
start("FindATender", func(ctx context.Context) error {
|
||||
return runFindATender(ctx, logger, &f.conf.FindATender, since)
|
||||
return runFindATender(ctx, logger, &f.conf.FindATender, f.save, since)
|
||||
})
|
||||
}
|
||||
if f.conf.FTSMock.Enabled {
|
||||
start("FTSMock", func(ctx context.Context) error {
|
||||
return runFTSMock(ctx, logger, &f.conf.FTSMock, since)
|
||||
return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
|
||||
})
|
||||
}
|
||||
if f.conf.Sleeper.Enabled {
|
||||
@@ -139,3 +157,30 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// save saves opportunities to the store given WithStore, and logs what
|
||||
// changed. Without a store it does nothing. A failure fails the worker's run,
|
||||
// so its checkpoint isn't moved on and the next run fetches them again.
|
||||
func (f *Fetcher) save(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
opportunities []*types.Opportunity,
|
||||
) error {
|
||||
if f.store == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := f.store.SaveOpportunities(ctx, opportunities)
|
||||
if err != nil {
|
||||
return fmt.Errorf("saving opportunities: %w", err)
|
||||
}
|
||||
|
||||
tagged(logger).Info(
|
||||
"saved opportunities",
|
||||
"new_opportunities", result.NewOpportunities,
|
||||
"updated_opportunities", result.UpdatedOpportunities,
|
||||
"new_releases", result.NewReleases,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user