Made the fetcher save what it fetches, with fetcher.WithStore

With WithStore, the Find a Tender and mock workers save each run's
opportunities with SaveOpportunities and log what changed; a failed save
fails the run, so the checkpoint stays put and the next run retries.
Without a store nothing is persisted. The mock's opportunities are saved
with Source "FTSMock" rather than the converter's "FindATender".

fetcher service and job-search serve open the store at database.file
(cli.OpenConfiguredStore), or warn and run without one. A missing file
isn't created, and a database needing migration says to run job-search
migrate.

Fixed upsertOpportunity, which never wrote the source column, so every
saved opportunity had an empty Source.
This commit is contained in:
bwilliams committed 2026-10-16 11:26:16 +01:00
1 parent eba6af680f
commit b3d20c1db0
10 files changed
+407 -38

No files matched your search

+19 -2
View File
@@ -2,6 +2,7 @@ package fetch
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
@@ -25,7 +26,7 @@ func commandService(opts *cli.Options) *cobra.Command {
PersistentPostRun: func(cmd *cobra.Command, args []string) {
slog.Debug("command persistent post-run", "command", "service")
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, args []string) (err error) {
slog.Debug("command run", "command", "service")
foreground, _ := cmd.Flags().GetBool("foreground")
@@ -53,7 +54,23 @@ func commandService(opts *cli.Options) *cobra.Command {
stop()
}()
return fetcher.New(&conf.Sources).Run(ctx, since)
var fetcherOptions []fetcher.Option
opportunityStore, openErr := cli.OpenConfiguredStore(ctx, conf)
switch {
case errors.Is(openErr, cli.ErrNoDatabase):
slog.Warn("not storing fetched opportunities", "reason", openErr)
case openErr != nil:
return openErr
default:
defer func() {
err = errors.Join(err, opportunityStore.Close())
}()
fetcherOptions = append(
fetcherOptions, fetcher.WithStore(opportunityStore),
)
}
return fetcher.New(&conf.Sources, fetcherOptions...).Run(ctx, since)
},
}
cmd.Flags().Time(