Made the fetcher save what it fetches, with fetcher.WithStore

With WithStore, the Find a Tender and mock workers save each run's
opportunities with SaveOpportunities and log what changed; a failed save
fails the run, so the checkpoint stays put and the next run retries.
Without a store nothing is persisted. The mock's opportunities are saved
with Source "FTSMock" rather than the converter's "FindATender".

fetcher service and job-search serve open the store at database.file
(cli.OpenConfiguredStore), or warn and run without one. A missing file
isn't created, and a database needing migration says to run job-search
migrate.

Fixed upsertOpportunity, which never wrote the source column, so every
saved opportunity had an empty Source.

internal/fetcher/fetcher.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
 	"git.avsdev.uk/Go/job-search/internal/util"
 )
 
@@ -27,6 +29,21 @@ func WithLogger(logger *slog.Logger) Option {
 	}
 }
 
+// WithStore makes the workers save the opportunities they fetch to
+// opportunityStore. Without it, nothing is persisted.
+func WithStore(opportunityStore *store.Store) Option {
+	return func(f *Fetcher) {
+		f.store = opportunityStore
+	}
+}
+
+// saveFunc saves the opportunities a worker fetched; see Fetcher.save.
+type saveFunc func(
+	ctx context.Context,
+	logger *slog.Logger,
+	opportunities []*types.Opportunity,
+) error
+
 // logger returns the logger Run logs to: the given one, or slog.Default().
 // It isn't tagged here, as it is also given to the clients the workers
 // create, which add their own component; see tagged.
@@ -48,6 +65,7 @@ func tagged(logger *slog.Logger) *slog.Logger {
 type Fetcher struct {
 	conf    *config.JSCSources
 	log     *slog.Logger
+	store   *store.Store // nil: nothing is persisted
 	started time.Time
 
 	// backgroundCtx is the context the workers run under. backgroundCancel
@@ -119,12 +137,12 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
 
 	if f.conf.FindATender.Enabled {
 		start("FindATender", func(ctx context.Context) error {
-			return runFindATender(ctx, logger, &f.conf.FindATender, since)
+			return runFindATender(ctx, logger, &f.conf.FindATender, f.save, since)
 		})
 	}
 	if f.conf.FTSMock.Enabled {
 		start("FTSMock", func(ctx context.Context) error {
-			return runFTSMock(ctx, logger, &f.conf.FTSMock, since)
+			return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
 		})
 	}
 	if f.conf.Sleeper.Enabled {
@@ -139,3 +157,30 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
 
 	return errors.Join(errs...)
 }
+
+// save saves opportunities to the store given WithStore, and logs what
+// changed. Without a store it does nothing. A failure fails the worker's run,
+// so its checkpoint isn't moved on and the next run fetches them again.
+func (f *Fetcher) save(
+	ctx context.Context,
+	logger *slog.Logger,
+	opportunities []*types.Opportunity,
+) error {
+	if f.store == nil {
+		return nil
+	}
+
+	result, err := f.store.SaveOpportunities(ctx, opportunities)
+	if err != nil {
+		return fmt.Errorf("saving opportunities: %w", err)
+	}
+
+	tagged(logger).Info(
+		"saved opportunities",
+		"new_opportunities", result.NewOpportunities,
+		"updated_opportunities", result.UpdatedOpportunities,
+		"new_releases", result.NewReleases,
+	)
+
+	return nil
+}

internal/fetcher/fetcher_test.go

@@ -15,6 +15,8 @@ import (
 	"time"
 
 	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
 	"git.avsdev.uk/Go/job-search/internal/util"
 )
 
@@ -59,20 +61,35 @@ func ftsServer(
 }
 
 // fetchFunc fetches from one source at apiRoot.
-type fetchFunc func(ctx context.Context, apiRoot string, since time.Time) error
+type fetchFunc func(
+	ctx context.Context, apiRoot string, since time.Time,
+) ([]*types.Opportunity, error)
 
 // fetchers are the sources' fetch functions, each given only its API root.
 var fetchers = map[string]fetchFunc{
-	"FindATender": func(ctx context.Context, apiRoot string, since time.Time) error {
-		conf := &config.JSCSourceFindATender{APIRoot: apiRoot}
+	"FindATender": func(
+		ctx context.Context, apiRoot string, since time.Time,
+	) ([]*types.Opportunity, error) {
+		conf := &config.JSCSourceFindATender{
+			APIRoot: apiRoot,
+		}
 		return fetchFindATender(ctx, slog.Default(), conf, since)
 	},
-	"FTSMock": func(ctx context.Context, apiRoot string, since time.Time) error {
-		conf := &config.JSCSourceFTSMock{APIRoot: apiRoot}
+	"FTSMock": func(
+		ctx context.Context, apiRoot string, since time.Time,
+	) ([]*types.Opportunity, error) {
+		conf := &config.JSCSourceFTSMock{
+			APIRoot: apiRoot,
+		}
 		return fetchFTSMock(ctx, slog.Default(), conf, since)
 	},
 }
 
+// noSave is a saveFunc that saves nothing.
+func noSave(context.Context, *slog.Logger, []*types.Opportunity) error {
+	return nil
+}
+
 func TestFetchFunctions(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -102,7 +119,7 @@ func TestFetchFunctions(t *testing.T) {
 		for _, tc := range tests {
 			t.Run(fname+"/"+tc.name, func(t *testing.T) {
 				srv, hits := ftsServer(t, tc.status, tc.body)
-				err := fetch(
+				_, err := fetch(
 					context.Background(), srv.URL, time.Now().Add(-time.Hour),
 				)
 				if !tc.check(err) {
@@ -162,7 +179,7 @@ func TestFetchCancelled(t *testing.T) {
 	cancel()
 	// Unroutable root: the request should never be waited on.
 	start := time.Now()
-	err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{})
+	_, err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{})
 	if !errors.Is(err, context.Canceled) {
 		t.Fatalf("err = %v, want context.Canceled", err)
 	}
@@ -196,7 +213,9 @@ func TestSleepCompletes(t *testing.T) {
 // one holds shutdown open until the worker's shutdown timeout.
 func TestRunSleeperShutdown(t *testing.T) {
 	for _, block := range []bool{false, true} {
-		conf := &config.JSCSourceSleeper{BlockShutdown: block}
+		conf := &config.JSCSourceSleeper{
+			BlockShutdown: block,
+		}
 		conf.Interval = time.Hour
 
 		ctx, cancel := context.WithCancel(context.Background())
@@ -225,12 +244,16 @@ func TestRunWorkersRejectZeroInterval(t *testing.T) {
 	ctx, logger := context.Background(), slog.Default()
 	runs := map[string]func() error{
 		"FindATender": func() error {
-			conf := &config.JSCSourceFindATender{APIRoot: "http://127.0.0.1:1"}
-			return runFindATender(ctx, logger, conf, time.Time{})
+			conf := &config.JSCSourceFindATender{
+				APIRoot: "http://127.0.0.1:1",
+			}
+			return runFindATender(ctx, logger, conf, noSave, time.Time{})
 		},
 		"FTSMock": func() error {
-			conf := &config.JSCSourceFTSMock{APIRoot: "http://127.0.0.1:1"}
-			return runFTSMock(ctx, logger, conf, time.Time{})
+			conf := &config.JSCSourceFTSMock{
+				APIRoot: "http://127.0.0.1:1",
+			}
+			return runFTSMock(ctx, logger, conf, noSave, time.Time{})
 		},
 		"Sleeper": func() error {
 			return runSleeper(ctx, logger, &config.JSCSourceSleeper{}, time.Time{})
@@ -402,3 +425,130 @@ func TestRunEnabledSources(t *testing.T) {
 		t.Fatal("Run did not return after cancellation")
 	}
 }
+
+// Each source's opportunities carry its own Source: the mock's are marked
+// FTSMock, though its converter marks them as Find a Tender's.
+func TestFetchSetsSource(t *testing.T) {
+	srv, _ := ftsServer(t, http.StatusOK, onePackage)
+	for name, want := range map[string]string{
+		"FindATender": "FindATender",
+		"FTSMock":     "FTSMock",
+	} {
+		opportunities, err := fetchers[name](
+			context.Background(), srv.URL, time.Now().Add(-time.Hour),
+		)
+		if err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if len(opportunities) != 1 || opportunities[0].Source != want {
+			t.Errorf(
+				"%s: opportunities = %+v; want one from %s",
+				name, opportunities, want,
+			)
+		}
+	}
+}
+
+// openTestStore returns a migrated store in a temporary file.
+func openTestStore(t *testing.T) *store.Store {
+	t.Helper()
+	path := t.TempDir() + "/fetcher.db"
+	opportunityStore, err := store.Open(
+		context.Background(), path, store.WithAutoMigrate(),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		opportunityStore.Close()
+	})
+	return opportunityStore
+}
+
+// WithStore makes the workers save what they fetch, each with its source.
+func TestRunSavesToStore(t *testing.T) {
+	opportunityStore := openTestStore(t)
+	fat, _ := ftsServer(t, http.StatusOK, onePackage)
+	mock, _ := ftsServer(
+		t, http.StatusOK, strings.ReplaceAll(onePackage, "ocds-1", "ocds-2"),
+	)
+	sources := &config.JSCSources{}
+	sources.FindATender.Enabled = true
+	sources.FindATender.Interval = time.Hour
+	sources.FindATender.APIRoot = fat.URL
+	sources.FTSMock.Enabled = true
+	sources.FTSMock.Interval = time.Hour
+	sources.FTSMock.APIRoot = mock.URL
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		done <- New(sources, WithStore(opportunityStore)).Run(
+			ctx, time.Now().Add(-time.Hour),
+		)
+	}()
+
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		stored, err := opportunityStore.ListOpportunities(
+			context.Background(), store.OpportunityFilter{},
+		)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(stored) == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("stored %d opportunities; want 2", len(stored))
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run = %v", err)
+	}
+
+	for uuid, want := range map[string]string{
+		"ocds-1": "FindATender",
+		"ocds-2": "FTSMock",
+	} {
+		opportunity, err := opportunityStore.GetOpportunity(
+			context.Background(), uuid,
+		)
+		if err != nil {
+			t.Fatalf("%s: %v", uuid, err)
+		}
+		if opportunity.Source != want {
+			t.Errorf("%s: Source = %q; want %q", uuid, opportunity.Source, want)
+		}
+	}
+}
+
+// Without a store, save does nothing; with one that fails, it returns the
+// error, so the worker's run fails and its checkpoint stays put.
+func TestSave(t *testing.T) {
+	srv, _ := ftsServer(t, http.StatusOK, onePackage)
+	opportunities, err := fetchers["FindATender"](
+		context.Background(), srv.URL, time.Now().Add(-time.Hour),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = New(&config.JSCSources{}).save(
+		context.Background(), slog.Default(), opportunities,
+	)
+	if err != nil {
+		t.Errorf("save without a store = %v; want nil", err)
+	}
+
+	closed := openTestStore(t)
+	closed.Close()
+	err = New(&config.JSCSources{}, WithStore(closed)).save(
+		context.Background(), slog.Default(), opportunities,
+	)
+	if err == nil || !strings.Contains(err.Error(), "saving opportunities") {
+		t.Errorf("save to a closed store = %v; want a saving error", err)
+	}
+}

internal/fetcher/worker_findatender.go

@@ -13,11 +13,12 @@ import (
 )
 
 // runFindATender polls Find a Tender every configured interval until ctx
-// is cancelled.
+// is cancelled, passing what each run fetches to save.
 func runFindATender(
 	ctx context.Context,
 	logger *slog.Logger,
 	conf *config.JSCSourceFindATender,
+	save saveFunc,
 	since time.Time,
 ) error {
 	if conf.APIRoot == "" {
@@ -29,7 +30,11 @@ func runFindATender(
 		Name:     "FindATender",
 		Interval: conf.Interval,
 		Task: func(ctx context.Context, since time.Time) error {
-			return fetchFindATender(ctx, logger, conf, since)
+			opportunities, err := fetchFindATender(ctx, logger, conf, since)
+			if err != nil {
+				return err
+			}
+			return save(ctx, logger, opportunities)
 		},
 	}
 	return worker.Run(ctx, since)
@@ -43,7 +48,7 @@ func fetchFindATender(
 	logger *slog.Logger,
 	conf *config.JSCSourceFindATender,
 	since time.Time,
-) error {
+) ([]*types.Opportunity, error) {
 	opportunities, err := util.AwaitTask(
 		ctx,
 		func() ([]*types.Opportunity, error) {
@@ -55,13 +60,13 @@ func fetchFindATender(
 		},
 	)
 	if errors.Is(err, findatender.ErrNoReleases) {
-		return ErrNoNewReleases
+		return nil, ErrNoNewReleases
 	}
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	tagged(logger).Info("fetched opportunities", "count", len(opportunities))
 
-	return nil
+	return opportunities, nil
 }

internal/fetcher/worker_ftsmock.go

@@ -12,12 +12,16 @@ import (
 	"git.avsdev.uk/Go/job-search/internal/util"
 )
 
+// ftsMockSource is the Source of the opportunities the mock source fetches.
+const ftsMockSource = "FTSMock"
+
 // runFTSMock polls the mock Find a Tender server every configured interval
-// until ctx is cancelled.
+// until ctx is cancelled, passing what each run fetches to save.
 func runFTSMock(
 	ctx context.Context,
 	logger *slog.Logger,
 	conf *config.JSCSourceFTSMock,
+	save saveFunc,
 	since time.Time,
 ) error {
 	if conf.APIRoot == "" {
@@ -26,24 +30,30 @@ func runFTSMock(
 
 	worker := util.Worker{
 		Logger:   tagged(logger),
-		Name:     "FTSMock",
+		Name:     ftsMockSource,
 		Interval: conf.Interval,
 		Task: func(ctx context.Context, since time.Time) error {
-			return fetchFTSMock(ctx, logger, conf, since)
+			opportunities, err := fetchFTSMock(ctx, logger, conf, since)
+			if err != nil {
+				return err
+			}
+			return save(ctx, logger, opportunities)
 		},
 	}
 	return worker.Run(ctx, since)
 }
 
 // fetchFTSMock fetches everything updated since since from the mock Find a
-// Tender server. Finding nothing returns ErrNoNewReleases, so the worker still
-// moves its checkpoint on.
+// Tender server. The mock serves the real API's format, so the converter
+// marks the opportunities as Find a Tender's; their Source is set to
+// ftsMockSource here instead. Finding nothing returns ErrNoNewReleases, so
+// the worker still moves its checkpoint on.
 func fetchFTSMock(
 	ctx context.Context,
 	logger *slog.Logger,
 	conf *config.JSCSourceFTSMock,
 	since time.Time,
-) error {
+) ([]*types.Opportunity, error) {
 	opportunities, err := util.AwaitTask(
 		ctx,
 		func() ([]*types.Opportunity, error) {
@@ -55,13 +65,17 @@ func fetchFTSMock(
 		},
 	)
 	if errors.Is(err, findatender.ErrNoReleases) {
-		return ErrNoNewReleases
+		return nil, ErrNoNewReleases
 	}
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	for _, opportunity := range opportunities {
+		opportunity.Source = ftsMockSource
 	}
 
 	tagged(logger).Info("fetched opportunities", "count", len(opportunities))
 
-	return nil
+	return opportunities, nil
 }