Made the fetcher save what it fetches, with fetcher.WithStore

With WithStore, the Find a Tender and mock workers save each run's opportunities with SaveOpportunities and log what changed; a failed save fails the run, so the checkpoint stays put and the next run retries. Without a store nothing is persisted. The mock's opportunities are saved with Source "FTSMock" rather than the converter's "FindATender". fetcher service and job-search serve open the store at database.file (cli.OpenConfiguredStore), or warn and run without one. A missing file isn't created, and a database needing migration says to run job-search migrate. Fixed upsertOpportunity, which never wrote the source column, so every saved opportunity had an empty Source.
2026-10-16 11:26:16 +01:00
1 parent eba6af680f
commit b3d20c1db0
@@ -15,6 +15,8 @@ import (
 	"time"

 	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
 	"git.avsdev.uk/Go/job-search/internal/util"
 )

@@ -59,20 +61,35 @@ func ftsServer(
 }

 // fetchFunc fetches from one source at apiRoot.
-type fetchFunc func(ctx context.Context, apiRoot string, since time.Time) error
+type fetchFunc func(
+	ctx context.Context, apiRoot string, since time.Time,
+) ([]*types.Opportunity, error)

 // fetchers are the sources' fetch functions, each given only its API root.
 var fetchers = map[string]fetchFunc{
-	"FindATender": func(ctx context.Context, apiRoot string, since time.Time) error {
-		conf := &config.JSCSourceFindATender{APIRoot: apiRoot}
+	"FindATender": func(
+		ctx context.Context, apiRoot string, since time.Time,
+	) ([]*types.Opportunity, error) {
+		conf := &config.JSCSourceFindATender{
+			APIRoot: apiRoot,
+		}
 		return fetchFindATender(ctx, slog.Default(), conf, since)
 	},
-	"FTSMock": func(ctx context.Context, apiRoot string, since time.Time) error {
-		conf := &config.JSCSourceFTSMock{APIRoot: apiRoot}
+	"FTSMock": func(
+		ctx context.Context, apiRoot string, since time.Time,
+	) ([]*types.Opportunity, error) {
+		conf := &config.JSCSourceFTSMock{
+			APIRoot: apiRoot,
+		}
 		return fetchFTSMock(ctx, slog.Default(), conf, since)
 	},
 }

+// noSave is a saveFunc that saves nothing.
+func noSave(context.Context, *slog.Logger, []*types.Opportunity) error {
+	return nil
+}
+
 func TestFetchFunctions(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -102,7 +119,7 @@ func TestFetchFunctions(t *testing.T) {
 		for _, tc := range tests {
 			t.Run(fname+"/"+tc.name, func(t *testing.T) {
 				srv, hits := ftsServer(t, tc.status, tc.body)
-				err := fetch(
+				_, err := fetch(
 					context.Background(), srv.URL, time.Now().Add(-time.Hour),
 				)
 				if !tc.check(err) {
@@ -162,7 +179,7 @@ func TestFetchCancelled(t *testing.T) {
 	cancel()
 	// Unroutable root: the request should never be waited on.
 	start := time.Now()
-	err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{})
+	_, err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{})
 	if !errors.Is(err, context.Canceled) {
 		t.Fatalf("err = %v, want context.Canceled", err)
 	}
@@ -196,7 +213,9 @@ func TestSleepCompletes(t *testing.T) {
 // one holds shutdown open until the worker's shutdown timeout.
 func TestRunSleeperShutdown(t *testing.T) {
 	for _, block := range []bool{false, true} {
-		conf := &config.JSCSourceSleeper{BlockShutdown: block}
+		conf := &config.JSCSourceSleeper{
+			BlockShutdown: block,
+		}
 		conf.Interval = time.Hour

 		ctx, cancel := context.WithCancel(context.Background())
@@ -225,12 +244,16 @@ func TestRunWorkersRejectZeroInterval(t *testing.T) {
 	ctx, logger := context.Background(), slog.Default()
 	runs := map[string]func() error{
 		"FindATender": func() error {
-			conf := &config.JSCSourceFindATender{APIRoot: "http://127.0.0.1:1"}
-			return runFindATender(ctx, logger, conf, time.Time{})
+			conf := &config.JSCSourceFindATender{
+				APIRoot: "http://127.0.0.1:1",
+			}
+			return runFindATender(ctx, logger, conf, noSave, time.Time{})
 		},
 		"FTSMock": func() error {
-			conf := &config.JSCSourceFTSMock{APIRoot: "http://127.0.0.1:1"}
-			return runFTSMock(ctx, logger, conf, time.Time{})
+			conf := &config.JSCSourceFTSMock{
+				APIRoot: "http://127.0.0.1:1",
+			}
+			return runFTSMock(ctx, logger, conf, noSave, time.Time{})
 		},
 		"Sleeper": func() error {
 			return runSleeper(ctx, logger, &config.JSCSourceSleeper{}, time.Time{})
@@ -402,3 +425,130 @@ func TestRunEnabledSources(t *testing.T) {
 		t.Fatal("Run did not return after cancellation")
 	}
 }
+
+// Each source's opportunities carry its own Source: the mock's are marked
+// FTSMock, though its converter marks them as Find a Tender's.
+func TestFetchSetsSource(t *testing.T) {
+	srv, _ := ftsServer(t, http.StatusOK, onePackage)
+	for name, want := range map[string]string{
+		"FindATender": "FindATender",
+		"FTSMock":     "FTSMock",
+	} {
+		opportunities, err := fetchers[name](
+			context.Background(), srv.URL, time.Now().Add(-time.Hour),
+		)
+		if err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if len(opportunities) != 1 || opportunities[0].Source != want {
+			t.Errorf(
+				"%s: opportunities = %+v; want one from %s",
+				name, opportunities, want,
+			)
+		}
+	}
+}
+
+// openTestStore returns a migrated store in a temporary file.
+func openTestStore(t *testing.T) *store.Store {
+	t.Helper()
+	path := t.TempDir() + "/fetcher.db"
+	opportunityStore, err := store.Open(
+		context.Background(), path, store.WithAutoMigrate(),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		opportunityStore.Close()
+	})
+	return opportunityStore
+}
+
+// WithStore makes the workers save what they fetch, each with its source.
+func TestRunSavesToStore(t *testing.T) {
+	opportunityStore := openTestStore(t)
+	fat, _ := ftsServer(t, http.StatusOK, onePackage)
+	mock, _ := ftsServer(
+		t, http.StatusOK, strings.ReplaceAll(onePackage, "ocds-1", "ocds-2"),
+	)
+	sources := &config.JSCSources{}
+	sources.FindATender.Enabled = true
+	sources.FindATender.Interval = time.Hour
+	sources.FindATender.APIRoot = fat.URL
+	sources.FTSMock.Enabled = true
+	sources.FTSMock.Interval = time.Hour
+	sources.FTSMock.APIRoot = mock.URL
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		done <- New(sources, WithStore(opportunityStore)).Run(
+			ctx, time.Now().Add(-time.Hour),
+		)
+	}()
+
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		stored, err := opportunityStore.ListOpportunities(
+			context.Background(), store.OpportunityFilter{},
+		)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(stored) == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("stored %d opportunities; want 2", len(stored))
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run = %v", err)
+	}
+
+	for uuid, want := range map[string]string{
+		"ocds-1": "FindATender",
+		"ocds-2": "FTSMock",
+	} {
+		opportunity, err := opportunityStore.GetOpportunity(
+			context.Background(), uuid,
+		)
+		if err != nil {
+			t.Fatalf("%s: %v", uuid, err)
+		}
+		if opportunity.Source != want {
+			t.Errorf("%s: Source = %q; want %q", uuid, opportunity.Source, want)
+		}
+	}
+}
+
+// Without a store, save does nothing; with one that fails, it returns the
+// error, so the worker's run fails and its checkpoint stays put.
+func TestSave(t *testing.T) {
+	srv, _ := ftsServer(t, http.StatusOK, onePackage)
+	opportunities, err := fetchers["FindATender"](
+		context.Background(), srv.URL, time.Now().Add(-time.Hour),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = New(&config.JSCSources{}).save(
+		context.Background(), slog.Default(), opportunities,
+	)
+	if err != nil {
+		t.Errorf("save without a store = %v; want nil", err)
+	}
+
+	closed := openTestStore(t)
+	closed.Close()
+	err = New(&config.JSCSources{}, WithStore(closed)).save(
+		context.Background(), slog.Default(), opportunities,
+	)
+	if err == nil || !strings.Contains(err.Error(), "saving opportunities") {
+		t.Errorf("save to a closed store = %v; want a saving error", err)
+	}
+}