Made the fetcher save what it fetches, with fetcher.WithStore

With WithStore, the Find a Tender and mock workers save each run's
opportunities with SaveOpportunities and log what changed; a failed save
fails the run, so the checkpoint stays put and the next run retries.
Without a store nothing is persisted. The mock's opportunities are saved
with Source "FTSMock" rather than the converter's "FindATender".

fetcher service and job-search serve open the store at database.file
(cli.OpenConfiguredStore), or warn and run without one. A missing file
isn't created, and a database needing migration says to run job-search
migrate.

Fixed upsertOpportunity, which never wrote the source column, so every
saved opportunity had an empty Source.

internal/cli/fetch/service.go

@@ -2,6 +2,7 @@ package fetch
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"os"
@@ -25,7 +26,7 @@ func commandService(opts *cli.Options) *cobra.Command {
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
 			slog.Debug("command persistent post-run", "command", "service")
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			slog.Debug("command run", "command", "service")
 
 			foreground, _ := cmd.Flags().GetBool("foreground")
@@ -53,7 +54,23 @@ func commandService(opts *cli.Options) *cobra.Command {
 				stop()
 			}()
 
-			return fetcher.New(&conf.Sources).Run(ctx, since)
+			var fetcherOptions []fetcher.Option
+			opportunityStore, openErr := cli.OpenConfiguredStore(ctx, conf)
+			switch {
+			case errors.Is(openErr, cli.ErrNoDatabase):
+				slog.Warn("not storing fetched opportunities", "reason", openErr)
+			case openErr != nil:
+				return openErr
+			default:
+				defer func() {
+					err = errors.Join(err, opportunityStore.Close())
+				}()
+				fetcherOptions = append(
+					fetcherOptions, fetcher.WithStore(opportunityStore),
+				)
+			}
+
+			return fetcher.New(&conf.Sources, fetcherOptions...).Run(ctx, since)
 		},
 	}
 	cmd.Flags().Time(

internal/cli/serve/command.go

@@ -29,9 +29,10 @@ func NewCommand(opts *cli.Options) *cobra.Command {
 		Short: "Serve the REST API (/api) and WebSocket (/ws), and fetch from sources",
 		Long: `Serve the REST API under /api and the WebSocket endpoint under /ws.
 
-It also runs a worker for every enabled source, as fetcher service does. A
-source that stops with an error stops the server too, and the server
-stopping stops the sources.`,
+It also runs a worker for every enabled source, as fetcher service does,
+saving what they fetch to database.file if it is set. A source that stops
+with an error stops the server too, and the server stopping stops the
+sources.`,
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			slog.Debug("command persistent pre-run", "command", "serve")
 		},
@@ -118,7 +119,23 @@ stopping stops the sources.`,
 			runCtx, cancelRun := context.WithCancel(ctx)
 			defer cancelRun()
 
-			sourceFetcher := fetcher.New(&conf.Sources)
+			var fetcherOptions []fetcher.Option
+			opportunityStore, openErr := cli.OpenConfiguredStore(ctx, conf)
+			switch {
+			case errors.Is(openErr, cli.ErrNoDatabase):
+				slog.Warn("not storing fetched opportunities", "reason", openErr)
+			case openErr != nil:
+				return openErr
+			default:
+				defer func() {
+					err = errors.Join(err, opportunityStore.Close())
+				}()
+				fetcherOptions = append(
+					fetcherOptions, fetcher.WithStore(opportunityStore),
+				)
+			}
+
+			sourceFetcher := fetcher.New(&conf.Sources, fetcherOptions...)
 			fetcherDone := make(chan error, 1)
 			go func() {
 				fetchErr := sourceFetcher.Run(runCtx, time.Time{})

internal/cli/store.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/store"
+)
+
+// ErrNoDatabase is returned by OpenConfiguredStore when database.file isn't
+// set.
+var ErrNoDatabase = errors.New("database.file is not set")
+
+// OpenConfiguredStore opens the store at database.file, or returns
+// ErrNoDatabase if none is configured. A missing file or a database that
+// needs migrating fails with a message saying how to create or migrate it; a
+// missing file isn't created, as store.Open would. The caller must close the
+// store.
+func OpenConfiguredStore(
+	ctx context.Context, conf *config.JobSearchConfig,
+) (*store.Store, error) {
+	file := conf.Database.File
+	if file == "" {
+		return nil, ErrNoDatabase
+	}
+
+	_, err := os.Stat(file)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf(
+			"database %s does not exist: run `job-search migrate up --create`",
+			file,
+		)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("database %s: %w", file, err)
+	}
+
+	opportunityStore, err := store.Open(ctx, file)
+	if errors.Is(err, store.ErrNeedsMigration) {
+		return nil, fmt.Errorf(
+			"database %s: %w: run `job-search migrate`", file, err,
+		)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("database %s: %w", file, err)
+	}
+
+	return opportunityStore, nil
+}

internal/cli/store_test.go

@@ -0,0 +1,66 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/store"
+)
+
+// No database.file means no store, reported as ErrNoDatabase.
+func TestOpenConfiguredStoreNone(t *testing.T) {
+	opportunityStore, err := OpenConfiguredStore(
+		context.Background(), &config.JobSearchConfig{},
+	)
+	if opportunityStore != nil || !errors.Is(err, ErrNoDatabase) {
+		t.Errorf(
+			"OpenConfiguredStore = %v, %v; want nil, ErrNoDatabase",
+			opportunityStore, err,
+		)
+	}
+}
+
+// A missing database isn't created; one that needs migrating says how to
+// migrate it; a migrated one opens.
+func TestOpenConfiguredStore(t *testing.T) {
+	conf := &config.JobSearchConfig{}
+	conf.Database.File = filepath.Join(t.TempDir(), "jobsearch.db")
+
+	_, err := OpenConfiguredStore(context.Background(), conf)
+	if err == nil || !strings.Contains(err.Error(), "migrate up --create") {
+		t.Fatalf("missing: err = %v; want a hint to create it", err)
+	}
+	_, statErr := os.Stat(conf.Database.File)
+	if !errors.Is(statErr, os.ErrNotExist) {
+		t.Fatal("the missing database file was created")
+	}
+
+	// An empty file is a database at version 0.
+	if err := os.WriteFile(conf.Database.File, nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err = OpenConfiguredStore(context.Background(), conf)
+	if !errors.Is(err, store.ErrNeedsMigration) ||
+		!strings.Contains(err.Error(), "run `job-search migrate`") {
+		t.Fatalf("unmigrated: err = %v; want ErrNeedsMigration and a hint", err)
+	}
+
+	migrated, err := store.Open(
+		context.Background(), conf.Database.File, store.WithAutoMigrate(),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	migrated.Close()
+
+	opportunityStore, err := OpenConfiguredStore(context.Background(), conf)
+	if err != nil {
+		t.Fatalf("migrated: err = %v", err)
+	}
+	opportunityStore.Close()
+}