job-search/internal/fetcher/fetcher_test.go

package fetcher

import (
	"bytes"
	"context"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"git.avsdev.uk/Go/job-search/internal/config"
	"git.avsdev.uk/Go/job-search/internal/events"
	"git.avsdev.uk/Go/job-search/internal/store"
	"git.avsdev.uk/Go/job-search/internal/types"
	"git.avsdev.uk/Go/job-search/internal/util"
)

func TestMain(m *testing.M) {
	// Keep worker and fetch log lines out of the test output.
	slog.SetDefault(slog.New(slog.DiscardHandler))
	os.Exit(m.Run())
}

const onePackage = `{
	"uri": "x",
	"releases": [{
		"ocid": "ocds-1",
		"id": "r1",
		"date": "2026-10-01T12:00:00Z",
		"tag": ["tender"],
		"tender": {"id": "t1", "title": "A tender"}
	}]
}`

const emptyPackage = `{"uri": "x", "releases": []}`

// ftsServer serves body (with status) for every request and counts requests.
func ftsServer(
	t *testing.T, status int, body string,
) (*httptest.Server, *atomic.Int32) {
	t.Helper()
	var hits atomic.Int32
	srv := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			hits.Add(1)
			if r.URL.Path != "/ocdsReleasePackages" {
				t.Errorf("unexpected path %q", r.URL.Path)
			}
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(status)
			io.WriteString(w, body)
		}),
	)
	t.Cleanup(srv.Close)
	return srv, &hits
}

// fetchFunc fetches from one source at apiRoot.
type fetchFunc func(
	ctx context.Context, apiRoot string, since time.Time,
) ([]*types.Opportunity, error)

// fetchers are the sources' fetch functions, each given only its API root.
var fetchers = map[string]fetchFunc{
	"FindATender": func(
		ctx context.Context, apiRoot string, since time.Time,
	) ([]*types.Opportunity, error) {
		conf := &config.JSCSourceFindATender{
			APIRoot: apiRoot,
		}
		return fetchFindATender(ctx, slog.Default(), conf, since)
	},
	"FTSMock": func(
		ctx context.Context, apiRoot string, since time.Time,
	) ([]*types.Opportunity, error) {
		conf := &config.JSCSourceFTSMock{
			APIRoot: apiRoot,
		}
		return fetchFTSMock(ctx, slog.Default(), conf, since)
	},
}

// noSave is a saveFunc that saves nothing.
func noSave(context.Context, *slog.Logger, []*types.Opportunity) error {
	return nil
}

func TestFetchFunctions(t *testing.T) {
	tests := []struct {
		name    string
		status  int
		body    string
		check   func(error) bool
		wantErr string
	}{
		{
			"releases found",
			http.StatusOK,
			onePackage,
			func(err error) bool {
				return err == nil
			},
			"nil",
		},
		{"no releases", http.StatusOK, emptyPackage, func(err error) bool {
			return errors.Is(err, ErrNoNewReleases) &&
				errors.Is(err, util.ErrWorkerNoWork)
		}, "ErrNoNewReleases wrapping ErrWorkerNoWork"},
		{"bad body", http.StatusInternalServerError, "oops", func(err error) bool {
			return err != nil && !errors.Is(err, util.ErrWorkerNoWork)
		}, "a failure error"},
	}
	for fname, fetch := range fetchers {
		for _, tc := range tests {
			t.Run(fname+"/"+tc.name, func(t *testing.T) {
				srv, hits := ftsServer(t, tc.status, tc.body)
				_, err := fetch(
					context.Background(), srv.URL, time.Now().Add(-time.Hour),
				)
				if !tc.check(err) {
					t.Fatalf("err = %v, want %s", err, tc.wantErr)
				}
				if hits.Load() == 0 {
					t.Fatal("server was not called")
				}
			})
		}
	}
}

// Run gives each worker its own source's configuration: with only one source
// enabled, only its API root is called.
func TestRunUsesEachSourcesAPIRoot(t *testing.T) {
	for _, enabled := range []string{"FindATender", "FTSMock"} {
		fat, fatHits := ftsServer(t, http.StatusOK, emptyPackage)
		mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage)
		sources := &config.JSCSources{}
		sources.FindATender.APIRoot = fat.URL
		sources.FindATender.Interval = time.Hour
		sources.FTSMock.APIRoot = mock.URL
		sources.FTSMock.Interval = time.Hour
		wanted, other := fatHits, mockHits
		if enabled == "FindATender" {
			sources.FindATender.Enabled = true
		} else {
			sources.FTSMock.Enabled = true
			wanted, other = mockHits, fatHits
		}

		ctx, cancel := context.WithCancel(context.Background())
		done := make(chan error, 1)
		go func() {
			done <- New(sources).Run(ctx, time.Time{})
		}()

		deadline := time.Now().Add(5 * time.Second)
		for wanted.Load() == 0 && time.Now().Before(deadline) {
			time.Sleep(5 * time.Millisecond)
		}
		cancel()
		<-done

		if wanted.Load() == 0 || other.Load() != 0 {
			t.Errorf(
				"%s enabled: its root hit %d times, the other's %d; want >0 and 0",
				enabled, wanted.Load(), other.Load(),
			)
		}
	}
}

func TestFetchCancelled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	// Unroutable root: the request should never be waited on.
	start := time.Now()
	_, err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{})
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("err = %v, want context.Canceled", err)
	}
	if d := time.Since(start); d > time.Second {
		t.Fatalf("cancelled fetch took %v", d)
	}
}

func TestSleepCancelled(t *testing.T) {
	for _, block := range []bool{false, true} {
		ctx, cancel := context.WithTimeout(
			context.Background(), 10*time.Millisecond,
		)
		defer cancel()
		err := sleep(ctx, block, time.Hour)
		if !errors.Is(err, context.DeadlineExceeded) {
			t.Fatalf("block=%v: err = %v, want context.DeadlineExceeded", block, err)
		}
	}
}

func TestSleepCompletes(t *testing.T) {
	for _, block := range []bool{false, true} {
		if err := sleep(context.Background(), block, time.Millisecond); err != nil {
			t.Fatalf("block=%v: err = %v", block, err)
		}
	}
}

// A non-blocking sleeper stops as soon as shutdown is signalled; a blocking
// one holds shutdown open until the worker's shutdown timeout.
func TestRunSleeperShutdown(t *testing.T) {
	for _, block := range []bool{false, true} {
		conf := &config.JSCSourceSleeper{
			BlockShutdown: block,
		}
		conf.Interval = time.Hour

		ctx, cancel := context.WithCancel(context.Background())
		done := make(chan struct{})
		go func() {
			runSleeper(ctx, slog.Default(), conf, time.Time{})
			close(done)
		}()
		time.Sleep(10 * time.Millisecond) // let the first sleep start
		cancel()

		select {
		case <-done:
			if block {
				t.Fatal("block=true: shutdown was not held open")
			}
		case <-time.After(time.Second):
			if !block {
				t.Fatal("block=false: shutdown was held open")
			}
		}
	}
}

func TestRunWorkersRejectZeroInterval(t *testing.T) {
	ctx, logger := context.Background(), slog.Default()
	runs := map[string]func() error{
		"FindATender": func() error {
			conf := &config.JSCSourceFindATender{
				APIRoot: "http://127.0.0.1:1",
			}
			return runFindATender(ctx, logger, conf, noSave, time.Time{})
		},
		"FTSMock": func() error {
			conf := &config.JSCSourceFTSMock{
				APIRoot: "http://127.0.0.1:1",
			}
			return runFTSMock(ctx, logger, conf, noSave, time.Time{})
		},
		"Sleeper": func() error {
			return runSleeper(ctx, logger, &config.JSCSourceSleeper{}, time.Time{})
		},
	}
	for name, run := range runs {
		err := run()
		if err == nil || !strings.Contains(err.Error(), "interval") {
			t.Errorf("%s: err = %v; want an error for a zero interval", name, err)
		}
	}
}

// A source that stops with an error has the error logged.
func TestRunLogsSourceErrors(t *testing.T) {
	var buf bytes.Buffer
	def := slog.Default()
	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil)))
	t.Cleanup(func() {
		slog.SetDefault(def)
	})

	// A zero interval makes each worker fail straight away.
	sources := &config.JSCSources{}
	sources.FindATender.Enabled = true
	sources.FindATender.APIRoot = "http://127.0.0.1:1"
	sources.FTSMock.Enabled = true
	sources.FTSMock.APIRoot = "http://127.0.0.1:1"
	sources.Sleeper.Enabled = true
	err := New(sources).Run(context.Background(), time.Time{})

	for _, name := range []string{"FindATender", "FTSMock", "Sleeper"} {
		if !strings.Contains(buf.String(), "source="+name) {
			t.Errorf("no error logged for %s: %s", name, &buf)
		}
		if err == nil || !strings.Contains(err.Error(), name+": ") {
			t.Errorf("returned error %v; want it to include %s", err, name)
		}
	}
	if n := strings.Count(
		buf.String(), "interval must be greater than 0",
	); n != 3 {
		t.Errorf("logged %d interval errors; want 3: %s", n, &buf)
	}
}

// An enabled Find a Tender or mock source with no api_root fails straight
// away, naming the setting.
func TestRunMissingAPIRoot(t *testing.T) {
	quiet := slog.New(slog.DiscardHandler)
	tests := []struct {
		name   string
		enable func(*config.JSCSources)
		key    string
	}{
		{"find_a_tender", func(sources *config.JSCSources) {
			sources.FindATender.Enabled = true
			sources.FindATender.Interval = time.Hour
		}, "sources.find_a_tender.api_root"},
		{"fts_mock", func(sources *config.JSCSources) {
			sources.FTSMock.Enabled = true
			sources.FTSMock.Interval = time.Hour
		}, "sources.fts_mock.api_root"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			sources := &config.JSCSources{}
			tt.enable(sources)

			err := New(sources, WithLogger(quiet)).Run(
				context.Background(), time.Time{},
			)
			if err == nil || !strings.Contains(err.Error(), tt.key) {
				t.Errorf("Run = %v; want an error naming %s", err, tt.key)
			}
		})
	}
}

func TestRunNoSourcesEnabled(t *testing.T) {
	done := make(chan struct{})
	go func() {
		err := New(&config.JSCSources{}).Run(context.Background(), time.Time{})
		if err != nil {
			t.Errorf("Run = %v; want nil", err)
		}
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("Run with no sources enabled did not return")
	}
}

// A worker that fails shuts the other workers down, without ctx being
// cancelled.
func TestRunShutsDownOnWorkerError(t *testing.T) {
	mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage)
	sources := &config.JSCSources{}
	sources.FTSMock.Enabled = true
	sources.FTSMock.Interval = time.Hour
	sources.FTSMock.APIRoot = mock.URL
	// The sleeper fails straight away: a zero interval is invalid.
	sources.Sleeper.Enabled = true

	done := make(chan error, 1)
	go func() {
		done <- New(sources).Run(context.Background(), time.Now().Add(-time.Hour))
	}()

	select {
	case err := <-done:
		if err == nil || !strings.Contains(err.Error(), "Sleeper: ") {
			t.Fatalf("Run = %v; want the Sleeper error", err)
		}
		if strings.Contains(err.Error(), "FTSMock") {
			t.Errorf("Run = %v; FTSMock should have stopped cleanly", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("Run did not shut down after a worker failed")
	}

	// FTSMock either polled once before the shutdown or never started a run.
	if n := mockHits.Load(); n > 1 {
		t.Errorf("FTSMock polled %d times; want at most 1", n)
	}
}

// Run starts each enabled source, and returns once ctx is cancelled.
func TestRunEnabledSources(t *testing.T) {
	fat, fatHits := ftsServer(t, http.StatusOK, onePackage)
	mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage)
	sources := &config.JSCSources{}
	sources.FindATender.Enabled = true
	sources.FindATender.Interval = time.Hour
	sources.FindATender.APIRoot = fat.URL
	sources.FTSMock.Enabled = true
	sources.FTSMock.Interval = time.Hour
	sources.FTSMock.APIRoot = mock.URL
	sources.Sleeper.Enabled = true
	sources.Sleeper.Interval = time.Hour

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() {
		err := New(sources).Run(ctx, time.Now().Add(-time.Hour))
		if err != nil {
			t.Errorf("Run = %v; want nil after cancellation", err)
		}
		close(done)
	}()

	deadline := time.Now().Add(5 * time.Second)
	for fatHits.Load() == 0 || mockHits.Load() == 0 {
		if time.Now().After(deadline) {
			t.Fatalf(
				"sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load(),
			)
		}
		time.Sleep(5 * time.Millisecond)
	}
	cancel()

	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("Run did not return after cancellation")
	}
}

// Each source's opportunities carry its own Source: the mock's are marked
// FTSMock, though its converter marks them as Find a Tender's.
func TestFetchSetsSource(t *testing.T) {
	srv, _ := ftsServer(t, http.StatusOK, onePackage)
	for name, want := range map[string]string{
		"FindATender": "FindATender",
		"FTSMock":     "FTSMock",
	} {
		opportunities, err := fetchers[name](
			context.Background(), srv.URL, time.Now().Add(-time.Hour),
		)
		if err != nil {
			t.Fatalf("%s: %v", name, err)
		}
		if len(opportunities) != 1 || opportunities[0].Source != want {
			t.Errorf(
				"%s: opportunities = %+v; want one from %s",
				name, opportunities, want,
			)
		}
	}
}

// openTestStore returns a migrated store in a temporary file.
func openTestStore(t *testing.T) *store.Store {
	t.Helper()
	path := t.TempDir() + "/fetcher.db"
	opportunityStore, err := store.Open(
		context.Background(), path, store.WithAutoMigrate(),
	)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		opportunityStore.Close()
	})
	return opportunityStore
}

// WithStore makes the workers save what they fetch, each with its source.
func TestRunSavesToStore(t *testing.T) {
	opportunityStore := openTestStore(t)
	fat, _ := ftsServer(t, http.StatusOK, onePackage)
	mock, _ := ftsServer(
		t, http.StatusOK, strings.ReplaceAll(onePackage, "ocds-1", "ocds-2"),
	)
	sources := &config.JSCSources{}
	sources.FindATender.Enabled = true
	sources.FindATender.Interval = time.Hour
	sources.FindATender.APIRoot = fat.URL
	sources.FTSMock.Enabled = true
	sources.FTSMock.Interval = time.Hour
	sources.FTSMock.APIRoot = mock.URL

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() {
		done <- New(sources, WithStore(opportunityStore)).Run(
			ctx, time.Now().Add(-time.Hour),
		)
	}()

	deadline := time.Now().Add(5 * time.Second)
	for {
		stored, err := opportunityStore.ListOpportunities(
			context.Background(), store.OpportunityFilter{},
		)
		if err != nil {
			t.Fatal(err)
		}
		if len(stored) == 2 {
			break
		}
		if time.Now().After(deadline) {
			t.Fatalf("stored %d opportunities; want 2", len(stored))
		}
		time.Sleep(5 * time.Millisecond)
	}
	cancel()
	if err := <-done; err != nil {
		t.Fatalf("Run = %v", err)
	}

	for uuid, want := range map[string]string{
		"ocds-1": "FindATender",
		"ocds-2": "FTSMock",
	} {
		opportunity, err := opportunityStore.GetOpportunity(
			context.Background(), uuid,
		)
		if err != nil {
			t.Fatalf("%s: %v", uuid, err)
		}
		if opportunity.Source != want {
			t.Errorf("%s: Source = %q; want %q", uuid, opportunity.Source, want)
		}
	}
}

// Without a store, save does nothing; with one that fails, it returns the
// error, so the worker's run fails and its checkpoint stays put.
func TestSave(t *testing.T) {
	srv, _ := ftsServer(t, http.StatusOK, onePackage)
	opportunities, err := fetchers["FindATender"](
		context.Background(), srv.URL, time.Now().Add(-time.Hour),
	)
	if err != nil {
		t.Fatal(err)
	}

	err = New(&config.JSCSources{}).save(
		context.Background(), slog.Default(), opportunities,
	)
	if err != nil {
		t.Errorf("save without a store = %v; want nil", err)
	}

	closed := openTestStore(t)
	closed.Close()
	err = New(&config.JSCSources{}, WithStore(closed)).save(
		context.Background(), slog.Default(), opportunities,
	)
	if err == nil || !strings.Contains(err.Error(), "saving opportunities") {
		t.Errorf("save to a closed store = %v; want a saving error", err)
	}
}

// updatedFromServer serves emptyPackage and sends the updatedFrom of each
// request to the returned channel.
func updatedFromServer(t *testing.T) (*httptest.Server, chan string) {
	t.Helper()
	updatedFrom := make(chan string, 10)
	srv := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			select {
			case updatedFrom <- r.URL.Query().Get("updatedFrom"):
			default:
			}
			io.WriteString(w, emptyPackage)
		}),
	)
	t.Cleanup(srv.Close)
	return srv, updatedFrom
}

// Without since, each source starts from its own newest stored release, or
// with nothing stored from the client's default (the last day); since
// overrides both.
func TestRunStartsFromNewestStoredRelease(t *testing.T) {
	opportunityStore := openTestStore(t)
	srv, _ := ftsServer(t, http.StatusOK, onePackage)
	stored, err := fetchers["FindATender"](
		context.Background(), srv.URL, time.Now().Add(-time.Hour),
	)
	if err != nil {
		t.Fatal(err)
	}
	_, err = opportunityStore.SaveOpportunities(context.Background(), stored)
	if err != nil {
		t.Fatal(err)
	}

	override := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
	tests := []struct {
		name     string
		since    time.Time
		wantFAT  string
		wantMock string
	}{
		// "" is the client's default: about a day ago.
		{"stored", time.Time{}, "2026-10-01T12:00:00", ""},
		{"override", override, "2026-09-01T00:00:00", "2026-09-01T00:00:00"},
	}
	for _, tt := range tests {
		fat, fatFrom := updatedFromServer(t)
		mock, mockFrom := updatedFromServer(t)
		sources := &config.JSCSources{}
		sources.FindATender.Enabled = true
		sources.FindATender.Interval = time.Hour
		sources.FindATender.APIRoot = fat.URL
		sources.FTSMock.Enabled = true
		sources.FTSMock.Interval = time.Hour
		sources.FTSMock.APIRoot = mock.URL

		ctx, cancel := context.WithCancel(context.Background())
		done := make(chan error, 1)
		go func() {
			done <- New(sources, WithStore(opportunityStore)).Run(ctx, tt.since)
		}()

		for source, got := range map[string]chan string{
			"FindATender": fatFrom,
			"FTSMock":     mockFrom,
		} {
			want := tt.wantFAT
			if source == "FTSMock" {
				want = tt.wantMock
			}
			select {
			case from := <-got:
				if want == "" {
					checkAboutADayAgo(t, from)
				} else if from != want {
					t.Errorf(
						"%s: %s updatedFrom = %s; want %s",
						tt.name, source, from, want,
					)
				}
			case <-time.After(5 * time.Second):
				t.Errorf("%s: %s was not polled", tt.name, source)
			}
		}
		cancel()
		<-done
	}
}

// checkAboutADayAgo fails the test unless updatedFrom, in local time as the
// client sends it, is about a day ago: within an hour, as a calendar day
// across a clock change isn't 24 hours.
func checkAboutADayAgo(t *testing.T, updatedFrom string) {
	t.Helper()
	from, err := time.ParseInLocation(
		"2006-01-02T15:04:05", updatedFrom, time.Local,
	)
	if err != nil {
		t.Fatal(err)
	}
	gap := time.Since(from) - 24*time.Hour
	if gap < -time.Hour || gap > time.Hour {
		t.Errorf("updatedFrom = %s; want about a day ago", updatedFrom)
	}
}

// fetched returns the opportunities converted from body, a release package.
func fetched(t *testing.T, body string) []*types.Opportunity {
	t.Helper()
	srv, _ := ftsServer(t, http.StatusOK, body)
	opportunities, err := fetchers["FindATender"](
		context.Background(), srv.URL, time.Now().Add(-time.Hour),
	)
	if err != nil {
		t.Fatal(err)
	}
	return opportunities
}

// A save publishes the new opportunities and those with new releases, and
// nothing when an opportunity is only fetched again.
func TestSavePublishesUpdates(t *testing.T) {
	var bus events.Bus
	var published []events.OpportunitiesUpdatedPayload
	bus.Subscribe(func(event events.Event) {
		if event.Type != events.OpportunitiesUpdated {
			t.Errorf("event type = %q", event.Type)
		}
		payload := event.Payload.(events.OpportunitiesUpdatedPayload)
		published = append(published, payload)
	})
	f := New(
		&config.JSCSources{},
		WithStore(openTestStore(t)),
		WithEvents(&bus),
	)
	save := func(body string) {
		t.Helper()
		err := f.save(context.Background(), slog.Default(), fetched(t, body))
		if err != nil {
			t.Fatal(err)
		}
	}
	newRelease := strings.NewReplacer(
		`"id": "r1"`, `"id": "r2"`,
		"2026-10-01T12:00:00Z", "2026-10-02T12:00:00Z",
	).Replace(onePackage)

	save(onePackage) // new
	save(onePackage) // fetched again: nothing published
	save(newRelease) // a new release of a stored opportunity

	want := []events.OpportunitiesUpdatedPayload{
		{
			New:     []string{"ocds-1"},
			Updated: []string{},
			Summary: events.OpportunitiesSummary{
				New: 1,
			},
		},
		{
			New:     []string{},
			Updated: []string{"ocds-1"},
			Summary: events.OpportunitiesSummary{
				Updated: 1,
			},
		},
	}
	if !reflect.DeepEqual(published, want) {
		t.Errorf("published %+v; want %+v", published, want)
	}
}

// Without WithEvents, a save publishes nothing and still succeeds.
func TestSaveWithoutEvents(t *testing.T) {
	f := New(&config.JSCSources{}, WithStore(openTestStore(t)))
	err := f.save(context.Background(), slog.Default(), fetched(t, onePackage))
	if err != nil {
		t.Fatal(err)
	}
}