job-search/internal/fetcher/fetcher_test.go

package fetcher

import (
	"bytes"
	"context"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"git.avsdev.uk/Go/job-search/internal/config"
	"git.avsdev.uk/Go/job-search/internal/util"
)

func TestMain(m *testing.M) {
	// Keep worker and fetch log lines out of the test output.
	slog.SetDefault(slog.New(slog.DiscardHandler))
	os.Exit(m.Run())
}

const onePackage = `{
	"uri": "x",
	"releases": [{
		"ocid": "ocds-1",
		"id": "r1",
		"date": "2026-10-01T12:00:00Z",
		"tag": ["tender"],
		"tender": {"id": "t1", "title": "A tender"}
	}]
}`

const emptyPackage = `{"uri": "x", "releases": []}`

// ftsServer serves body (with status) for every request and counts requests.
func ftsServer(
	t *testing.T, status int, body string,
) (*httptest.Server, *atomic.Int32) {
	t.Helper()
	var hits atomic.Int32
	srv := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			hits.Add(1)
			if r.URL.Path != "/ocdsReleasePackages" {
				t.Errorf("unexpected path %q", r.URL.Path)
			}
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(status)
			io.WriteString(w, body)
		}),
	)
	t.Cleanup(srv.Close)
	return srv, &hits
}

// fetchFunc fetches from one source at apiRoot.
type fetchFunc func(ctx context.Context, apiRoot string, since time.Time) error

// fetchers are the sources' fetch functions, each given only its API root.
var fetchers = map[string]fetchFunc{
	"FindATender": func(ctx context.Context, apiRoot string, since time.Time) error {
		conf := &config.JSCSourceFindATender{APIRoot: apiRoot}
		return fetchFindATender(ctx, slog.Default(), conf, since)
	},
	"FTSMock": func(ctx context.Context, apiRoot string, since time.Time) error {
		conf := &config.JSCSourceFTSMock{APIRoot: apiRoot}
		return fetchFTSMock(ctx, slog.Default(), conf, since)
	},
}

func TestFetchFunctions(t *testing.T) {
	tests := []struct {
		name    string
		status  int
		body    string
		check   func(error) bool
		wantErr string
	}{
		{
			"releases found",
			http.StatusOK,
			onePackage,
			func(err error) bool {
				return err == nil
			},
			"nil",
		},
		{"no releases", http.StatusOK, emptyPackage, func(err error) bool {
			return errors.Is(err, ErrNoNewReleases) &&
				errors.Is(err, util.ErrWorkerNoWork)
		}, "ErrNoNewReleases wrapping ErrWorkerNoWork"},
		{"bad body", http.StatusInternalServerError, "oops", func(err error) bool {
			return err != nil && !errors.Is(err, util.ErrWorkerNoWork)
		}, "a failure error"},
	}
	for fname, fetch := range fetchers {
		for _, tc := range tests {
			t.Run(fname+"/"+tc.name, func(t *testing.T) {
				srv, hits := ftsServer(t, tc.status, tc.body)
				err := fetch(
					context.Background(), srv.URL, time.Now().Add(-time.Hour),
				)
				if !tc.check(err) {
					t.Fatalf("err = %v, want %s", err, tc.wantErr)
				}
				if hits.Load() == 0 {
					t.Fatal("server was not called")
				}
			})
		}
	}
}

// Run gives each worker its own source's configuration: with only one source
// enabled, only its API root is called.
func TestRunUsesEachSourcesAPIRoot(t *testing.T) {
	for _, enabled := range []string{"FindATender", "FTSMock"} {
		fat, fatHits := ftsServer(t, http.StatusOK, emptyPackage)
		mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage)
		sources := &config.JSCSources{}
		sources.FindATender.APIRoot = fat.URL
		sources.FindATender.Interval = time.Hour
		sources.FTSMock.APIRoot = mock.URL
		sources.FTSMock.Interval = time.Hour
		wanted, other := fatHits, mockHits
		if enabled == "FindATender" {
			sources.FindATender.Enabled = true
		} else {
			sources.FTSMock.Enabled = true
			wanted, other = mockHits, fatHits
		}

		ctx, cancel := context.WithCancel(context.Background())
		done := make(chan error, 1)
		go func() {
			done <- New(sources).Run(ctx, time.Time{})
		}()

		deadline := time.Now().Add(5 * time.Second)
		for wanted.Load() == 0 && time.Now().Before(deadline) {
			time.Sleep(5 * time.Millisecond)
		}
		cancel()
		<-done

		if wanted.Load() == 0 || other.Load() != 0 {
			t.Errorf(
				"%s enabled: its root hit %d times, the other's %d; want >0 and 0",
				enabled, wanted.Load(), other.Load(),
			)
		}
	}
}

func TestFetchCancelled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	// Unroutable root: the request should never be waited on.
	start := time.Now()
	err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{})
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("err = %v, want context.Canceled", err)
	}
	if d := time.Since(start); d > time.Second {
		t.Fatalf("cancelled fetch took %v", d)
	}
}

func TestSleepCancelled(t *testing.T) {
	for _, block := range []bool{false, true} {
		ctx, cancel := context.WithTimeout(
			context.Background(), 10*time.Millisecond,
		)
		defer cancel()
		err := sleep(ctx, block, time.Hour)
		if !errors.Is(err, context.DeadlineExceeded) {
			t.Fatalf("block=%v: err = %v, want context.DeadlineExceeded", block, err)
		}
	}
}

func TestSleepCompletes(t *testing.T) {
	for _, block := range []bool{false, true} {
		if err := sleep(context.Background(), block, time.Millisecond); err != nil {
			t.Fatalf("block=%v: err = %v", block, err)
		}
	}
}

// A non-blocking sleeper stops as soon as shutdown is signalled; a blocking
// one holds shutdown open until the worker's shutdown timeout.
func TestRunSleeperShutdown(t *testing.T) {
	for _, block := range []bool{false, true} {
		conf := &config.JSCSourceSleeper{BlockShutdown: block}
		conf.Interval = time.Hour

		ctx, cancel := context.WithCancel(context.Background())
		done := make(chan struct{})
		go func() {
			runSleeper(ctx, slog.Default(), conf, time.Time{})
			close(done)
		}()
		time.Sleep(10 * time.Millisecond) // let the first sleep start
		cancel()

		select {
		case <-done:
			if block {
				t.Fatal("block=true: shutdown was not held open")
			}
		case <-time.After(time.Second):
			if !block {
				t.Fatal("block=false: shutdown was held open")
			}
		}
	}
}

func TestRunWorkersRejectZeroInterval(t *testing.T) {
	ctx, logger := context.Background(), slog.Default()
	runs := map[string]func() error{
		"FindATender": func() error {
			conf := &config.JSCSourceFindATender{APIRoot: "http://127.0.0.1:1"}
			return runFindATender(ctx, logger, conf, time.Time{})
		},
		"FTSMock": func() error {
			conf := &config.JSCSourceFTSMock{APIRoot: "http://127.0.0.1:1"}
			return runFTSMock(ctx, logger, conf, time.Time{})
		},
		"Sleeper": func() error {
			return runSleeper(ctx, logger, &config.JSCSourceSleeper{}, time.Time{})
		},
	}
	for name, run := range runs {
		err := run()
		if err == nil || !strings.Contains(err.Error(), "interval") {
			t.Errorf("%s: err = %v; want an error for a zero interval", name, err)
		}
	}
}

// A source that stops with an error has the error logged.
func TestRunLogsSourceErrors(t *testing.T) {
	var buf bytes.Buffer
	def := slog.Default()
	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil)))
	t.Cleanup(func() {
		slog.SetDefault(def)
	})

	// A zero interval makes each worker fail straight away.
	sources := &config.JSCSources{}
	sources.FindATender.Enabled = true
	sources.FindATender.APIRoot = "http://127.0.0.1:1"
	sources.FTSMock.Enabled = true
	sources.FTSMock.APIRoot = "http://127.0.0.1:1"
	sources.Sleeper.Enabled = true
	err := New(sources).Run(context.Background(), time.Time{})

	for _, name := range []string{"FindATender", "FTSMock", "Sleeper"} {
		if !strings.Contains(buf.String(), "source="+name) {
			t.Errorf("no error logged for %s: %s", name, &buf)
		}
		if err == nil || !strings.Contains(err.Error(), name+": ") {
			t.Errorf("returned error %v; want it to include %s", err, name)
		}
	}
	if n := strings.Count(
		buf.String(), "interval must be greater than 0",
	); n != 3 {
		t.Errorf("logged %d interval errors; want 3: %s", n, &buf)
	}
}

// An enabled Find a Tender or mock source with no api_root fails straight
// away, naming the setting.
func TestRunMissingAPIRoot(t *testing.T) {
	quiet := slog.New(slog.DiscardHandler)
	tests := []struct {
		name   string
		enable func(*config.JSCSources)
		key    string
	}{
		{"find_a_tender", func(sources *config.JSCSources) {
			sources.FindATender.Enabled = true
			sources.FindATender.Interval = time.Hour
		}, "sources.find_a_tender.api_root"},
		{"fts_mock", func(sources *config.JSCSources) {
			sources.FTSMock.Enabled = true
			sources.FTSMock.Interval = time.Hour
		}, "sources.fts_mock.api_root"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			sources := &config.JSCSources{}
			tt.enable(sources)

			err := New(sources, WithLogger(quiet)).Run(
				context.Background(), time.Time{},
			)
			if err == nil || !strings.Contains(err.Error(), tt.key) {
				t.Errorf("Run = %v; want an error naming %s", err, tt.key)
			}
		})
	}
}

func TestRunNoSourcesEnabled(t *testing.T) {
	done := make(chan struct{})
	go func() {
		err := New(&config.JSCSources{}).Run(context.Background(), time.Time{})
		if err != nil {
			t.Errorf("Run = %v; want nil", err)
		}
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("Run with no sources enabled did not return")
	}
}

// A worker that fails shuts the other workers down, without ctx being
// cancelled.
func TestRunShutsDownOnWorkerError(t *testing.T) {
	mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage)
	sources := &config.JSCSources{}
	sources.FTSMock.Enabled = true
	sources.FTSMock.Interval = time.Hour
	sources.FTSMock.APIRoot = mock.URL
	// The sleeper fails straight away: a zero interval is invalid.
	sources.Sleeper.Enabled = true

	done := make(chan error, 1)
	go func() {
		done <- New(sources).Run(context.Background(), time.Now().Add(-time.Hour))
	}()

	select {
	case err := <-done:
		if err == nil || !strings.Contains(err.Error(), "Sleeper: ") {
			t.Fatalf("Run = %v; want the Sleeper error", err)
		}
		if strings.Contains(err.Error(), "FTSMock") {
			t.Errorf("Run = %v; FTSMock should have stopped cleanly", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("Run did not shut down after a worker failed")
	}

	// FTSMock either polled once before the shutdown or never started a run.
	if n := mockHits.Load(); n > 1 {
		t.Errorf("FTSMock polled %d times; want at most 1", n)
	}
}

// Run starts each enabled source, and returns once ctx is cancelled.
func TestRunEnabledSources(t *testing.T) {
	fat, fatHits := ftsServer(t, http.StatusOK, onePackage)
	mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage)
	sources := &config.JSCSources{}
	sources.FindATender.Enabled = true
	sources.FindATender.Interval = time.Hour
	sources.FindATender.APIRoot = fat.URL
	sources.FTSMock.Enabled = true
	sources.FTSMock.Interval = time.Hour
	sources.FTSMock.APIRoot = mock.URL
	sources.Sleeper.Enabled = true
	sources.Sleeper.Interval = time.Hour

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() {
		err := New(sources).Run(ctx, time.Now().Add(-time.Hour))
		if err != nil {
			t.Errorf("Run = %v; want nil after cancellation", err)
		}
		close(done)
	}()

	deadline := time.Now().Add(5 * time.Second)
	for fatHits.Load() == 0 || mockHits.Load() == 0 {
		if time.Now().After(deadline) {
			t.Fatalf(
				"sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load(),
			)
		}
		time.Sleep(5 * time.Millisecond)
	}
	cancel()

	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("Run did not return after cancellation")
	}
}