package fetcher import ( "bytes" "context" "errors" "io" "log/slog" "net/http" "net/http/httptest" "os" "strings" "sync/atomic" "testing" "time" "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/util" ) func TestMain(m *testing.M) { // Keep worker and fetch log lines out of the test output. slog.SetDefault(slog.New(slog.DiscardHandler)) os.Exit(m.Run()) } const onePackage = `{ "uri": "x", "releases": [{ "ocid": "ocds-1", "id": "r1", "date": "2026-10-01T12:00:00Z", "tag": ["tender"], "tender": {"id": "t1", "title": "A tender"} }] }` const emptyPackage = `{"uri": "x", "releases": []}` // ftsServer serves body (with status) for every request and counts requests. func ftsServer( t *testing.T, status int, body string, ) (*httptest.Server, *atomic.Int32) { t.Helper() var hits atomic.Int32 srv := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) if r.URL.Path != "/ocdsReleasePackages" { t.Errorf("unexpected path %q", r.URL.Path) } w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) io.WriteString(w, body) }), ) t.Cleanup(srv.Close) return srv, &hits } // fetchFunc fetches from one source at apiRoot. type fetchFunc func(ctx context.Context, apiRoot string, since time.Time) error // fetchers are the sources' fetch functions, each given only its API root. var fetchers = map[string]fetchFunc{ "FindATender": func(ctx context.Context, apiRoot string, since time.Time) error { conf := &config.JSCSourceFindATender{APIRoot: apiRoot} return fetchFindATender(ctx, slog.Default(), conf, since) }, "FTSMock": func(ctx context.Context, apiRoot string, since time.Time) error { conf := &config.JSCSourceFTSMock{APIRoot: apiRoot} return fetchFTSMock(ctx, slog.Default(), conf, since) }, } func TestFetchFunctions(t *testing.T) { tests := []struct { name string status int body string check func(error) bool wantErr string }{ { "releases found", http.StatusOK, onePackage, func(err error) bool { return err == nil }, "nil", }, {"no releases", http.StatusOK, emptyPackage, func(err error) bool { return errors.Is(err, ErrNoNewReleases) && errors.Is(err, util.ErrWorkerNoWork) }, "ErrNoNewReleases wrapping ErrWorkerNoWork"}, {"bad body", http.StatusInternalServerError, "oops", func(err error) bool { return err != nil && !errors.Is(err, util.ErrWorkerNoWork) }, "a failure error"}, } for fname, fetch := range fetchers { for _, tc := range tests { t.Run(fname+"/"+tc.name, func(t *testing.T) { srv, hits := ftsServer(t, tc.status, tc.body) err := fetch( context.Background(), srv.URL, time.Now().Add(-time.Hour), ) if !tc.check(err) { t.Fatalf("err = %v, want %s", err, tc.wantErr) } if hits.Load() == 0 { t.Fatal("server was not called") } }) } } } // Run gives each worker its own source's configuration: with only one source // enabled, only its API root is called. func TestRunUsesEachSourcesAPIRoot(t *testing.T) { for _, enabled := range []string{"FindATender", "FTSMock"} { fat, fatHits := ftsServer(t, http.StatusOK, emptyPackage) mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage) sources := &config.JSCSources{} sources.FindATender.APIRoot = fat.URL sources.FindATender.Interval = time.Hour sources.FTSMock.APIRoot = mock.URL sources.FTSMock.Interval = time.Hour wanted, other := fatHits, mockHits if enabled == "FindATender" { sources.FindATender.Enabled = true } else { sources.FTSMock.Enabled = true wanted, other = mockHits, fatHits } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- New(sources).Run(ctx, time.Time{}) }() deadline := time.Now().Add(5 * time.Second) for wanted.Load() == 0 && time.Now().Before(deadline) { time.Sleep(5 * time.Millisecond) } cancel() <-done if wanted.Load() == 0 || other.Load() != 0 { t.Errorf( "%s enabled: its root hit %d times, the other's %d; want >0 and 0", enabled, wanted.Load(), other.Load(), ) } } } func TestFetchCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // Unroutable root: the request should never be waited on. start := time.Now() err := fetchers["FTSMock"](ctx, "http://192.0.2.1:9", time.Time{}) if !errors.Is(err, context.Canceled) { t.Fatalf("err = %v, want context.Canceled", err) } if d := time.Since(start); d > time.Second { t.Fatalf("cancelled fetch took %v", d) } } func TestSleepCancelled(t *testing.T) { for _, block := range []bool{false, true} { ctx, cancel := context.WithTimeout( context.Background(), 10*time.Millisecond, ) defer cancel() err := sleep(ctx, block, time.Hour) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("block=%v: err = %v, want context.DeadlineExceeded", block, err) } } } func TestSleepCompletes(t *testing.T) { for _, block := range []bool{false, true} { if err := sleep(context.Background(), block, time.Millisecond); err != nil { t.Fatalf("block=%v: err = %v", block, err) } } } // A non-blocking sleeper stops as soon as shutdown is signalled; a blocking // one holds shutdown open until the worker's shutdown timeout. func TestRunSleeperShutdown(t *testing.T) { for _, block := range []bool{false, true} { conf := &config.JSCSourceSleeper{BlockShutdown: block} conf.Interval = time.Hour ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { runSleeper(ctx, slog.Default(), conf, time.Time{}) close(done) }() time.Sleep(10 * time.Millisecond) // let the first sleep start cancel() select { case <-done: if block { t.Fatal("block=true: shutdown was not held open") } case <-time.After(time.Second): if !block { t.Fatal("block=false: shutdown was held open") } } } } func TestRunWorkersRejectZeroInterval(t *testing.T) { ctx, logger := context.Background(), slog.Default() runs := map[string]func() error{ "FindATender": func() error { conf := &config.JSCSourceFindATender{APIRoot: "http://127.0.0.1:1"} return runFindATender(ctx, logger, conf, time.Time{}) }, "FTSMock": func() error { conf := &config.JSCSourceFTSMock{APIRoot: "http://127.0.0.1:1"} return runFTSMock(ctx, logger, conf, time.Time{}) }, "Sleeper": func() error { return runSleeper(ctx, logger, &config.JSCSourceSleeper{}, time.Time{}) }, } for name, run := range runs { err := run() if err == nil || !strings.Contains(err.Error(), "interval") { t.Errorf("%s: err = %v; want an error for a zero interval", name, err) } } } // A source that stops with an error has the error logged. func TestRunLogsSourceErrors(t *testing.T) { var buf bytes.Buffer def := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) t.Cleanup(func() { slog.SetDefault(def) }) // A zero interval makes each worker fail straight away. sources := &config.JSCSources{} sources.FindATender.Enabled = true sources.FindATender.APIRoot = "http://127.0.0.1:1" sources.FTSMock.Enabled = true sources.FTSMock.APIRoot = "http://127.0.0.1:1" sources.Sleeper.Enabled = true err := New(sources).Run(context.Background(), time.Time{}) for _, name := range []string{"FindATender", "FTSMock", "Sleeper"} { if !strings.Contains(buf.String(), "source="+name) { t.Errorf("no error logged for %s: %s", name, &buf) } if err == nil || !strings.Contains(err.Error(), name+": ") { t.Errorf("returned error %v; want it to include %s", err, name) } } if n := strings.Count( buf.String(), "interval must be greater than 0", ); n != 3 { t.Errorf("logged %d interval errors; want 3: %s", n, &buf) } } // An enabled Find a Tender or mock source with no api_root fails straight // away, naming the setting. func TestRunMissingAPIRoot(t *testing.T) { quiet := slog.New(slog.DiscardHandler) tests := []struct { name string enable func(*config.JSCSources) key string }{ {"find_a_tender", func(sources *config.JSCSources) { sources.FindATender.Enabled = true sources.FindATender.Interval = time.Hour }, "sources.find_a_tender.api_root"}, {"fts_mock", func(sources *config.JSCSources) { sources.FTSMock.Enabled = true sources.FTSMock.Interval = time.Hour }, "sources.fts_mock.api_root"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sources := &config.JSCSources{} tt.enable(sources) err := New(sources, WithLogger(quiet)).Run( context.Background(), time.Time{}, ) if err == nil || !strings.Contains(err.Error(), tt.key) { t.Errorf("Run = %v; want an error naming %s", err, tt.key) } }) } } func TestRunNoSourcesEnabled(t *testing.T) { done := make(chan struct{}) go func() { err := New(&config.JSCSources{}).Run(context.Background(), time.Time{}) if err != nil { t.Errorf("Run = %v; want nil", err) } close(done) }() select { case <-done: case <-time.After(time.Second): t.Fatal("Run with no sources enabled did not return") } } // A worker that fails shuts the other workers down, without ctx being // cancelled. func TestRunShutsDownOnWorkerError(t *testing.T) { mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage) sources := &config.JSCSources{} sources.FTSMock.Enabled = true sources.FTSMock.Interval = time.Hour sources.FTSMock.APIRoot = mock.URL // The sleeper fails straight away: a zero interval is invalid. sources.Sleeper.Enabled = true done := make(chan error, 1) go func() { done <- New(sources).Run(context.Background(), time.Now().Add(-time.Hour)) }() select { case err := <-done: if err == nil || !strings.Contains(err.Error(), "Sleeper: ") { t.Fatalf("Run = %v; want the Sleeper error", err) } if strings.Contains(err.Error(), "FTSMock") { t.Errorf("Run = %v; FTSMock should have stopped cleanly", err) } case <-time.After(5 * time.Second): t.Fatal("Run did not shut down after a worker failed") } // FTSMock either polled once before the shutdown or never started a run. if n := mockHits.Load(); n > 1 { t.Errorf("FTSMock polled %d times; want at most 1", n) } } // Run starts each enabled source, and returns once ctx is cancelled. func TestRunEnabledSources(t *testing.T) { fat, fatHits := ftsServer(t, http.StatusOK, onePackage) mock, mockHits := ftsServer(t, http.StatusOK, emptyPackage) sources := &config.JSCSources{} sources.FindATender.Enabled = true sources.FindATender.Interval = time.Hour sources.FindATender.APIRoot = fat.URL sources.FTSMock.Enabled = true sources.FTSMock.Interval = time.Hour sources.FTSMock.APIRoot = mock.URL sources.Sleeper.Enabled = true sources.Sleeper.Interval = time.Hour ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { err := New(sources).Run(ctx, time.Now().Add(-time.Hour)) if err != nil { t.Errorf("Run = %v; want nil after cancellation", err) } close(done) }() deadline := time.Now().Add(5 * time.Second) for fatHits.Load() == 0 || mockHits.Load() == 0 { if time.Now().After(deadline) { t.Fatalf( "sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load(), ) } time.Sleep(5 * time.Millisecond) } cancel() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("Run did not return after cancellation") } }