package fetcher import ( "context" "errors" "fmt" "log/slog" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/eusupply" "git.avsdev.uk/Go/job-search/internal/store" "git.avsdev.uk/Go/job-search/internal/types" ) var errTestStore = errors.New("test store failed") // testTender is one row of a fake EU Supply search page, and its page. type testTender struct { id, pid, title string published string // "21/10/2026 12:45:00" buyer string description string } // euSupplySite is a fake EU Supply portal: one page of search results with // tenders, and a page for each of them. It counts the tender pages read. type euSupplySite struct { mu sync.Mutex tenders []testTender detailed []string // the PIDs whose pages were read failPID string // a PID whose page fails } func (s *euSupplySite) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.mu.Lock() defer s.mu.Unlock() if r.Method == http.MethodPost { var rows strings.Builder for _, tender := range s.tenders { fmt.Fprintf( &rows, `
%s
`, description, ) } // fakeTenderStore is a tenderStore with some releases stored and other // sources' opportunities. type fakeTenderStore struct { stored map[string]bool // uuid "/" release id others []*types.Opportunity titlesErr error } func (s *fakeTenderStore) ReleaseExists( ctx context.Context, uuid, releaseID string, ) (bool, error) { return s.stored[uuid+"/"+releaseID], nil } func (s *fakeTenderStore) ListOtherSourcesTitles( ctx context.Context, source string, from, before time.Time, ) ([]store.OpportunityTitle, error) { if s.titlesErr != nil { return nil, s.titlesErr } var titles []store.OpportunityTitle for _, other := range s.others { if !other.LatestReleaseDate.Before(from) && other.LatestReleaseDate.Before(before) { titles = append(titles, store.OpportunityTitle{ UUID: other.UUID, Title: other.Title, LatestReleaseDate: other.LatestReleaseDate, }) } } return titles, nil } func (s *fakeTenderStore) GetOpportunity( ctx context.Context, uuid string, ) (*types.Opportunity, error) { for _, other := range s.others { if other.UUID == uuid { return other, nil } } return nil, store.ErrNotFound } // newEUSupplyClient returns a client for srv that doesn't wait. func newEUSupplyClient(srv *httptest.Server) *eusupply.Client { return eusupply.New(srv.URL, eusupply.WithRequestGap(0, 0)) } // Only tenders not stored are read and returned. The ones another source // has within the window, with a matching title and a matching buyer or // description, are marked Hidden and Skipped. func TestFetchEUSupply(t *testing.T) { const day = "20/10/2026 10:00:00" site := &euSupplySite{ tenders: []testTender{ {"1", "11", "Already stored", day, "", ""}, // The same generic title from another buyer: not a repeat. {"2", "12", "Pest Control Services", day, "Yorkshire Housing Limited", "Pest control across our homes in Leeds"}, // The same buyer, named differently: a repeat. {"3", "13", "Google Reseller Framework", day, "Jisc", ""}, // An agent buying for the buyer, but the same description. {"4", "14", "Fairhive - Materials Procurement", day, "Echelon Consultancy", "Supply of building materials for Fairhive housing repairs"}, // Alike, but too long before. {"5", "15", "Housing related support", day, "Hull City Council", ""}, }, } srv := httptest.NewServer(site) t.Cleanup(srv.Close) published := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC) other := func( uuid, title, buyer, description string, released time.Time, ) *types.Opportunity { return &types.Opportunity{ UUID: uuid, Title: title, BuyerName: buyer, Description: description, LatestReleaseDate: released, } } lookup := &fakeTenderStore{ stored: map[string]bool{ "eusupply-uk-1/1": true, }, others: []*types.Opportunity{ other("ocds-pest", "PEST CONTROL SERVICE", "Anchor Hanover Group", "Pest control for retirement housing", published), other("ocds-jisc", "Google Reseller Framework", "JISC SERVICES LIMITED", "", published.Add(time.Hour)), other("ocds-fairhive", "Fairhive - Materials Procurement", "Fairhive Housing Limited", "Fairhive seeks supply of building materials for housing repairs", published), other("ocds-old", "Housing related support", "Hull City Council", "", published.Add(-40*24*time.Hour)), }, } opportunities, err := fetchEUSupply( context.Background(), slog.Default(), newEUSupplyClient(srv), lookup, published.Add(-24*time.Hour), time.Time{}, ) if err != nil { t.Fatal(err) } if got := strings.Join(site.detailed, ","); got != "12,13,14,15" { t.Errorf("pages read = %s; want the new tenders' only", got) } marks := map[string]bool{} for _, opportunity := range opportunities { if opportunity.Source != "EU Supply - UK" { t.Errorf("%s: Source = %q", opportunity.UUID, opportunity.Source) } if opportunity.Marks.Hidden != opportunity.Marks.Skipped { t.Errorf("%s: marks = %+v", opportunity.UUID, opportunity.Marks) } marks[opportunity.UUID] = opportunity.Marks.Hidden } want := map[string]bool{ "eusupply-uk-2": false, "eusupply-uk-3": true, "eusupply-uk-4": true, "eusupply-uk-5": false, } if fmt.Sprint(marks) != fmt.Sprint(want) { t.Errorf("hidden = %v; want %v", marks, want) } } // A tender page that fails stops the run, keeping the tenders read before // it; a failed title lookup returns nothing, so nothing is saved unmarked. func TestFetchEUSupplyErrors(t *testing.T) { site := &euSupplySite{ tenders: []testTender{ {"1", "11", "First", "20/10/2026 10:00:00", "", ""}, {"2", "12", "Second", "20/10/2026 10:00:00", "", ""}, {"3", "13", "Third", "20/10/2026 10:00:00", "", ""}, }, failPID: "12", } srv := httptest.NewServer(site) t.Cleanup(srv.Close) client := newEUSupplyClient(srv) since := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC) opportunities, err := fetchEUSupply( context.Background(), slog.Default(), client, nil, since, time.Time{}, ) if !errors.Is(err, eusupply.ErrUnexpectedStatus) { t.Errorf("err = %v; want ErrUnexpectedStatus", err) } if len(opportunities) != 1 || opportunities[0].UUID != "eusupply-uk-1" { t.Errorf("opportunities = %v; want the first tender", opportunities) } site.failPID = "" lookup := &fakeTenderStore{ titlesErr: errTestStore, } opportunities, err = fetchEUSupply( context.Background(), slog.Default(), client, lookup, since, time.Time{}, ) if !errors.Is(err, errTestStore) || opportunities != nil { t.Errorf( "err = %v, %d opportunities; want the store's error and none", err, len(opportunities), ) } site.tenders = nil _, err = fetchEUSupply( context.Background(), slog.Default(), client, lookup, since, time.Time{}, ) if !errors.Is(err, ErrNoNewReleases) { t.Errorf("no tenders: err = %v; want ErrNoNewReleases", err) } } // Run starts the EU Supply source when it is enabled, and saves its // opportunities with their Source. func TestRunEUSupply(t *testing.T) { site := &euSupplySite{ tenders: []testTender{ { "1", "11", "A tender", time.Now().Format("02/01/2006 15:04:05"), "", "", }, }, } srv := httptest.NewServer(site) t.Cleanup(srv.Close) opportunityStore := openTestStore(t) sources := &config.JSCSources{} sources.EUSupply.Enabled = true sources.EUSupply.Interval = time.Hour sources.EUSupply.BaseURL = srv.URL sources.EUSupply.RequestMinGap = time.Millisecond sources.EUSupply.RequestMaxGap = time.Millisecond ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- New(sources, WithStore(opportunityStore)).Run(ctx, time.Time{}) }() deadline := time.Now().Add(5 * time.Second) var stored []string for time.Now().Before(deadline) { opportunity, err := opportunityStore.GetOpportunity( context.Background(), "eusupply-uk-1", ) if err == nil { stored = append(stored, opportunity.Source) break } time.Sleep(5 * time.Millisecond) } cancel() if err := <-done; err != nil { t.Fatalf("Run = %v", err) } if len(stored) != 1 || stored[0] != "EU Supply - UK" { t.Errorf("stored sources = %v; want EU Supply - UK", stored) } }