job-search/internal/eusupply/eusupply_test.go

package eusupply

import (
	"context"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strings"
	"sync"
	"testing"
	"time"
)

// readFixture returns a file in testdata.
func readFixture(t *testing.T, name string) []byte {
	t.Helper()
	body, err := os.ReadFile("testdata/" + name) //nolint:gosec // G304: a test fixture
	if err != nil {
		t.Fatal(err)
	}
	return body
}

// newTestClient returns a client for srv that doesn't wait between
// requests and logs nothing.
func newTestClient(srv *httptest.Server) *Client {
	return New(
		srv.URL,
		WithRequestGap(0, 0),
		WithLogger(slog.New(slog.DiscardHandler)),
	)
}

func TestParseSearchPage(t *testing.T) {
	c := New("https://uk.example", WithLogger(slog.New(slog.DiscardHandler)))
	tenders, pages, err := c.parseSearchPage(readFixture(t, "search.html"))
	if err != nil {
		t.Fatal(err)
	}
	if pages != 2 {
		t.Errorf("pages = %d; want 2", pages)
	}
	// The row with no link is skipped.
	if len(tenders) != 2 {
		t.Fatalf("tenders = %d; want 2: %+v", len(tenders), tenders)
	}

	first := tenders[0]
	wantPublished := time.Date(2026, 10, 21, 11, 45, 0, 0, time.UTC) // BST
	wantDeadline := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)    // GMT
	switch {
	case first.ID != "100114946", first.PID != "109912",
		first.Reference != "Proc2852":
		t.Errorf("ids = %q, %q, %q", first.ID, first.PID, first.Reference)
	case first.Title != "Jisc Google for Education Reseller Framework Agreement":
		t.Errorf("title = %q", first.Title)
	case first.URL != "https://uk.example/ctm/Supplier/PublicPurchase/109912/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK":
		t.Errorf("url = %q", first.URL)
	case !first.Published.Equal(wantPublished):
		t.Errorf("published = %v; want %v", first.Published, wantPublished)
	case first.Deadline == nil || !first.Deadline.Equal(wantDeadline):
		t.Errorf("deadline = %v; want %v", first.Deadline, wantDeadline)
	case first.Process != "PA23 - Competitive Flexible", first.Buyer != "Jisc",
		first.Countries != "United Kingdom":
		t.Errorf("cells = %q, %q, %q", first.Process, first.Buyer, first.Countries)
	case first.HasDetails:
		t.Error("HasDetails set from the search page")
	}

	second := tenders[1]
	if second.Reference != "" || second.Deadline != nil {
		t.Errorf("second = %+v; want no reference or deadline", second)
	}
	// Zero width spaces are dropped from titles.
	if second.Title != "York and North Yorkshire Transport Sub-strategies" {
		t.Errorf("second title = %q", second.Title)
	}
}

func TestParseSearchPageChanged(t *testing.T) {
	c := New("", WithLogger(slog.New(slog.DiscardHandler)))
	page := []byte("<html><body>Maintenance</body></html>")
	_, _, err := c.parseSearchPage(page)
	if !errors.Is(err, ErrNoResults) {
		t.Errorf("err = %v; want ErrNoResults", err)
	}
}

func TestParseDetails(t *testing.T) {
	tender := Tender{
		ID: "100114946",
	}
	if err := parseDetails(readFixture(t, "tender.html"), &tender); err != nil {
		t.Fatal(err)
	}

	wantDescription := "This opportunity is for a framework agreement.\n" +
		"The Supplier(s) shall manage the full licence lifecycle (the 'Agreement')."
	if tender.Description != wantDescription {
		t.Errorf("description = %q", tender.Description)
	}
	if tender.ContractType != "Supplies" {
		t.Errorf("contract type = %q", tender.ContractType)
	}
	if len(tender.CPVCodes) != 2 ||
		tender.CPVCodes[0].Code != "48000000-8" ||
		tender.CPVCodes[0].Description !=
			"Software package and information systems" {
		t.Errorf("cpv codes = %+v", tender.CPVCodes)
	}
	wantAddress := "Jisc|4 Portwall Lane|BS1 6NB|Bristol|United Kingdom"
	if got := strings.Join(tender.BuyerAddress, "|"); got != wantAddress {
		t.Errorf("buyer address = %q", got)
	}
	// The search had no deadline, so the page's is used.
	if tender.Deadline == nil ||
		!tender.Deadline.Equal(time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)) {
		t.Errorf("deadline = %v", tender.Deadline)
	}
	if tender.Lots != 2 || !tender.HasDetails {
		t.Errorf("lots = %d, HasDetails = %v", tender.Lots, tender.HasDetails)
	}

	err := parseDetails([]byte("<html><body>Not found</body></html>"), &tender)
	if !errors.Is(err, ErrNoDetails) {
		t.Errorf("err = %v; want ErrNoDetails", err)
	}
}

// Search posts the portal's form for each page, and keeps the pages read
// before one fails.
func TestSearch(t *testing.T) {
	page := readFixture(t, "search.html")
	var (
		mu    sync.Mutex
		forms []url.Values
	)
	fail := false
	srv := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if r.Method != http.MethodPost || r.URL.Path != searchPath {
				t.Errorf("request = %s %s", r.Method, r.URL.Path)
			}
			r.ParseForm()
			mu.Lock()
			forms = append(forms, r.PostForm)
			number := len(forms)
			mu.Unlock()
			if fail && number == 2 {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			w.Write(page)
		}),
	)
	t.Cleanup(srv.Close)
	c := newTestClient(srv)

	from := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC) // 15th in the UK
	to := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
	tenders, err := c.Search(context.Background(), from, to)
	if err != nil {
		t.Fatal(err)
	}
	if len(tenders) != 4 || len(forms) != 2 {
		t.Fatalf(
			"%d tenders from %d pages; want 4 from 2", len(tenders), len(forms),
		)
	}
	for i, form := range forms {
		wantPage := []string{"1", "2"}[i]
		if form.Get("SearchFilter.FromDate") != "15/10/2026" ||
			form.Get("SearchFilter.ToDate") != "21/10/2026" ||
			form.Get("SearchFilter.PagingInfo.PageNumber") != wantPage ||
			form.Get("SearchFilter.PagingInfo.PageSize") != "100" ||
			form.Get("Branding") != "UK" {
			t.Errorf("form %d = %v", i+1, form)
		}
	}

	forms = nil
	fail = true
	tenders, err = c.Search(context.Background(), from, to)
	if !errors.Is(err, ErrUnexpectedStatus) {
		t.Errorf("err = %v; want ErrUnexpectedStatus", err)
	}
	if len(tenders) != 2 {
		t.Errorf("%d tenders; want the first page's 2", len(tenders))
	}
}

func TestDetails(t *testing.T) {
	var path string
	srv := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			path = r.URL.RequestURI()
			w.Write(readFixture(t, "tender.html"))
		}),
	)
	t.Cleanup(srv.Close)

	tender := Tender{
		ID:  "100114946",
		PID: "109912",
	}
	err := newTestClient(srv).Details(context.Background(), &tender)
	if err != nil {
		t.Fatal(err)
	}
	if path != tenderPath("109912") || !tender.HasDetails {
		t.Errorf("path = %q, HasDetails = %v", path, tender.HasDetails)
	}
}

// The client waits from minGap to maxGap between requests, and a cancelled
// context stops the wait.
func TestRequestGap(t *testing.T) {
	var (
		mu    sync.Mutex
		times []time.Time
	)
	srv := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			mu.Lock()
			times = append(times, time.Now())
			mu.Unlock()
			io.WriteString(w, "ok")
		}),
	)
	t.Cleanup(srv.Close)

	const minGap, maxGap = 50 * time.Millisecond, 80 * time.Millisecond
	c := New(
		srv.URL,
		WithRequestGap(minGap, maxGap),
		WithLogger(slog.New(slog.DiscardHandler)),
	)
	for range 3 {
		if _, err := c.get(context.Background(), "/"); err != nil {
			t.Fatal(err)
		}
	}
	for i := 1; i < len(times); i++ {
		if gap := times[i].Sub(times[i-1]); gap < minGap {
			t.Errorf("gap %d = %v; want at least %v", i, gap, minGap)
		}
	}

	slow := New(
		srv.URL,
		WithRequestGap(time.Hour, time.Hour),
		WithLogger(slog.New(slog.DiscardHandler)),
	)
	slow.get(context.Background(), "/")
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
	defer cancel()
	start := time.Now()
	if _, err := slow.get(ctx, "/"); !errors.Is(err, context.DeadlineExceeded) {
		t.Errorf("err = %v; want context.DeadlineExceeded", err)
	}
	if time.Since(start) > 5*time.Second {
		t.Error("cancelling did not stop the wait")
	}
}

func TestConvert(t *testing.T) {
	published := time.Date(2026, 10, 21, 11, 45, 0, 0, time.UTC)
	tender := Tender{
		ID:           "100114946",
		PID:          "109912",
		Title:        "Hazard Management Tool",
		Published:    published,
		Buyer:        "UK Atomic Energy Authority",
		HasDetails:   true,
		Description:  "A tool",
		ContractType: "Services",
		CPVCodes: []CPVCode{
			{
				Code:        "48000000-8",
				Description: "Software package and information systems",
			},
		},
		Lots: 2,
	}

	opportunities := Convert([]Tender{tender})
	if len(opportunities) != 1 {
		t.Fatalf("%d opportunities; want 1", len(opportunities))
	}
	opportunity := opportunities[0]
	switch {
	case opportunity.UUID != "eusupply-uk-100114946",
		opportunity.Source != Source:
		t.Errorf("uuid, source = %q, %q", opportunity.UUID, opportunity.Source)
	case opportunity.Title != tender.Title, opportunity.Description != "A tool",
		opportunity.BuyerName != tender.Buyer, opportunity.Status != "active":
		t.Errorf("opportunity = %+v", opportunity)
	case opportunity.MainCategory != "services", !opportunity.MultiLot:
		t.Errorf(
			"category, multi-lot = %q, %v",
			opportunity.MainCategory, opportunity.MultiLot,
		)
	case len(opportunity.Classifications) != 1 ||
		opportunity.Classifications[0].Code != "48000000" ||
		!opportunity.Classifications[0].IsCPV:
		t.Errorf("classifications = %+v", opportunity.Classifications)
	case !opportunity.LatestReleaseDate.Equal(published),
		opportunity.StartDate == nil || !opportunity.StartDate.Equal(published):
		t.Errorf(
			"dates = %v, %v", opportunity.LatestReleaseDate, opportunity.StartDate,
		)
	case opportunity.Marks.Hidden || opportunity.Marks.Skipped:
		t.Error("Convert set marks")
	}

	release := opportunity.Releases[0]
	if len(opportunity.Releases) != 1 || release.ReleaseID != "100114946" ||
		len(release.Tags) != 1 || release.Tags[0].Tag != "tender" ||
		!strings.Contains(release.JSON, `"pid":"109912"`) {
		t.Errorf("releases = %+v", opportunity.Releases)
	}
}