job-search/internal/fetcher/worker_eusupply_test.go

package fetcher

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"

	"git.avsdev.uk/Go/job-search/internal/config"
	"git.avsdev.uk/Go/job-search/internal/eusupply"
	"git.avsdev.uk/Go/job-search/internal/store"
	"git.avsdev.uk/Go/job-search/internal/types"
)

var errTestStore = errors.New("test store failed")

// testTender is one row of a fake EU Supply search page, and its page.
type testTender struct {
	id, pid, title string
	published      string // "21/10/2026 12:45:00"
	buyer          string
	description    string
}

// euSupplySite is a fake EU Supply portal: one page of search results with
// tenders, and a page for each of them. It counts the tender pages read.
type euSupplySite struct {
	mu       sync.Mutex
	tenders  []testTender
	detailed []string // the PIDs whose pages were read
	failPID  string   // a PID whose page fails
}

func (s *euSupplySite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if r.Method == http.MethodPost {
		var rows strings.Builder
		for _, tender := range s.tenders {
			fmt.Fprintf(
				&rows,
				`<tr><td><div class="js-tooltip-content">%s</div></td>`+
					`<td><div class="js-tooltip-content"></div></td>`+
					`<td><a href="/app/rfq/rwlentrance_s.asp?PID=%s">%s</a></td>`+
					`<td><div class="js-tooltip-content" title="%s"></div></td>`+
					`<td></td><td></td><td>%s</td><td></td></tr>`,
				tender.id, tender.pid, tender.title, tender.published, tender.buyer,
			)
		}
		fmt.Fprintf(
			w,
			`<html><body><div id="searchResultContainer"><table><tbody>%s`+
				`</tbody></table></div></body></html>`,
			rows.String(),
		)
		return
	}

	pid := strings.Split(strings.TrimPrefix(
		r.URL.Path, "/ctm/Supplier/PublicPurchase/",
	), "/")[0]
	s.detailed = append(s.detailed, pid)
	if pid == s.failPID {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	description := ""
	for _, tender := range s.tenders {
		if tender.pid == pid {
			description = tender.description
		}
	}
	fmt.Fprintf(
		w,
		`<html><body><div class="ctm-content-label">Detailed description:</div>`+
			`<p>%s</p></body></html>`,
		description,
	)
}

// fakeTenderStore is a tenderStore with some releases stored and other
// sources' opportunities.
type fakeTenderStore struct {
	stored    map[string]bool // uuid "/" release id
	others    []*types.Opportunity
	titlesErr error
}

func (s *fakeTenderStore) ReleaseExists(
	ctx context.Context, uuid, releaseID string,
) (bool, error) {
	return s.stored[uuid+"/"+releaseID], nil
}

func (s *fakeTenderStore) ListOtherSourcesTitles(
	ctx context.Context, source string, from, before time.Time,
) ([]store.OpportunityTitle, error) {
	if s.titlesErr != nil {
		return nil, s.titlesErr
	}
	var titles []store.OpportunityTitle
	for _, other := range s.others {
		if !other.LatestReleaseDate.Before(from) &&
			other.LatestReleaseDate.Before(before) {
			titles = append(titles, store.OpportunityTitle{
				UUID:              other.UUID,
				Title:             other.Title,
				LatestReleaseDate: other.LatestReleaseDate,
			})
		}
	}
	return titles, nil
}

func (s *fakeTenderStore) GetOpportunity(
	ctx context.Context, uuid string,
) (*types.Opportunity, error) {
	for _, other := range s.others {
		if other.UUID == uuid {
			return other, nil
		}
	}
	return nil, store.ErrNotFound
}

// newEUSupplyClient returns a client for srv that doesn't wait.
func newEUSupplyClient(srv *httptest.Server) *eusupply.Client {
	return eusupply.New(srv.URL, eusupply.WithRequestGap(0, 0))
}

// Only tenders not stored are read and returned. The ones another source
// has within the window, with a matching title and a matching buyer or
// description, are marked Hidden and Skipped.
func TestFetchEUSupply(t *testing.T) {
	const day = "20/10/2026 10:00:00"
	site := &euSupplySite{
		tenders: []testTender{
			{"1", "11", "Already stored", day, "", ""},
			// The same generic title from another buyer: not a repeat.
			{"2", "12", "Pest Control Services", day, "Yorkshire Housing Limited",
				"Pest control across our homes in Leeds"},
			// The same buyer, named differently: a repeat.
			{"3", "13", "Google Reseller Framework", day, "Jisc", ""},
			// An agent buying for the buyer, but the same description.
			{"4", "14", "Fairhive - Materials Procurement", day,
				"Echelon Consultancy",
				"Supply of building materials for Fairhive housing repairs"},
			// Alike, but too long before.
			{"5", "15", "Housing related support", day, "Hull City Council", ""},
		},
	}
	srv := httptest.NewServer(site)
	t.Cleanup(srv.Close)

	published := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
	other := func(
		uuid, title, buyer, description string, released time.Time,
	) *types.Opportunity {
		return &types.Opportunity{
			UUID:              uuid,
			Title:             title,
			BuyerName:         buyer,
			Description:       description,
			LatestReleaseDate: released,
		}
	}
	lookup := &fakeTenderStore{
		stored: map[string]bool{
			"eusupply-uk-1/1": true,
		},
		others: []*types.Opportunity{
			other("ocds-pest", "PEST CONTROL SERVICE", "Anchor Hanover Group",
				"Pest control for retirement housing", published),
			other("ocds-jisc", "Google Reseller Framework",
				"JISC SERVICES LIMITED", "", published.Add(time.Hour)),
			other("ocds-fairhive", "Fairhive - Materials Procurement",
				"Fairhive Housing Limited",
				"Fairhive seeks supply of building materials for housing repairs",
				published),
			other("ocds-old", "Housing related support", "Hull City Council", "",
				published.Add(-40*24*time.Hour)),
		},
	}

	opportunities, err := fetchEUSupply(
		context.Background(),
		slog.Default(),
		newEUSupplyClient(srv),
		lookup,
		published.Add(-24*time.Hour),
		time.Time{},
	)
	if err != nil {
		t.Fatal(err)
	}

	if got := strings.Join(site.detailed, ","); got != "12,13,14,15" {
		t.Errorf("pages read = %s; want the new tenders' only", got)
	}
	marks := map[string]bool{}
	for _, opportunity := range opportunities {
		if opportunity.Source != "EU Supply - UK" {
			t.Errorf("%s: Source = %q", opportunity.UUID, opportunity.Source)
		}
		if opportunity.Marks.Hidden != opportunity.Marks.Skipped {
			t.Errorf("%s: marks = %+v", opportunity.UUID, opportunity.Marks)
		}
		marks[opportunity.UUID] = opportunity.Marks.Hidden
	}
	want := map[string]bool{
		"eusupply-uk-2": false,
		"eusupply-uk-3": true,
		"eusupply-uk-4": true,
		"eusupply-uk-5": false,
	}
	if fmt.Sprint(marks) != fmt.Sprint(want) {
		t.Errorf("hidden = %v; want %v", marks, want)
	}
}

// A tender page that fails stops the run, keeping the tenders read before
// it; a failed title lookup returns nothing, so nothing is saved unmarked.
func TestFetchEUSupplyErrors(t *testing.T) {
	site := &euSupplySite{
		tenders: []testTender{
			{"1", "11", "First", "20/10/2026 10:00:00", "", ""},
			{"2", "12", "Second", "20/10/2026 10:00:00", "", ""},
			{"3", "13", "Third", "20/10/2026 10:00:00", "", ""},
		},
		failPID: "12",
	}
	srv := httptest.NewServer(site)
	t.Cleanup(srv.Close)
	client := newEUSupplyClient(srv)
	since := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)

	opportunities, err := fetchEUSupply(
		context.Background(), slog.Default(), client, nil, since, time.Time{},
	)
	if !errors.Is(err, eusupply.ErrUnexpectedStatus) {
		t.Errorf("err = %v; want ErrUnexpectedStatus", err)
	}
	if len(opportunities) != 1 || opportunities[0].UUID != "eusupply-uk-1" {
		t.Errorf("opportunities = %v; want the first tender", opportunities)
	}

	site.failPID = ""
	lookup := &fakeTenderStore{
		titlesErr: errTestStore,
	}
	opportunities, err = fetchEUSupply(
		context.Background(), slog.Default(), client, lookup, since, time.Time{},
	)
	if !errors.Is(err, errTestStore) || opportunities != nil {
		t.Errorf(
			"err = %v, %d opportunities; want the store's error and none",
			err, len(opportunities),
		)
	}

	site.tenders = nil
	_, err = fetchEUSupply(
		context.Background(), slog.Default(), client, lookup, since, time.Time{},
	)
	if !errors.Is(err, ErrNoNewReleases) {
		t.Errorf("no tenders: err = %v; want ErrNoNewReleases", err)
	}
}

// Run starts the EU Supply source when it is enabled, and saves its
// opportunities with their Source.
func TestRunEUSupply(t *testing.T) {
	site := &euSupplySite{
		tenders: []testTender{
			{
				"1", "11", "A tender",
				time.Now().Format("02/01/2006 15:04:05"), "", "",
			},
		},
	}
	srv := httptest.NewServer(site)
	t.Cleanup(srv.Close)

	opportunityStore := openTestStore(t)
	sources := &config.JSCSources{}
	sources.EUSupply.Enabled = true
	sources.EUSupply.Interval = time.Hour
	sources.EUSupply.BaseURL = srv.URL
	sources.EUSupply.RequestMinGap = time.Millisecond
	sources.EUSupply.RequestMaxGap = time.Millisecond

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() {
		done <- New(sources, WithStore(opportunityStore)).Run(ctx, time.Time{})
	}()

	deadline := time.Now().Add(5 * time.Second)
	var stored []string
	for time.Now().Before(deadline) {
		opportunity, err := opportunityStore.GetOpportunity(
			context.Background(), "eusupply-uk-1",
		)
		if err == nil {
			stored = append(stored, opportunity.Source)
			break
		}
		time.Sleep(5 * time.Millisecond)
	}
	cancel()
	if err := <-done; err != nil {
		t.Fatalf("Run = %v", err)
	}
	if len(stored) != 1 || stored[0] != "EU Supply - UK" {
		t.Errorf("stored sources = %v; want EU Supply - UK", stored)
	}
}