Added an EU Supply source, scraping the UK portal's public tenders

internal/eusupply replaces the old Node scraper (eu-supply.js) for
uk.eu-supply.com. The site has no API, so it posts the public search
form, which now needs the form's current defaults and reads the HTML
with golang.org/x/net/html. Each tender not stored yet has its own
page read too, for its description, contract type, CPV codes and
buyer address. Tenders are stored with the source "EU Supply - UK"
as eusupply-uk-<tender id>, with one release holding the tender as
read.

The site publishes no rate limit, so the client sets its own: one
request at a time, with a random wait between
sources.eu_supply.request_min_gap and request_max_gap (2s to 10s).
The source has the same 7-day backfill as Find a Tender's.

Most of its tenders are also on Find a Tender, so a new one is stored
Hidden and Skipped when another source's opportunity released within
30 days has a title at least 90% alike, and its buyer or description
matches too (80% of the shorter one's significant words).

internal/fetcher/fetcher.go

@@ -185,6 +185,18 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
 			return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
 		})
 	}
+	if f.conf.EUSupply.Enabled {
+		// A nil *store.Store in the interface would not be nil.
+		var lookup tenderStore
+		if f.store != nil {
+			lookup = f.store
+		}
+		start(euSupplySource, func(ctx context.Context, since time.Time) error {
+			return runEUSupply(
+				ctx, logger, &f.conf.EUSupply, lookup, f.save, since,
+			)
+		})
+	}
 	if f.conf.Sleeper.Enabled {
 		start("Sleeper", func(ctx context.Context, since time.Time) error {
 			return runSleeper(ctx, logger, &f.conf.Sleeper, since)

internal/fetcher/match.go

@@ -0,0 +1,252 @@
+package fetcher
+
+import (
+	"math"
+	"strings"
+	"unicode"
+
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+// titleMatchThreshold is how alike two titles must be (see titleSimilarity)
+// for an EU Supply tender to be taken as one already stored from another
+// source, such as the Find a Tender notice the portal publishes for most of
+// its tenders (the user's rule: a 90% match).
+const titleMatchThreshold = 0.9
+
+// The other thresholds of a match (see confirmedRepeat). The user's rule
+// (2026-10-21): a title match must also be backed by the buyer or the
+// description, at a ratio of Claude's choosing; 0.8 kept every real repeat
+// in a live run that either could check, and rejected the one false match.
+const (
+	// buyerMatchThreshold is how much of the shorter buyer name's
+	// significant words must be in the other's (see buyerSimilarity).
+	buyerMatchThreshold = 0.8
+	// descriptionMatchThreshold is how much of the shorter description's
+	// words must be in the other's (see descriptionSimilarity).
+	descriptionMatchThreshold = 0.8
+	// wordMatchThreshold is how alike two words of buyer names must be to
+	// count as the same word, so a typo or a plural still matches.
+	wordMatchThreshold = 0.8
+	// minDescriptionWordLength is the shortest word a description is
+	// compared by; shorter ones ("the", "and", "for") say little.
+	minDescriptionWordLength = 4
+)
+
+// genericBuyerWords are left out of buyer names before comparing them:
+// legal forms and kinds of body, which the two sites give differently
+// ("Jisc" and "JISC SERVICES LIMITED", "The Borough Council of Calderdale"
+// and "Borough Council of Calderdale") and which say nothing about which
+// buyer it is.
+var genericBuyerWords = map[string]bool{
+	"the": true, "of": true, "and": true, "for": true,
+	"limited": true, "ltd": true, "plc": true, "llp": true, "cic": true,
+	"council": true, "borough": true, "metropolitan": true, "city": true,
+	"county": true, "district": true, "authority": true, "services": true,
+	"group": true, "trust": true,
+}
+
+// confirmedRepeat reports whether other, whose title already matches
+// opportunity's (see similarTitles), is the same tender: their buyers or
+// their descriptions must match too. Generic titles ("Pest control
+// services") are bought by many buyers, so the title alone isn't enough.
+func confirmedRepeat(opportunity, other *types.Opportunity) bool {
+	buyer, buyerKnown := buyerSimilarity(
+		opportunity.BuyerName, other.BuyerName,
+	)
+	if buyerKnown && buyer >= buyerMatchThreshold {
+		return true
+	}
+	description, descriptionKnown := descriptionSimilarity(
+		opportunity.Description, other.Description,
+	)
+	return descriptionKnown && description >= descriptionMatchThreshold
+}
+
+// buyerSimilarity returns the share of the shorter buyer name's significant
+// words (all but genericBuyerWords) that are in the other name, each at
+// least wordMatchThreshold alike to one of its words. It is false if either
+// name has no significant words, when the buyers can't be compared.
+func buyerSimilarity(first, second string) (float64, bool) {
+	firstWords := significantWords(first)
+	secondWords := significantWords(second)
+	if len(firstWords) == 0 || len(secondWords) == 0 {
+		return 0, false
+	}
+	if len(secondWords) < len(firstWords) {
+		firstWords, secondWords = secondWords, firstWords
+	}
+
+	found := 0
+	for _, word := range firstWords {
+		for _, other := range secondWords {
+			if similarWords(word, other) {
+				found++
+				break
+			}
+		}
+	}
+	return float64(found) / float64(len(firstWords)), true
+}
+
+// significantWords returns the words of a buyer name, normalised as titles
+// are, without genericBuyerWords.
+func significantWords(name string) []string {
+	var words []string
+	for word := range strings.FieldsSeq(normaliseTitle(name)) {
+		if !genericBuyerWords[word] {
+			words = append(words, word)
+		}
+	}
+	return words
+}
+
+// similarWords reports whether two words are at least wordMatchThreshold
+// alike (see titleSimilarity).
+func similarWords(first, second string) bool {
+	firstRunes, secondRunes := []rune(first), []rune(second)
+	longest := max(len(firstRunes), len(secondRunes))
+	limit := int(math.Floor((1-wordMatchThreshold)*float64(longest) + 1e-9))
+	_, within := editDistance(firstRunes, secondRunes, limit)
+	return within
+}
+
+// descriptionSimilarity returns the share of the shorter description's
+// distinct words (of minDescriptionWordLength letters or more) that are in
+// the other. It is false if either has no such words: EU Supply leaves
+// some descriptions empty.
+func descriptionSimilarity(first, second string) (float64, bool) {
+	firstWords := descriptionWords(first)
+	secondWords := descriptionWords(second)
+	if len(firstWords) == 0 || len(secondWords) == 0 {
+		return 0, false
+	}
+	if len(secondWords) < len(firstWords) {
+		firstWords, secondWords = secondWords, firstWords
+	}
+
+	found := 0
+	for word := range firstWords {
+		if secondWords[word] {
+			found++
+		}
+	}
+	return float64(found) / float64(len(firstWords)), true
+}
+
+// descriptionWords returns the distinct words of a description, normalised
+// as titles are, of minDescriptionWordLength letters or more.
+func descriptionWords(description string) map[string]bool {
+	words := make(map[string]bool)
+	for word := range strings.FieldsSeq(normaliseTitle(description)) {
+		if len([]rune(word)) >= minDescriptionWordLength {
+			words[word] = true
+		}
+	}
+	return words
+}
+
+// normaliseTitle returns title as titles are compared: lower case, with
+// each run of anything but letters and numbers (punctuation, white space,
+// zero width spaces) made one space, and trimmed.
+func normaliseTitle(title string) string {
+	var normalised strings.Builder
+	space := false
+	for _, character := range strings.ToLower(title) {
+		if unicode.IsLetter(character) || unicode.IsNumber(character) {
+			if space && normalised.Len() > 0 {
+				normalised.WriteByte(' ')
+			}
+			normalised.WriteRune(character)
+			space = false
+			continue
+		}
+		space = true
+	}
+	return normalised.String()
+}
+
+// titleSimilarity returns how alike two normalised titles (see
+// normaliseTitle) are, from 0 to 1: one less their edit distance (the
+// characters inserted, deleted or changed to turn one into the other) over
+// the longer one's length. Two empty titles are not alike (0).
+func titleSimilarity(first, second string) float64 {
+	firstRunes, secondRunes := []rune(first), []rune(second)
+	longest := max(len(firstRunes), len(secondRunes))
+	if longest == 0 {
+		return 0
+	}
+	distance, _ := editDistance(firstRunes, secondRunes, longest)
+	return 1 - float64(distance)/float64(longest)
+}
+
+// similarTitles reports whether two normalised titles are at least
+// titleMatchThreshold alike. It gives up as soon as the distance can't be
+// small enough, so it is much faster than titleSimilarity for titles that
+// aren't.
+func similarTitles(first, second string) bool {
+	firstRunes, secondRunes := []rune(first), []rune(second)
+	longest := max(len(firstRunes), len(secondRunes))
+	if longest == 0 {
+		return false
+	}
+	// The epsilon stops (1-0.9)*10 = 0.99999... rounding down to 0.
+	limit := int(math.Floor((1-titleMatchThreshold)*float64(longest) + 1e-9))
+	_, within := editDistance(firstRunes, secondRunes, limit)
+	return within
+}
+
+// editDistance returns the Levenshtein distance between source and target,
+// and whether it is at most limit. Only the cells within limit of the
+// diagonal are computed, and it stops once a whole row exceeds limit, when
+// the distance returned is limit+1.
+func editDistance(source, target []rune, limit int) (int, bool) {
+	if abs(len(source)-len(target)) > limit {
+		return limit + 1, false
+	}
+
+	exceeded := limit + 1
+	previous := make([]int, len(target)+1)
+	current := make([]int, len(target)+1)
+	for column := range previous {
+		previous[column] = min(column, exceeded)
+	}
+
+	for row := 1; row <= len(source); row++ {
+		low, high := max(1, row-limit), min(len(target), row+limit)
+		current[0] = min(row, exceeded)
+		if low > 1 {
+			current[low-1] = exceeded
+		}
+		rowMin := current[0]
+		for column := low; column <= high; column++ {
+			cost := 1
+			if source[row-1] == target[column-1] {
+				cost = 0
+			}
+			value := min(
+				previous[column]+1, current[column-1]+1, previous[column-1]+cost,
+			)
+			current[column] = min(value, exceeded)
+			rowMin = min(rowMin, current[column])
+		}
+		if high < len(target) {
+			current[high+1] = exceeded
+		}
+		if rowMin > limit {
+			return exceeded, false
+		}
+		previous, current = current, previous
+	}
+
+	distance := previous[len(target)]
+	return distance, distance <= limit
+}
+
+// abs returns the absolute value of value.
+func abs(value int) int {
+	if value < 0 {
+		return -value
+	}
+	return value
+}

internal/fetcher/match_test.go

@@ -0,0 +1,208 @@
+package fetcher
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+func TestNormaliseTitle(t *testing.T) {
+	tests := []struct {
+		title string
+		want  string
+	}{
+		{"Pest Control Services", "pest control services"},
+		{"  RRE-01284 - For the provision  ", "rre 01284 for the provision"},
+		{"\u200b\u200bThe supply 3 kg of Li₂TiO₃", "the supply 3 kg of li₂tio₃"},
+		{"Children & Young People's", "children young people s"},
+		{"---", ""},
+	}
+	for _, tt := range tests {
+		if got := normaliseTitle(tt.title); got != tt.want {
+			t.Errorf("normaliseTitle(%q) = %q; want %q", tt.title, got, tt.want)
+		}
+	}
+}
+
+func TestTitleSimilarity(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          float64
+	}{
+		{"pest control services", "pest control services", 1},
+		{"abcdefghij", "abcdefghix", 0.9},
+		{"abcdefghij", "abcdefghxx", 0.8},
+		{"kitten", "sitting", 1 - 3.0/7},
+		{"", "", 0},
+		{"abc", "", 0},
+	}
+	for _, tt := range tests {
+		got := titleSimilarity(tt.first, tt.second)
+		if got < tt.want-1e-9 || got > tt.want+1e-9 {
+			t.Errorf(
+				"titleSimilarity(%q, %q) = %v; want %v",
+				tt.first, tt.second, got, tt.want,
+			)
+		}
+	}
+}
+
+func TestSimilarTitles(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          bool
+	}{
+		{"housing related support", "housing related support", true},
+		// One edit in ten is exactly 90%.
+		{"abcdefghij", "abcdefghix", true},
+		{"abcdefghij", "abcdefghxx", false},
+		{"pest control services", "pest control service", true},
+		{"alternative provision", "alternative provision for schools", false},
+		{"", "", false},
+	}
+	for _, tt := range tests {
+		if got := similarTitles(tt.first, tt.second); got != tt.want {
+			t.Errorf(
+				"similarTitles(%q, %q) = %v; want %v",
+				tt.first, tt.second, got, tt.want,
+			)
+		}
+	}
+}
+
+// similarTitles' banded, early-stopping distance agrees with the full one.
+func TestSimilarTitlesAgreesWithTitleSimilarity(t *testing.T) {
+	//nolint:gosec // G404: seeded math/rand keeps the test deterministic
+	rng := rand.New(rand.NewSource(1))
+	alphabet := []rune("abc d")
+	random := func() string {
+		runes := make([]rune, 5+rng.Intn(30))
+		for i := range runes {
+			runes[i] = alphabet[rng.Intn(len(alphabet))]
+		}
+		return string(runes)
+	}
+	mutate := func(title string) string {
+		runes := []rune(title)
+		for range rng.Intn(5) {
+			i := rng.Intn(len(runes))
+			switch rng.Intn(3) {
+			case 0:
+				runes[i] = alphabet[rng.Intn(len(alphabet))]
+			case 1:
+				runes = append(runes[:i], runes[i+1:]...)
+			default:
+				runes = append(runes[:i], append([]rune{'x'}, runes[i:]...)...)
+			}
+			if len(runes) == 0 {
+				runes = []rune("a")
+			}
+		}
+		return string(runes)
+	}
+
+	for range 5000 {
+		first := random()
+		second := mutate(first)
+		if rng.Intn(4) == 0 {
+			second = random()
+		}
+		want := titleSimilarity(first, second) >= titleMatchThreshold-1e-9
+		if got := similarTitles(first, second); got != want {
+			t.Fatalf(
+				"similarTitles(%q, %q) = %v; titleSimilarity = %v",
+				first, second, got, titleSimilarity(first, second),
+			)
+		}
+	}
+}
+
+func TestBuyerSimilarity(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          float64
+		known         bool
+	}{
+		{"Jisc", "JISC SERVICES LIMITED", 1, true},
+		{"The Borough Council of Calderdale", "Borough Council of Calderdale", 1, true},
+		{"Wakefield Council", "The Council of the City of Wakefield", 1, true},
+		{"City of York Council", "York and North Yorkshire Combined Authority", 1, true},
+		{"UK Atomic Energy Authority", "UK Atomic Energy Authoritys", 1, true},
+		{"Yorkshire Housing Limited", "Anchor Hanover Group", 0, true},
+		// Only the shorter name's words count, so a name within another
+		// matches it, rightly for York above, wrongly here: the title and
+		// date must match too, so this is accepted.
+		{"North East Lincolnshire", "North Lincolnshire Council", 1, true},
+		{"Hull City Council", "East Riding of Yorkshire Council", 0, true},
+		{"The Council", "Jisc", 0, false},
+		{"", "Jisc", 0, false},
+	}
+	for _, tt := range tests {
+		got, known := buyerSimilarity(tt.first, tt.second)
+		if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
+			t.Errorf(
+				"buyerSimilarity(%q, %q) = %v, %v; want %v, %v",
+				tt.first, tt.second, got, known, tt.want, tt.known,
+			)
+		}
+	}
+}
+
+func TestDescriptionSimilarity(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          float64
+		known         bool
+	}{
+		{"Supply of pest control.", "Pest control: the supply", 1, true},
+		{"Building materials for repairs", "Building works", 0.5, true},
+		{"", "Anything at all", 0, false},
+		{"a to be", "Anything at all", 0, false},
+	}
+	for _, tt := range tests {
+		got, known := descriptionSimilarity(tt.first, tt.second)
+		if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
+			t.Errorf(
+				"descriptionSimilarity(%q, %q) = %v, %v; want %v, %v",
+				tt.first, tt.second, got, known, tt.want, tt.known,
+			)
+		}
+	}
+}
+
+// A title match is confirmed by the buyer or, failing that, the
+// description; with neither to compare, it isn't.
+func TestConfirmedRepeat(t *testing.T) {
+	opportunity := func(buyer, description string) *types.Opportunity {
+		return &types.Opportunity{
+			BuyerName:   buyer,
+			Description: description,
+		}
+	}
+	tests := []struct {
+		name        string
+		first, last *types.Opportunity
+		want        bool
+	}{
+		{"buyer", opportunity("Jisc", ""), opportunity("JISC LIMITED", ""), true},
+		{
+			"description, not buyer",
+			opportunity("Echelon", "building materials supply"),
+			opportunity("Fairhive", "supply of building materials"),
+			true,
+		},
+		{
+			"neither",
+			opportunity("Yorkshire Housing", "pest control in Leeds homes"),
+			opportunity("Anchor Hanover", "pest control for retirement housing"),
+			false,
+		},
+		{"nothing to compare", opportunity("", ""), opportunity("", ""), false},
+	}
+	for _, tt := range tests {
+		if got := confirmedRepeat(tt.first, tt.last); got != tt.want {
+			t.Errorf("%s: confirmedRepeat = %v; want %v", tt.name, got, tt.want)
+		}
+	}
+}

internal/fetcher/worker_eusupply.go

@@ -0,0 +1,249 @@
+package fetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/eusupply"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+// euSupplySource is the Source of the opportunities the EU Supply source
+// fetches.
+const euSupplySource = eusupply.Source
+
+// repeatWindow is how far apart an EU Supply tender's publication and
+// another source's opportunity's latest release may be for the two to be
+// the same (see markRepeats).
+const repeatWindow = 30 * 24 * time.Hour
+
+// tenderStore is what the EU Supply source reads from the store, to skip
+// tenders already stored and to find the ones another source has.
+type tenderStore interface {
+	ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error)
+	ListOtherSourcesTitles(
+		ctx context.Context, source string, from, before time.Time,
+	) ([]store.OpportunityTitle, error)
+	GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error)
+}
+
+// runEUSupply scrapes EU Supply's UK portal every configured interval until
+// ctx is cancelled, passing what each run fetches to save, even when the run
+// fails partway. A long window is backfilled first (see backfill). One
+// client is used throughout, so its gap between requests holds across runs.
+// lookup is nil without a store: every tender is then new, and none is
+// checked for repeats.
+func runEUSupply(
+	ctx context.Context,
+	logger *slog.Logger,
+	conf *config.JSCSourceEUSupply,
+	lookup tenderStore,
+	save saveFunc,
+	since time.Time,
+) error {
+	client := eusupply.New(
+		conf.BaseURL,
+		eusupply.WithLogger(logger),
+		eusupply.WithRequestGap(conf.RequestMinGap, conf.RequestMaxGap),
+	)
+	fetch := func(
+		ctx context.Context, since, until time.Time,
+	) ([]*types.Opportunity, error) {
+		return fetchEUSupply(ctx, logger, client, lookup, since, until)
+	}
+
+	since = backfill(
+		ctx,
+		logger,
+		euSupplySource,
+		conf.Interval,
+		fetch,
+		save,
+		since,
+	)
+
+	worker := util.Worker{
+		Logger:   tagged(logger),
+		Name:     euSupplySource,
+		Interval: conf.Interval,
+		Task: func(ctx context.Context, since time.Time) error {
+			opportunities, err := fetch(ctx, since, time.Time{})
+			return saveFetched(ctx, logger, save, opportunities, err)
+		},
+	}
+	return worker.Run(ctx, since)
+}
+
+// fetchEUSupply fetches the tenders published from since up to until (zero:
+// today) that aren't stored yet, with their own pages read, and marks the
+// ones another source already has Hidden and Skipped (see markRepeats). If
+// it fails partway, the tenders read in full before the error are returned
+// along with it. Finding nothing new returns ErrNoNewReleases.
+func fetchEUSupply(
+	ctx context.Context,
+	logger *slog.Logger,
+	client *eusupply.Client,
+	lookup tenderStore,
+	since, until time.Time,
+) ([]*types.Opportunity, error) {
+	tenders, fetchErr := client.Search(ctx, since, until)
+
+	fresh, err := newTenders(ctx, lookup, tenders)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range fresh {
+		if err := client.Details(ctx, &fresh[i]); err != nil {
+			fresh = fresh[:i]
+			fetchErr = errors.Join(fetchErr, err)
+			break
+		}
+	}
+
+	opportunities := eusupply.Convert(fresh)
+	if err := markRepeats(ctx, logger, lookup, opportunities); err != nil {
+		// Saved unmarked, a repeat would stay visible, as marks are only
+		// stored for a new opportunity: fetch them again next run instead.
+		return nil, errors.Join(fetchErr, err)
+	}
+
+	tagged(logger).Info(
+		"fetched opportunities",
+		"source", euSupplySource,
+		"listed", len(tenders),
+		"count", len(opportunities),
+	)
+
+	if len(opportunities) == 0 && fetchErr == nil {
+		return nil, ErrNoNewReleases
+	}
+	return opportunities, fetchErr
+}
+
+// newTenders returns the tenders whose release isn't stored yet, so only
+// their pages are read. Saving a stored one again would replace its row
+// with the search's fields alone. With no lookup, every tender is new.
+func newTenders(
+	ctx context.Context, lookup tenderStore, tenders []eusupply.Tender,
+) ([]eusupply.Tender, error) {
+	if lookup == nil {
+		return tenders, nil
+	}
+
+	var fresh []eusupply.Tender
+	for _, tender := range tenders {
+		exists, err := lookup.ReleaseExists(
+			ctx, eusupply.UUID(tender.ID), eusupply.ReleaseID(tender),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("checking tender %s: %w", tender.ID, err)
+		}
+		if !exists {
+			fresh = append(fresh, tender)
+		}
+	}
+
+	return fresh, nil
+}
+
+// markRepeats marks Hidden and Skipped each opportunity that repeats one
+// from another source, most often its Find a Tender notice: its title is at
+// least titleMatchThreshold alike (see similarTitles), its latest release
+// is within repeatWindow, and its buyer or description matches too (see
+// confirmedRepeat). Only the opportunities whose titles match are read in
+// full. The store keeps the marks of an opportunity it already has, so only
+// new ones are marked. With no lookup it does nothing.
+func markRepeats(
+	ctx context.Context,
+	logger *slog.Logger,
+	lookup tenderStore,
+	opportunities []*types.Opportunity,
+) error {
+	if lookup == nil || len(opportunities) == 0 {
+		return nil
+	}
+
+	earliest := opportunities[0].LatestReleaseDate
+	latest := earliest
+	for _, opportunity := range opportunities {
+		earliest = minTime(earliest, opportunity.LatestReleaseDate)
+		latest = maxTime(latest, opportunity.LatestReleaseDate)
+	}
+	others, err := lookup.ListOtherSourcesTitles(
+		ctx,
+		euSupplySource,
+		earliest.Add(-repeatWindow),
+		latest.Add(repeatWindow+time.Nanosecond),
+	)
+	if err != nil {
+		return err
+	}
+
+	normalised := make([]string, len(others))
+	for i, other := range others {
+		normalised[i] = normaliseTitle(other.Title)
+	}
+
+	repeats := 0
+	for _, opportunity := range opportunities {
+		title := normaliseTitle(opportunity.Title)
+		for i, other := range others {
+			gap := other.LatestReleaseDate.Sub(opportunity.LatestReleaseDate)
+			if gap.Abs() > repeatWindow || !similarTitles(title, normalised[i]) {
+				continue
+			}
+			repeated, err := lookup.GetOpportunity(ctx, other.UUID)
+			if err != nil {
+				return fmt.Errorf("reading opportunity %s: %w", other.UUID, err)
+			}
+			if !confirmedRepeat(opportunity, repeated) {
+				tagged(logger).Debug(
+					"title matched, but not buyer or description",
+					"uuid", opportunity.UUID,
+					"title_matches", other.UUID,
+				)
+				continue
+			}
+			opportunity.Marks.Hidden = true
+			opportunity.Marks.Skipped = true
+			repeats++
+			tagged(logger).Debug(
+				"hid repeated opportunity",
+				"uuid", opportunity.UUID,
+				"repeats", other.UUID,
+			)
+			break
+		}
+	}
+
+	tagged(logger).Info(
+		"hid repeated opportunities",
+		"source", euSupplySource,
+		"count", repeats,
+	)
+
+	return nil
+}
+
+// minTime returns the earlier of two times.
+func minTime(first, second time.Time) time.Time {
+	if second.Before(first) {
+		return second
+	}
+	return first
+}
+
+// maxTime returns the later of two times.
+func maxTime(first, second time.Time) time.Time {
+	if second.After(first) {
+		return second
+	}
+	return first
+}

internal/fetcher/worker_eusupply_test.go

@@ -0,0 +1,324 @@
+package fetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/eusupply"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+var errTestStore = errors.New("test store failed")
+
+// testTender is one row of a fake EU Supply search page, and its page.
+type testTender struct {
+	id, pid, title string
+	published      string // "21/10/2026 12:45:00"
+	buyer          string
+	description    string
+}
+
+// euSupplySite is a fake EU Supply portal: one page of search results with
+// tenders, and a page for each of them. It counts the tender pages read.
+type euSupplySite struct {
+	mu       sync.Mutex
+	tenders  []testTender
+	detailed []string // the PIDs whose pages were read
+	failPID  string   // a PID whose page fails
+}
+
+func (s *euSupplySite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if r.Method == http.MethodPost {
+		var rows strings.Builder
+		for _, tender := range s.tenders {
+			fmt.Fprintf(
+				&rows,
+				`<tr><td><div class="js-tooltip-content">%s</div></td>`+
+					`<td><div class="js-tooltip-content"></div></td>`+
+					`<td><a href="/app/rfq/rwlentrance_s.asp?PID=%s">%s</a></td>`+
+					`<td><div class="js-tooltip-content" title="%s"></div></td>`+
+					`<td></td><td></td><td>%s</td><td></td></tr>`,
+				tender.id, tender.pid, tender.title, tender.published, tender.buyer,
+			)
+		}
+		fmt.Fprintf(
+			w,
+			`<html><body><div id="searchResultContainer"><table><tbody>%s`+
+				`</tbody></table></div></body></html>`,
+			rows.String(),
+		)
+		return
+	}
+
+	pid := strings.Split(strings.TrimPrefix(
+		r.URL.Path, "/ctm/Supplier/PublicPurchase/",
+	), "/")[0]
+	s.detailed = append(s.detailed, pid)
+	if pid == s.failPID {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	description := ""
+	for _, tender := range s.tenders {
+		if tender.pid == pid {
+			description = tender.description
+		}
+	}
+	fmt.Fprintf(
+		w,
+		`<html><body><div class="ctm-content-label">Detailed description:</div>`+
+			`<p>%s</p></body></html>`,
+		description,
+	)
+}
+
+// fakeTenderStore is a tenderStore with some releases stored and other
+// sources' opportunities.
+type fakeTenderStore struct {
+	stored    map[string]bool // uuid "/" release id
+	others    []*types.Opportunity
+	titlesErr error
+}
+
+func (s *fakeTenderStore) ReleaseExists(
+	ctx context.Context, uuid, releaseID string,
+) (bool, error) {
+	return s.stored[uuid+"/"+releaseID], nil
+}
+
+func (s *fakeTenderStore) ListOtherSourcesTitles(
+	ctx context.Context, source string, from, before time.Time,
+) ([]store.OpportunityTitle, error) {
+	if s.titlesErr != nil {
+		return nil, s.titlesErr
+	}
+	var titles []store.OpportunityTitle
+	for _, other := range s.others {
+		if !other.LatestReleaseDate.Before(from) &&
+			other.LatestReleaseDate.Before(before) {
+			titles = append(titles, store.OpportunityTitle{
+				UUID:              other.UUID,
+				Title:             other.Title,
+				LatestReleaseDate: other.LatestReleaseDate,
+			})
+		}
+	}
+	return titles, nil
+}
+
+func (s *fakeTenderStore) GetOpportunity(
+	ctx context.Context, uuid string,
+) (*types.Opportunity, error) {
+	for _, other := range s.others {
+		if other.UUID == uuid {
+			return other, nil
+		}
+	}
+	return nil, store.ErrNotFound
+}
+
+// newEUSupplyClient returns a client for srv that doesn't wait.
+func newEUSupplyClient(srv *httptest.Server) *eusupply.Client {
+	return eusupply.New(srv.URL, eusupply.WithRequestGap(0, 0))
+}
+
+// Only tenders not stored are read and returned. The ones another source
+// has within the window, with a matching title and a matching buyer or
+// description, are marked Hidden and Skipped.
+func TestFetchEUSupply(t *testing.T) {
+	const day = "20/10/2026 10:00:00"
+	site := &euSupplySite{
+		tenders: []testTender{
+			{"1", "11", "Already stored", day, "", ""},
+			// The same generic title from another buyer: not a repeat.
+			{"2", "12", "Pest Control Services", day, "Yorkshire Housing Limited",
+				"Pest control across our homes in Leeds"},
+			// The same buyer, named differently: a repeat.
+			{"3", "13", "Google Reseller Framework", day, "Jisc", ""},
+			// An agent buying for the buyer, but the same description.
+			{"4", "14", "Fairhive - Materials Procurement", day,
+				"Echelon Consultancy",
+				"Supply of building materials for Fairhive housing repairs"},
+			// Alike, but too long before.
+			{"5", "15", "Housing related support", day, "Hull City Council", ""},
+		},
+	}
+	srv := httptest.NewServer(site)
+	t.Cleanup(srv.Close)
+
+	published := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
+	other := func(
+		uuid, title, buyer, description string, released time.Time,
+	) *types.Opportunity {
+		return &types.Opportunity{
+			UUID:              uuid,
+			Title:             title,
+			BuyerName:         buyer,
+			Description:       description,
+			LatestReleaseDate: released,
+		}
+	}
+	lookup := &fakeTenderStore{
+		stored: map[string]bool{
+			"eusupply-uk-1/1": true,
+		},
+		others: []*types.Opportunity{
+			other("ocds-pest", "PEST CONTROL SERVICE", "Anchor Hanover Group",
+				"Pest control for retirement housing", published),
+			other("ocds-jisc", "Google Reseller Framework",
+				"JISC SERVICES LIMITED", "", published.Add(time.Hour)),
+			other("ocds-fairhive", "Fairhive - Materials Procurement",
+				"Fairhive Housing Limited",
+				"Fairhive seeks supply of building materials for housing repairs",
+				published),
+			other("ocds-old", "Housing related support", "Hull City Council", "",
+				published.Add(-40*24*time.Hour)),
+		},
+	}
+
+	opportunities, err := fetchEUSupply(
+		context.Background(),
+		slog.Default(),
+		newEUSupplyClient(srv),
+		lookup,
+		published.Add(-24*time.Hour),
+		time.Time{},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got := strings.Join(site.detailed, ","); got != "12,13,14,15" {
+		t.Errorf("pages read = %s; want the new tenders' only", got)
+	}
+	marks := map[string]bool{}
+	for _, opportunity := range opportunities {
+		if opportunity.Source != "EU Supply - UK" {
+			t.Errorf("%s: Source = %q", opportunity.UUID, opportunity.Source)
+		}
+		if opportunity.Marks.Hidden != opportunity.Marks.Skipped {
+			t.Errorf("%s: marks = %+v", opportunity.UUID, opportunity.Marks)
+		}
+		marks[opportunity.UUID] = opportunity.Marks.Hidden
+	}
+	want := map[string]bool{
+		"eusupply-uk-2": false,
+		"eusupply-uk-3": true,
+		"eusupply-uk-4": true,
+		"eusupply-uk-5": false,
+	}
+	if fmt.Sprint(marks) != fmt.Sprint(want) {
+		t.Errorf("hidden = %v; want %v", marks, want)
+	}
+}
+
+// A tender page that fails stops the run, keeping the tenders read before
+// it; a failed title lookup returns nothing, so nothing is saved unmarked.
+func TestFetchEUSupplyErrors(t *testing.T) {
+	site := &euSupplySite{
+		tenders: []testTender{
+			{"1", "11", "First", "20/10/2026 10:00:00", "", ""},
+			{"2", "12", "Second", "20/10/2026 10:00:00", "", ""},
+			{"3", "13", "Third", "20/10/2026 10:00:00", "", ""},
+		},
+		failPID: "12",
+	}
+	srv := httptest.NewServer(site)
+	t.Cleanup(srv.Close)
+	client := newEUSupplyClient(srv)
+	since := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
+
+	opportunities, err := fetchEUSupply(
+		context.Background(), slog.Default(), client, nil, since, time.Time{},
+	)
+	if !errors.Is(err, eusupply.ErrUnexpectedStatus) {
+		t.Errorf("err = %v; want ErrUnexpectedStatus", err)
+	}
+	if len(opportunities) != 1 || opportunities[0].UUID != "eusupply-uk-1" {
+		t.Errorf("opportunities = %v; want the first tender", opportunities)
+	}
+
+	site.failPID = ""
+	lookup := &fakeTenderStore{
+		titlesErr: errTestStore,
+	}
+	opportunities, err = fetchEUSupply(
+		context.Background(), slog.Default(), client, lookup, since, time.Time{},
+	)
+	if !errors.Is(err, errTestStore) || opportunities != nil {
+		t.Errorf(
+			"err = %v, %d opportunities; want the store's error and none",
+			err, len(opportunities),
+		)
+	}
+
+	site.tenders = nil
+	_, err = fetchEUSupply(
+		context.Background(), slog.Default(), client, lookup, since, time.Time{},
+	)
+	if !errors.Is(err, ErrNoNewReleases) {
+		t.Errorf("no tenders: err = %v; want ErrNoNewReleases", err)
+	}
+}
+
+// Run starts the EU Supply source when it is enabled, and saves its
+// opportunities with their Source.
+func TestRunEUSupply(t *testing.T) {
+	site := &euSupplySite{
+		tenders: []testTender{
+			{
+				"1", "11", "A tender",
+				time.Now().Format("02/01/2006 15:04:05"), "", "",
+			},
+		},
+	}
+	srv := httptest.NewServer(site)
+	t.Cleanup(srv.Close)
+
+	opportunityStore := openTestStore(t)
+	sources := &config.JSCSources{}
+	sources.EUSupply.Enabled = true
+	sources.EUSupply.Interval = time.Hour
+	sources.EUSupply.BaseURL = srv.URL
+	sources.EUSupply.RequestMinGap = time.Millisecond
+	sources.EUSupply.RequestMaxGap = time.Millisecond
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		done <- New(sources, WithStore(opportunityStore)).Run(ctx, time.Time{})
+	}()
+
+	deadline := time.Now().Add(5 * time.Second)
+	var stored []string
+	for time.Now().Before(deadline) {
+		opportunity, err := opportunityStore.GetOpportunity(
+			context.Background(), "eusupply-uk-1",
+		)
+		if err == nil {
+			stored = append(stored, opportunity.Source)
+			break
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run = %v", err)
+	}
+	if len(stored) != 1 || stored[0] != "EU Supply - UK" {
+		t.Errorf("stored sources = %v; want EU Supply - UK", stored)
+	}
+}