Added an EU Supply source, scraping the UK portal's public tenders

internal/eusupply replaces the old Node scraper (eu-supply.js) for
uk.eu-supply.com. The site has no API, so it posts the public search
form, which now needs the form's current defaults and reads the HTML
with golang.org/x/net/html. Each tender not stored yet has its own
page read too, for its description, contract type, CPV codes and
buyer address. Tenders are stored with the source "EU Supply - UK"
as eusupply-uk-<tender id>, with one release holding the tender as
read.

The site publishes no rate limit, so the client sets its own: one
request at a time, with a random wait between
sources.eu_supply.request_min_gap and request_max_gap (2s to 10s).
The source has the same 7-day backfill as Find a Tender's.

Most of its tenders are also on Find a Tender, so a new one is stored
Hidden and Skipped when another source's opportunity released within
30 days has a title at least 90% alike, and its buyer or description
matches too (80% of the shorter one's significant words).

README.md

@@ -120,7 +120,8 @@ nothing stored, Find a Tender's client fetches the last day. `--since`
 overrides that for every source.
 
 When the start is more than 7 days ago (an early `--since`, or a long time
-offline), the Find a Tender sources first backfill it in 7-day windows,
+offline), the Find a Tender and EU Supply sources first backfill it in 7-day
+windows,
 oldest first, saving each before fetching the next. A window that fails is
 saved as far as it got and fetched again after the source's interval. The
 API returns the newest releases first, so if a backfill is stopped partway
@@ -132,6 +133,18 @@ A rate limit is waited out, as the API asks, up to
 `sources.<source>.rate_limit_retries` times per request. A failed fetch
 keeps what it fetched before the failure.
 
+### EU Supply
+The `eu_supply` source scrapes the public tenders on EU Supply's UK portal
+(`uk.eu-supply.com`), stored with the source `EU Supply - UK`. The site has
+no API and publishes no rate limit, so the client waits a random time from
+`request_min_gap` to `request_max_gap` (2s to 10s) between its requests. Each
+tender not stored yet has its own page read too, for its description and
+CPV codes. Most of its tenders are also published on Find a Tender: a new
+one is stored Hidden and Skipped when another source's opportunity released
+within 30 days of it has a title at least 90% alike, and its buyer or its
+description matches too (80% of the shorter one's words). See
+[internal/eusupply](internal/eusupply/README.md).
+
 `fts-once` can be limited with `--planning`, `--opportunities` or
 `--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead
 of waiting.
@@ -191,6 +204,7 @@ second one fails with "address already in use".
 | `internal/skiprules` | The rules that mark new opportunities Skipped (see [internal/skiprules](internal/skiprules/README.md)) |
 | `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) |
 | `internal/findatender` | FTS API client and conversion from OCDS to JobSearch types |
+| `internal/eusupply` | EU Supply UK portal scraper and conversion to JobSearch types (see [internal/eusupply](internal/eusupply/README.md)) |
 | `internal/types` | JobSearch's own opportunity types |
 | `internal/util` | The periodic `Worker` used by the fetcher |
 | `internal/logging` | slog setup and the HTTP access log |

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	go.yaml.in/yaml/v3 v3.0.5
+	golang.org/x/net v0.58.0
 	golang.org/x/time v0.16.0
 	resty.dev/v3 v3.0.0-rc.4
 )
@@ -34,7 +35,6 @@ require (
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/net v0.58.0 // indirect
 	golang.org/x/sync v0.22.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.41.0 // indirect

go.sum

@@ -10,7 +10,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
 github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
-github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
 github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE=
 github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
@@ -28,7 +27,6 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
 github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -93,7 +91,6 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
 golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
 golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
-golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
 golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/config/README.md

@@ -75,9 +75,13 @@ supported.
 | `sources.find_a_tender.rate_limit_retries` | int | `5` | How many times a request refused for the API's rate limit is waited out (for its `Retry-After`) and retried. 0 fails at the first; can't be negative. `fetcher fts-once --ignore-rate-limit` uses 0 |
 | `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`). Empty: the source fails to start if enabled |
 | `sources.fts_mock.rate_limit_retries` | int | `5` | As for `find_a_tender` |
+| `sources.eu_supply.base_url` | string | `https://uk.eu-supply.com` | Root of EU Supply's UK portal (see `internal/eusupply`) |
+| `sources.eu_supply.request_min_gap` | duration | `2s` | The shortest wait between two requests to the portal. Must be greater than 0: the site publishes no rate limit |
+| `sources.eu_supply.request_max_gap` | duration | `10s` | The longest wait between two requests; each wait is random, from `request_min_gap`. Must be greater than 0; less than `request_min_gap` is read as `request_min_gap` |
 | `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown |
 
-The current sources are `find_a_tender`, `fts_mock` and `sleeper`. See
+The current sources are `find_a_tender`, `fts_mock`, `eu_supply` and
+`sleeper`. See
 "Addition of a new opportunities source" in the top level README for how to add
 a new source.
 

internal/config/conf_sections_test.go

@@ -288,6 +288,35 @@ func TestConcreteSourceSections(t *testing.T) {
 	}
 }
 
+func TestEUSupplySection(t *testing.T) {
+	conf := defaultConfig(t)
+	got := conf.Sources.EUSupply
+	if got.Enabled || got.Interval != 10*time.Minute ||
+		got.BaseURL != "https://uk.eu-supply.com" ||
+		got.RequestMinGap != 2*time.Second ||
+		got.RequestMaxGap != 10*time.Second {
+		t.Errorf("defaults = %+v", got)
+	}
+
+	c := JSCSourceEUSupply{}
+	if err := c.SetValue("request_min_gap", "5s"); err != nil ||
+		c.RequestMinGap != 5*time.Second {
+		t.Errorf("SetValue(request_min_gap, 5s): %v, %v", err, c.RequestMinGap)
+	}
+	for _, key := range []string{"request_min_gap", "request_max_gap"} {
+		if err := c.SetValue(key, "0s"); err == nil {
+			t.Errorf("SetValue(%s, 0s) succeeded", key)
+		}
+	}
+	// Its own validate must still check interval, as JSCSourcesSource's does.
+	if err := c.SetValue("interval", "0s"); err == nil {
+		t.Error("SetValue(interval, 0s) succeeded")
+	}
+	if v, err := c.ValueOf("base_url"); err != nil || v != "" {
+		t.Errorf("ValueOf(base_url) = %v, %v", v, err)
+	}
+}
+
 func TestSourcesSection(t *testing.T) {
 	var c JSCSources
 	if err := c.SetValue("fts_mock.api_root", "http://mock"); err != nil {
@@ -317,8 +346,8 @@ func TestSourcesSection(t *testing.T) {
 	if flat["sources.fts_mock.api_root"] != "http://mock" {
 		t.Errorf("Flatten missing sources.fts_mock.api_root: %v", flat)
 	}
-	if len(flat) != 12 {
+	if len(flat) != 17 {
-		t.Errorf("Flatten has %d keys; want 12: %v", len(flat), flat)
+		t.Errorf("Flatten has %d keys; want 17: %v", len(flat), flat)
 	}
 
 	if err := c.SetValue("skip_rules_file", "rules.json"); err != nil ||

internal/config/conf_source_eusupply.go

@@ -0,0 +1,62 @@
+package config
+
+import (
+	"errors"
+	"reflect"
+	"time"
+)
+
+// JSCKSourceEUSupply is the EU Supply source's configuration key.
+const JSCKSourceEUSupply string = "eu_supply"
+
+// JSCSourceEUSupply configures scraping the public tenders on EU Supply's
+// UK portal at BaseURL (see internal/eusupply). The client waits a random
+// time from RequestMinGap to RequestMaxGap after each request; both must be
+// greater than 0, as the site's own limit isn't published, and a
+// RequestMaxGap less than RequestMinGap is read as RequestMinGap.
+type JSCSourceEUSupply struct {
+	JSCSourcesSource
+	BaseURL       string        `cfg:"base_url"        default:"https://uk.eu-supply.com"`
+	RequestMinGap time.Duration `cfg:"request_min_gap" default:"2s"`
+	RequestMaxGap time.Duration `cfg:"request_max_gap" default:"10s"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+// Map implements Section.
+func (c JSCSourceEUSupply) Map() map[string]any { return mapOf(c) }
+
+// Flatten implements Section.
+func (c JSCSourceEUSupply) Flatten() map[string]any {
+	return flattenOf(JSCKSourceEUSupply, c)
+}
+
+// TypeOf implements Section.
+func (c JSCSourceEUSupply) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+// ValueOf implements Section.
+func (c JSCSourceEUSupply) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+// SetValue implements Section.
+func (c *JSCSourceEUSupply) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}
+
+// ---------- Section specific methods ----------
+
+// validate checks values before they are stored (see validator), as
+// JSCSourcesSource's does, which this one hides. request_min_gap and
+// request_max_gap must be greater than 0.
+func (c *JSCSourceEUSupply) validate(key string, value any) error {
+	if (key == "request_min_gap" || key == "request_max_gap") &&
+		value.(time.Duration) <= 0 {
+		return errors.New(
+			"config: request_min_gap and request_max_gap must be greater than 0, as EU Supply publishes no rate limit",
+		)
+	}
+	return c.JSCSourcesSource.validate(key, value)
+}

internal/config/conf_sources.go

@@ -20,6 +20,7 @@ type JSCSources struct {
 	SkipRulesFile string               `cfg:"skip_rules_file"`
 	FindATender   JSCSourceFindATender `cfg:"find_a_tender"`
 	FTSMock       JSCSourceFTSMock     `cfg:"fts_mock"`
+	EUSupply      JSCSourceEUSupply    `cfg:"eu_supply"`
 	Sleeper       JSCSourceSleeper     `cfg:"sleeper"`
 }
 

internal/eusupply/README.md

@@ -0,0 +1,47 @@
+# EU Supply
+
+A scraper for the public tenders on EU Supply's UK portal,
+[uk.eu-supply.com](https://uk.eu-supply.com). The site (Mercell CTM) has no
+API, so the client posts its public search form and reads the HTML. It
+replaces an old Node scraper, which also covered the Sellafield and Shared
+Systems portals; only the UK portal is still used.
+
+## Requests
+
+| Request | Path | Gives |
+|---|---|---|
+| Search | `POST /ctm/Supplier/PublicTenders` | Up to 100 tenders a page: id, reference, title, publication date, response deadline, process, buyers, countries |
+| Tender page | `GET /ctm/Supplier/PublicPurchase/<PID>/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK` | Description, type of contract, CPV codes, buyer address, lots |
+
+- The search filters on the **publication date**, by whole UK days
+  (`SearchFilter.FromDate`/`ToDate` as `DD/MM/YYYY`, both included). Only
+  issued tenders are searched (`SearchFilter.PublishType` 1), not awarded
+  ones.
+- The form must be posted whole, with the portal's own defaults
+  (`searchForm`). With the old scraper's values (`OperatorId` 1 and an
+  empty `SavedCategoryId`), the portal answers 500.
+- Times are UK time. Each date cell's tooltip `title` holds the full time
+  (`21/10/2026 12:45:00`); the cell's text has only the date.
+- A tender's PID (its pages' id) is only in its search result's link
+  (`rwlentrance_s.asp?PID=…`), which redirects to its page.
+- A tender page's fields are each a `div.ctm-content-label` (`CPV codes:`)
+  followed by a `<p>` with the value.
+- The system certificate authorities are enough: the old scraper's own CA
+  file isn't needed.
+
+## Rate limit
+
+The site publishes no rate limit, so the client sets its own: it makes one
+request at a time, waiting a random time from the minimum to the maximum gap
+(`WithRequestGap`; the source's `request_min_gap` and `request_max_gap`,
+2s to 10s by default) after each before making the next.
+
+## Opportunities
+
+`Convert` makes one opportunity per tender, with the source `EU Supply - UK`
+and the UUID `eusupply-uk-<tender id>`. It has one release (id: the tender
+id), dated its publication, tagged `tender`, whose JSON is the `Tender` as
+read. CPV codes are stored without their check digit, as Find a Tender's
+are. The package sets no marks: the fetcher marks the tenders it takes as
+repeats of another source's opportunities Hidden and Skipped (see
+`internal/fetcher/match.go`).

internal/eusupply/convert.go

@@ -0,0 +1,101 @@
+package eusupply
+
+import (
+	"encoding/json"
+	"strings"
+
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+// The opportunities Convert makes.
+const (
+	// UUIDPrefix starts the UUID of every opportunity from the UK portal;
+	// the tender's id follows it.
+	UUIDPrefix = "eusupply-uk-"
+	// uuidType is their UUIDType.
+	uuidType = "eusupply"
+	// releaseTag is their one release's tag: the portal lists open tenders.
+	releaseTag = "tender"
+)
+
+// mainCategories maps the portal's types of contract to OCDS's
+// mainProcurementCategory, as Find a Tender's opportunities have.
+var mainCategories = map[string]string{
+	"Supplies": "goods",
+	"Services": "services",
+	"Works":    "works",
+}
+
+// UUID returns the UUID of the opportunity for the tender with id.
+func UUID(id string) string {
+	return UUIDPrefix + id
+}
+
+// ReleaseID returns the id of the one release of the opportunity for
+// tender: its id, as the portal gives each tender one page.
+func ReleaseID(tender Tender) string {
+	return tender.ID
+}
+
+// Convert returns the opportunity for each tender, with one release whose
+// JSON is the tender as read.
+func Convert(tenders []Tender) []*types.Opportunity {
+	opportunities := make([]*types.Opportunity, 0, len(tenders))
+	for _, tender := range tenders {
+		opportunities = append(opportunities, convertOne(tender))
+	}
+	return opportunities
+}
+
+// convertOne builds the opportunity for one tender.
+func convertOne(tender Tender) *types.Opportunity {
+	published := tender.Published
+	opportunity := &types.Opportunity{
+		UUID:              UUID(tender.ID),
+		UUIDType:          uuidType,
+		Title:             tender.Title,
+		Status:            "active",
+		Description:       tender.Description,
+		BuyerName:         tender.Buyer,
+		Classifications:   classifications(tender),
+		MainCategory:      mainCategories[tender.ContractType],
+		MultiLot:          tender.Lots > 1,
+		StartDate:         &published,
+		EndDate:           tender.Deadline,
+		Source:            Source,
+		LatestReleaseDate: tender.Published,
+	}
+
+	release := types.OpportunityRelease{
+		UUID:        opportunity.UUID,
+		ReleaseID:   ReleaseID(tender),
+		ReleaseDate: tender.Published,
+		Tags: []types.OpportunityReleaseTag{
+			{
+				Tag: releaseTag,
+			},
+		},
+	}
+	// A Tender is plain data, so marshalling it can't fail.
+	if encoded, err := json.Marshal(tender); err == nil {
+		release.JSON = string(encoded)
+	}
+	opportunity.Releases = []types.OpportunityRelease{release}
+
+	return opportunity
+}
+
+// classifications returns the tender's CPV codes as Find a Tender's are
+// stored: without the check digit ("48000000", not "48000000-8").
+func classifications(tender Tender) []types.ClassificationCode {
+	codes := make([]types.ClassificationCode, 0, len(tender.CPVCodes))
+	for _, cpv := range tender.CPVCodes {
+		code, _, _ := strings.Cut(cpv.Code, "-")
+		codes = append(codes, types.ClassificationCode{
+			IsCPV:          true,
+			Code:           code,
+			Classification: cpv.Description,
+		})
+	}
+	return codes
+}

internal/eusupply/details.go

@@ -0,0 +1,108 @@
+package eusupply
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// ErrNoDetails means a tender's page had none of the labelled fields the
+// public page always has: the page has changed, or isn't a tender's.
+var ErrNoDetails = errors.New("eusupply: no tender details on the page")
+
+// Details reads tender's own page and fills in its description, contract
+// type, CPV codes, buyer address and number of lots, and sets HasDetails.
+func (c *Client) Details(ctx context.Context, tender *Tender) error {
+	body, err := c.get(ctx, tenderPath(tender.PID))
+	if err != nil {
+		return fmt.Errorf("reading tender %s: %w", tender.ID, err)
+	}
+
+	if err := parseDetails(body, tender); err != nil {
+		return fmt.Errorf("reading tender %s: %w", tender.ID, err)
+	}
+
+	return nil
+}
+
+// parseDetails reads a tender's page into tender. Each field is a
+// "ctm-content-label" div ("CPV codes:") followed by a <p> holding its
+// value.
+func parseDetails(body []byte, tender *Tender) error {
+	document, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+
+	values := make(map[string]*html.Node)
+	labels := findAll(document, hasClass("div", "ctm-content-label"))
+	for _, label := range labels {
+		value := nextElement(label)
+		if value == nil || value.Data != "p" {
+			continue
+		}
+		name := strings.TrimSuffix(text(label), ":")
+		if _, found := values[name]; !found {
+			values[name] = value
+		}
+	}
+	if len(values) == 0 {
+		return ErrNoDetails
+	}
+
+	if value := values["Detailed description"]; value != nil {
+		tender.Description = strings.TrimSpace(rawText(value))
+	}
+	if value := values["Type of contract"]; value != nil {
+		tender.ContractType = text(value)
+	}
+	if value := values["CPV codes"]; value != nil {
+		tender.CPVCodes = parseCPVCodes(value)
+	}
+	if value := values["Buyer"]; value != nil {
+		tender.BuyerAddress = buyerAddress(value)
+	}
+	if deadline := values["Response deadline (UK time)"]; deadline != nil &&
+		tender.Deadline == nil {
+		if parsed, err := parseUKTime(text(deadline)); err == nil {
+			tender.Deadline = &parsed
+		}
+	}
+	tender.Lots = len(findAll(document, hasClass("div", "package-container")))
+	tender.HasDetails = true
+
+	return nil
+}
+
+// parseCPVCodes reads the CPV codes listed in value, one per <span>: the
+// code, two non-breaking spaces, then its description.
+func parseCPVCodes(value *html.Node) []CPVCode {
+	var codes []CPVCode
+	for _, span := range findAll(value, isTag("span")) {
+		code, description, _ := strings.Cut(text(span), " ")
+		if code == "" {
+			continue
+		}
+		codes = append(codes, CPVCode{
+			Code:        code,
+			Description: description,
+		})
+	}
+	return codes
+}
+
+// buyerAddress reads the buyer's name and address, one line each, leaving
+// out the "View profile" link.
+func buyerAddress(value *html.Node) []string {
+	var address []string
+	for _, line := range lines(value) {
+		if line != "View profile" {
+			address = append(address, line)
+		}
+	}
+	return address
+}

internal/eusupply/eusupply.go

@@ -0,0 +1,208 @@
+// Package eusupply scrapes the public tenders listed on EU Supply's UK
+// portal (uk.eu-supply.com), and converts them to opportunities.
+//
+// The site has no API: the client posts the portal's search form and reads
+// the HTML it returns, then reads each tender's own page for its
+// description and CPV codes. The client limits its own request rate (see
+// WithRequestGap), as the site publishes no limit to wait out.
+package eusupply
+
+import (
+	"cmp"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"math/rand/v2"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+	_ "time/tzdata" // the portal's times are UK time, wherever this runs
+)
+
+// The defaults of the client's options.
+const (
+	// DefaultBaseURL is the UK portal's root.
+	DefaultBaseURL = "https://uk.eu-supply.com"
+	// DefaultMinRequestGap and DefaultMaxRequestGap bound the random wait
+	// between two requests (see WithRequestGap).
+	DefaultMinRequestGap = 2 * time.Second
+	DefaultMaxRequestGap = 10 * time.Second
+	// requestTimeout is how long one request may take.
+	requestTimeout = 60 * time.Second
+)
+
+// Source is the Source of the opportunities Convert makes.
+const Source = "EU Supply - UK"
+
+// ErrUnexpectedStatus wraps a response status other than 200 OK.
+var ErrUnexpectedStatus = errors.New("eusupply: unexpected response status")
+
+// ukTime is the zone of the portal's dates and times.
+var ukTime = mustLoadLocation("Europe/London")
+
+// Client scrapes one EU Supply portal. Create it with New; it is safe for
+// concurrent use, though its requests are made one at a time.
+type Client struct {
+	baseURL    string
+	httpClient *http.Client
+	log        *slog.Logger
+
+	minGap, maxGap time.Duration
+
+	// mu serialises requests, so the gap between them holds; lastRequest is
+	// when the previous one finished (zero: none yet).
+	mu          sync.Mutex
+	lastRequest time.Time
+}
+
+// Option configures a Client.
+type Option func(*Client)
+
+// WithLogger makes the client log to logger instead of slog.Default().
+func WithLogger(logger *slog.Logger) Option {
+	return func(c *Client) {
+		c.log = logger
+	}
+}
+
+// WithRequestGap makes the client wait a random time from minGap to maxGap
+// after each request before making the next. A maxGap less than minGap is
+// read as minGap; 0 or less for both means no wait, for tests.
+func WithRequestGap(minGap, maxGap time.Duration) Option {
+	return func(c *Client) {
+		c.minGap = minGap
+		c.maxGap = max(minGap, maxGap)
+	}
+}
+
+// New returns a client for the portal at baseURL (DefaultBaseURL if empty).
+// It waits DefaultMinRequestGap to DefaultMaxRequestGap between requests
+// unless given WithRequestGap, and logs to slog.Default() unless given
+// WithLogger.
+func New(baseURL string, opts ...Option) *Client {
+	jar, _ := cookiejar.New(nil) // cookiejar.New never returns an error
+	client := &Client{
+		baseURL: strings.TrimSuffix(cmp.Or(baseURL, DefaultBaseURL), "/"),
+		httpClient: &http.Client{
+			Timeout: requestTimeout,
+			Jar:     jar,
+		},
+		minGap: DefaultMinRequestGap,
+		maxGap: DefaultMaxRequestGap,
+	}
+	for _, opt := range opts {
+		opt(client)
+	}
+
+	return client
+}
+
+// logger returns the logger the client logs to, tagged with its component.
+func (c *Client) logger() *slog.Logger {
+	log := c.log
+	if log == nil {
+		log = slog.Default()
+	}
+	return log.With("component", "eusupply")
+}
+
+// do makes one request, once the gap since the previous request has passed,
+// and returns its body. A status other than 200 OK fails with
+// ErrUnexpectedStatus. It returns the request context's error if that is
+// cancelled while it waits.
+func (c *Client) do(req *http.Request) ([]byte, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if err := c.wait(req.Context()); err != nil {
+		return nil, err
+	}
+	defer func() {
+		c.lastRequest = time.Now()
+	}()
+
+	c.logger().Info("fetching", "method", req.Method, "uri", req.URL.String())
+
+	res, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close() //nolint:errcheck // read only
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w: %s", ErrUnexpectedStatus, res.Status)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading %s: %w", req.URL, err)
+	}
+
+	return body, nil
+}
+
+// wait sleeps until a random gap (see WithRequestGap) has passed since the
+// previous request, or ctx is cancelled. c.mu must be held.
+func (c *Client) wait(ctx context.Context) error {
+	if c.lastRequest.IsZero() || c.maxGap <= 0 {
+		return ctx.Err()
+	}
+
+	gap := c.minGap
+	if c.maxGap > c.minGap {
+		//nolint:gosec // G404: a jitter, not a secret
+		gap += rand.N(c.maxGap - c.minGap + 1)
+	}
+
+	timer := time.NewTimer(time.Until(c.lastRequest.Add(gap)))
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// get makes a GET request for the portal's path.
+func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
+	req, err := http.NewRequestWithContext(
+		ctx, http.MethodGet, c.baseURL+path, nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return c.do(req)
+}
+
+// post posts form to the portal's path.
+func (c *Client) post(
+	ctx context.Context, path string, form url.Values,
+) ([]byte, error) {
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		c.baseURL+path,
+		strings.NewReader(form.Encode()),
+	)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	return c.do(req)
+}
+
+// mustLoadLocation loads a time zone known to be in the embedded tzdata.
+func mustLoadLocation(name string) *time.Location {
+	location, err := time.LoadLocation(name)
+	if err != nil {
+		panic(err)
+	}
+	return location
+}

internal/eusupply/eusupply_test.go

@@ -0,0 +1,326 @@
+package eusupply
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// readFixture returns a file in testdata.
+func readFixture(t *testing.T, name string) []byte {
+	t.Helper()
+	body, err := os.ReadFile("testdata/" + name) //nolint:gosec // G304: a test fixture
+	if err != nil {
+		t.Fatal(err)
+	}
+	return body
+}
+
+// newTestClient returns a client for srv that doesn't wait between
+// requests and logs nothing.
+func newTestClient(srv *httptest.Server) *Client {
+	return New(
+		srv.URL,
+		WithRequestGap(0, 0),
+		WithLogger(slog.New(slog.DiscardHandler)),
+	)
+}
+
+func TestParseSearchPage(t *testing.T) {
+	c := New("https://uk.example", WithLogger(slog.New(slog.DiscardHandler)))
+	tenders, pages, err := c.parseSearchPage(readFixture(t, "search.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pages != 2 {
+		t.Errorf("pages = %d; want 2", pages)
+	}
+	// The row with no link is skipped.
+	if len(tenders) != 2 {
+		t.Fatalf("tenders = %d; want 2: %+v", len(tenders), tenders)
+	}
+
+	first := tenders[0]
+	wantPublished := time.Date(2026, 10, 21, 11, 45, 0, 0, time.UTC) // BST
+	wantDeadline := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)    // GMT
+	switch {
+	case first.ID != "100114946", first.PID != "109912",
+		first.Reference != "Proc2852":
+		t.Errorf("ids = %q, %q, %q", first.ID, first.PID, first.Reference)
+	case first.Title != "Jisc Google for Education Reseller Framework Agreement":
+		t.Errorf("title = %q", first.Title)
+	case first.URL != "https://uk.example/ctm/Supplier/PublicPurchase/109912/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK":
+		t.Errorf("url = %q", first.URL)
+	case !first.Published.Equal(wantPublished):
+		t.Errorf("published = %v; want %v", first.Published, wantPublished)
+	case first.Deadline == nil || !first.Deadline.Equal(wantDeadline):
+		t.Errorf("deadline = %v; want %v", first.Deadline, wantDeadline)
+	case first.Process != "PA23 - Competitive Flexible", first.Buyer != "Jisc",
+		first.Countries != "United Kingdom":
+		t.Errorf("cells = %q, %q, %q", first.Process, first.Buyer, first.Countries)
+	case first.HasDetails:
+		t.Error("HasDetails set from the search page")
+	}
+
+	second := tenders[1]
+	if second.Reference != "" || second.Deadline != nil {
+		t.Errorf("second = %+v; want no reference or deadline", second)
+	}
+	// Zero width spaces are dropped from titles.
+	if second.Title != "York and North Yorkshire Transport Sub-strategies" {
+		t.Errorf("second title = %q", second.Title)
+	}
+}
+
+func TestParseSearchPageChanged(t *testing.T) {
+	c := New("", WithLogger(slog.New(slog.DiscardHandler)))
+	page := []byte("<html><body>Maintenance</body></html>")
+	_, _, err := c.parseSearchPage(page)
+	if !errors.Is(err, ErrNoResults) {
+		t.Errorf("err = %v; want ErrNoResults", err)
+	}
+}
+
+func TestParseDetails(t *testing.T) {
+	tender := Tender{
+		ID: "100114946",
+	}
+	if err := parseDetails(readFixture(t, "tender.html"), &tender); err != nil {
+		t.Fatal(err)
+	}
+
+	wantDescription := "This opportunity is for a framework agreement.\n" +
+		"The Supplier(s) shall manage the full licence lifecycle (the 'Agreement')."
+	if tender.Description != wantDescription {
+		t.Errorf("description = %q", tender.Description)
+	}
+	if tender.ContractType != "Supplies" {
+		t.Errorf("contract type = %q", tender.ContractType)
+	}
+	if len(tender.CPVCodes) != 2 ||
+		tender.CPVCodes[0].Code != "48000000-8" ||
+		tender.CPVCodes[0].Description !=
+			"Software package and information systems" {
+		t.Errorf("cpv codes = %+v", tender.CPVCodes)
+	}
+	wantAddress := "Jisc|4 Portwall Lane|BS1 6NB|Bristol|United Kingdom"
+	if got := strings.Join(tender.BuyerAddress, "|"); got != wantAddress {
+		t.Errorf("buyer address = %q", got)
+	}
+	// The search had no deadline, so the page's is used.
+	if tender.Deadline == nil ||
+		!tender.Deadline.Equal(time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)) {
+		t.Errorf("deadline = %v", tender.Deadline)
+	}
+	if tender.Lots != 2 || !tender.HasDetails {
+		t.Errorf("lots = %d, HasDetails = %v", tender.Lots, tender.HasDetails)
+	}
+
+	err := parseDetails([]byte("<html><body>Not found</body></html>"), &tender)
+	if !errors.Is(err, ErrNoDetails) {
+		t.Errorf("err = %v; want ErrNoDetails", err)
+	}
+}
+
+// Search posts the portal's form for each page, and keeps the pages read
+// before one fails.
+func TestSearch(t *testing.T) {
+	page := readFixture(t, "search.html")
+	var (
+		mu    sync.Mutex
+		forms []url.Values
+	)
+	fail := false
+	srv := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method != http.MethodPost || r.URL.Path != searchPath {
+				t.Errorf("request = %s %s", r.Method, r.URL.Path)
+			}
+			r.ParseForm()
+			mu.Lock()
+			forms = append(forms, r.PostForm)
+			number := len(forms)
+			mu.Unlock()
+			if fail && number == 2 {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			w.Write(page)
+		}),
+	)
+	t.Cleanup(srv.Close)
+	c := newTestClient(srv)
+
+	from := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC) // 15th in the UK
+	to := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+	tenders, err := c.Search(context.Background(), from, to)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(tenders) != 4 || len(forms) != 2 {
+		t.Fatalf(
+			"%d tenders from %d pages; want 4 from 2", len(tenders), len(forms),
+		)
+	}
+	for i, form := range forms {
+		wantPage := []string{"1", "2"}[i]
+		if form.Get("SearchFilter.FromDate") != "15/10/2026" ||
+			form.Get("SearchFilter.ToDate") != "21/10/2026" ||
+			form.Get("SearchFilter.PagingInfo.PageNumber") != wantPage ||
+			form.Get("SearchFilter.PagingInfo.PageSize") != "100" ||
+			form.Get("Branding") != "UK" {
+			t.Errorf("form %d = %v", i+1, form)
+		}
+	}
+
+	forms = nil
+	fail = true
+	tenders, err = c.Search(context.Background(), from, to)
+	if !errors.Is(err, ErrUnexpectedStatus) {
+		t.Errorf("err = %v; want ErrUnexpectedStatus", err)
+	}
+	if len(tenders) != 2 {
+		t.Errorf("%d tenders; want the first page's 2", len(tenders))
+	}
+}
+
+func TestDetails(t *testing.T) {
+	var path string
+	srv := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			path = r.URL.RequestURI()
+			w.Write(readFixture(t, "tender.html"))
+		}),
+	)
+	t.Cleanup(srv.Close)
+
+	tender := Tender{
+		ID:  "100114946",
+		PID: "109912",
+	}
+	err := newTestClient(srv).Details(context.Background(), &tender)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if path != tenderPath("109912") || !tender.HasDetails {
+		t.Errorf("path = %q, HasDetails = %v", path, tender.HasDetails)
+	}
+}
+
+// The client waits from minGap to maxGap between requests, and a cancelled
+// context stops the wait.
+func TestRequestGap(t *testing.T) {
+	var (
+		mu    sync.Mutex
+		times []time.Time
+	)
+	srv := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			mu.Lock()
+			times = append(times, time.Now())
+			mu.Unlock()
+			io.WriteString(w, "ok")
+		}),
+	)
+	t.Cleanup(srv.Close)
+
+	const minGap, maxGap = 50 * time.Millisecond, 80 * time.Millisecond
+	c := New(
+		srv.URL,
+		WithRequestGap(minGap, maxGap),
+		WithLogger(slog.New(slog.DiscardHandler)),
+	)
+	for range 3 {
+		if _, err := c.get(context.Background(), "/"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for i := 1; i < len(times); i++ {
+		if gap := times[i].Sub(times[i-1]); gap < minGap {
+			t.Errorf("gap %d = %v; want at least %v", i, gap, minGap)
+		}
+	}
+
+	slow := New(
+		srv.URL,
+		WithRequestGap(time.Hour, time.Hour),
+		WithLogger(slog.New(slog.DiscardHandler)),
+	)
+	slow.get(context.Background(), "/")
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	if _, err := slow.get(ctx, "/"); !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("err = %v; want context.DeadlineExceeded", err)
+	}
+	if time.Since(start) > 5*time.Second {
+		t.Error("cancelling did not stop the wait")
+	}
+}
+
+func TestConvert(t *testing.T) {
+	published := time.Date(2026, 10, 21, 11, 45, 0, 0, time.UTC)
+	tender := Tender{
+		ID:           "100114946",
+		PID:          "109912",
+		Title:        "Hazard Management Tool",
+		Published:    published,
+		Buyer:        "UK Atomic Energy Authority",
+		HasDetails:   true,
+		Description:  "A tool",
+		ContractType: "Services",
+		CPVCodes: []CPVCode{
+			{
+				Code:        "48000000-8",
+				Description: "Software package and information systems",
+			},
+		},
+		Lots: 2,
+	}
+
+	opportunities := Convert([]Tender{tender})
+	if len(opportunities) != 1 {
+		t.Fatalf("%d opportunities; want 1", len(opportunities))
+	}
+	opportunity := opportunities[0]
+	switch {
+	case opportunity.UUID != "eusupply-uk-100114946",
+		opportunity.Source != Source:
+		t.Errorf("uuid, source = %q, %q", opportunity.UUID, opportunity.Source)
+	case opportunity.Title != tender.Title, opportunity.Description != "A tool",
+		opportunity.BuyerName != tender.Buyer, opportunity.Status != "active":
+		t.Errorf("opportunity = %+v", opportunity)
+	case opportunity.MainCategory != "services", !opportunity.MultiLot:
+		t.Errorf(
+			"category, multi-lot = %q, %v",
+			opportunity.MainCategory, opportunity.MultiLot,
+		)
+	case len(opportunity.Classifications) != 1 ||
+		opportunity.Classifications[0].Code != "48000000" ||
+		!opportunity.Classifications[0].IsCPV:
+		t.Errorf("classifications = %+v", opportunity.Classifications)
+	case !opportunity.LatestReleaseDate.Equal(published),
+		opportunity.StartDate == nil || !opportunity.StartDate.Equal(published):
+		t.Errorf(
+			"dates = %v, %v", opportunity.LatestReleaseDate, opportunity.StartDate,
+		)
+	case opportunity.Marks.Hidden || opportunity.Marks.Skipped:
+		t.Error("Convert set marks")
+	}
+
+	release := opportunity.Releases[0]
+	if len(opportunity.Releases) != 1 || release.ReleaseID != "100114946" ||
+		len(release.Tags) != 1 || release.Tags[0].Tag != "tender" ||
+		!strings.Contains(release.JSON, `"pid":"109912"`) {
+		t.Errorf("releases = %+v", opportunity.Releases)
+	}
+}

internal/eusupply/html.go

@@ -0,0 +1,139 @@
+package eusupply
+
+import (
+	"slices"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// The helpers below read the parsed pages. The site's HTML is server
+// rendered, so the elements are found by their ids and classes, as the old
+// Node scraper found them with CSS selectors.
+
+// findAll returns the elements under node (node included) that match, in
+// document order.
+func findAll(node *html.Node, match func(*html.Node) bool) []*html.Node {
+	var found []*html.Node
+	var walk func(*html.Node)
+	walk = func(current *html.Node) {
+		if current.Type == html.ElementNode && match(current) {
+			found = append(found, current)
+		}
+		for child := current.FirstChild; child != nil; child = child.NextSibling {
+			walk(child)
+		}
+	}
+	walk(node)
+	return found
+}
+
+// findFirst returns the first element under node that matches, or nil.
+func findFirst(node *html.Node, match func(*html.Node) bool) *html.Node {
+	found := findAll(node, match)
+	if len(found) == 0 {
+		return nil
+	}
+	return found[0]
+}
+
+// isTag matches elements with the tag name.
+func isTag(tag string) func(*html.Node) bool {
+	return func(node *html.Node) bool {
+		return node.Data == tag
+	}
+}
+
+// hasID matches the element with the id.
+func hasID(id string) func(*html.Node) bool {
+	return func(node *html.Node) bool {
+		return attr(node, "id") == id
+	}
+}
+
+// hasClass matches elements with tag (any, if empty) and the class among
+// their classes.
+func hasClass(tag, class string) func(*html.Node) bool {
+	return func(node *html.Node) bool {
+		if tag != "" && node.Data != tag {
+			return false
+		}
+		return slices.Contains(strings.Fields(attr(node, "class")), class)
+	}
+}
+
+// children returns node's element children with the tag name.
+func children(node *html.Node, tag string) []*html.Node {
+	var found []*html.Node
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if child.Type == html.ElementNode && child.Data == tag {
+			found = append(found, child)
+		}
+	}
+	return found
+}
+
+// nextElement returns the element after node among its siblings, or nil.
+func nextElement(node *html.Node) *html.Node {
+	sibling := node.NextSibling
+	for sibling != nil && sibling.Type != html.ElementNode {
+		sibling = sibling.NextSibling
+	}
+	return sibling
+}
+
+// attr returns the value of node's attribute key, or "".
+func attr(node *html.Node, key string) string {
+	for _, attribute := range node.Attr {
+		if attribute.Key == key {
+			return attribute.Val
+		}
+	}
+	return ""
+}
+
+// rawText returns the text under node as written, with a newline for each
+// <br>.
+func rawText(node *html.Node) string {
+	var text strings.Builder
+	var walk func(*html.Node)
+	walk = func(current *html.Node) {
+		switch {
+		case current.Type == html.TextNode:
+			text.WriteString(current.Data)
+		case current.Type == html.ElementNode && current.Data == "br":
+			text.WriteString("\n")
+		}
+		for child := current.FirstChild; child != nil; child = child.NextSibling {
+			walk(child)
+		}
+	}
+	walk(node)
+	return text.String()
+}
+
+// text returns the text under node with its white space collapsed: runs of
+// spaces, non-breaking spaces and line breaks become one space, and the ends
+// are trimmed.
+func text(node *html.Node) string {
+	return collapseSpace(rawText(node))
+}
+
+// lines returns the non-empty lines of the text under node, each with its
+// white space collapsed.
+func lines(node *html.Node) []string {
+	var found []string
+	for line := range strings.SplitSeq(rawText(node), "\n") {
+		if collapsed := collapseSpace(line); collapsed != "" {
+			found = append(found, collapsed)
+		}
+	}
+	return found
+}
+
+// collapseSpace collapses runs of white space (non-breaking spaces and zero
+// width spaces included) into one space, and trims the ends.
+func collapseSpace(value string) string {
+	value = strings.ReplaceAll(value, "\u200b", "")
+	return strings.Join(strings.Fields(value), " ")
+}

internal/eusupply/search.go

@@ -0,0 +1,277 @@
+package eusupply
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+// The search form's endpoint and paging.
+const (
+	// searchPath is where the portal's public search form posts to.
+	searchPath = "/ctm/Supplier/PublicTenders"
+	// pageSize is the most results per page the form offers.
+	pageSize = 100
+	// defaultSearchDays is how far back a search with no from date goes,
+	// as the old scraper's first search did.
+	defaultSearchDays = 7
+	// dateLayout is the form's and the results' date layout; dateTimeLayout
+	// adds the time of day the results give in their cells' titles.
+	dateLayout     = "02/01/2006"
+	dateTimeLayout = "02/01/2006 15:04:05"
+)
+
+// ErrNoResults means a search page had no results table, which the portal
+// always returns, even for no tenders: the page has changed.
+var ErrNoResults = errors.New("eusupply: no search results table on the page")
+
+// pidPattern finds a tender's portal id (PID) in its link.
+var pidPattern = regexp.MustCompile(`[?&]PID=(\d+)`)
+
+// pagesPattern reads the number of pages from the pager's "of N".
+var pagesPattern = regexp.MustCompile(`^of (\d+)$`)
+
+// Tender is one public tender on the portal. The fields up to Countries are
+// from the search results; the rest are from the tender's own page (see
+// Client.Details), and are empty until it is read.
+type Tender struct {
+	// ID is the quote/tender id the portal shows ("100114946").
+	ID string `json:"id"`
+	// PID is the portal's internal id, which its pages are addressed by.
+	PID       string     `json:"pid"`
+	Reference string     `json:"reference"`
+	Title     string     `json:"title"`
+	URL       string     `json:"url"`
+	Published time.Time  `json:"published"`
+	Deadline  *time.Time `json:"deadline"`
+	Process   string     `json:"process"`
+	Buyer     string     `json:"buyer"`
+	Countries string     `json:"countries"`
+
+	// HasDetails is set once the tender's own page has been read.
+	HasDetails   bool      `json:"has_details"`
+	Description  string    `json:"description,omitempty"`
+	ContractType string    `json:"contract_type,omitempty"`
+	CPVCodes     []CPVCode `json:"cpv_codes,omitempty"`
+	BuyerAddress []string  `json:"buyer_address,omitempty"`
+	Lots         int       `json:"lots,omitempty"`
+}
+
+// CPVCode is one of a tender's CPV codes, as its page lists them: the code
+// with its check digit ("48000000-8") and its description.
+type CPVCode struct {
+	Code        string `json:"code"`
+	Description string `json:"description"`
+}
+
+// Search returns the tenders published from the date of from to the date
+// of until (UK dates, both included), reading every page of results. A
+// zero from searches the last 7 days; a zero until searches up to today. If a page
+// fails, the tenders of the pages before it are returned with the error.
+func (c *Client) Search(
+	ctx context.Context, from, until time.Time,
+) ([]Tender, error) {
+	if until.IsZero() {
+		until = time.Now()
+	}
+	if from.IsZero() {
+		from = until.AddDate(0, 0, -defaultSearchDays)
+	}
+
+	var tenders []Tender
+	for page := 1; ; page++ {
+		body, err := c.post(ctx, searchPath, searchForm(from, until, page))
+		if err != nil {
+			return tenders, fmt.Errorf("searching page %d: %w", page, err)
+		}
+
+		found, pages, err := c.parseSearchPage(body)
+		if err != nil {
+			return tenders, fmt.Errorf("reading search page %d: %w", page, err)
+		}
+		tenders = append(tenders, found...)
+
+		if page >= pages {
+			return tenders, nil
+		}
+	}
+}
+
+// searchForm returns the search form's fields for one page of the tenders
+// published from from to until. They are the fields the portal's own form
+// posts, with its defaults; the portal answers 500 if one is missing or
+// isn't valid.
+func searchForm(from, until time.Time, page int) url.Values {
+	return url.Values{
+		"SearchFilter.SortField":             {"None"},
+		"SearchFilter.SortDirection":         {"None"},
+		"SearchFilter.ShortDescription":      {""},
+		"SearchFilter.Reference":             {""},
+		"SearchFilter.TenderId":              {"0"},
+		"SearchFilter.OperatorId":            {"0"},
+		"Branding":                           {"UK"},
+		"SavedCategoryId":                    {"0"},
+		"SavedUnitAndName":                   {""},
+		"SearchFilter.PublishType":           {"1"}, // issued, not awarded
+		"SearchFilter.BrandingCode":          {""},
+		"TextFilter":                         {""},
+		"SearchFilter.FromDate":              {from.In(ukTime).Format(dateLayout)},
+		"SearchFilter.ToDate":                {until.In(ukTime).Format(dateLayout)},
+		"CpvContainer.CpvCodes":              {""},
+		"CpvContainer.CpvMain":               {""},
+		"CpvContainer.ContractType":          {""},
+		"CpvContainer.CpvIds":                {""},
+		"CpvContainer.IsMandatory":           {"True"},
+		"SearchFilter.ShowExpiredRft":        {"false"},
+		"SearchFilter.PagingInfo.PageNumber": {strconv.Itoa(page)},
+		"SearchFilter.PagingInfo.PageSize":   {strconv.Itoa(pageSize)},
+	}
+}
+
+// parseSearchPage reads the tenders on a page of search results, and the
+// number of pages of results. A row that can't be read is logged and
+// skipped.
+func (c *Client) parseSearchPage(body []byte) ([]Tender, int, error) {
+	document, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return nil, 0, err
+	}
+
+	container := findFirst(document, hasID("searchResultContainer"))
+	if container == nil {
+		return nil, 0, ErrNoResults
+	}
+	table := findFirst(container, isTag("table"))
+	if table == nil {
+		return nil, 0, ErrNoResults
+	}
+
+	var tenders []Tender
+	for _, body := range children(table, "tbody") {
+		for _, row := range children(body, "tr") {
+			tender, err := c.parseResultRow(row)
+			if err != nil {
+				c.logger().Warn(
+					"skipping unreadable search result",
+					"row", text(row),
+					"error", err,
+				)
+				continue
+			}
+			tenders = append(tenders, tender)
+		}
+	}
+
+	return tenders, searchPages(document), nil
+}
+
+// parseResultRow reads one row of search results. Its cells are the id,
+// reference, title (a link to the tender), publication date, response
+// deadline, process, buyers and countries; each date cell's title has the
+// full time.
+func (c *Client) parseResultRow(row *html.Node) (Tender, error) {
+	cells := children(row, "td")
+	if len(cells) < 8 {
+		return Tender{}, fmt.Errorf(
+			"eusupply: a result row has %d cells, not 8", len(cells),
+		)
+	}
+
+	link := findFirst(cells[2], isTag("a"))
+	if link == nil {
+		return Tender{}, errors.New("eusupply: a result row has no link")
+	}
+	match := pidPattern.FindStringSubmatch(attr(link, "href"))
+	if match == nil {
+		return Tender{}, fmt.Errorf(
+			"eusupply: no PID in the link %q", attr(link, "href"),
+		)
+	}
+
+	published, err := cellTime(cells[3])
+	if err != nil {
+		return Tender{}, fmt.Errorf("publication date: %w", err)
+	}
+
+	tender := Tender{
+		ID:        text(cells[0]),
+		PID:       match[1],
+		Reference: text(cells[1]),
+		Title:     text(link),
+		URL:       c.tenderURL(match[1]),
+		Published: published,
+		Process:   text(cells[5]),
+		Buyer:     text(cells[6]),
+		Countries: text(cells[7]),
+	}
+	if tender.ID == "" {
+		return Tender{}, errors.New("eusupply: a result row has no id")
+	}
+
+	// A tender may have no deadline yet (e.g. market engagement).
+	if deadline, err := cellTime(cells[4]); err == nil {
+		tender.Deadline = &deadline
+	}
+
+	return tender, nil
+}
+
+// tenderURL returns the address of the public page of the tender with pid.
+func (c *Client) tenderURL(pid string) string {
+	return c.baseURL + tenderPath(pid)
+}
+
+// tenderPath returns the path of the public page of the tender with pid,
+// which the search results' links redirect to.
+func tenderPath(pid string) string {
+	return "/ctm/Supplier/PublicPurchase/" + url.PathEscape(pid) +
+		"/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK"
+}
+
+// cellTime reads the UK time in the title of a result cell's tooltip.
+func cellTime(cell *html.Node) (time.Time, error) {
+	tooltip := findFirst(cell, hasClass("div", "js-tooltip-content"))
+	if tooltip == nil {
+		return time.Time{}, errors.New("eusupply: no time in the cell")
+	}
+	return parseUKTime(attr(tooltip, "title"))
+}
+
+// parseUKTime reads a portal time ("21/10/2026 12:45:00"; the seconds may
+// be left out) in UK time.
+func parseUKTime(value string) (time.Time, error) {
+	value = strings.TrimSpace(value)
+	parsed, err := time.ParseInLocation(dateTimeLayout, value, ukTime)
+	if err != nil {
+		parsed, err = time.ParseInLocation("02/01/2006 15:04", value, ukTime)
+	}
+	if err != nil {
+		return time.Time{}, fmt.Errorf("eusupply: time %q: %w", value, err)
+	}
+	return parsed, nil
+}
+
+// searchPages reads the number of pages from the pager ("Page 1 of 3"), or
+// 1 if there is no pager.
+func searchPages(document *html.Node) int {
+	for _, pager := range findAll(document, hasClass("ul", "ctm-pager-info")) {
+		for _, item := range children(pager, "li") {
+			match := pagesPattern.FindStringSubmatch(text(item))
+			if match == nil {
+				continue
+			}
+			if pages, err := strconv.Atoi(match[1]); err == nil && pages > 0 {
+				return pages
+			}
+		}
+	}
+	return 1
+}

internal/eusupply/testdata/search.html

@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<html>
+<body>
+<form action="/ctm/Supplier/PublicTenders" id="searchForm" method="post">
+<div id="searchResultContainer">
+<table class="table table-striped">
+<thead>
+<tr><th>Quote/tender Id</th><th>Reference</th><th>Name</th><th>Date of publication</th><th>Response deadline (UK time)</th><th>Process</th><th>Buyers</th><th>Countries</th></tr>
+</thead>
+<tbody>
+<tr>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="100114946"> 100114946 </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="Proc2852"> Proc2852 </div> </td>
+<td> <span data="OLD URL: /app/rfq/publicpurchase_frameset.asp?PID=109912&PP=ctm/Supplier/PublicTenders&B=UK,&PS=1&NR=1"></span> <a href="/app/rfq/rwlentrance_s.asp?PID=109912&amp;PP=ctm/Supplier/PublicTenders&amp;B=UK&amp;PS=1" target="_blank">Jisc Google for Education Reseller Framework Agreement</a> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="21/10/2026 12:45:00"> 21/10/2026 </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="04/11/2026 12:00:00"> 04/11/2026 12:00 </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="PA23 - Competitive Flexible "> PA23 - Competitive Flexible </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="Jisc"> Jisc </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="United Kingdom"> United Kingdom </div> </td>
+</tr>
+<tr>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="100115168"> 100115168 </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content"> </div> </td>
+<td> <a href="/app/rfq/rwlentrance_s.asp?PID=110143&amp;PP=ctm/Supplier/PublicTenders&amp;B=UK&amp;PS=1" target="_blank">&#8203;&#8203;York and North Yorkshire Transport Sub-strategies</a> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="20/10/2026 12:34:00"> 20/10/2026 </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content"> </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="PA23 - Preliminary Market Engagement"> PA23 - Preliminary Market Engagement </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="York and North Yorkshire Combined Authority"> York and North Yorkshire Combined Authority </div> </td>
+<td> <div data-toggle="tooltip" class="js-tooltip-content" title="United Kingdom"> United Kingdom </div> </td>
+</tr>
+<tr>
+<td> <div class="js-tooltip-content" title="100115999"> 100115999 </div> </td>
+<td> <div class="js-tooltip-content"> </div> </td>
+<td> A row with no link </td>
+<td> <div class="js-tooltip-content" title="20/10/2026 09:00:00"> 20/10/2026 </div> </td>
+<td></td><td></td><td></td><td></td>
+</tr>
+</tbody>
+</table>
+<ul class="ctm-pager-info"><li><span><label for="SearchFilter_PagingInfo_PageNumber">Page</label></span></li><li><span class="ctm-pager-input"><input id="SearchFilter_PagingInfo_PageNumber" name="SearchFilter.PagingInfo.PageNumber" type="text" value="1" /></span></li><li><span>of 2</span></li></ul>
+<ul class="ctm-pager-info pull-right"><li><span>items per page</span></li></ul>
+</div>
+</form>
+</body>
+</html>

internal/eusupply/testdata/tender.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div id="tenderInfoSection" class="ctm-section-box">
+<div class="row-fluid">
+<div class="span9 ctm-divider-right">
+<div class="ctm-content-label">Detailed description:</div>
+<p class="ctm-multi-line">This opportunity is for a framework agreement.
+The Supplier(s) shall manage the full licence lifecycle (the &#39;Agreement&#39;).</p>
+<div class="ctm-content-label">Type of contract:</div>
+<p>Supplies</p>
+<div class="ctm-content-label">Response deadline (UK time):</div>
+<p>04/11/2026 12:00:00</p>
+<div class="ctm-content-label">CPV codes:</div>
+<p>
+<span class="text-underline">48000000-8&nbsp;&nbsp;Software package and information systems</span><br />
+<span class="">72220000-3&nbsp;&nbsp;Systems and technical consultancy services</span><br />
+</p>
+</div>
+<div class="span3">
+<div class="ctm-content-label">Buyer:</div>
+<p>
+<br />
+Jisc        <br />
+4 Portwall Lane        <br />
+BS1 6NB        <br />
+Bristol        <br />
+United Kingdom
+<br />
+<a href="/ctm/Supplier/CompanyInformation/Index/87175" class="openPopup">
+View profile
+</a>
+</p>
+<div class="ctm-content-label">Contact:</div>
+<p><br /><a href="https://www.jisc.ac.uk" target="_blank">https://www.jisc.ac.uk</a></p>
+</div>
+</div>
+</div>
+<div class="package-container"><div class="row-fluid">Lot 1</div></div>
+<div class="package-container"><div class="row-fluid">Lot 2</div></div>
+</body>
+</html>

internal/fetcher/fetcher.go

@@ -185,6 +185,18 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error {
 			return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since)
 		})
 	}
+	if f.conf.EUSupply.Enabled {
+		// A nil *store.Store in the interface would not be nil.
+		var lookup tenderStore
+		if f.store != nil {
+			lookup = f.store
+		}
+		start(euSupplySource, func(ctx context.Context, since time.Time) error {
+			return runEUSupply(
+				ctx, logger, &f.conf.EUSupply, lookup, f.save, since,
+			)
+		})
+	}
 	if f.conf.Sleeper.Enabled {
 		start("Sleeper", func(ctx context.Context, since time.Time) error {
 			return runSleeper(ctx, logger, &f.conf.Sleeper, since)

internal/fetcher/match.go

@@ -0,0 +1,252 @@
+package fetcher
+
+import (
+	"math"
+	"strings"
+	"unicode"
+
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+// titleMatchThreshold is how alike two titles must be (see titleSimilarity)
+// for an EU Supply tender to be taken as one already stored from another
+// source, such as the Find a Tender notice the portal publishes for most of
+// its tenders (the user's rule: a 90% match).
+const titleMatchThreshold = 0.9
+
+// The other thresholds of a match (see confirmedRepeat). The user's rule
+// (2026-10-21): a title match must also be backed by the buyer or the
+// description, at a ratio of Claude's choosing; 0.8 kept every real repeat
+// in a live run that either could check, and rejected the one false match.
+const (
+	// buyerMatchThreshold is how much of the shorter buyer name's
+	// significant words must be in the other's (see buyerSimilarity).
+	buyerMatchThreshold = 0.8
+	// descriptionMatchThreshold is how much of the shorter description's
+	// words must be in the other's (see descriptionSimilarity).
+	descriptionMatchThreshold = 0.8
+	// wordMatchThreshold is how alike two words of buyer names must be to
+	// count as the same word, so a typo or a plural still matches.
+	wordMatchThreshold = 0.8
+	// minDescriptionWordLength is the shortest word a description is
+	// compared by; shorter ones ("the", "and", "for") say little.
+	minDescriptionWordLength = 4
+)
+
+// genericBuyerWords are left out of buyer names before comparing them:
+// legal forms and kinds of body, which the two sites give differently
+// ("Jisc" and "JISC SERVICES LIMITED", "The Borough Council of Calderdale"
+// and "Borough Council of Calderdale") and which say nothing about which
+// buyer it is.
+var genericBuyerWords = map[string]bool{
+	"the": true, "of": true, "and": true, "for": true,
+	"limited": true, "ltd": true, "plc": true, "llp": true, "cic": true,
+	"council": true, "borough": true, "metropolitan": true, "city": true,
+	"county": true, "district": true, "authority": true, "services": true,
+	"group": true, "trust": true,
+}
+
+// confirmedRepeat reports whether other, whose title already matches
+// opportunity's (see similarTitles), is the same tender: their buyers or
+// their descriptions must match too. Generic titles ("Pest control
+// services") are bought by many buyers, so the title alone isn't enough.
+func confirmedRepeat(opportunity, other *types.Opportunity) bool {
+	buyer, buyerKnown := buyerSimilarity(
+		opportunity.BuyerName, other.BuyerName,
+	)
+	if buyerKnown && buyer >= buyerMatchThreshold {
+		return true
+	}
+	description, descriptionKnown := descriptionSimilarity(
+		opportunity.Description, other.Description,
+	)
+	return descriptionKnown && description >= descriptionMatchThreshold
+}
+
+// buyerSimilarity returns the share of the shorter buyer name's significant
+// words (all but genericBuyerWords) that are in the other name, each at
+// least wordMatchThreshold alike to one of its words. It is false if either
+// name has no significant words, when the buyers can't be compared.
+func buyerSimilarity(first, second string) (float64, bool) {
+	firstWords := significantWords(first)
+	secondWords := significantWords(second)
+	if len(firstWords) == 0 || len(secondWords) == 0 {
+		return 0, false
+	}
+	if len(secondWords) < len(firstWords) {
+		firstWords, secondWords = secondWords, firstWords
+	}
+
+	found := 0
+	for _, word := range firstWords {
+		for _, other := range secondWords {
+			if similarWords(word, other) {
+				found++
+				break
+			}
+		}
+	}
+	return float64(found) / float64(len(firstWords)), true
+}
+
+// significantWords returns the words of a buyer name, normalised as titles
+// are, without genericBuyerWords.
+func significantWords(name string) []string {
+	var words []string
+	for word := range strings.FieldsSeq(normaliseTitle(name)) {
+		if !genericBuyerWords[word] {
+			words = append(words, word)
+		}
+	}
+	return words
+}
+
+// similarWords reports whether two words are at least wordMatchThreshold
+// alike (see titleSimilarity).
+func similarWords(first, second string) bool {
+	firstRunes, secondRunes := []rune(first), []rune(second)
+	longest := max(len(firstRunes), len(secondRunes))
+	limit := int(math.Floor((1-wordMatchThreshold)*float64(longest) + 1e-9))
+	_, within := editDistance(firstRunes, secondRunes, limit)
+	return within
+}
+
+// descriptionSimilarity returns the share of the shorter description's
+// distinct words (of minDescriptionWordLength letters or more) that are in
+// the other. It is false if either has no such words: EU Supply leaves
+// some descriptions empty.
+func descriptionSimilarity(first, second string) (float64, bool) {
+	firstWords := descriptionWords(first)
+	secondWords := descriptionWords(second)
+	if len(firstWords) == 0 || len(secondWords) == 0 {
+		return 0, false
+	}
+	if len(secondWords) < len(firstWords) {
+		firstWords, secondWords = secondWords, firstWords
+	}
+
+	found := 0
+	for word := range firstWords {
+		if secondWords[word] {
+			found++
+		}
+	}
+	return float64(found) / float64(len(firstWords)), true
+}
+
+// descriptionWords returns the distinct words of a description, normalised
+// as titles are, of minDescriptionWordLength letters or more.
+func descriptionWords(description string) map[string]bool {
+	words := make(map[string]bool)
+	for word := range strings.FieldsSeq(normaliseTitle(description)) {
+		if len([]rune(word)) >= minDescriptionWordLength {
+			words[word] = true
+		}
+	}
+	return words
+}
+
+// normaliseTitle returns title as titles are compared: lower case, with
+// each run of anything but letters and numbers (punctuation, white space,
+// zero width spaces) made one space, and trimmed.
+func normaliseTitle(title string) string {
+	var normalised strings.Builder
+	space := false
+	for _, character := range strings.ToLower(title) {
+		if unicode.IsLetter(character) || unicode.IsNumber(character) {
+			if space && normalised.Len() > 0 {
+				normalised.WriteByte(' ')
+			}
+			normalised.WriteRune(character)
+			space = false
+			continue
+		}
+		space = true
+	}
+	return normalised.String()
+}
+
+// titleSimilarity returns how alike two normalised titles (see
+// normaliseTitle) are, from 0 to 1: one less their edit distance (the
+// characters inserted, deleted or changed to turn one into the other) over
+// the longer one's length. Two empty titles are not alike (0).
+func titleSimilarity(first, second string) float64 {
+	firstRunes, secondRunes := []rune(first), []rune(second)
+	longest := max(len(firstRunes), len(secondRunes))
+	if longest == 0 {
+		return 0
+	}
+	distance, _ := editDistance(firstRunes, secondRunes, longest)
+	return 1 - float64(distance)/float64(longest)
+}
+
+// similarTitles reports whether two normalised titles are at least
+// titleMatchThreshold alike. It gives up as soon as the distance can't be
+// small enough, so it is much faster than titleSimilarity for titles that
+// aren't.
+func similarTitles(first, second string) bool {
+	firstRunes, secondRunes := []rune(first), []rune(second)
+	longest := max(len(firstRunes), len(secondRunes))
+	if longest == 0 {
+		return false
+	}
+	// The epsilon stops (1-0.9)*10 = 0.99999... rounding down to 0.
+	limit := int(math.Floor((1-titleMatchThreshold)*float64(longest) + 1e-9))
+	_, within := editDistance(firstRunes, secondRunes, limit)
+	return within
+}
+
+// editDistance returns the Levenshtein distance between source and target,
+// and whether it is at most limit. Only the cells within limit of the
+// diagonal are computed, and it stops once a whole row exceeds limit, when
+// the distance returned is limit+1.
+func editDistance(source, target []rune, limit int) (int, bool) {
+	if abs(len(source)-len(target)) > limit {
+		return limit + 1, false
+	}
+
+	exceeded := limit + 1
+	previous := make([]int, len(target)+1)
+	current := make([]int, len(target)+1)
+	for column := range previous {
+		previous[column] = min(column, exceeded)
+	}
+
+	for row := 1; row <= len(source); row++ {
+		low, high := max(1, row-limit), min(len(target), row+limit)
+		current[0] = min(row, exceeded)
+		if low > 1 {
+			current[low-1] = exceeded
+		}
+		rowMin := current[0]
+		for column := low; column <= high; column++ {
+			cost := 1
+			if source[row-1] == target[column-1] {
+				cost = 0
+			}
+			value := min(
+				previous[column]+1, current[column-1]+1, previous[column-1]+cost,
+			)
+			current[column] = min(value, exceeded)
+			rowMin = min(rowMin, current[column])
+		}
+		if high < len(target) {
+			current[high+1] = exceeded
+		}
+		if rowMin > limit {
+			return exceeded, false
+		}
+		previous, current = current, previous
+	}
+
+	distance := previous[len(target)]
+	return distance, distance <= limit
+}
+
+// abs returns the absolute value of value.
+func abs(value int) int {
+	if value < 0 {
+		return -value
+	}
+	return value
+}

internal/fetcher/match_test.go

@@ -0,0 +1,208 @@
+package fetcher
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+func TestNormaliseTitle(t *testing.T) {
+	tests := []struct {
+		title string
+		want  string
+	}{
+		{"Pest Control Services", "pest control services"},
+		{"  RRE-01284 - For the provision  ", "rre 01284 for the provision"},
+		{"\u200b\u200bThe supply 3 kg of Li₂TiO₃", "the supply 3 kg of li₂tio₃"},
+		{"Children & Young People's", "children young people s"},
+		{"---", ""},
+	}
+	for _, tt := range tests {
+		if got := normaliseTitle(tt.title); got != tt.want {
+			t.Errorf("normaliseTitle(%q) = %q; want %q", tt.title, got, tt.want)
+		}
+	}
+}
+
+func TestTitleSimilarity(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          float64
+	}{
+		{"pest control services", "pest control services", 1},
+		{"abcdefghij", "abcdefghix", 0.9},
+		{"abcdefghij", "abcdefghxx", 0.8},
+		{"kitten", "sitting", 1 - 3.0/7},
+		{"", "", 0},
+		{"abc", "", 0},
+	}
+	for _, tt := range tests {
+		got := titleSimilarity(tt.first, tt.second)
+		if got < tt.want-1e-9 || got > tt.want+1e-9 {
+			t.Errorf(
+				"titleSimilarity(%q, %q) = %v; want %v",
+				tt.first, tt.second, got, tt.want,
+			)
+		}
+	}
+}
+
+func TestSimilarTitles(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          bool
+	}{
+		{"housing related support", "housing related support", true},
+		// One edit in ten is exactly 90%.
+		{"abcdefghij", "abcdefghix", true},
+		{"abcdefghij", "abcdefghxx", false},
+		{"pest control services", "pest control service", true},
+		{"alternative provision", "alternative provision for schools", false},
+		{"", "", false},
+	}
+	for _, tt := range tests {
+		if got := similarTitles(tt.first, tt.second); got != tt.want {
+			t.Errorf(
+				"similarTitles(%q, %q) = %v; want %v",
+				tt.first, tt.second, got, tt.want,
+			)
+		}
+	}
+}
+
+// similarTitles' banded, early-stopping distance agrees with the full one.
+func TestSimilarTitlesAgreesWithTitleSimilarity(t *testing.T) {
+	//nolint:gosec // G404: seeded math/rand keeps the test deterministic
+	rng := rand.New(rand.NewSource(1))
+	alphabet := []rune("abc d")
+	random := func() string {
+		runes := make([]rune, 5+rng.Intn(30))
+		for i := range runes {
+			runes[i] = alphabet[rng.Intn(len(alphabet))]
+		}
+		return string(runes)
+	}
+	mutate := func(title string) string {
+		runes := []rune(title)
+		for range rng.Intn(5) {
+			i := rng.Intn(len(runes))
+			switch rng.Intn(3) {
+			case 0:
+				runes[i] = alphabet[rng.Intn(len(alphabet))]
+			case 1:
+				runes = append(runes[:i], runes[i+1:]...)
+			default:
+				runes = append(runes[:i], append([]rune{'x'}, runes[i:]...)...)
+			}
+			if len(runes) == 0 {
+				runes = []rune("a")
+			}
+		}
+		return string(runes)
+	}
+
+	for range 5000 {
+		first := random()
+		second := mutate(first)
+		if rng.Intn(4) == 0 {
+			second = random()
+		}
+		want := titleSimilarity(first, second) >= titleMatchThreshold-1e-9
+		if got := similarTitles(first, second); got != want {
+			t.Fatalf(
+				"similarTitles(%q, %q) = %v; titleSimilarity = %v",
+				first, second, got, titleSimilarity(first, second),
+			)
+		}
+	}
+}
+
+func TestBuyerSimilarity(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          float64
+		known         bool
+	}{
+		{"Jisc", "JISC SERVICES LIMITED", 1, true},
+		{"The Borough Council of Calderdale", "Borough Council of Calderdale", 1, true},
+		{"Wakefield Council", "The Council of the City of Wakefield", 1, true},
+		{"City of York Council", "York and North Yorkshire Combined Authority", 1, true},
+		{"UK Atomic Energy Authority", "UK Atomic Energy Authoritys", 1, true},
+		{"Yorkshire Housing Limited", "Anchor Hanover Group", 0, true},
+		// Only the shorter name's words count, so a name within another
+		// matches it, rightly for York above, wrongly here: the title and
+		// date must match too, so this is accepted.
+		{"North East Lincolnshire", "North Lincolnshire Council", 1, true},
+		{"Hull City Council", "East Riding of Yorkshire Council", 0, true},
+		{"The Council", "Jisc", 0, false},
+		{"", "Jisc", 0, false},
+	}
+	for _, tt := range tests {
+		got, known := buyerSimilarity(tt.first, tt.second)
+		if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
+			t.Errorf(
+				"buyerSimilarity(%q, %q) = %v, %v; want %v, %v",
+				tt.first, tt.second, got, known, tt.want, tt.known,
+			)
+		}
+	}
+}
+
+func TestDescriptionSimilarity(t *testing.T) {
+	tests := []struct {
+		first, second string
+		want          float64
+		known         bool
+	}{
+		{"Supply of pest control.", "Pest control: the supply", 1, true},
+		{"Building materials for repairs", "Building works", 0.5, true},
+		{"", "Anything at all", 0, false},
+		{"a to be", "Anything at all", 0, false},
+	}
+	for _, tt := range tests {
+		got, known := descriptionSimilarity(tt.first, tt.second)
+		if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
+			t.Errorf(
+				"descriptionSimilarity(%q, %q) = %v, %v; want %v, %v",
+				tt.first, tt.second, got, known, tt.want, tt.known,
+			)
+		}
+	}
+}
+
+// A title match is confirmed by the buyer or, failing that, the
+// description; with neither to compare, it isn't.
+func TestConfirmedRepeat(t *testing.T) {
+	opportunity := func(buyer, description string) *types.Opportunity {
+		return &types.Opportunity{
+			BuyerName:   buyer,
+			Description: description,
+		}
+	}
+	tests := []struct {
+		name        string
+		first, last *types.Opportunity
+		want        bool
+	}{
+		{"buyer", opportunity("Jisc", ""), opportunity("JISC LIMITED", ""), true},
+		{
+			"description, not buyer",
+			opportunity("Echelon", "building materials supply"),
+			opportunity("Fairhive", "supply of building materials"),
+			true,
+		},
+		{
+			"neither",
+			opportunity("Yorkshire Housing", "pest control in Leeds homes"),
+			opportunity("Anchor Hanover", "pest control for retirement housing"),
+			false,
+		},
+		{"nothing to compare", opportunity("", ""), opportunity("", ""), false},
+	}
+	for _, tt := range tests {
+		if got := confirmedRepeat(tt.first, tt.last); got != tt.want {
+			t.Errorf("%s: confirmedRepeat = %v; want %v", tt.name, got, tt.want)
+		}
+	}
+}

internal/fetcher/worker_eusupply.go

@@ -0,0 +1,249 @@
+package fetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/eusupply"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
+	"git.avsdev.uk/Go/job-search/internal/util"
+)
+
+// euSupplySource is the Source of the opportunities the EU Supply source
+// fetches.
+const euSupplySource = eusupply.Source
+
+// repeatWindow is how far apart an EU Supply tender's publication and
+// another source's opportunity's latest release may be for the two to be
+// the same (see markRepeats).
+const repeatWindow = 30 * 24 * time.Hour
+
+// tenderStore is what the EU Supply source reads from the store, to skip
+// tenders already stored and to find the ones another source has.
+type tenderStore interface {
+	ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error)
+	ListOtherSourcesTitles(
+		ctx context.Context, source string, from, before time.Time,
+	) ([]store.OpportunityTitle, error)
+	GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error)
+}
+
+// runEUSupply scrapes EU Supply's UK portal every configured interval until
+// ctx is cancelled, passing what each run fetches to save, even when the run
+// fails partway. A long window is backfilled first (see backfill). One
+// client is used throughout, so its gap between requests holds across runs.
+// lookup is nil without a store: every tender is then new, and none is
+// checked for repeats.
+func runEUSupply(
+	ctx context.Context,
+	logger *slog.Logger,
+	conf *config.JSCSourceEUSupply,
+	lookup tenderStore,
+	save saveFunc,
+	since time.Time,
+) error {
+	client := eusupply.New(
+		conf.BaseURL,
+		eusupply.WithLogger(logger),
+		eusupply.WithRequestGap(conf.RequestMinGap, conf.RequestMaxGap),
+	)
+	fetch := func(
+		ctx context.Context, since, until time.Time,
+	) ([]*types.Opportunity, error) {
+		return fetchEUSupply(ctx, logger, client, lookup, since, until)
+	}
+
+	since = backfill(
+		ctx,
+		logger,
+		euSupplySource,
+		conf.Interval,
+		fetch,
+		save,
+		since,
+	)
+
+	worker := util.Worker{
+		Logger:   tagged(logger),
+		Name:     euSupplySource,
+		Interval: conf.Interval,
+		Task: func(ctx context.Context, since time.Time) error {
+			opportunities, err := fetch(ctx, since, time.Time{})
+			return saveFetched(ctx, logger, save, opportunities, err)
+		},
+	}
+	return worker.Run(ctx, since)
+}
+
+// fetchEUSupply fetches the tenders published from since up to until (zero:
+// today) that aren't stored yet, with their own pages read, and marks the
+// ones another source already has Hidden and Skipped (see markRepeats). If
+// it fails partway, the tenders read in full before the error are returned
+// along with it. Finding nothing new returns ErrNoNewReleases.
+func fetchEUSupply(
+	ctx context.Context,
+	logger *slog.Logger,
+	client *eusupply.Client,
+	lookup tenderStore,
+	since, until time.Time,
+) ([]*types.Opportunity, error) {
+	tenders, fetchErr := client.Search(ctx, since, until)
+
+	fresh, err := newTenders(ctx, lookup, tenders)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range fresh {
+		if err := client.Details(ctx, &fresh[i]); err != nil {
+			fresh = fresh[:i]
+			fetchErr = errors.Join(fetchErr, err)
+			break
+		}
+	}
+
+	opportunities := eusupply.Convert(fresh)
+	if err := markRepeats(ctx, logger, lookup, opportunities); err != nil {
+		// Saved unmarked, a repeat would stay visible, as marks are only
+		// stored for a new opportunity: fetch them again next run instead.
+		return nil, errors.Join(fetchErr, err)
+	}
+
+	tagged(logger).Info(
+		"fetched opportunities",
+		"source", euSupplySource,
+		"listed", len(tenders),
+		"count", len(opportunities),
+	)
+
+	if len(opportunities) == 0 && fetchErr == nil {
+		return nil, ErrNoNewReleases
+	}
+	return opportunities, fetchErr
+}
+
+// newTenders returns the tenders whose release isn't stored yet, so only
+// their pages are read. Saving a stored one again would replace its row
+// with the search's fields alone. With no lookup, every tender is new.
+func newTenders(
+	ctx context.Context, lookup tenderStore, tenders []eusupply.Tender,
+) ([]eusupply.Tender, error) {
+	if lookup == nil {
+		return tenders, nil
+	}
+
+	var fresh []eusupply.Tender
+	for _, tender := range tenders {
+		exists, err := lookup.ReleaseExists(
+			ctx, eusupply.UUID(tender.ID), eusupply.ReleaseID(tender),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("checking tender %s: %w", tender.ID, err)
+		}
+		if !exists {
+			fresh = append(fresh, tender)
+		}
+	}
+
+	return fresh, nil
+}
+
+// markRepeats marks Hidden and Skipped each opportunity that repeats one
+// from another source, most often its Find a Tender notice: its title is at
+// least titleMatchThreshold alike (see similarTitles), its latest release
+// is within repeatWindow, and its buyer or description matches too (see
+// confirmedRepeat). Only the opportunities whose titles match are read in
+// full. The store keeps the marks of an opportunity it already has, so only
+// new ones are marked. With no lookup it does nothing.
+func markRepeats(
+	ctx context.Context,
+	logger *slog.Logger,
+	lookup tenderStore,
+	opportunities []*types.Opportunity,
+) error {
+	if lookup == nil || len(opportunities) == 0 {
+		return nil
+	}
+
+	earliest := opportunities[0].LatestReleaseDate
+	latest := earliest
+	for _, opportunity := range opportunities {
+		earliest = minTime(earliest, opportunity.LatestReleaseDate)
+		latest = maxTime(latest, opportunity.LatestReleaseDate)
+	}
+	others, err := lookup.ListOtherSourcesTitles(
+		ctx,
+		euSupplySource,
+		earliest.Add(-repeatWindow),
+		latest.Add(repeatWindow+time.Nanosecond),
+	)
+	if err != nil {
+		return err
+	}
+
+	normalised := make([]string, len(others))
+	for i, other := range others {
+		normalised[i] = normaliseTitle(other.Title)
+	}
+
+	repeats := 0
+	for _, opportunity := range opportunities {
+		title := normaliseTitle(opportunity.Title)
+		for i, other := range others {
+			gap := other.LatestReleaseDate.Sub(opportunity.LatestReleaseDate)
+			if gap.Abs() > repeatWindow || !similarTitles(title, normalised[i]) {
+				continue
+			}
+			repeated, err := lookup.GetOpportunity(ctx, other.UUID)
+			if err != nil {
+				return fmt.Errorf("reading opportunity %s: %w", other.UUID, err)
+			}
+			if !confirmedRepeat(opportunity, repeated) {
+				tagged(logger).Debug(
+					"title matched, but not buyer or description",
+					"uuid", opportunity.UUID,
+					"title_matches", other.UUID,
+				)
+				continue
+			}
+			opportunity.Marks.Hidden = true
+			opportunity.Marks.Skipped = true
+			repeats++
+			tagged(logger).Debug(
+				"hid repeated opportunity",
+				"uuid", opportunity.UUID,
+				"repeats", other.UUID,
+			)
+			break
+		}
+	}
+
+	tagged(logger).Info(
+		"hid repeated opportunities",
+		"source", euSupplySource,
+		"count", repeats,
+	)
+
+	return nil
+}
+
+// minTime returns the earlier of two times.
+func minTime(first, second time.Time) time.Time {
+	if second.Before(first) {
+		return second
+	}
+	return first
+}
+
+// maxTime returns the later of two times.
+func maxTime(first, second time.Time) time.Time {
+	if second.After(first) {
+		return second
+	}
+	return first
+}

internal/fetcher/worker_eusupply_test.go

@@ -0,0 +1,324 @@
+package fetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.avsdev.uk/Go/job-search/internal/config"
+	"git.avsdev.uk/Go/job-search/internal/eusupply"
+	"git.avsdev.uk/Go/job-search/internal/store"
+	"git.avsdev.uk/Go/job-search/internal/types"
+)
+
+var errTestStore = errors.New("test store failed")
+
+// testTender is one row of a fake EU Supply search page, and its page.
+type testTender struct {
+	id, pid, title string
+	published      string // "21/10/2026 12:45:00"
+	buyer          string
+	description    string
+}
+
+// euSupplySite is a fake EU Supply portal: one page of search results with
+// tenders, and a page for each of them. It counts the tender pages read.
+type euSupplySite struct {
+	mu       sync.Mutex
+	tenders  []testTender
+	detailed []string // the PIDs whose pages were read
+	failPID  string   // a PID whose page fails
+}
+
+func (s *euSupplySite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if r.Method == http.MethodPost {
+		var rows strings.Builder
+		for _, tender := range s.tenders {
+			fmt.Fprintf(
+				&rows,
+				`<tr><td><div class="js-tooltip-content">%s</div></td>`+
+					`<td><div class="js-tooltip-content"></div></td>`+
+					`<td><a href="/app/rfq/rwlentrance_s.asp?PID=%s">%s</a></td>`+
+					`<td><div class="js-tooltip-content" title="%s"></div></td>`+
+					`<td></td><td></td><td>%s</td><td></td></tr>`,
+				tender.id, tender.pid, tender.title, tender.published, tender.buyer,
+			)
+		}
+		fmt.Fprintf(
+			w,
+			`<html><body><div id="searchResultContainer"><table><tbody>%s`+
+				`</tbody></table></div></body></html>`,
+			rows.String(),
+		)
+		return
+	}
+
+	pid := strings.Split(strings.TrimPrefix(
+		r.URL.Path, "/ctm/Supplier/PublicPurchase/",
+	), "/")[0]
+	s.detailed = append(s.detailed, pid)
+	if pid == s.failPID {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	description := ""
+	for _, tender := range s.tenders {
+		if tender.pid == pid {
+			description = tender.description
+		}
+	}
+	fmt.Fprintf(
+		w,
+		`<html><body><div class="ctm-content-label">Detailed description:</div>`+
+			`<p>%s</p></body></html>`,
+		description,
+	)
+}
+
+// fakeTenderStore is a tenderStore with some releases stored and other
+// sources' opportunities.
+type fakeTenderStore struct {
+	stored    map[string]bool // uuid "/" release id
+	others    []*types.Opportunity
+	titlesErr error
+}
+
+func (s *fakeTenderStore) ReleaseExists(
+	ctx context.Context, uuid, releaseID string,
+) (bool, error) {
+	return s.stored[uuid+"/"+releaseID], nil
+}
+
+func (s *fakeTenderStore) ListOtherSourcesTitles(
+	ctx context.Context, source string, from, before time.Time,
+) ([]store.OpportunityTitle, error) {
+	if s.titlesErr != nil {
+		return nil, s.titlesErr
+	}
+	var titles []store.OpportunityTitle
+	for _, other := range s.others {
+		if !other.LatestReleaseDate.Before(from) &&
+			other.LatestReleaseDate.Before(before) {
+			titles = append(titles, store.OpportunityTitle{
+				UUID:              other.UUID,
+				Title:             other.Title,
+				LatestReleaseDate: other.LatestReleaseDate,
+			})
+		}
+	}
+	return titles, nil
+}
+
+func (s *fakeTenderStore) GetOpportunity(
+	ctx context.Context, uuid string,
+) (*types.Opportunity, error) {
+	for _, other := range s.others {
+		if other.UUID == uuid {
+			return other, nil
+		}
+	}
+	return nil, store.ErrNotFound
+}
+
+// newEUSupplyClient returns a client for srv that doesn't wait.
+func newEUSupplyClient(srv *httptest.Server) *eusupply.Client {
+	return eusupply.New(srv.URL, eusupply.WithRequestGap(0, 0))
+}
+
+// Only tenders not stored are read and returned. The ones another source
+// has within the window, with a matching title and a matching buyer or
+// description, are marked Hidden and Skipped.
+func TestFetchEUSupply(t *testing.T) {
+	const day = "20/10/2026 10:00:00"
+	site := &euSupplySite{
+		tenders: []testTender{
+			{"1", "11", "Already stored", day, "", ""},
+			// The same generic title from another buyer: not a repeat.
+			{"2", "12", "Pest Control Services", day, "Yorkshire Housing Limited",
+				"Pest control across our homes in Leeds"},
+			// The same buyer, named differently: a repeat.
+			{"3", "13", "Google Reseller Framework", day, "Jisc", ""},
+			// An agent buying for the buyer, but the same description.
+			{"4", "14", "Fairhive - Materials Procurement", day,
+				"Echelon Consultancy",
+				"Supply of building materials for Fairhive housing repairs"},
+			// Alike, but too long before.
+			{"5", "15", "Housing related support", day, "Hull City Council", ""},
+		},
+	}
+	srv := httptest.NewServer(site)
+	t.Cleanup(srv.Close)
+
+	published := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
+	other := func(
+		uuid, title, buyer, description string, released time.Time,
+	) *types.Opportunity {
+		return &types.Opportunity{
+			UUID:              uuid,
+			Title:             title,
+			BuyerName:         buyer,
+			Description:       description,
+			LatestReleaseDate: released,
+		}
+	}
+	lookup := &fakeTenderStore{
+		stored: map[string]bool{
+			"eusupply-uk-1/1": true,
+		},
+		others: []*types.Opportunity{
+			other("ocds-pest", "PEST CONTROL SERVICE", "Anchor Hanover Group",
+				"Pest control for retirement housing", published),
+			other("ocds-jisc", "Google Reseller Framework",
+				"JISC SERVICES LIMITED", "", published.Add(time.Hour)),
+			other("ocds-fairhive", "Fairhive - Materials Procurement",
+				"Fairhive Housing Limited",
+				"Fairhive seeks supply of building materials for housing repairs",
+				published),
+			other("ocds-old", "Housing related support", "Hull City Council", "",
+				published.Add(-40*24*time.Hour)),
+		},
+	}
+
+	opportunities, err := fetchEUSupply(
+		context.Background(),
+		slog.Default(),
+		newEUSupplyClient(srv),
+		lookup,
+		published.Add(-24*time.Hour),
+		time.Time{},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got := strings.Join(site.detailed, ","); got != "12,13,14,15" {
+		t.Errorf("pages read = %s; want the new tenders' only", got)
+	}
+	marks := map[string]bool{}
+	for _, opportunity := range opportunities {
+		if opportunity.Source != "EU Supply - UK" {
+			t.Errorf("%s: Source = %q", opportunity.UUID, opportunity.Source)
+		}
+		if opportunity.Marks.Hidden != opportunity.Marks.Skipped {
+			t.Errorf("%s: marks = %+v", opportunity.UUID, opportunity.Marks)
+		}
+		marks[opportunity.UUID] = opportunity.Marks.Hidden
+	}
+	want := map[string]bool{
+		"eusupply-uk-2": false,
+		"eusupply-uk-3": true,
+		"eusupply-uk-4": true,
+		"eusupply-uk-5": false,
+	}
+	if fmt.Sprint(marks) != fmt.Sprint(want) {
+		t.Errorf("hidden = %v; want %v", marks, want)
+	}
+}
+
+// A tender page that fails stops the run, keeping the tenders read before
+// it; a failed title lookup returns nothing, so nothing is saved unmarked.
+func TestFetchEUSupplyErrors(t *testing.T) {
+	site := &euSupplySite{
+		tenders: []testTender{
+			{"1", "11", "First", "20/10/2026 10:00:00", "", ""},
+			{"2", "12", "Second", "20/10/2026 10:00:00", "", ""},
+			{"3", "13", "Third", "20/10/2026 10:00:00", "", ""},
+		},
+		failPID: "12",
+	}
+	srv := httptest.NewServer(site)
+	t.Cleanup(srv.Close)
+	client := newEUSupplyClient(srv)
+	since := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
+
+	opportunities, err := fetchEUSupply(
+		context.Background(), slog.Default(), client, nil, since, time.Time{},
+	)
+	if !errors.Is(err, eusupply.ErrUnexpectedStatus) {
+		t.Errorf("err = %v; want ErrUnexpectedStatus", err)
+	}
+	if len(opportunities) != 1 || opportunities[0].UUID != "eusupply-uk-1" {
+		t.Errorf("opportunities = %v; want the first tender", opportunities)
+	}
+
+	site.failPID = ""
+	lookup := &fakeTenderStore{
+		titlesErr: errTestStore,
+	}
+	opportunities, err = fetchEUSupply(
+		context.Background(), slog.Default(), client, lookup, since, time.Time{},
+	)
+	if !errors.Is(err, errTestStore) || opportunities != nil {
+		t.Errorf(
+			"err = %v, %d opportunities; want the store's error and none",
+			err, len(opportunities),
+		)
+	}
+
+	site.tenders = nil
+	_, err = fetchEUSupply(
+		context.Background(), slog.Default(), client, lookup, since, time.Time{},
+	)
+	if !errors.Is(err, ErrNoNewReleases) {
+		t.Errorf("no tenders: err = %v; want ErrNoNewReleases", err)
+	}
+}
+
+// Run starts the EU Supply source when it is enabled, and saves its
+// opportunities with their Source.
+func TestRunEUSupply(t *testing.T) {
+	site := &euSupplySite{
+		tenders: []testTender{
+			{
+				"1", "11", "A tender",
+				time.Now().Format("02/01/2006 15:04:05"), "", "",
+			},
+		},
+	}
+	srv := httptest.NewServer(site)
+	t.Cleanup(srv.Close)
+
+	opportunityStore := openTestStore(t)
+	sources := &config.JSCSources{}
+	sources.EUSupply.Enabled = true
+	sources.EUSupply.Interval = time.Hour
+	sources.EUSupply.BaseURL = srv.URL
+	sources.EUSupply.RequestMinGap = time.Millisecond
+	sources.EUSupply.RequestMaxGap = time.Millisecond
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		done <- New(sources, WithStore(opportunityStore)).Run(ctx, time.Time{})
+	}()
+
+	deadline := time.Now().Add(5 * time.Second)
+	var stored []string
+	for time.Now().Before(deadline) {
+		opportunity, err := opportunityStore.GetOpportunity(
+			context.Background(), "eusupply-uk-1",
+		)
+		if err == nil {
+			stored = append(stored, opportunity.Source)
+			break
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run = %v", err)
+	}
+	if len(stored) != 1 || stored[0] != "EU Supply - UK" {
+		t.Errorf("stored sources = %v; want EU Supply - UK", stored)
+	}
+}

internal/store/queries.go

@@ -149,6 +149,13 @@ LIMIT ? OFFSET ?`
 )`
 )
 
+// selectOtherSourcesTitles lists the uuid, title and latest_release_date of
+// the opportunities not from one source whose latest_release_date is in
+// [from, before); it walks the latest_release_date index.
+const selectOtherSourcesTitles = `
+SELECT uuid, title, latest_release_date FROM opportunities
+WHERE source <> ? AND latest_release_date >= ? AND latest_release_date < ?`
+
 // ---------- opportunity_releases ----------
 
 // insertRelease stores a release; 0 rows affected means it was already

internal/store/queries_test.go

@@ -36,6 +36,9 @@ var checks = map[string]queryCheck{
 	"selectSourceLatestReleaseDate": {
 		sql: selectSourceLatestReleaseDate,
 	},
+	"selectOtherSourcesTitles": {
+		sql: selectOtherSourcesTitles,
+	},
 	"upsertOpportunity":       {sql: upsertOpportunity, named: true},
 	"selectOpportunityByUUID": {sql: selectOpportunityByUUID},
 	"listOpportunitiesSelect": {

internal/store/store.go

@@ -1041,6 +1041,37 @@ func (s *Store) LatestReleaseDate(
 	return latest, nil
 }
 
+// OpportunityTitle is an opportunity's UUID, title and latest release date,
+// as ListOtherSourcesTitles lists them.
+type OpportunityTitle struct {
+	UUID              string    `db:"uuid"`
+	Title             string    `db:"title"`
+	LatestReleaseDate time.Time `db:"latest_release_date"`
+}
+
+// ListOtherSourcesTitles returns the UUID, title and latest release date of
+// every opportunity not from source whose latest release is from from
+// (inclusive) to before (exclusive), so a fetcher can find the ones a new
+// opportunity repeats.
+func (s *Store) ListOtherSourcesTitles(
+	ctx context.Context, source string, from, before time.Time,
+) ([]OpportunityTitle, error) {
+	titles := []OpportunityTitle{}
+	err := s.db.SelectContext(
+		ctx,
+		&titles,
+		selectOtherSourcesTitles,
+		source,
+		from.UTC(),
+		before.UTC(),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("listing titles of sources but %s: %w", source, err)
+	}
+
+	return titles, nil
+}
+
 // ReleaseExists reports whether a release is already stored, so a fetcher
 // can skip converting it.
 func (s *Store) ReleaseExists(

internal/store/store_test.go

@@ -866,6 +866,32 @@ func TestLatestReleaseDate(t *testing.T) {
 	}
 }
 
+// ListOtherSourcesTitles lists only other sources' opportunities, with
+// their latest release in [from, before).
+func TestListOtherSourcesTitles(t *testing.T) {
+	s := openStore(t)
+	ctx := context.Background()
+
+	mustSave(t, s, opportunity("ocds-1", "Too early", 1, "r1"))
+	mustSave(t, s, opportunity("ocds-2", "In the window", 3, "r1"))
+	mustSave(t, s, opportunity("ocds-3", "At the end", 5, "r1"))
+	own := opportunity("eusupply-uk-1", "Its own source", 3, "r1")
+	own.Source = "EU Supply - UK"
+	mustSave(t, s, own)
+
+	titles, err := s.ListOtherSourcesTitles(
+		ctx, "EU Supply - UK", day(2), day(5),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(titles) != 1 || titles[0].UUID != "ocds-2" ||
+		titles[0].Title != "In the window" ||
+		!titles[0].LatestReleaseDate.Equal(day(3)) {
+		t.Errorf("titles = %+v; want only ocds-2", titles)
+	}
+}
+
 // NewOpportunity is set only for an opportunity that wasn't already stored,
 // by both SaveOpportunity and SaveOpportunities.
 func TestSaveResultNewOpportunity(t *testing.T) {