From 3a06132071ff9c7b1555c0648d9003d2fa0d34b7 Mon Sep 17 00:00:00 2001 From: Craig Williams Date: Wed, 21 Oct 2026 13:57:54 +0100 Subject: [PATCH] Added an EU Supply source, scraping the UK portal's public tenders internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words). --- README.md | 16 +- go.mod | 2 +- go.sum | 3 - internal/config/README.md | 6 +- internal/config/conf_sections_test.go | 33 ++- internal/config/conf_source_eusupply.go | 62 +++++ internal/config/conf_sources.go | 1 + internal/eusupply/README.md | 47 ++++ internal/eusupply/convert.go | 101 +++++++ internal/eusupply/details.go | 108 ++++++++ internal/eusupply/eusupply.go | 208 +++++++++++++++ internal/eusupply/eusupply_test.go | 326 +++++++++++++++++++++++ internal/eusupply/html.go | 139 ++++++++++ internal/eusupply/search.go | 277 +++++++++++++++++++ internal/eusupply/testdata/search.html | 45 ++++ internal/eusupply/testdata/tender.html | 42 +++ internal/fetcher/fetcher.go | 12 + internal/fetcher/match.go | 252 ++++++++++++++++++ internal/fetcher/match_test.go | 208 +++++++++++++++ internal/fetcher/worker_eusupply.go | 249 +++++++++++++++++ internal/fetcher/worker_eusupply_test.go | 324 ++++++++++++++++++++++ internal/store/queries.go | 7 + internal/store/queries_test.go | 3 + internal/store/store.go | 31 +++ internal/store/store_test.go | 26 ++ 25 files changed, 2520 insertions(+), 8 deletions(-) create mode 100644 internal/config/conf_source_eusupply.go create mode 100644 internal/eusupply/README.md create mode 100644 internal/eusupply/convert.go create mode 100644 internal/eusupply/details.go create mode 100644 internal/eusupply/eusupply.go create mode 100644 internal/eusupply/eusupply_test.go create mode 100644 internal/eusupply/html.go create mode 100644 internal/eusupply/search.go create mode 100644 internal/eusupply/testdata/search.html create mode 100644 internal/eusupply/testdata/tender.html create mode 100644 internal/fetcher/match.go create mode 100644 internal/fetcher/match_test.go create mode 100644 internal/fetcher/worker_eusupply.go create mode 100644 internal/fetcher/worker_eusupply_test.go diff --git a/README.md b/README.md index b6dbb8c..78c0ba6 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,8 @@ nothing stored, Find a Tender's client fetches the last day. `--since` overrides that for every source. When the start is more than 7 days ago (an early `--since`, or a long time -offline), the Find a Tender sources first backfill it in 7-day windows, +offline), the Find a Tender and EU Supply sources first backfill it in 7-day +windows, oldest first, saving each before fetching the next. A window that fails is saved as far as it got and fetched again after the source's interval. The API returns the newest releases first, so if a backfill is stopped partway @@ -132,6 +133,18 @@ A rate limit is waited out, as the API asks, up to `sources..rate_limit_retries` times per request. A failed fetch keeps what it fetched before the failure. +### EU Supply +The `eu_supply` source scrapes the public tenders on EU Supply's UK portal +(`uk.eu-supply.com`), stored with the source `EU Supply - UK`. The site has +no API and publishes no rate limit, so the client waits a random time from +`request_min_gap` to `request_max_gap` (2s to 10s) between its requests. Each +tender not stored yet has its own page read too, for its description and +CPV codes. Most of its tenders are also published on Find a Tender: a new +one is stored Hidden and Skipped when another source's opportunity released +within 30 days of it has a title at least 90% alike, and its buyer or its +description matches too (80% of the shorter one's words). See +[internal/eusupply](internal/eusupply/README.md). + `fts-once` can be limited with `--planning`, `--opportunities` or `--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead of waiting. @@ -191,6 +204,7 @@ second one fails with "address already in use". | `internal/skiprules` | The rules that mark new opportunities Skipped (see [internal/skiprules](internal/skiprules/README.md)) | | `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) | | `internal/findatender` | FTS API client and conversion from OCDS to JobSearch types | +| `internal/eusupply` | EU Supply UK portal scraper and conversion to JobSearch types (see [internal/eusupply](internal/eusupply/README.md)) | | `internal/types` | JobSearch's own opportunity types | | `internal/util` | The periodic `Worker` used by the fetcher | | `internal/logging` | slog setup and the HTTP access log | diff --git a/go.mod b/go.mod index f795ee3..74bf02e 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 go.yaml.in/yaml/v3 v3.0.5 + golang.org/x/net v0.58.0 golang.org/x/time v0.16.0 resty.dev/v3 v3.0.0-rc.4 ) @@ -34,7 +35,6 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect diff --git a/go.sum b/go.sum index cbc68f8..8ec7971 100644 --- a/go.sum +++ b/go.sum @@ -10,7 +10,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= -github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE= github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= @@ -28,7 +27,6 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -93,7 +91,6 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/config/README.md b/internal/config/README.md index f537920..2ed436c 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -75,9 +75,13 @@ supported. | `sources.find_a_tender.rate_limit_retries` | int | `5` | How many times a request refused for the API's rate limit is waited out (for its `Retry-After`) and retried. 0 fails at the first; can't be negative. `fetcher fts-once --ignore-rate-limit` uses 0 | | `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`). Empty: the source fails to start if enabled | | `sources.fts_mock.rate_limit_retries` | int | `5` | As for `find_a_tender` | +| `sources.eu_supply.base_url` | string | `https://uk.eu-supply.com` | Root of EU Supply's UK portal (see `internal/eusupply`) | +| `sources.eu_supply.request_min_gap` | duration | `2s` | The shortest wait between two requests to the portal. Must be greater than 0: the site publishes no rate limit | +| `sources.eu_supply.request_max_gap` | duration | `10s` | The longest wait between two requests; each wait is random, from `request_min_gap`. Must be greater than 0; less than `request_min_gap` is read as `request_min_gap` | | `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown | -The current sources are `find_a_tender`, `fts_mock` and `sleeper`. See +The current sources are `find_a_tender`, `fts_mock`, `eu_supply` and +`sleeper`. See "Addition of a new opportunities source" in the top level README for how to add a new source. diff --git a/internal/config/conf_sections_test.go b/internal/config/conf_sections_test.go index da2c88a..9b8ffaa 100644 --- a/internal/config/conf_sections_test.go +++ b/internal/config/conf_sections_test.go @@ -288,6 +288,35 @@ func TestConcreteSourceSections(t *testing.T) { } } +func TestEUSupplySection(t *testing.T) { + conf := defaultConfig(t) + got := conf.Sources.EUSupply + if got.Enabled || got.Interval != 10*time.Minute || + got.BaseURL != "https://uk.eu-supply.com" || + got.RequestMinGap != 2*time.Second || + got.RequestMaxGap != 10*time.Second { + t.Errorf("defaults = %+v", got) + } + + c := JSCSourceEUSupply{} + if err := c.SetValue("request_min_gap", "5s"); err != nil || + c.RequestMinGap != 5*time.Second { + t.Errorf("SetValue(request_min_gap, 5s): %v, %v", err, c.RequestMinGap) + } + for _, key := range []string{"request_min_gap", "request_max_gap"} { + if err := c.SetValue(key, "0s"); err == nil { + t.Errorf("SetValue(%s, 0s) succeeded", key) + } + } + // Its own validate must still check interval, as JSCSourcesSource's does. + if err := c.SetValue("interval", "0s"); err == nil { + t.Error("SetValue(interval, 0s) succeeded") + } + if v, err := c.ValueOf("base_url"); err != nil || v != "" { + t.Errorf("ValueOf(base_url) = %v, %v", v, err) + } +} + func TestSourcesSection(t *testing.T) { var c JSCSources if err := c.SetValue("fts_mock.api_root", "http://mock"); err != nil { @@ -317,8 +346,8 @@ func TestSourcesSection(t *testing.T) { if flat["sources.fts_mock.api_root"] != "http://mock" { t.Errorf("Flatten missing sources.fts_mock.api_root: %v", flat) } - if len(flat) != 12 { - t.Errorf("Flatten has %d keys; want 12: %v", len(flat), flat) + if len(flat) != 17 { + t.Errorf("Flatten has %d keys; want 17: %v", len(flat), flat) } if err := c.SetValue("skip_rules_file", "rules.json"); err != nil || diff --git a/internal/config/conf_source_eusupply.go b/internal/config/conf_source_eusupply.go new file mode 100644 index 0000000..abbb286 --- /dev/null +++ b/internal/config/conf_source_eusupply.go @@ -0,0 +1,62 @@ +package config + +import ( + "errors" + "reflect" + "time" +) + +// JSCKSourceEUSupply is the EU Supply source's configuration key. +const JSCKSourceEUSupply string = "eu_supply" + +// JSCSourceEUSupply configures scraping the public tenders on EU Supply's +// UK portal at BaseURL (see internal/eusupply). The client waits a random +// time from RequestMinGap to RequestMaxGap after each request; both must be +// greater than 0, as the site's own limit isn't published, and a +// RequestMaxGap less than RequestMinGap is read as RequestMinGap. +type JSCSourceEUSupply struct { + JSCSourcesSource + BaseURL string `cfg:"base_url" default:"https://uk.eu-supply.com"` + RequestMinGap time.Duration `cfg:"request_min_gap" default:"2s"` + RequestMaxGap time.Duration `cfg:"request_max_gap" default:"10s"` +} + +// ---------- Section methods (see Section) ---------- + +// Map implements Section. +func (c JSCSourceEUSupply) Map() map[string]any { return mapOf(c) } + +// Flatten implements Section. +func (c JSCSourceEUSupply) Flatten() map[string]any { + return flattenOf(JSCKSourceEUSupply, c) +} + +// TypeOf implements Section. +func (c JSCSourceEUSupply) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +// ValueOf implements Section. +func (c JSCSourceEUSupply) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +// SetValue implements Section. +func (c *JSCSourceEUSupply) SetValue(key string, value any) error { + return setValue(c, key, value) +} + +// ---------- Section specific methods ---------- + +// validate checks values before they are stored (see validator), as +// JSCSourcesSource's does, which this one hides. request_min_gap and +// request_max_gap must be greater than 0. +func (c *JSCSourceEUSupply) validate(key string, value any) error { + if (key == "request_min_gap" || key == "request_max_gap") && + value.(time.Duration) <= 0 { + return errors.New( + "config: request_min_gap and request_max_gap must be greater than 0, as EU Supply publishes no rate limit", + ) + } + return c.JSCSourcesSource.validate(key, value) +} diff --git a/internal/config/conf_sources.go b/internal/config/conf_sources.go index 1b917ec..ccddf4c 100644 --- a/internal/config/conf_sources.go +++ b/internal/config/conf_sources.go @@ -20,6 +20,7 @@ type JSCSources struct { SkipRulesFile string `cfg:"skip_rules_file"` FindATender JSCSourceFindATender `cfg:"find_a_tender"` FTSMock JSCSourceFTSMock `cfg:"fts_mock"` + EUSupply JSCSourceEUSupply `cfg:"eu_supply"` Sleeper JSCSourceSleeper `cfg:"sleeper"` } diff --git a/internal/eusupply/README.md b/internal/eusupply/README.md new file mode 100644 index 0000000..17304b7 --- /dev/null +++ b/internal/eusupply/README.md @@ -0,0 +1,47 @@ +# EU Supply + +A scraper for the public tenders on EU Supply's UK portal, +[uk.eu-supply.com](https://uk.eu-supply.com). The site (Mercell CTM) has no +API, so the client posts its public search form and reads the HTML. It +replaces an old Node scraper, which also covered the Sellafield and Shared +Systems portals; only the UK portal is still used. + +## Requests + +| Request | Path | Gives | +|---|---|---| +| Search | `POST /ctm/Supplier/PublicTenders` | Up to 100 tenders a page: id, reference, title, publication date, response deadline, process, buyers, countries | +| Tender page | `GET /ctm/Supplier/PublicPurchase//0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK` | Description, type of contract, CPV codes, buyer address, lots | + +- The search filters on the **publication date**, by whole UK days + (`SearchFilter.FromDate`/`ToDate` as `DD/MM/YYYY`, both included). Only + issued tenders are searched (`SearchFilter.PublishType` 1), not awarded + ones. +- The form must be posted whole, with the portal's own defaults + (`searchForm`). With the old scraper's values (`OperatorId` 1 and an + empty `SavedCategoryId`), the portal answers 500. +- Times are UK time. Each date cell's tooltip `title` holds the full time + (`21/10/2026 12:45:00`); the cell's text has only the date. +- A tender's PID (its pages' id) is only in its search result's link + (`rwlentrance_s.asp?PID=…`), which redirects to its page. +- A tender page's fields are each a `div.ctm-content-label` (`CPV codes:`) + followed by a `

` with the value. +- The system certificate authorities are enough: the old scraper's own CA + file isn't needed. + +## Rate limit + +The site publishes no rate limit, so the client sets its own: it makes one +request at a time, waiting a random time from the minimum to the maximum gap +(`WithRequestGap`; the source's `request_min_gap` and `request_max_gap`, +2s to 10s by default) after each before making the next. + +## Opportunities + +`Convert` makes one opportunity per tender, with the source `EU Supply - UK` +and the UUID `eusupply-uk-`. It has one release (id: the tender +id), dated its publication, tagged `tender`, whose JSON is the `Tender` as +read. CPV codes are stored without their check digit, as Find a Tender's +are. The package sets no marks: the fetcher marks the tenders it takes as +repeats of another source's opportunities Hidden and Skipped (see +`internal/fetcher/match.go`). diff --git a/internal/eusupply/convert.go b/internal/eusupply/convert.go new file mode 100644 index 0000000..ecd00d8 --- /dev/null +++ b/internal/eusupply/convert.go @@ -0,0 +1,101 @@ +package eusupply + +import ( + "encoding/json" + "strings" + + "git.avsdev.uk/Go/job-search/internal/types" +) + +// The opportunities Convert makes. +const ( + // UUIDPrefix starts the UUID of every opportunity from the UK portal; + // the tender's id follows it. + UUIDPrefix = "eusupply-uk-" + // uuidType is their UUIDType. + uuidType = "eusupply" + // releaseTag is their one release's tag: the portal lists open tenders. + releaseTag = "tender" +) + +// mainCategories maps the portal's types of contract to OCDS's +// mainProcurementCategory, as Find a Tender's opportunities have. +var mainCategories = map[string]string{ + "Supplies": "goods", + "Services": "services", + "Works": "works", +} + +// UUID returns the UUID of the opportunity for the tender with id. +func UUID(id string) string { + return UUIDPrefix + id +} + +// ReleaseID returns the id of the one release of the opportunity for +// tender: its id, as the portal gives each tender one page. +func ReleaseID(tender Tender) string { + return tender.ID +} + +// Convert returns the opportunity for each tender, with one release whose +// JSON is the tender as read. +func Convert(tenders []Tender) []*types.Opportunity { + opportunities := make([]*types.Opportunity, 0, len(tenders)) + for _, tender := range tenders { + opportunities = append(opportunities, convertOne(tender)) + } + return opportunities +} + +// convertOne builds the opportunity for one tender. +func convertOne(tender Tender) *types.Opportunity { + published := tender.Published + opportunity := &types.Opportunity{ + UUID: UUID(tender.ID), + UUIDType: uuidType, + Title: tender.Title, + Status: "active", + Description: tender.Description, + BuyerName: tender.Buyer, + Classifications: classifications(tender), + MainCategory: mainCategories[tender.ContractType], + MultiLot: tender.Lots > 1, + StartDate: &published, + EndDate: tender.Deadline, + Source: Source, + LatestReleaseDate: tender.Published, + } + + release := types.OpportunityRelease{ + UUID: opportunity.UUID, + ReleaseID: ReleaseID(tender), + ReleaseDate: tender.Published, + Tags: []types.OpportunityReleaseTag{ + { + Tag: releaseTag, + }, + }, + } + // A Tender is plain data, so marshalling it can't fail. + if encoded, err := json.Marshal(tender); err == nil { + release.JSON = string(encoded) + } + opportunity.Releases = []types.OpportunityRelease{release} + + return opportunity +} + +// classifications returns the tender's CPV codes as Find a Tender's are +// stored: without the check digit ("48000000", not "48000000-8"). +func classifications(tender Tender) []types.ClassificationCode { + codes := make([]types.ClassificationCode, 0, len(tender.CPVCodes)) + for _, cpv := range tender.CPVCodes { + code, _, _ := strings.Cut(cpv.Code, "-") + codes = append(codes, types.ClassificationCode{ + IsCPV: true, + Code: code, + Classification: cpv.Description, + }) + } + return codes +} diff --git a/internal/eusupply/details.go b/internal/eusupply/details.go new file mode 100644 index 0000000..2e961e5 --- /dev/null +++ b/internal/eusupply/details.go @@ -0,0 +1,108 @@ +package eusupply + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + + "golang.org/x/net/html" +) + +// ErrNoDetails means a tender's page had none of the labelled fields the +// public page always has: the page has changed, or isn't a tender's. +var ErrNoDetails = errors.New("eusupply: no tender details on the page") + +// Details reads tender's own page and fills in its description, contract +// type, CPV codes, buyer address and number of lots, and sets HasDetails. +func (c *Client) Details(ctx context.Context, tender *Tender) error { + body, err := c.get(ctx, tenderPath(tender.PID)) + if err != nil { + return fmt.Errorf("reading tender %s: %w", tender.ID, err) + } + + if err := parseDetails(body, tender); err != nil { + return fmt.Errorf("reading tender %s: %w", tender.ID, err) + } + + return nil +} + +// parseDetails reads a tender's page into tender. Each field is a +// "ctm-content-label" div ("CPV codes:") followed by a

holding its +// value. +func parseDetails(body []byte, tender *Tender) error { + document, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return err + } + + values := make(map[string]*html.Node) + labels := findAll(document, hasClass("div", "ctm-content-label")) + for _, label := range labels { + value := nextElement(label) + if value == nil || value.Data != "p" { + continue + } + name := strings.TrimSuffix(text(label), ":") + if _, found := values[name]; !found { + values[name] = value + } + } + if len(values) == 0 { + return ErrNoDetails + } + + if value := values["Detailed description"]; value != nil { + tender.Description = strings.TrimSpace(rawText(value)) + } + if value := values["Type of contract"]; value != nil { + tender.ContractType = text(value) + } + if value := values["CPV codes"]; value != nil { + tender.CPVCodes = parseCPVCodes(value) + } + if value := values["Buyer"]; value != nil { + tender.BuyerAddress = buyerAddress(value) + } + if deadline := values["Response deadline (UK time)"]; deadline != nil && + tender.Deadline == nil { + if parsed, err := parseUKTime(text(deadline)); err == nil { + tender.Deadline = &parsed + } + } + tender.Lots = len(findAll(document, hasClass("div", "package-container"))) + tender.HasDetails = true + + return nil +} + +// parseCPVCodes reads the CPV codes listed in value, one per : the +// code, two non-breaking spaces, then its description. +func parseCPVCodes(value *html.Node) []CPVCode { + var codes []CPVCode + for _, span := range findAll(value, isTag("span")) { + code, description, _ := strings.Cut(text(span), " ") + if code == "" { + continue + } + codes = append(codes, CPVCode{ + Code: code, + Description: description, + }) + } + return codes +} + +// buyerAddress reads the buyer's name and address, one line each, leaving +// out the "View profile" link. +func buyerAddress(value *html.Node) []string { + var address []string + for _, line := range lines(value) { + if line != "View profile" { + address = append(address, line) + } + } + return address +} diff --git a/internal/eusupply/eusupply.go b/internal/eusupply/eusupply.go new file mode 100644 index 0000000..1ff3081 --- /dev/null +++ b/internal/eusupply/eusupply.go @@ -0,0 +1,208 @@ +// Package eusupply scrapes the public tenders listed on EU Supply's UK +// portal (uk.eu-supply.com), and converts them to opportunities. +// +// The site has no API: the client posts the portal's search form and reads +// the HTML it returns, then reads each tender's own page for its +// description and CPV codes. The client limits its own request rate (see +// WithRequestGap), as the site publishes no limit to wait out. +package eusupply + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "sync" + "time" + _ "time/tzdata" // the portal's times are UK time, wherever this runs +) + +// The defaults of the client's options. +const ( + // DefaultBaseURL is the UK portal's root. + DefaultBaseURL = "https://uk.eu-supply.com" + // DefaultMinRequestGap and DefaultMaxRequestGap bound the random wait + // between two requests (see WithRequestGap). + DefaultMinRequestGap = 2 * time.Second + DefaultMaxRequestGap = 10 * time.Second + // requestTimeout is how long one request may take. + requestTimeout = 60 * time.Second +) + +// Source is the Source of the opportunities Convert makes. +const Source = "EU Supply - UK" + +// ErrUnexpectedStatus wraps a response status other than 200 OK. +var ErrUnexpectedStatus = errors.New("eusupply: unexpected response status") + +// ukTime is the zone of the portal's dates and times. +var ukTime = mustLoadLocation("Europe/London") + +// Client scrapes one EU Supply portal. Create it with New; it is safe for +// concurrent use, though its requests are made one at a time. +type Client struct { + baseURL string + httpClient *http.Client + log *slog.Logger + + minGap, maxGap time.Duration + + // mu serialises requests, so the gap between them holds; lastRequest is + // when the previous one finished (zero: none yet). + mu sync.Mutex + lastRequest time.Time +} + +// Option configures a Client. +type Option func(*Client) + +// WithLogger makes the client log to logger instead of slog.Default(). +func WithLogger(logger *slog.Logger) Option { + return func(c *Client) { + c.log = logger + } +} + +// WithRequestGap makes the client wait a random time from minGap to maxGap +// after each request before making the next. A maxGap less than minGap is +// read as minGap; 0 or less for both means no wait, for tests. +func WithRequestGap(minGap, maxGap time.Duration) Option { + return func(c *Client) { + c.minGap = minGap + c.maxGap = max(minGap, maxGap) + } +} + +// New returns a client for the portal at baseURL (DefaultBaseURL if empty). +// It waits DefaultMinRequestGap to DefaultMaxRequestGap between requests +// unless given WithRequestGap, and logs to slog.Default() unless given +// WithLogger. +func New(baseURL string, opts ...Option) *Client { + jar, _ := cookiejar.New(nil) // cookiejar.New never returns an error + client := &Client{ + baseURL: strings.TrimSuffix(cmp.Or(baseURL, DefaultBaseURL), "/"), + httpClient: &http.Client{ + Timeout: requestTimeout, + Jar: jar, + }, + minGap: DefaultMinRequestGap, + maxGap: DefaultMaxRequestGap, + } + for _, opt := range opts { + opt(client) + } + + return client +} + +// logger returns the logger the client logs to, tagged with its component. +func (c *Client) logger() *slog.Logger { + log := c.log + if log == nil { + log = slog.Default() + } + return log.With("component", "eusupply") +} + +// do makes one request, once the gap since the previous request has passed, +// and returns its body. A status other than 200 OK fails with +// ErrUnexpectedStatus. It returns the request context's error if that is +// cancelled while it waits. +func (c *Client) do(req *http.Request) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if err := c.wait(req.Context()); err != nil { + return nil, err + } + defer func() { + c.lastRequest = time.Now() + }() + + c.logger().Info("fetching", "method", req.Method, "uri", req.URL.String()) + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() //nolint:errcheck // read only + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: %s", ErrUnexpectedStatus, res.Status) + } + + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", req.URL, err) + } + + return body, nil +} + +// wait sleeps until a random gap (see WithRequestGap) has passed since the +// previous request, or ctx is cancelled. c.mu must be held. +func (c *Client) wait(ctx context.Context) error { + if c.lastRequest.IsZero() || c.maxGap <= 0 { + return ctx.Err() + } + + gap := c.minGap + if c.maxGap > c.minGap { + //nolint:gosec // G404: a jitter, not a secret + gap += rand.N(c.maxGap - c.minGap + 1) + } + + timer := time.NewTimer(time.Until(c.lastRequest.Add(gap))) + defer timer.Stop() + + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// get makes a GET request for the portal's path. +func (c *Client) get(ctx context.Context, path string) ([]byte, error) { + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, c.baseURL+path, nil, + ) + if err != nil { + return nil, err + } + return c.do(req) +} + +// post posts form to the portal's path. +func (c *Client) post( + ctx context.Context, path string, form url.Values, +) ([]byte, error) { + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + c.baseURL+path, + strings.NewReader(form.Encode()), + ) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return c.do(req) +} + +// mustLoadLocation loads a time zone known to be in the embedded tzdata. +func mustLoadLocation(name string) *time.Location { + location, err := time.LoadLocation(name) + if err != nil { + panic(err) + } + return location +} diff --git a/internal/eusupply/eusupply_test.go b/internal/eusupply/eusupply_test.go new file mode 100644 index 0000000..38570a1 --- /dev/null +++ b/internal/eusupply/eusupply_test.go @@ -0,0 +1,326 @@ +package eusupply + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" +) + +// readFixture returns a file in testdata. +func readFixture(t *testing.T, name string) []byte { + t.Helper() + body, err := os.ReadFile("testdata/" + name) //nolint:gosec // G304: a test fixture + if err != nil { + t.Fatal(err) + } + return body +} + +// newTestClient returns a client for srv that doesn't wait between +// requests and logs nothing. +func newTestClient(srv *httptest.Server) *Client { + return New( + srv.URL, + WithRequestGap(0, 0), + WithLogger(slog.New(slog.DiscardHandler)), + ) +} + +func TestParseSearchPage(t *testing.T) { + c := New("https://uk.example", WithLogger(slog.New(slog.DiscardHandler))) + tenders, pages, err := c.parseSearchPage(readFixture(t, "search.html")) + if err != nil { + t.Fatal(err) + } + if pages != 2 { + t.Errorf("pages = %d; want 2", pages) + } + // The row with no link is skipped. + if len(tenders) != 2 { + t.Fatalf("tenders = %d; want 2: %+v", len(tenders), tenders) + } + + first := tenders[0] + wantPublished := time.Date(2026, 10, 21, 11, 45, 0, 0, time.UTC) // BST + wantDeadline := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) // GMT + switch { + case first.ID != "100114946", first.PID != "109912", + first.Reference != "Proc2852": + t.Errorf("ids = %q, %q, %q", first.ID, first.PID, first.Reference) + case first.Title != "Jisc Google for Education Reseller Framework Agreement": + t.Errorf("title = %q", first.Title) + case first.URL != "https://uk.example/ctm/Supplier/PublicPurchase/109912/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK": + t.Errorf("url = %q", first.URL) + case !first.Published.Equal(wantPublished): + t.Errorf("published = %v; want %v", first.Published, wantPublished) + case first.Deadline == nil || !first.Deadline.Equal(wantDeadline): + t.Errorf("deadline = %v; want %v", first.Deadline, wantDeadline) + case first.Process != "PA23 - Competitive Flexible", first.Buyer != "Jisc", + first.Countries != "United Kingdom": + t.Errorf("cells = %q, %q, %q", first.Process, first.Buyer, first.Countries) + case first.HasDetails: + t.Error("HasDetails set from the search page") + } + + second := tenders[1] + if second.Reference != "" || second.Deadline != nil { + t.Errorf("second = %+v; want no reference or deadline", second) + } + // Zero width spaces are dropped from titles. + if second.Title != "York and North Yorkshire Transport Sub-strategies" { + t.Errorf("second title = %q", second.Title) + } +} + +func TestParseSearchPageChanged(t *testing.T) { + c := New("", WithLogger(slog.New(slog.DiscardHandler))) + page := []byte("Maintenance") + _, _, err := c.parseSearchPage(page) + if !errors.Is(err, ErrNoResults) { + t.Errorf("err = %v; want ErrNoResults", err) + } +} + +func TestParseDetails(t *testing.T) { + tender := Tender{ + ID: "100114946", + } + if err := parseDetails(readFixture(t, "tender.html"), &tender); err != nil { + t.Fatal(err) + } + + wantDescription := "This opportunity is for a framework agreement.\n" + + "The Supplier(s) shall manage the full licence lifecycle (the 'Agreement')." + if tender.Description != wantDescription { + t.Errorf("description = %q", tender.Description) + } + if tender.ContractType != "Supplies" { + t.Errorf("contract type = %q", tender.ContractType) + } + if len(tender.CPVCodes) != 2 || + tender.CPVCodes[0].Code != "48000000-8" || + tender.CPVCodes[0].Description != + "Software package and information systems" { + t.Errorf("cpv codes = %+v", tender.CPVCodes) + } + wantAddress := "Jisc|4 Portwall Lane|BS1 6NB|Bristol|United Kingdom" + if got := strings.Join(tender.BuyerAddress, "|"); got != wantAddress { + t.Errorf("buyer address = %q", got) + } + // The search had no deadline, so the page's is used. + if tender.Deadline == nil || + !tender.Deadline.Equal(time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)) { + t.Errorf("deadline = %v", tender.Deadline) + } + if tender.Lots != 2 || !tender.HasDetails { + t.Errorf("lots = %d, HasDetails = %v", tender.Lots, tender.HasDetails) + } + + err := parseDetails([]byte("Not found"), &tender) + if !errors.Is(err, ErrNoDetails) { + t.Errorf("err = %v; want ErrNoDetails", err) + } +} + +// Search posts the portal's form for each page, and keeps the pages read +// before one fails. +func TestSearch(t *testing.T) { + page := readFixture(t, "search.html") + var ( + mu sync.Mutex + forms []url.Values + ) + fail := false + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != searchPath { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + } + r.ParseForm() + mu.Lock() + forms = append(forms, r.PostForm) + number := len(forms) + mu.Unlock() + if fail && number == 2 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Write(page) + }), + ) + t.Cleanup(srv.Close) + c := newTestClient(srv) + + from := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC) // 15th in the UK + to := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) + tenders, err := c.Search(context.Background(), from, to) + if err != nil { + t.Fatal(err) + } + if len(tenders) != 4 || len(forms) != 2 { + t.Fatalf( + "%d tenders from %d pages; want 4 from 2", len(tenders), len(forms), + ) + } + for i, form := range forms { + wantPage := []string{"1", "2"}[i] + if form.Get("SearchFilter.FromDate") != "15/10/2026" || + form.Get("SearchFilter.ToDate") != "21/10/2026" || + form.Get("SearchFilter.PagingInfo.PageNumber") != wantPage || + form.Get("SearchFilter.PagingInfo.PageSize") != "100" || + form.Get("Branding") != "UK" { + t.Errorf("form %d = %v", i+1, form) + } + } + + forms = nil + fail = true + tenders, err = c.Search(context.Background(), from, to) + if !errors.Is(err, ErrUnexpectedStatus) { + t.Errorf("err = %v; want ErrUnexpectedStatus", err) + } + if len(tenders) != 2 { + t.Errorf("%d tenders; want the first page's 2", len(tenders)) + } +} + +func TestDetails(t *testing.T) { + var path string + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.RequestURI() + w.Write(readFixture(t, "tender.html")) + }), + ) + t.Cleanup(srv.Close) + + tender := Tender{ + ID: "100114946", + PID: "109912", + } + err := newTestClient(srv).Details(context.Background(), &tender) + if err != nil { + t.Fatal(err) + } + if path != tenderPath("109912") || !tender.HasDetails { + t.Errorf("path = %q, HasDetails = %v", path, tender.HasDetails) + } +} + +// The client waits from minGap to maxGap between requests, and a cancelled +// context stops the wait. +func TestRequestGap(t *testing.T) { + var ( + mu sync.Mutex + times []time.Time + ) + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + times = append(times, time.Now()) + mu.Unlock() + io.WriteString(w, "ok") + }), + ) + t.Cleanup(srv.Close) + + const minGap, maxGap = 50 * time.Millisecond, 80 * time.Millisecond + c := New( + srv.URL, + WithRequestGap(minGap, maxGap), + WithLogger(slog.New(slog.DiscardHandler)), + ) + for range 3 { + if _, err := c.get(context.Background(), "/"); err != nil { + t.Fatal(err) + } + } + for i := 1; i < len(times); i++ { + if gap := times[i].Sub(times[i-1]); gap < minGap { + t.Errorf("gap %d = %v; want at least %v", i, gap, minGap) + } + } + + slow := New( + srv.URL, + WithRequestGap(time.Hour, time.Hour), + WithLogger(slog.New(slog.DiscardHandler)), + ) + slow.get(context.Background(), "/") + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := slow.get(ctx, "/"); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v; want context.DeadlineExceeded", err) + } + if time.Since(start) > 5*time.Second { + t.Error("cancelling did not stop the wait") + } +} + +func TestConvert(t *testing.T) { + published := time.Date(2026, 10, 21, 11, 45, 0, 0, time.UTC) + tender := Tender{ + ID: "100114946", + PID: "109912", + Title: "Hazard Management Tool", + Published: published, + Buyer: "UK Atomic Energy Authority", + HasDetails: true, + Description: "A tool", + ContractType: "Services", + CPVCodes: []CPVCode{ + { + Code: "48000000-8", + Description: "Software package and information systems", + }, + }, + Lots: 2, + } + + opportunities := Convert([]Tender{tender}) + if len(opportunities) != 1 { + t.Fatalf("%d opportunities; want 1", len(opportunities)) + } + opportunity := opportunities[0] + switch { + case opportunity.UUID != "eusupply-uk-100114946", + opportunity.Source != Source: + t.Errorf("uuid, source = %q, %q", opportunity.UUID, opportunity.Source) + case opportunity.Title != tender.Title, opportunity.Description != "A tool", + opportunity.BuyerName != tender.Buyer, opportunity.Status != "active": + t.Errorf("opportunity = %+v", opportunity) + case opportunity.MainCategory != "services", !opportunity.MultiLot: + t.Errorf( + "category, multi-lot = %q, %v", + opportunity.MainCategory, opportunity.MultiLot, + ) + case len(opportunity.Classifications) != 1 || + opportunity.Classifications[0].Code != "48000000" || + !opportunity.Classifications[0].IsCPV: + t.Errorf("classifications = %+v", opportunity.Classifications) + case !opportunity.LatestReleaseDate.Equal(published), + opportunity.StartDate == nil || !opportunity.StartDate.Equal(published): + t.Errorf( + "dates = %v, %v", opportunity.LatestReleaseDate, opportunity.StartDate, + ) + case opportunity.Marks.Hidden || opportunity.Marks.Skipped: + t.Error("Convert set marks") + } + + release := opportunity.Releases[0] + if len(opportunity.Releases) != 1 || release.ReleaseID != "100114946" || + len(release.Tags) != 1 || release.Tags[0].Tag != "tender" || + !strings.Contains(release.JSON, `"pid":"109912"`) { + t.Errorf("releases = %+v", opportunity.Releases) + } +} diff --git a/internal/eusupply/html.go b/internal/eusupply/html.go new file mode 100644 index 0000000..4a76f33 --- /dev/null +++ b/internal/eusupply/html.go @@ -0,0 +1,139 @@ +package eusupply + +import ( + "slices" + "strings" + + "golang.org/x/net/html" +) + +// The helpers below read the parsed pages. The site's HTML is server +// rendered, so the elements are found by their ids and classes, as the old +// Node scraper found them with CSS selectors. + +// findAll returns the elements under node (node included) that match, in +// document order. +func findAll(node *html.Node, match func(*html.Node) bool) []*html.Node { + var found []*html.Node + var walk func(*html.Node) + walk = func(current *html.Node) { + if current.Type == html.ElementNode && match(current) { + found = append(found, current) + } + for child := current.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(node) + return found +} + +// findFirst returns the first element under node that matches, or nil. +func findFirst(node *html.Node, match func(*html.Node) bool) *html.Node { + found := findAll(node, match) + if len(found) == 0 { + return nil + } + return found[0] +} + +// isTag matches elements with the tag name. +func isTag(tag string) func(*html.Node) bool { + return func(node *html.Node) bool { + return node.Data == tag + } +} + +// hasID matches the element with the id. +func hasID(id string) func(*html.Node) bool { + return func(node *html.Node) bool { + return attr(node, "id") == id + } +} + +// hasClass matches elements with tag (any, if empty) and the class among +// their classes. +func hasClass(tag, class string) func(*html.Node) bool { + return func(node *html.Node) bool { + if tag != "" && node.Data != tag { + return false + } + return slices.Contains(strings.Fields(attr(node, "class")), class) + } +} + +// children returns node's element children with the tag name. +func children(node *html.Node, tag string) []*html.Node { + var found []*html.Node + for child := node.FirstChild; child != nil; child = child.NextSibling { + if child.Type == html.ElementNode && child.Data == tag { + found = append(found, child) + } + } + return found +} + +// nextElement returns the element after node among its siblings, or nil. +func nextElement(node *html.Node) *html.Node { + sibling := node.NextSibling + for sibling != nil && sibling.Type != html.ElementNode { + sibling = sibling.NextSibling + } + return sibling +} + +// attr returns the value of node's attribute key, or "". +func attr(node *html.Node, key string) string { + for _, attribute := range node.Attr { + if attribute.Key == key { + return attribute.Val + } + } + return "" +} + +// rawText returns the text under node as written, with a newline for each +//
. +func rawText(node *html.Node) string { + var text strings.Builder + var walk func(*html.Node) + walk = func(current *html.Node) { + switch { + case current.Type == html.TextNode: + text.WriteString(current.Data) + case current.Type == html.ElementNode && current.Data == "br": + text.WriteString("\n") + } + for child := current.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(node) + return text.String() +} + +// text returns the text under node with its white space collapsed: runs of +// spaces, non-breaking spaces and line breaks become one space, and the ends +// are trimmed. +func text(node *html.Node) string { + return collapseSpace(rawText(node)) +} + +// lines returns the non-empty lines of the text under node, each with its +// white space collapsed. +func lines(node *html.Node) []string { + var found []string + for line := range strings.SplitSeq(rawText(node), "\n") { + if collapsed := collapseSpace(line); collapsed != "" { + found = append(found, collapsed) + } + } + return found +} + +// collapseSpace collapses runs of white space (non-breaking spaces and zero +// width spaces included) into one space, and trims the ends. +func collapseSpace(value string) string { + value = strings.ReplaceAll(value, "\u200b", "") + return strings.Join(strings.Fields(value), " ") +} diff --git a/internal/eusupply/search.go b/internal/eusupply/search.go new file mode 100644 index 0000000..c97411c --- /dev/null +++ b/internal/eusupply/search.go @@ -0,0 +1,277 @@ +package eusupply + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "golang.org/x/net/html" +) + +// The search form's endpoint and paging. +const ( + // searchPath is where the portal's public search form posts to. + searchPath = "/ctm/Supplier/PublicTenders" + // pageSize is the most results per page the form offers. + pageSize = 100 + // defaultSearchDays is how far back a search with no from date goes, + // as the old scraper's first search did. + defaultSearchDays = 7 + // dateLayout is the form's and the results' date layout; dateTimeLayout + // adds the time of day the results give in their cells' titles. + dateLayout = "02/01/2006" + dateTimeLayout = "02/01/2006 15:04:05" +) + +// ErrNoResults means a search page had no results table, which the portal +// always returns, even for no tenders: the page has changed. +var ErrNoResults = errors.New("eusupply: no search results table on the page") + +// pidPattern finds a tender's portal id (PID) in its link. +var pidPattern = regexp.MustCompile(`[?&]PID=(\d+)`) + +// pagesPattern reads the number of pages from the pager's "of N". +var pagesPattern = regexp.MustCompile(`^of (\d+)$`) + +// Tender is one public tender on the portal. The fields up to Countries are +// from the search results; the rest are from the tender's own page (see +// Client.Details), and are empty until it is read. +type Tender struct { + // ID is the quote/tender id the portal shows ("100114946"). + ID string `json:"id"` + // PID is the portal's internal id, which its pages are addressed by. + PID string `json:"pid"` + Reference string `json:"reference"` + Title string `json:"title"` + URL string `json:"url"` + Published time.Time `json:"published"` + Deadline *time.Time `json:"deadline"` + Process string `json:"process"` + Buyer string `json:"buyer"` + Countries string `json:"countries"` + + // HasDetails is set once the tender's own page has been read. + HasDetails bool `json:"has_details"` + Description string `json:"description,omitempty"` + ContractType string `json:"contract_type,omitempty"` + CPVCodes []CPVCode `json:"cpv_codes,omitempty"` + BuyerAddress []string `json:"buyer_address,omitempty"` + Lots int `json:"lots,omitempty"` +} + +// CPVCode is one of a tender's CPV codes, as its page lists them: the code +// with its check digit ("48000000-8") and its description. +type CPVCode struct { + Code string `json:"code"` + Description string `json:"description"` +} + +// Search returns the tenders published from the date of from to the date +// of until (UK dates, both included), reading every page of results. A +// zero from searches the last 7 days; a zero until searches up to today. If a page +// fails, the tenders of the pages before it are returned with the error. +func (c *Client) Search( + ctx context.Context, from, until time.Time, +) ([]Tender, error) { + if until.IsZero() { + until = time.Now() + } + if from.IsZero() { + from = until.AddDate(0, 0, -defaultSearchDays) + } + + var tenders []Tender + for page := 1; ; page++ { + body, err := c.post(ctx, searchPath, searchForm(from, until, page)) + if err != nil { + return tenders, fmt.Errorf("searching page %d: %w", page, err) + } + + found, pages, err := c.parseSearchPage(body) + if err != nil { + return tenders, fmt.Errorf("reading search page %d: %w", page, err) + } + tenders = append(tenders, found...) + + if page >= pages { + return tenders, nil + } + } +} + +// searchForm returns the search form's fields for one page of the tenders +// published from from to until. They are the fields the portal's own form +// posts, with its defaults; the portal answers 500 if one is missing or +// isn't valid. +func searchForm(from, until time.Time, page int) url.Values { + return url.Values{ + "SearchFilter.SortField": {"None"}, + "SearchFilter.SortDirection": {"None"}, + "SearchFilter.ShortDescription": {""}, + "SearchFilter.Reference": {""}, + "SearchFilter.TenderId": {"0"}, + "SearchFilter.OperatorId": {"0"}, + "Branding": {"UK"}, + "SavedCategoryId": {"0"}, + "SavedUnitAndName": {""}, + "SearchFilter.PublishType": {"1"}, // issued, not awarded + "SearchFilter.BrandingCode": {""}, + "TextFilter": {""}, + "SearchFilter.FromDate": {from.In(ukTime).Format(dateLayout)}, + "SearchFilter.ToDate": {until.In(ukTime).Format(dateLayout)}, + "CpvContainer.CpvCodes": {""}, + "CpvContainer.CpvMain": {""}, + "CpvContainer.ContractType": {""}, + "CpvContainer.CpvIds": {""}, + "CpvContainer.IsMandatory": {"True"}, + "SearchFilter.ShowExpiredRft": {"false"}, + "SearchFilter.PagingInfo.PageNumber": {strconv.Itoa(page)}, + "SearchFilter.PagingInfo.PageSize": {strconv.Itoa(pageSize)}, + } +} + +// parseSearchPage reads the tenders on a page of search results, and the +// number of pages of results. A row that can't be read is logged and +// skipped. +func (c *Client) parseSearchPage(body []byte) ([]Tender, int, error) { + document, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return nil, 0, err + } + + container := findFirst(document, hasID("searchResultContainer")) + if container == nil { + return nil, 0, ErrNoResults + } + table := findFirst(container, isTag("table")) + if table == nil { + return nil, 0, ErrNoResults + } + + var tenders []Tender + for _, body := range children(table, "tbody") { + for _, row := range children(body, "tr") { + tender, err := c.parseResultRow(row) + if err != nil { + c.logger().Warn( + "skipping unreadable search result", + "row", text(row), + "error", err, + ) + continue + } + tenders = append(tenders, tender) + } + } + + return tenders, searchPages(document), nil +} + +// parseResultRow reads one row of search results. Its cells are the id, +// reference, title (a link to the tender), publication date, response +// deadline, process, buyers and countries; each date cell's title has the +// full time. +func (c *Client) parseResultRow(row *html.Node) (Tender, error) { + cells := children(row, "td") + if len(cells) < 8 { + return Tender{}, fmt.Errorf( + "eusupply: a result row has %d cells, not 8", len(cells), + ) + } + + link := findFirst(cells[2], isTag("a")) + if link == nil { + return Tender{}, errors.New("eusupply: a result row has no link") + } + match := pidPattern.FindStringSubmatch(attr(link, "href")) + if match == nil { + return Tender{}, fmt.Errorf( + "eusupply: no PID in the link %q", attr(link, "href"), + ) + } + + published, err := cellTime(cells[3]) + if err != nil { + return Tender{}, fmt.Errorf("publication date: %w", err) + } + + tender := Tender{ + ID: text(cells[0]), + PID: match[1], + Reference: text(cells[1]), + Title: text(link), + URL: c.tenderURL(match[1]), + Published: published, + Process: text(cells[5]), + Buyer: text(cells[6]), + Countries: text(cells[7]), + } + if tender.ID == "" { + return Tender{}, errors.New("eusupply: a result row has no id") + } + + // A tender may have no deadline yet (e.g. market engagement). + if deadline, err := cellTime(cells[4]); err == nil { + tender.Deadline = &deadline + } + + return tender, nil +} + +// tenderURL returns the address of the public page of the tender with pid. +func (c *Client) tenderURL(pid string) string { + return c.baseURL + tenderPath(pid) +} + +// tenderPath returns the path of the public page of the tender with pid, +// which the search results' links redirect to. +func tenderPath(pid string) string { + return "/ctm/Supplier/PublicPurchase/" + url.PathEscape(pid) + + "/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK" +} + +// cellTime reads the UK time in the title of a result cell's tooltip. +func cellTime(cell *html.Node) (time.Time, error) { + tooltip := findFirst(cell, hasClass("div", "js-tooltip-content")) + if tooltip == nil { + return time.Time{}, errors.New("eusupply: no time in the cell") + } + return parseUKTime(attr(tooltip, "title")) +} + +// parseUKTime reads a portal time ("21/10/2026 12:45:00"; the seconds may +// be left out) in UK time. +func parseUKTime(value string) (time.Time, error) { + value = strings.TrimSpace(value) + parsed, err := time.ParseInLocation(dateTimeLayout, value, ukTime) + if err != nil { + parsed, err = time.ParseInLocation("02/01/2006 15:04", value, ukTime) + } + if err != nil { + return time.Time{}, fmt.Errorf("eusupply: time %q: %w", value, err) + } + return parsed, nil +} + +// searchPages reads the number of pages from the pager ("Page 1 of 3"), or +// 1 if there is no pager. +func searchPages(document *html.Node) int { + for _, pager := range findAll(document, hasClass("ul", "ctm-pager-info")) { + for _, item := range children(pager, "li") { + match := pagesPattern.FindStringSubmatch(text(item)) + if match == nil { + continue + } + if pages, err := strconv.Atoi(match[1]); err == nil && pages > 0 { + return pages + } + } + } + return 1 +} diff --git a/internal/eusupply/testdata/search.html b/internal/eusupply/testdata/search.html new file mode 100644 index 0000000..6e0721a --- /dev/null +++ b/internal/eusupply/testdata/search.html @@ -0,0 +1,45 @@ + + + +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Quote/tender IdReferenceNameDate of publicationResponse deadline (UK time)ProcessBuyersCountries
100114946
Proc2852
Jisc Google for Education Reseller Framework Agreement
21/10/2026
04/11/2026 12:00
PA23 - Competitive Flexible
Jisc
United Kingdom
100115168
​​York and North Yorkshire Transport Sub-strategies
20/10/2026
PA23 - Preliminary Market Engagement
York and North Yorkshire Combined Authority
United Kingdom
100115999
A row with no link
20/10/2026
+
  • of 2
+
  • items per page
+
+
+ + diff --git a/internal/eusupply/testdata/tender.html b/internal/eusupply/testdata/tender.html new file mode 100644 index 0000000..c97f12f --- /dev/null +++ b/internal/eusupply/testdata/tender.html @@ -0,0 +1,42 @@ + + + +
+
+
+
Detailed description:
+

This opportunity is for a framework agreement. +The Supplier(s) shall manage the full licence lifecycle (the 'Agreement').

+
Type of contract:
+

Supplies

+
Response deadline (UK time):
+

04/11/2026 12:00:00

+
CPV codes:
+

+48000000-8  Software package and information systems
+72220000-3  Systems and technical consultancy services
+

+
+
+
Buyer:
+

+
+Jisc
+4 Portwall Lane
+BS1 6NB
+Bristol
+United Kingdom +
+ +View profile + +

+
Contact:
+


https://www.jisc.ac.uk

+
+
+
+
Lot 1
+
Lot 2
+ + diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index 27b0394..0cb2e23 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -185,6 +185,18 @@ func (f *Fetcher) Run(ctx context.Context, since time.Time) error { return runFTSMock(ctx, logger, &f.conf.FTSMock, f.save, since) }) } + if f.conf.EUSupply.Enabled { + // A nil *store.Store in the interface would not be nil. + var lookup tenderStore + if f.store != nil { + lookup = f.store + } + start(euSupplySource, func(ctx context.Context, since time.Time) error { + return runEUSupply( + ctx, logger, &f.conf.EUSupply, lookup, f.save, since, + ) + }) + } if f.conf.Sleeper.Enabled { start("Sleeper", func(ctx context.Context, since time.Time) error { return runSleeper(ctx, logger, &f.conf.Sleeper, since) diff --git a/internal/fetcher/match.go b/internal/fetcher/match.go new file mode 100644 index 0000000..cfbc434 --- /dev/null +++ b/internal/fetcher/match.go @@ -0,0 +1,252 @@ +package fetcher + +import ( + "math" + "strings" + "unicode" + + "git.avsdev.uk/Go/job-search/internal/types" +) + +// titleMatchThreshold is how alike two titles must be (see titleSimilarity) +// for an EU Supply tender to be taken as one already stored from another +// source, such as the Find a Tender notice the portal publishes for most of +// its tenders (the user's rule: a 90% match). +const titleMatchThreshold = 0.9 + +// The other thresholds of a match (see confirmedRepeat). The user's rule +// (2026-10-21): a title match must also be backed by the buyer or the +// description, at a ratio of Claude's choosing; 0.8 kept every real repeat +// in a live run that either could check, and rejected the one false match. +const ( + // buyerMatchThreshold is how much of the shorter buyer name's + // significant words must be in the other's (see buyerSimilarity). + buyerMatchThreshold = 0.8 + // descriptionMatchThreshold is how much of the shorter description's + // words must be in the other's (see descriptionSimilarity). + descriptionMatchThreshold = 0.8 + // wordMatchThreshold is how alike two words of buyer names must be to + // count as the same word, so a typo or a plural still matches. + wordMatchThreshold = 0.8 + // minDescriptionWordLength is the shortest word a description is + // compared by; shorter ones ("the", "and", "for") say little. + minDescriptionWordLength = 4 +) + +// genericBuyerWords are left out of buyer names before comparing them: +// legal forms and kinds of body, which the two sites give differently +// ("Jisc" and "JISC SERVICES LIMITED", "The Borough Council of Calderdale" +// and "Borough Council of Calderdale") and which say nothing about which +// buyer it is. +var genericBuyerWords = map[string]bool{ + "the": true, "of": true, "and": true, "for": true, + "limited": true, "ltd": true, "plc": true, "llp": true, "cic": true, + "council": true, "borough": true, "metropolitan": true, "city": true, + "county": true, "district": true, "authority": true, "services": true, + "group": true, "trust": true, +} + +// confirmedRepeat reports whether other, whose title already matches +// opportunity's (see similarTitles), is the same tender: their buyers or +// their descriptions must match too. Generic titles ("Pest control +// services") are bought by many buyers, so the title alone isn't enough. +func confirmedRepeat(opportunity, other *types.Opportunity) bool { + buyer, buyerKnown := buyerSimilarity( + opportunity.BuyerName, other.BuyerName, + ) + if buyerKnown && buyer >= buyerMatchThreshold { + return true + } + description, descriptionKnown := descriptionSimilarity( + opportunity.Description, other.Description, + ) + return descriptionKnown && description >= descriptionMatchThreshold +} + +// buyerSimilarity returns the share of the shorter buyer name's significant +// words (all but genericBuyerWords) that are in the other name, each at +// least wordMatchThreshold alike to one of its words. It is false if either +// name has no significant words, when the buyers can't be compared. +func buyerSimilarity(first, second string) (float64, bool) { + firstWords := significantWords(first) + secondWords := significantWords(second) + if len(firstWords) == 0 || len(secondWords) == 0 { + return 0, false + } + if len(secondWords) < len(firstWords) { + firstWords, secondWords = secondWords, firstWords + } + + found := 0 + for _, word := range firstWords { + for _, other := range secondWords { + if similarWords(word, other) { + found++ + break + } + } + } + return float64(found) / float64(len(firstWords)), true +} + +// significantWords returns the words of a buyer name, normalised as titles +// are, without genericBuyerWords. +func significantWords(name string) []string { + var words []string + for word := range strings.FieldsSeq(normaliseTitle(name)) { + if !genericBuyerWords[word] { + words = append(words, word) + } + } + return words +} + +// similarWords reports whether two words are at least wordMatchThreshold +// alike (see titleSimilarity). +func similarWords(first, second string) bool { + firstRunes, secondRunes := []rune(first), []rune(second) + longest := max(len(firstRunes), len(secondRunes)) + limit := int(math.Floor((1-wordMatchThreshold)*float64(longest) + 1e-9)) + _, within := editDistance(firstRunes, secondRunes, limit) + return within +} + +// descriptionSimilarity returns the share of the shorter description's +// distinct words (of minDescriptionWordLength letters or more) that are in +// the other. It is false if either has no such words: EU Supply leaves +// some descriptions empty. +func descriptionSimilarity(first, second string) (float64, bool) { + firstWords := descriptionWords(first) + secondWords := descriptionWords(second) + if len(firstWords) == 0 || len(secondWords) == 0 { + return 0, false + } + if len(secondWords) < len(firstWords) { + firstWords, secondWords = secondWords, firstWords + } + + found := 0 + for word := range firstWords { + if secondWords[word] { + found++ + } + } + return float64(found) / float64(len(firstWords)), true +} + +// descriptionWords returns the distinct words of a description, normalised +// as titles are, of minDescriptionWordLength letters or more. +func descriptionWords(description string) map[string]bool { + words := make(map[string]bool) + for word := range strings.FieldsSeq(normaliseTitle(description)) { + if len([]rune(word)) >= minDescriptionWordLength { + words[word] = true + } + } + return words +} + +// normaliseTitle returns title as titles are compared: lower case, with +// each run of anything but letters and numbers (punctuation, white space, +// zero width spaces) made one space, and trimmed. +func normaliseTitle(title string) string { + var normalised strings.Builder + space := false + for _, character := range strings.ToLower(title) { + if unicode.IsLetter(character) || unicode.IsNumber(character) { + if space && normalised.Len() > 0 { + normalised.WriteByte(' ') + } + normalised.WriteRune(character) + space = false + continue + } + space = true + } + return normalised.String() +} + +// titleSimilarity returns how alike two normalised titles (see +// normaliseTitle) are, from 0 to 1: one less their edit distance (the +// characters inserted, deleted or changed to turn one into the other) over +// the longer one's length. Two empty titles are not alike (0). +func titleSimilarity(first, second string) float64 { + firstRunes, secondRunes := []rune(first), []rune(second) + longest := max(len(firstRunes), len(secondRunes)) + if longest == 0 { + return 0 + } + distance, _ := editDistance(firstRunes, secondRunes, longest) + return 1 - float64(distance)/float64(longest) +} + +// similarTitles reports whether two normalised titles are at least +// titleMatchThreshold alike. It gives up as soon as the distance can't be +// small enough, so it is much faster than titleSimilarity for titles that +// aren't. +func similarTitles(first, second string) bool { + firstRunes, secondRunes := []rune(first), []rune(second) + longest := max(len(firstRunes), len(secondRunes)) + if longest == 0 { + return false + } + // The epsilon stops (1-0.9)*10 = 0.99999... rounding down to 0. + limit := int(math.Floor((1-titleMatchThreshold)*float64(longest) + 1e-9)) + _, within := editDistance(firstRunes, secondRunes, limit) + return within +} + +// editDistance returns the Levenshtein distance between source and target, +// and whether it is at most limit. Only the cells within limit of the +// diagonal are computed, and it stops once a whole row exceeds limit, when +// the distance returned is limit+1. +func editDistance(source, target []rune, limit int) (int, bool) { + if abs(len(source)-len(target)) > limit { + return limit + 1, false + } + + exceeded := limit + 1 + previous := make([]int, len(target)+1) + current := make([]int, len(target)+1) + for column := range previous { + previous[column] = min(column, exceeded) + } + + for row := 1; row <= len(source); row++ { + low, high := max(1, row-limit), min(len(target), row+limit) + current[0] = min(row, exceeded) + if low > 1 { + current[low-1] = exceeded + } + rowMin := current[0] + for column := low; column <= high; column++ { + cost := 1 + if source[row-1] == target[column-1] { + cost = 0 + } + value := min( + previous[column]+1, current[column-1]+1, previous[column-1]+cost, + ) + current[column] = min(value, exceeded) + rowMin = min(rowMin, current[column]) + } + if high < len(target) { + current[high+1] = exceeded + } + if rowMin > limit { + return exceeded, false + } + previous, current = current, previous + } + + distance := previous[len(target)] + return distance, distance <= limit +} + +// abs returns the absolute value of value. +func abs(value int) int { + if value < 0 { + return -value + } + return value +} diff --git a/internal/fetcher/match_test.go b/internal/fetcher/match_test.go new file mode 100644 index 0000000..d0a39e5 --- /dev/null +++ b/internal/fetcher/match_test.go @@ -0,0 +1,208 @@ +package fetcher + +import ( + "math/rand" + "testing" + + "git.avsdev.uk/Go/job-search/internal/types" +) + +func TestNormaliseTitle(t *testing.T) { + tests := []struct { + title string + want string + }{ + {"Pest Control Services", "pest control services"}, + {" RRE-01284 - For the provision ", "rre 01284 for the provision"}, + {"\u200b\u200bThe supply 3 kg of Li₂TiO₃", "the supply 3 kg of li₂tio₃"}, + {"Children & Young People's", "children young people s"}, + {"---", ""}, + } + for _, tt := range tests { + if got := normaliseTitle(tt.title); got != tt.want { + t.Errorf("normaliseTitle(%q) = %q; want %q", tt.title, got, tt.want) + } + } +} + +func TestTitleSimilarity(t *testing.T) { + tests := []struct { + first, second string + want float64 + }{ + {"pest control services", "pest control services", 1}, + {"abcdefghij", "abcdefghix", 0.9}, + {"abcdefghij", "abcdefghxx", 0.8}, + {"kitten", "sitting", 1 - 3.0/7}, + {"", "", 0}, + {"abc", "", 0}, + } + for _, tt := range tests { + got := titleSimilarity(tt.first, tt.second) + if got < tt.want-1e-9 || got > tt.want+1e-9 { + t.Errorf( + "titleSimilarity(%q, %q) = %v; want %v", + tt.first, tt.second, got, tt.want, + ) + } + } +} + +func TestSimilarTitles(t *testing.T) { + tests := []struct { + first, second string + want bool + }{ + {"housing related support", "housing related support", true}, + // One edit in ten is exactly 90%. + {"abcdefghij", "abcdefghix", true}, + {"abcdefghij", "abcdefghxx", false}, + {"pest control services", "pest control service", true}, + {"alternative provision", "alternative provision for schools", false}, + {"", "", false}, + } + for _, tt := range tests { + if got := similarTitles(tt.first, tt.second); got != tt.want { + t.Errorf( + "similarTitles(%q, %q) = %v; want %v", + tt.first, tt.second, got, tt.want, + ) + } + } +} + +// similarTitles' banded, early-stopping distance agrees with the full one. +func TestSimilarTitlesAgreesWithTitleSimilarity(t *testing.T) { + //nolint:gosec // G404: seeded math/rand keeps the test deterministic + rng := rand.New(rand.NewSource(1)) + alphabet := []rune("abc d") + random := func() string { + runes := make([]rune, 5+rng.Intn(30)) + for i := range runes { + runes[i] = alphabet[rng.Intn(len(alphabet))] + } + return string(runes) + } + mutate := func(title string) string { + runes := []rune(title) + for range rng.Intn(5) { + i := rng.Intn(len(runes)) + switch rng.Intn(3) { + case 0: + runes[i] = alphabet[rng.Intn(len(alphabet))] + case 1: + runes = append(runes[:i], runes[i+1:]...) + default: + runes = append(runes[:i], append([]rune{'x'}, runes[i:]...)...) + } + if len(runes) == 0 { + runes = []rune("a") + } + } + return string(runes) + } + + for range 5000 { + first := random() + second := mutate(first) + if rng.Intn(4) == 0 { + second = random() + } + want := titleSimilarity(first, second) >= titleMatchThreshold-1e-9 + if got := similarTitles(first, second); got != want { + t.Fatalf( + "similarTitles(%q, %q) = %v; titleSimilarity = %v", + first, second, got, titleSimilarity(first, second), + ) + } + } +} + +func TestBuyerSimilarity(t *testing.T) { + tests := []struct { + first, second string + want float64 + known bool + }{ + {"Jisc", "JISC SERVICES LIMITED", 1, true}, + {"The Borough Council of Calderdale", "Borough Council of Calderdale", 1, true}, + {"Wakefield Council", "The Council of the City of Wakefield", 1, true}, + {"City of York Council", "York and North Yorkshire Combined Authority", 1, true}, + {"UK Atomic Energy Authority", "UK Atomic Energy Authoritys", 1, true}, + {"Yorkshire Housing Limited", "Anchor Hanover Group", 0, true}, + // Only the shorter name's words count, so a name within another + // matches it, rightly for York above, wrongly here: the title and + // date must match too, so this is accepted. + {"North East Lincolnshire", "North Lincolnshire Council", 1, true}, + {"Hull City Council", "East Riding of Yorkshire Council", 0, true}, + {"The Council", "Jisc", 0, false}, + {"", "Jisc", 0, false}, + } + for _, tt := range tests { + got, known := buyerSimilarity(tt.first, tt.second) + if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 { + t.Errorf( + "buyerSimilarity(%q, %q) = %v, %v; want %v, %v", + tt.first, tt.second, got, known, tt.want, tt.known, + ) + } + } +} + +func TestDescriptionSimilarity(t *testing.T) { + tests := []struct { + first, second string + want float64 + known bool + }{ + {"Supply of pest control.", "Pest control: the supply", 1, true}, + {"Building materials for repairs", "Building works", 0.5, true}, + {"", "Anything at all", 0, false}, + {"a to be", "Anything at all", 0, false}, + } + for _, tt := range tests { + got, known := descriptionSimilarity(tt.first, tt.second) + if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 { + t.Errorf( + "descriptionSimilarity(%q, %q) = %v, %v; want %v, %v", + tt.first, tt.second, got, known, tt.want, tt.known, + ) + } + } +} + +// A title match is confirmed by the buyer or, failing that, the +// description; with neither to compare, it isn't. +func TestConfirmedRepeat(t *testing.T) { + opportunity := func(buyer, description string) *types.Opportunity { + return &types.Opportunity{ + BuyerName: buyer, + Description: description, + } + } + tests := []struct { + name string + first, last *types.Opportunity + want bool + }{ + {"buyer", opportunity("Jisc", ""), opportunity("JISC LIMITED", ""), true}, + { + "description, not buyer", + opportunity("Echelon", "building materials supply"), + opportunity("Fairhive", "supply of building materials"), + true, + }, + { + "neither", + opportunity("Yorkshire Housing", "pest control in Leeds homes"), + opportunity("Anchor Hanover", "pest control for retirement housing"), + false, + }, + {"nothing to compare", opportunity("", ""), opportunity("", ""), false}, + } + for _, tt := range tests { + if got := confirmedRepeat(tt.first, tt.last); got != tt.want { + t.Errorf("%s: confirmedRepeat = %v; want %v", tt.name, got, tt.want) + } + } +} diff --git a/internal/fetcher/worker_eusupply.go b/internal/fetcher/worker_eusupply.go new file mode 100644 index 0000000..6f23ffe --- /dev/null +++ b/internal/fetcher/worker_eusupply.go @@ -0,0 +1,249 @@ +package fetcher + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "git.avsdev.uk/Go/job-search/internal/config" + "git.avsdev.uk/Go/job-search/internal/eusupply" + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" + "git.avsdev.uk/Go/job-search/internal/util" +) + +// euSupplySource is the Source of the opportunities the EU Supply source +// fetches. +const euSupplySource = eusupply.Source + +// repeatWindow is how far apart an EU Supply tender's publication and +// another source's opportunity's latest release may be for the two to be +// the same (see markRepeats). +const repeatWindow = 30 * 24 * time.Hour + +// tenderStore is what the EU Supply source reads from the store, to skip +// tenders already stored and to find the ones another source has. +type tenderStore interface { + ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) + ListOtherSourcesTitles( + ctx context.Context, source string, from, before time.Time, + ) ([]store.OpportunityTitle, error) + GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) +} + +// runEUSupply scrapes EU Supply's UK portal every configured interval until +// ctx is cancelled, passing what each run fetches to save, even when the run +// fails partway. A long window is backfilled first (see backfill). One +// client is used throughout, so its gap between requests holds across runs. +// lookup is nil without a store: every tender is then new, and none is +// checked for repeats. +func runEUSupply( + ctx context.Context, + logger *slog.Logger, + conf *config.JSCSourceEUSupply, + lookup tenderStore, + save saveFunc, + since time.Time, +) error { + client := eusupply.New( + conf.BaseURL, + eusupply.WithLogger(logger), + eusupply.WithRequestGap(conf.RequestMinGap, conf.RequestMaxGap), + ) + fetch := func( + ctx context.Context, since, until time.Time, + ) ([]*types.Opportunity, error) { + return fetchEUSupply(ctx, logger, client, lookup, since, until) + } + + since = backfill( + ctx, + logger, + euSupplySource, + conf.Interval, + fetch, + save, + since, + ) + + worker := util.Worker{ + Logger: tagged(logger), + Name: euSupplySource, + Interval: conf.Interval, + Task: func(ctx context.Context, since time.Time) error { + opportunities, err := fetch(ctx, since, time.Time{}) + return saveFetched(ctx, logger, save, opportunities, err) + }, + } + return worker.Run(ctx, since) +} + +// fetchEUSupply fetches the tenders published from since up to until (zero: +// today) that aren't stored yet, with their own pages read, and marks the +// ones another source already has Hidden and Skipped (see markRepeats). If +// it fails partway, the tenders read in full before the error are returned +// along with it. Finding nothing new returns ErrNoNewReleases. +func fetchEUSupply( + ctx context.Context, + logger *slog.Logger, + client *eusupply.Client, + lookup tenderStore, + since, until time.Time, +) ([]*types.Opportunity, error) { + tenders, fetchErr := client.Search(ctx, since, until) + + fresh, err := newTenders(ctx, lookup, tenders) + if err != nil { + return nil, err + } + + for i := range fresh { + if err := client.Details(ctx, &fresh[i]); err != nil { + fresh = fresh[:i] + fetchErr = errors.Join(fetchErr, err) + break + } + } + + opportunities := eusupply.Convert(fresh) + if err := markRepeats(ctx, logger, lookup, opportunities); err != nil { + // Saved unmarked, a repeat would stay visible, as marks are only + // stored for a new opportunity: fetch them again next run instead. + return nil, errors.Join(fetchErr, err) + } + + tagged(logger).Info( + "fetched opportunities", + "source", euSupplySource, + "listed", len(tenders), + "count", len(opportunities), + ) + + if len(opportunities) == 0 && fetchErr == nil { + return nil, ErrNoNewReleases + } + return opportunities, fetchErr +} + +// newTenders returns the tenders whose release isn't stored yet, so only +// their pages are read. Saving a stored one again would replace its row +// with the search's fields alone. With no lookup, every tender is new. +func newTenders( + ctx context.Context, lookup tenderStore, tenders []eusupply.Tender, +) ([]eusupply.Tender, error) { + if lookup == nil { + return tenders, nil + } + + var fresh []eusupply.Tender + for _, tender := range tenders { + exists, err := lookup.ReleaseExists( + ctx, eusupply.UUID(tender.ID), eusupply.ReleaseID(tender), + ) + if err != nil { + return nil, fmt.Errorf("checking tender %s: %w", tender.ID, err) + } + if !exists { + fresh = append(fresh, tender) + } + } + + return fresh, nil +} + +// markRepeats marks Hidden and Skipped each opportunity that repeats one +// from another source, most often its Find a Tender notice: its title is at +// least titleMatchThreshold alike (see similarTitles), its latest release +// is within repeatWindow, and its buyer or description matches too (see +// confirmedRepeat). Only the opportunities whose titles match are read in +// full. The store keeps the marks of an opportunity it already has, so only +// new ones are marked. With no lookup it does nothing. +func markRepeats( + ctx context.Context, + logger *slog.Logger, + lookup tenderStore, + opportunities []*types.Opportunity, +) error { + if lookup == nil || len(opportunities) == 0 { + return nil + } + + earliest := opportunities[0].LatestReleaseDate + latest := earliest + for _, opportunity := range opportunities { + earliest = minTime(earliest, opportunity.LatestReleaseDate) + latest = maxTime(latest, opportunity.LatestReleaseDate) + } + others, err := lookup.ListOtherSourcesTitles( + ctx, + euSupplySource, + earliest.Add(-repeatWindow), + latest.Add(repeatWindow+time.Nanosecond), + ) + if err != nil { + return err + } + + normalised := make([]string, len(others)) + for i, other := range others { + normalised[i] = normaliseTitle(other.Title) + } + + repeats := 0 + for _, opportunity := range opportunities { + title := normaliseTitle(opportunity.Title) + for i, other := range others { + gap := other.LatestReleaseDate.Sub(opportunity.LatestReleaseDate) + if gap.Abs() > repeatWindow || !similarTitles(title, normalised[i]) { + continue + } + repeated, err := lookup.GetOpportunity(ctx, other.UUID) + if err != nil { + return fmt.Errorf("reading opportunity %s: %w", other.UUID, err) + } + if !confirmedRepeat(opportunity, repeated) { + tagged(logger).Debug( + "title matched, but not buyer or description", + "uuid", opportunity.UUID, + "title_matches", other.UUID, + ) + continue + } + opportunity.Marks.Hidden = true + opportunity.Marks.Skipped = true + repeats++ + tagged(logger).Debug( + "hid repeated opportunity", + "uuid", opportunity.UUID, + "repeats", other.UUID, + ) + break + } + } + + tagged(logger).Info( + "hid repeated opportunities", + "source", euSupplySource, + "count", repeats, + ) + + return nil +} + +// minTime returns the earlier of two times. +func minTime(first, second time.Time) time.Time { + if second.Before(first) { + return second + } + return first +} + +// maxTime returns the later of two times. +func maxTime(first, second time.Time) time.Time { + if second.After(first) { + return second + } + return first +} diff --git a/internal/fetcher/worker_eusupply_test.go b/internal/fetcher/worker_eusupply_test.go new file mode 100644 index 0000000..d9ef7eb --- /dev/null +++ b/internal/fetcher/worker_eusupply_test.go @@ -0,0 +1,324 @@ +package fetcher + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/config" + "git.avsdev.uk/Go/job-search/internal/eusupply" + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" +) + +var errTestStore = errors.New("test store failed") + +// testTender is one row of a fake EU Supply search page, and its page. +type testTender struct { + id, pid, title string + published string // "21/10/2026 12:45:00" + buyer string + description string +} + +// euSupplySite is a fake EU Supply portal: one page of search results with +// tenders, and a page for each of them. It counts the tender pages read. +type euSupplySite struct { + mu sync.Mutex + tenders []testTender + detailed []string // the PIDs whose pages were read + failPID string // a PID whose page fails +} + +func (s *euSupplySite) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if r.Method == http.MethodPost { + var rows strings.Builder + for _, tender := range s.tenders { + fmt.Fprintf( + &rows, + `
%s
`+ + `
`+ + `%s`+ + `
`+ + `%s`, + tender.id, tender.pid, tender.title, tender.published, tender.buyer, + ) + } + fmt.Fprintf( + w, + `
%s`+ + `
`, + rows.String(), + ) + return + } + + pid := strings.Split(strings.TrimPrefix( + r.URL.Path, "/ctm/Supplier/PublicPurchase/", + ), "/")[0] + s.detailed = append(s.detailed, pid) + if pid == s.failPID { + w.WriteHeader(http.StatusInternalServerError) + return + } + description := "" + for _, tender := range s.tenders { + if tender.pid == pid { + description = tender.description + } + } + fmt.Fprintf( + w, + `
Detailed description:
`+ + `

%s

`, + description, + ) +} + +// fakeTenderStore is a tenderStore with some releases stored and other +// sources' opportunities. +type fakeTenderStore struct { + stored map[string]bool // uuid "/" release id + others []*types.Opportunity + titlesErr error +} + +func (s *fakeTenderStore) ReleaseExists( + ctx context.Context, uuid, releaseID string, +) (bool, error) { + return s.stored[uuid+"/"+releaseID], nil +} + +func (s *fakeTenderStore) ListOtherSourcesTitles( + ctx context.Context, source string, from, before time.Time, +) ([]store.OpportunityTitle, error) { + if s.titlesErr != nil { + return nil, s.titlesErr + } + var titles []store.OpportunityTitle + for _, other := range s.others { + if !other.LatestReleaseDate.Before(from) && + other.LatestReleaseDate.Before(before) { + titles = append(titles, store.OpportunityTitle{ + UUID: other.UUID, + Title: other.Title, + LatestReleaseDate: other.LatestReleaseDate, + }) + } + } + return titles, nil +} + +func (s *fakeTenderStore) GetOpportunity( + ctx context.Context, uuid string, +) (*types.Opportunity, error) { + for _, other := range s.others { + if other.UUID == uuid { + return other, nil + } + } + return nil, store.ErrNotFound +} + +// newEUSupplyClient returns a client for srv that doesn't wait. +func newEUSupplyClient(srv *httptest.Server) *eusupply.Client { + return eusupply.New(srv.URL, eusupply.WithRequestGap(0, 0)) +} + +// Only tenders not stored are read and returned. The ones another source +// has within the window, with a matching title and a matching buyer or +// description, are marked Hidden and Skipped. +func TestFetchEUSupply(t *testing.T) { + const day = "20/10/2026 10:00:00" + site := &euSupplySite{ + tenders: []testTender{ + {"1", "11", "Already stored", day, "", ""}, + // The same generic title from another buyer: not a repeat. + {"2", "12", "Pest Control Services", day, "Yorkshire Housing Limited", + "Pest control across our homes in Leeds"}, + // The same buyer, named differently: a repeat. + {"3", "13", "Google Reseller Framework", day, "Jisc", ""}, + // An agent buying for the buyer, but the same description. + {"4", "14", "Fairhive - Materials Procurement", day, + "Echelon Consultancy", + "Supply of building materials for Fairhive housing repairs"}, + // Alike, but too long before. + {"5", "15", "Housing related support", day, "Hull City Council", ""}, + }, + } + srv := httptest.NewServer(site) + t.Cleanup(srv.Close) + + published := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC) + other := func( + uuid, title, buyer, description string, released time.Time, + ) *types.Opportunity { + return &types.Opportunity{ + UUID: uuid, + Title: title, + BuyerName: buyer, + Description: description, + LatestReleaseDate: released, + } + } + lookup := &fakeTenderStore{ + stored: map[string]bool{ + "eusupply-uk-1/1": true, + }, + others: []*types.Opportunity{ + other("ocds-pest", "PEST CONTROL SERVICE", "Anchor Hanover Group", + "Pest control for retirement housing", published), + other("ocds-jisc", "Google Reseller Framework", + "JISC SERVICES LIMITED", "", published.Add(time.Hour)), + other("ocds-fairhive", "Fairhive - Materials Procurement", + "Fairhive Housing Limited", + "Fairhive seeks supply of building materials for housing repairs", + published), + other("ocds-old", "Housing related support", "Hull City Council", "", + published.Add(-40*24*time.Hour)), + }, + } + + opportunities, err := fetchEUSupply( + context.Background(), + slog.Default(), + newEUSupplyClient(srv), + lookup, + published.Add(-24*time.Hour), + time.Time{}, + ) + if err != nil { + t.Fatal(err) + } + + if got := strings.Join(site.detailed, ","); got != "12,13,14,15" { + t.Errorf("pages read = %s; want the new tenders' only", got) + } + marks := map[string]bool{} + for _, opportunity := range opportunities { + if opportunity.Source != "EU Supply - UK" { + t.Errorf("%s: Source = %q", opportunity.UUID, opportunity.Source) + } + if opportunity.Marks.Hidden != opportunity.Marks.Skipped { + t.Errorf("%s: marks = %+v", opportunity.UUID, opportunity.Marks) + } + marks[opportunity.UUID] = opportunity.Marks.Hidden + } + want := map[string]bool{ + "eusupply-uk-2": false, + "eusupply-uk-3": true, + "eusupply-uk-4": true, + "eusupply-uk-5": false, + } + if fmt.Sprint(marks) != fmt.Sprint(want) { + t.Errorf("hidden = %v; want %v", marks, want) + } +} + +// A tender page that fails stops the run, keeping the tenders read before +// it; a failed title lookup returns nothing, so nothing is saved unmarked. +func TestFetchEUSupplyErrors(t *testing.T) { + site := &euSupplySite{ + tenders: []testTender{ + {"1", "11", "First", "20/10/2026 10:00:00", "", ""}, + {"2", "12", "Second", "20/10/2026 10:00:00", "", ""}, + {"3", "13", "Third", "20/10/2026 10:00:00", "", ""}, + }, + failPID: "12", + } + srv := httptest.NewServer(site) + t.Cleanup(srv.Close) + client := newEUSupplyClient(srv) + since := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC) + + opportunities, err := fetchEUSupply( + context.Background(), slog.Default(), client, nil, since, time.Time{}, + ) + if !errors.Is(err, eusupply.ErrUnexpectedStatus) { + t.Errorf("err = %v; want ErrUnexpectedStatus", err) + } + if len(opportunities) != 1 || opportunities[0].UUID != "eusupply-uk-1" { + t.Errorf("opportunities = %v; want the first tender", opportunities) + } + + site.failPID = "" + lookup := &fakeTenderStore{ + titlesErr: errTestStore, + } + opportunities, err = fetchEUSupply( + context.Background(), slog.Default(), client, lookup, since, time.Time{}, + ) + if !errors.Is(err, errTestStore) || opportunities != nil { + t.Errorf( + "err = %v, %d opportunities; want the store's error and none", + err, len(opportunities), + ) + } + + site.tenders = nil + _, err = fetchEUSupply( + context.Background(), slog.Default(), client, lookup, since, time.Time{}, + ) + if !errors.Is(err, ErrNoNewReleases) { + t.Errorf("no tenders: err = %v; want ErrNoNewReleases", err) + } +} + +// Run starts the EU Supply source when it is enabled, and saves its +// opportunities with their Source. +func TestRunEUSupply(t *testing.T) { + site := &euSupplySite{ + tenders: []testTender{ + { + "1", "11", "A tender", + time.Now().Format("02/01/2006 15:04:05"), "", "", + }, + }, + } + srv := httptest.NewServer(site) + t.Cleanup(srv.Close) + + opportunityStore := openTestStore(t) + sources := &config.JSCSources{} + sources.EUSupply.Enabled = true + sources.EUSupply.Interval = time.Hour + sources.EUSupply.BaseURL = srv.URL + sources.EUSupply.RequestMinGap = time.Millisecond + sources.EUSupply.RequestMaxGap = time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- New(sources, WithStore(opportunityStore)).Run(ctx, time.Time{}) + }() + + deadline := time.Now().Add(5 * time.Second) + var stored []string + for time.Now().Before(deadline) { + opportunity, err := opportunityStore.GetOpportunity( + context.Background(), "eusupply-uk-1", + ) + if err == nil { + stored = append(stored, opportunity.Source) + break + } + time.Sleep(5 * time.Millisecond) + } + cancel() + if err := <-done; err != nil { + t.Fatalf("Run = %v", err) + } + if len(stored) != 1 || stored[0] != "EU Supply - UK" { + t.Errorf("stored sources = %v; want EU Supply - UK", stored) + } +} diff --git a/internal/store/queries.go b/internal/store/queries.go index 457321f..58e5a37 100644 --- a/internal/store/queries.go +++ b/internal/store/queries.go @@ -149,6 +149,13 @@ LIMIT ? OFFSET ?` )` ) +// selectOtherSourcesTitles lists the uuid, title and latest_release_date of +// the opportunities not from one source whose latest_release_date is in +// [from, before); it walks the latest_release_date index. +const selectOtherSourcesTitles = ` +SELECT uuid, title, latest_release_date FROM opportunities +WHERE source <> ? AND latest_release_date >= ? AND latest_release_date < ?` + // ---------- opportunity_releases ---------- // insertRelease stores a release; 0 rows affected means it was already diff --git a/internal/store/queries_test.go b/internal/store/queries_test.go index 82196c6..db9557a 100644 --- a/internal/store/queries_test.go +++ b/internal/store/queries_test.go @@ -36,6 +36,9 @@ var checks = map[string]queryCheck{ "selectSourceLatestReleaseDate": { sql: selectSourceLatestReleaseDate, }, + "selectOtherSourcesTitles": { + sql: selectOtherSourcesTitles, + }, "upsertOpportunity": {sql: upsertOpportunity, named: true}, "selectOpportunityByUUID": {sql: selectOpportunityByUUID}, "listOpportunitiesSelect": { diff --git a/internal/store/store.go b/internal/store/store.go index ad3f4ce..ea89dfd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1041,6 +1041,37 @@ func (s *Store) LatestReleaseDate( return latest, nil } +// OpportunityTitle is an opportunity's UUID, title and latest release date, +// as ListOtherSourcesTitles lists them. +type OpportunityTitle struct { + UUID string `db:"uuid"` + Title string `db:"title"` + LatestReleaseDate time.Time `db:"latest_release_date"` +} + +// ListOtherSourcesTitles returns the UUID, title and latest release date of +// every opportunity not from source whose latest release is from from +// (inclusive) to before (exclusive), so a fetcher can find the ones a new +// opportunity repeats. +func (s *Store) ListOtherSourcesTitles( + ctx context.Context, source string, from, before time.Time, +) ([]OpportunityTitle, error) { + titles := []OpportunityTitle{} + err := s.db.SelectContext( + ctx, + &titles, + selectOtherSourcesTitles, + source, + from.UTC(), + before.UTC(), + ) + if err != nil { + return nil, fmt.Errorf("listing titles of sources but %s: %w", source, err) + } + + return titles, nil +} + // ReleaseExists reports whether a release is already stored, so a fetcher // can skip converting it. func (s *Store) ReleaseExists( diff --git a/internal/store/store_test.go b/internal/store/store_test.go index bbd4f5c..46f459b 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -866,6 +866,32 @@ func TestLatestReleaseDate(t *testing.T) { } } +// ListOtherSourcesTitles lists only other sources' opportunities, with +// their latest release in [from, before). +func TestListOtherSourcesTitles(t *testing.T) { + s := openStore(t) + ctx := context.Background() + + mustSave(t, s, opportunity("ocds-1", "Too early", 1, "r1")) + mustSave(t, s, opportunity("ocds-2", "In the window", 3, "r1")) + mustSave(t, s, opportunity("ocds-3", "At the end", 5, "r1")) + own := opportunity("eusupply-uk-1", "Its own source", 3, "r1") + own.Source = "EU Supply - UK" + mustSave(t, s, own) + + titles, err := s.ListOtherSourcesTitles( + ctx, "EU Supply - UK", day(2), day(5), + ) + if err != nil { + t.Fatal(err) + } + if len(titles) != 1 || titles[0].UUID != "ocds-2" || + titles[0].Title != "In the window" || + !titles[0].LatestReleaseDate.Equal(day(3)) { + t.Errorf("titles = %+v; want only ocds-2", titles) + } +} + // NewOpportunity is set only for an opportunity that wasn't already stored, // by both SaveOpportunity and SaveOpportunities. func TestSaveResultNewOpportunity(t *testing.T) {