Added an EU Supply source, scraping the UK portal's public tenders

internal/eusupply replaces the old Node scraper (eu-supply.js) for
uk.eu-supply.com. The site has no API, so it posts the public search
form, which now needs the form's current defaults and reads the HTML
with golang.org/x/net/html. Each tender not stored yet has its own
page read too, for its description, contract type, CPV codes and
buyer address. Tenders are stored with the source "EU Supply - UK"
as eusupply-uk-<tender id>, with one release holding the tender as
read.

The site publishes no rate limit, so the client sets its own: one
request at a time, with a random wait between
sources.eu_supply.request_min_gap and request_max_gap (2s to 10s).
The source has the same 7-day backfill as Find a Tender's.

Most of its tenders are also on Find a Tender, so a new one is stored
Hidden and Skipped when another source's opportunity released within
30 days has a title at least 90% alike, and its buyer or description
matches too (80% of the shorter one's significant words).

internal/config/README.md

@@ -75,9 +75,13 @@ supported.
 | `sources.find_a_tender.rate_limit_retries` | int | `5` | How many times a request refused for the API's rate limit is waited out (for its `Retry-After`) and retried. 0 fails at the first; can't be negative. `fetcher fts-once --ignore-rate-limit` uses 0 |
 | `sources.fts_mock.api_root` | string | | URL of the mock FTS server (see `internal/mock`). Empty: the source fails to start if enabled |
 | `sources.fts_mock.rate_limit_retries` | int | `5` | As for `find_a_tender` |
+| `sources.eu_supply.base_url` | string | `https://uk.eu-supply.com` | Root of EU Supply's UK portal (see `internal/eusupply`) |
+| `sources.eu_supply.request_min_gap` | duration | `2s` | The shortest wait between two requests to the portal. Must be greater than 0: the site publishes no rate limit |
+| `sources.eu_supply.request_max_gap` | duration | `10s` | The longest wait between two requests; each wait is random, from `request_min_gap`. Must be greater than 0; less than `request_min_gap` is read as `request_min_gap` |
 | `sources.sleeper.block_shutdown` | bool | `false` | Test source: if true, its sleep ignores shutdown |
 
-The current sources are `find_a_tender`, `fts_mock` and `sleeper`. See
+The current sources are `find_a_tender`, `fts_mock`, `eu_supply` and
+`sleeper`. See
 "Addition of a new opportunities source" in the top level README for how to add
 a new source.
 

internal/config/conf_sections_test.go

@@ -288,6 +288,35 @@ func TestConcreteSourceSections(t *testing.T) {
 	}
 }
 
+func TestEUSupplySection(t *testing.T) {
+	conf := defaultConfig(t)
+	got := conf.Sources.EUSupply
+	if got.Enabled || got.Interval != 10*time.Minute ||
+		got.BaseURL != "https://uk.eu-supply.com" ||
+		got.RequestMinGap != 2*time.Second ||
+		got.RequestMaxGap != 10*time.Second {
+		t.Errorf("defaults = %+v", got)
+	}
+
+	c := JSCSourceEUSupply{}
+	if err := c.SetValue("request_min_gap", "5s"); err != nil ||
+		c.RequestMinGap != 5*time.Second {
+		t.Errorf("SetValue(request_min_gap, 5s): %v, %v", err, c.RequestMinGap)
+	}
+	for _, key := range []string{"request_min_gap", "request_max_gap"} {
+		if err := c.SetValue(key, "0s"); err == nil {
+			t.Errorf("SetValue(%s, 0s) succeeded", key)
+		}
+	}
+	// Its own validate must still check interval, as JSCSourcesSource's does.
+	if err := c.SetValue("interval", "0s"); err == nil {
+		t.Error("SetValue(interval, 0s) succeeded")
+	}
+	if v, err := c.ValueOf("base_url"); err != nil || v != "" {
+		t.Errorf("ValueOf(base_url) = %v, %v", v, err)
+	}
+}
+
 func TestSourcesSection(t *testing.T) {
 	var c JSCSources
 	if err := c.SetValue("fts_mock.api_root", "http://mock"); err != nil {
@@ -317,8 +346,8 @@ func TestSourcesSection(t *testing.T) {
 	if flat["sources.fts_mock.api_root"] != "http://mock" {
 		t.Errorf("Flatten missing sources.fts_mock.api_root: %v", flat)
 	}
-	if len(flat) != 12 {
-		t.Errorf("Flatten has %d keys; want 12: %v", len(flat), flat)
+	if len(flat) != 17 {
+		t.Errorf("Flatten has %d keys; want 17: %v", len(flat), flat)
 	}
 
 	if err := c.SetValue("skip_rules_file", "rules.json"); err != nil ||

internal/config/conf_source_eusupply.go

@@ -0,0 +1,62 @@
+package config
+
+import (
+	"errors"
+	"reflect"
+	"time"
+)
+
+// JSCKSourceEUSupply is the EU Supply source's configuration key.
+const JSCKSourceEUSupply string = "eu_supply"
+
+// JSCSourceEUSupply configures scraping the public tenders on EU Supply's
+// UK portal at BaseURL (see internal/eusupply). The client waits a random
+// time from RequestMinGap to RequestMaxGap after each request; both must be
+// greater than 0, as the site's own limit isn't published, and a
+// RequestMaxGap less than RequestMinGap is read as RequestMinGap.
+type JSCSourceEUSupply struct {
+	JSCSourcesSource
+	BaseURL       string        `cfg:"base_url"        default:"https://uk.eu-supply.com"`
+	RequestMinGap time.Duration `cfg:"request_min_gap" default:"2s"`
+	RequestMaxGap time.Duration `cfg:"request_max_gap" default:"10s"`
+}
+
+// ---------- Section methods (see Section) ----------
+
+// Map implements Section.
+func (c JSCSourceEUSupply) Map() map[string]any { return mapOf(c) }
+
+// Flatten implements Section.
+func (c JSCSourceEUSupply) Flatten() map[string]any {
+	return flattenOf(JSCKSourceEUSupply, c)
+}
+
+// TypeOf implements Section.
+func (c JSCSourceEUSupply) TypeOf(key string) (reflect.Kind, error) {
+	return typeOf(c, key)
+}
+
+// ValueOf implements Section.
+func (c JSCSourceEUSupply) ValueOf(key string) (any, error) {
+	return valueOf(c, key)
+}
+
+// SetValue implements Section.
+func (c *JSCSourceEUSupply) SetValue(key string, value any) error {
+	return setValue(c, key, value)
+}
+
+// ---------- Section specific methods ----------
+
+// validate checks values before they are stored (see validator), as
+// JSCSourcesSource's does, which this one hides. request_min_gap and
+// request_max_gap must be greater than 0.
+func (c *JSCSourceEUSupply) validate(key string, value any) error {
+	if (key == "request_min_gap" || key == "request_max_gap") &&
+		value.(time.Duration) <= 0 {
+		return errors.New(
+			"config: request_min_gap and request_max_gap must be greater than 0, as EU Supply publishes no rate limit",
+		)
+	}
+	return c.JSCSourcesSource.validate(key, value)
+}

internal/config/conf_sources.go

@@ -20,6 +20,7 @@ type JSCSources struct {
 	SkipRulesFile string               `cfg:"skip_rules_file"`
 	FindATender   JSCSourceFindATender `cfg:"find_a_tender"`
 	FTSMock       JSCSourceFTSMock     `cfg:"fts_mock"`
+	EUSupply      JSCSourceEUSupply    `cfg:"eu_supply"`
 	Sleeper       JSCSourceSleeper     `cfg:"sleeper"`
 }