Added an EU Supply source, scraping the UK portal's public tenders

internal/eusupply replaces the old Node scraper (eu-supply.js) for
uk.eu-supply.com. The site has no API, so it posts the public search
form, which now needs the form's current defaults and reads the HTML
with golang.org/x/net/html. Each tender not stored yet has its own
page read too, for its description, contract type, CPV codes and
buyer address. Tenders are stored with the source "EU Supply - UK"
as eusupply-uk-<tender id>, with one release holding the tender as
read.

The site publishes no rate limit, so the client sets its own: one
request at a time, with a random wait between
sources.eu_supply.request_min_gap and request_max_gap (2s to 10s).
The source has the same 7-day backfill as Find a Tender's.

Most of its tenders are also on Find a Tender, so a new one is stored
Hidden and Skipped when another source's opportunity released within
30 days has a title at least 90% alike, and its buyer or description
matches too (80% of the shorter one's significant words).

internal/store/queries.go

@@ -149,6 +149,13 @@ LIMIT ? OFFSET ?`
 )`
 )
 
+// selectOtherSourcesTitles lists the uuid, title and latest_release_date of
+// the opportunities not from one source whose latest_release_date is in
+// [from, before); it walks the latest_release_date index.
+const selectOtherSourcesTitles = `
+SELECT uuid, title, latest_release_date FROM opportunities
+WHERE source <> ? AND latest_release_date >= ? AND latest_release_date < ?`
+
 // ---------- opportunity_releases ----------
 
 // insertRelease stores a release; 0 rows affected means it was already

internal/store/queries_test.go

@@ -36,6 +36,9 @@ var checks = map[string]queryCheck{
 	"selectSourceLatestReleaseDate": {
 		sql: selectSourceLatestReleaseDate,
 	},
+	"selectOtherSourcesTitles": {
+		sql: selectOtherSourcesTitles,
+	},
 	"upsertOpportunity":       {sql: upsertOpportunity, named: true},
 	"selectOpportunityByUUID": {sql: selectOpportunityByUUID},
 	"listOpportunitiesSelect": {

internal/store/store.go

@@ -1041,6 +1041,37 @@ func (s *Store) LatestReleaseDate(
 	return latest, nil
 }
 
+// OpportunityTitle is an opportunity's UUID, title and latest release date,
+// as ListOtherSourcesTitles lists them.
+type OpportunityTitle struct {
+	UUID              string    `db:"uuid"`
+	Title             string    `db:"title"`
+	LatestReleaseDate time.Time `db:"latest_release_date"`
+}
+
+// ListOtherSourcesTitles returns the UUID, title and latest release date of
+// every opportunity not from source whose latest release is from from
+// (inclusive) to before (exclusive), so a fetcher can find the ones a new
+// opportunity repeats.
+func (s *Store) ListOtherSourcesTitles(
+	ctx context.Context, source string, from, before time.Time,
+) ([]OpportunityTitle, error) {
+	titles := []OpportunityTitle{}
+	err := s.db.SelectContext(
+		ctx,
+		&titles,
+		selectOtherSourcesTitles,
+		source,
+		from.UTC(),
+		before.UTC(),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("listing titles of sources but %s: %w", source, err)
+	}
+
+	return titles, nil
+}
+
 // ReleaseExists reports whether a release is already stored, so a fetcher
 // can skip converting it.
 func (s *Store) ReleaseExists(

internal/store/store_test.go

@@ -866,6 +866,32 @@ func TestLatestReleaseDate(t *testing.T) {
 	}
 }
 
+// ListOtherSourcesTitles lists only other sources' opportunities, with
+// their latest release in [from, before).
+func TestListOtherSourcesTitles(t *testing.T) {
+	s := openStore(t)
+	ctx := context.Background()
+
+	mustSave(t, s, opportunity("ocds-1", "Too early", 1, "r1"))
+	mustSave(t, s, opportunity("ocds-2", "In the window", 3, "r1"))
+	mustSave(t, s, opportunity("ocds-3", "At the end", 5, "r1"))
+	own := opportunity("eusupply-uk-1", "Its own source", 3, "r1")
+	own.Source = "EU Supply - UK"
+	mustSave(t, s, own)
+
+	titles, err := s.ListOtherSourcesTitles(
+		ctx, "EU Supply - UK", day(2), day(5),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(titles) != 1 || titles[0].UUID != "ocds-2" ||
+		titles[0].Title != "In the window" ||
+		!titles[0].LatestReleaseDate.Equal(day(3)) {
+		t.Errorf("titles = %+v; want only ocds-2", titles)
+	}
+}
+
 // NewOpportunity is set only for an opportunity that wasn't already stored,
 // by both SaveOpportunity and SaveOpportunities.
 func TestSaveResultNewOpportunity(t *testing.T) {