Added an EU Supply source, scraping the UK portal's public tenders

internal/eusupply replaces the old Node scraper (eu-supply.js) for
uk.eu-supply.com. The site has no API, so it posts the public search
form, which now needs the form's current defaults and reads the HTML
with golang.org/x/net/html. Each tender not stored yet has its own
page read too, for its description, contract type, CPV codes and
buyer address. Tenders are stored with the source "EU Supply - UK"
as eusupply-uk-<tender id>, with one release holding the tender as
read.

The site publishes no rate limit, so the client sets its own: one
request at a time, with a random wait between
sources.eu_supply.request_min_gap and request_max_gap (2s to 10s).
The source has the same 7-day backfill as Find a Tender's.

Most of its tenders are also on Find a Tender, so a new one is stored
Hidden and Skipped when another source's opportunity released within
30 days has a title at least 90% alike, and its buyer or description
matches too (80% of the shorter one's significant words).

internal/eusupply/search.go

@@ -0,0 +1,277 @@
+package eusupply
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+// The search form's endpoint and paging.
+const (
+	// searchPath is where the portal's public search form posts to.
+	searchPath = "/ctm/Supplier/PublicTenders"
+	// pageSize is the most results per page the form offers.
+	pageSize = 100
+	// defaultSearchDays is how far back a search with no from date goes,
+	// as the old scraper's first search did.
+	defaultSearchDays = 7
+	// dateLayout is the form's and the results' date layout; dateTimeLayout
+	// adds the time of day the results give in their cells' titles.
+	dateLayout     = "02/01/2006"
+	dateTimeLayout = "02/01/2006 15:04:05"
+)
+
+// ErrNoResults means a search page had no results table, which the portal
+// always returns, even for no tenders: the page has changed.
+var ErrNoResults = errors.New("eusupply: no search results table on the page")
+
+// pidPattern finds a tender's portal id (PID) in its link.
+var pidPattern = regexp.MustCompile(`[?&]PID=(\d+)`)
+
+// pagesPattern reads the number of pages from the pager's "of N".
+var pagesPattern = regexp.MustCompile(`^of (\d+)$`)
+
+// Tender is one public tender on the portal. The fields up to Countries are
+// from the search results; the rest are from the tender's own page (see
+// Client.Details), and are empty until it is read.
+type Tender struct {
+	// ID is the quote/tender id the portal shows ("100114946").
+	ID string `json:"id"`
+	// PID is the portal's internal id, which its pages are addressed by.
+	PID       string     `json:"pid"`
+	Reference string     `json:"reference"`
+	Title     string     `json:"title"`
+	URL       string     `json:"url"`
+	Published time.Time  `json:"published"`
+	Deadline  *time.Time `json:"deadline"`
+	Process   string     `json:"process"`
+	Buyer     string     `json:"buyer"`
+	Countries string     `json:"countries"`
+
+	// HasDetails is set once the tender's own page has been read.
+	HasDetails   bool      `json:"has_details"`
+	Description  string    `json:"description,omitempty"`
+	ContractType string    `json:"contract_type,omitempty"`
+	CPVCodes     []CPVCode `json:"cpv_codes,omitempty"`
+	BuyerAddress []string  `json:"buyer_address,omitempty"`
+	Lots         int       `json:"lots,omitempty"`
+}
+
+// CPVCode is one of a tender's CPV codes, as its page lists them: the code
+// with its check digit ("48000000-8") and its description.
+type CPVCode struct {
+	Code        string `json:"code"`
+	Description string `json:"description"`
+}
+
+// Search returns the tenders published from the date of from to the date
+// of until (UK dates, both included), reading every page of results. A
+// zero from searches the last 7 days; a zero until searches up to today. If a page
+// fails, the tenders of the pages before it are returned with the error.
+func (c *Client) Search(
+	ctx context.Context, from, until time.Time,
+) ([]Tender, error) {
+	if until.IsZero() {
+		until = time.Now()
+	}
+	if from.IsZero() {
+		from = until.AddDate(0, 0, -defaultSearchDays)
+	}
+
+	var tenders []Tender
+	for page := 1; ; page++ {
+		body, err := c.post(ctx, searchPath, searchForm(from, until, page))
+		if err != nil {
+			return tenders, fmt.Errorf("searching page %d: %w", page, err)
+		}
+
+		found, pages, err := c.parseSearchPage(body)
+		if err != nil {
+			return tenders, fmt.Errorf("reading search page %d: %w", page, err)
+		}
+		tenders = append(tenders, found...)
+
+		if page >= pages {
+			return tenders, nil
+		}
+	}
+}
+
+// searchForm returns the search form's fields for one page of the tenders
+// published from from to until. They are the fields the portal's own form
+// posts, with its defaults; the portal answers 500 if one is missing or
+// isn't valid.
+func searchForm(from, until time.Time, page int) url.Values {
+	return url.Values{
+		"SearchFilter.SortField":             {"None"},
+		"SearchFilter.SortDirection":         {"None"},
+		"SearchFilter.ShortDescription":      {""},
+		"SearchFilter.Reference":             {""},
+		"SearchFilter.TenderId":              {"0"},
+		"SearchFilter.OperatorId":            {"0"},
+		"Branding":                           {"UK"},
+		"SavedCategoryId":                    {"0"},
+		"SavedUnitAndName":                   {""},
+		"SearchFilter.PublishType":           {"1"}, // issued, not awarded
+		"SearchFilter.BrandingCode":          {""},
+		"TextFilter":                         {""},
+		"SearchFilter.FromDate":              {from.In(ukTime).Format(dateLayout)},
+		"SearchFilter.ToDate":                {until.In(ukTime).Format(dateLayout)},
+		"CpvContainer.CpvCodes":              {""},
+		"CpvContainer.CpvMain":               {""},
+		"CpvContainer.ContractType":          {""},
+		"CpvContainer.CpvIds":                {""},
+		"CpvContainer.IsMandatory":           {"True"},
+		"SearchFilter.ShowExpiredRft":        {"false"},
+		"SearchFilter.PagingInfo.PageNumber": {strconv.Itoa(page)},
+		"SearchFilter.PagingInfo.PageSize":   {strconv.Itoa(pageSize)},
+	}
+}
+
+// parseSearchPage reads the tenders on a page of search results, and the
+// number of pages of results. A row that can't be read is logged and
+// skipped.
+func (c *Client) parseSearchPage(body []byte) ([]Tender, int, error) {
+	document, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return nil, 0, err
+	}
+
+	container := findFirst(document, hasID("searchResultContainer"))
+	if container == nil {
+		return nil, 0, ErrNoResults
+	}
+	table := findFirst(container, isTag("table"))
+	if table == nil {
+		return nil, 0, ErrNoResults
+	}
+
+	var tenders []Tender
+	for _, body := range children(table, "tbody") {
+		for _, row := range children(body, "tr") {
+			tender, err := c.parseResultRow(row)
+			if err != nil {
+				c.logger().Warn(
+					"skipping unreadable search result",
+					"row", text(row),
+					"error", err,
+				)
+				continue
+			}
+			tenders = append(tenders, tender)
+		}
+	}
+
+	return tenders, searchPages(document), nil
+}
+
+// parseResultRow reads one row of search results. Its cells are the id,
+// reference, title (a link to the tender), publication date, response
+// deadline, process, buyers and countries; each date cell's title has the
+// full time.
+func (c *Client) parseResultRow(row *html.Node) (Tender, error) {
+	cells := children(row, "td")
+	if len(cells) < 8 {
+		return Tender{}, fmt.Errorf(
+			"eusupply: a result row has %d cells, not 8", len(cells),
+		)
+	}
+
+	link := findFirst(cells[2], isTag("a"))
+	if link == nil {
+		return Tender{}, errors.New("eusupply: a result row has no link")
+	}
+	match := pidPattern.FindStringSubmatch(attr(link, "href"))
+	if match == nil {
+		return Tender{}, fmt.Errorf(
+			"eusupply: no PID in the link %q", attr(link, "href"),
+		)
+	}
+
+	published, err := cellTime(cells[3])
+	if err != nil {
+		return Tender{}, fmt.Errorf("publication date: %w", err)
+	}
+
+	tender := Tender{
+		ID:        text(cells[0]),
+		PID:       match[1],
+		Reference: text(cells[1]),
+		Title:     text(link),
+		URL:       c.tenderURL(match[1]),
+		Published: published,
+		Process:   text(cells[5]),
+		Buyer:     text(cells[6]),
+		Countries: text(cells[7]),
+	}
+	if tender.ID == "" {
+		return Tender{}, errors.New("eusupply: a result row has no id")
+	}
+
+	// A tender may have no deadline yet (e.g. market engagement).
+	if deadline, err := cellTime(cells[4]); err == nil {
+		tender.Deadline = &deadline
+	}
+
+	return tender, nil
+}
+
+// tenderURL returns the address of the public page of the tender with pid.
+func (c *Client) tenderURL(pid string) string {
+	return c.baseURL + tenderPath(pid)
+}
+
+// tenderPath returns the path of the public page of the tender with pid,
+// which the search results' links redirect to.
+func tenderPath(pid string) string {
+	return "/ctm/Supplier/PublicPurchase/" + url.PathEscape(pid) +
+		"/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK"
+}
+
+// cellTime reads the UK time in the title of a result cell's tooltip.
+func cellTime(cell *html.Node) (time.Time, error) {
+	tooltip := findFirst(cell, hasClass("div", "js-tooltip-content"))
+	if tooltip == nil {
+		return time.Time{}, errors.New("eusupply: no time in the cell")
+	}
+	return parseUKTime(attr(tooltip, "title"))
+}
+
+// parseUKTime reads a portal time ("21/10/2026 12:45:00"; the seconds may
+// be left out) in UK time.
+func parseUKTime(value string) (time.Time, error) {
+	value = strings.TrimSpace(value)
+	parsed, err := time.ParseInLocation(dateTimeLayout, value, ukTime)
+	if err != nil {
+		parsed, err = time.ParseInLocation("02/01/2006 15:04", value, ukTime)
+	}
+	if err != nil {
+		return time.Time{}, fmt.Errorf("eusupply: time %q: %w", value, err)
+	}
+	return parsed, nil
+}
+
+// searchPages reads the number of pages from the pager ("Page 1 of 3"), or
+// 1 if there is no pager.
+func searchPages(document *html.Node) int {
+	for _, pager := range findAll(document, hasClass("ul", "ctm-pager-info")) {
+		for _, item := range children(pager, "li") {
+			match := pagesPattern.FindStringSubmatch(text(item))
+			if match == nil {
+				continue
+			}
+			if pages, err := strconv.Atoi(match[1]); err == nil && pages > 0 {
+				return pages
+			}
+		}
+	}
+	return 1
+}