job-search/internal/eusupply/search.go

package eusupply

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/html"
)

// The search form's endpoint and paging.
const (
	// searchPath is where the portal's public search form posts to.
	searchPath = "/ctm/Supplier/PublicTenders"
	// pageSize is the most results per page the form offers.
	pageSize = 100
	// defaultSearchDays is how far back a search with no from date goes,
	// as the old scraper's first search did.
	defaultSearchDays = 7
	// dateLayout is the form's and the results' date layout; dateTimeLayout
	// adds the time of day the results give in their cells' titles.
	dateLayout     = "02/01/2006"
	dateTimeLayout = "02/01/2006 15:04:05"
)

// ErrNoResults means a search page had no results table, which the portal
// always returns, even for no tenders: the page has changed.
var ErrNoResults = errors.New("eusupply: no search results table on the page")

// pidPattern finds a tender's portal id (PID) in its link.
var pidPattern = regexp.MustCompile(`[?&]PID=(\d+)`)

// pagesPattern reads the number of pages from the pager's "of N".
var pagesPattern = regexp.MustCompile(`^of (\d+)$`)

// Tender is one public tender on the portal. The fields up to Countries are
// from the search results; the rest are from the tender's own page (see
// Client.Details), and are empty until it is read.
type Tender struct {
	// ID is the quote/tender id the portal shows ("100114946").
	ID string `json:"id"`
	// PID is the portal's internal id, which its pages are addressed by.
	PID       string     `json:"pid"`
	Reference string     `json:"reference"`
	Title     string     `json:"title"`
	URL       string     `json:"url"`
	Published time.Time  `json:"published"`
	Deadline  *time.Time `json:"deadline"`
	Process   string     `json:"process"`
	Buyer     string     `json:"buyer"`
	Countries string     `json:"countries"`

	// HasDetails is set once the tender's own page has been read.
	HasDetails   bool      `json:"has_details"`
	Description  string    `json:"description,omitempty"`
	ContractType string    `json:"contract_type,omitempty"`
	CPVCodes     []CPVCode `json:"cpv_codes,omitempty"`
	BuyerAddress []string  `json:"buyer_address,omitempty"`
	Lots         int       `json:"lots,omitempty"`
}

// CPVCode is one of a tender's CPV codes, as its page lists them: the code
// with its check digit ("48000000-8") and its description.
type CPVCode struct {
	Code        string `json:"code"`
	Description string `json:"description"`
}

// Search returns the tenders published from the date of from to the date
// of until (UK dates, both included), reading every page of results. A
// zero from searches the last 7 days; a zero until searches up to today. If a page
// fails, the tenders of the pages before it are returned with the error.
func (c *Client) Search(
	ctx context.Context, from, until time.Time,
) ([]Tender, error) {
	if until.IsZero() {
		until = time.Now()
	}
	if from.IsZero() {
		from = until.AddDate(0, 0, -defaultSearchDays)
	}

	var tenders []Tender
	for page := 1; ; page++ {
		body, err := c.post(ctx, searchPath, searchForm(from, until, page))
		if err != nil {
			return tenders, fmt.Errorf("searching page %d: %w", page, err)
		}

		found, pages, err := c.parseSearchPage(body)
		if err != nil {
			return tenders, fmt.Errorf("reading search page %d: %w", page, err)
		}
		tenders = append(tenders, found...)

		if page >= pages {
			return tenders, nil
		}
	}
}

// searchForm returns the search form's fields for one page of the tenders
// published from from to until. They are the fields the portal's own form
// posts, with its defaults; the portal answers 500 if one is missing or
// isn't valid.
func searchForm(from, until time.Time, page int) url.Values {
	return url.Values{
		"SearchFilter.SortField":             {"None"},
		"SearchFilter.SortDirection":         {"None"},
		"SearchFilter.ShortDescription":      {""},
		"SearchFilter.Reference":             {""},
		"SearchFilter.TenderId":              {"0"},
		"SearchFilter.OperatorId":            {"0"},
		"Branding":                           {"UK"},
		"SavedCategoryId":                    {"0"},
		"SavedUnitAndName":                   {""},
		"SearchFilter.PublishType":           {"1"}, // issued, not awarded
		"SearchFilter.BrandingCode":          {""},
		"TextFilter":                         {""},
		"SearchFilter.FromDate":              {from.In(ukTime).Format(dateLayout)},
		"SearchFilter.ToDate":                {until.In(ukTime).Format(dateLayout)},
		"CpvContainer.CpvCodes":              {""},
		"CpvContainer.CpvMain":               {""},
		"CpvContainer.ContractType":          {""},
		"CpvContainer.CpvIds":                {""},
		"CpvContainer.IsMandatory":           {"True"},
		"SearchFilter.ShowExpiredRft":        {"false"},
		"SearchFilter.PagingInfo.PageNumber": {strconv.Itoa(page)},
		"SearchFilter.PagingInfo.PageSize":   {strconv.Itoa(pageSize)},
	}
}

// parseSearchPage reads the tenders on a page of search results, and the
// number of pages of results. A row that can't be read is logged and
// skipped.
func (c *Client) parseSearchPage(body []byte) ([]Tender, int, error) {
	document, err := html.Parse(bytes.NewReader(body))
	if err != nil {
		return nil, 0, err
	}

	container := findFirst(document, hasID("searchResultContainer"))
	if container == nil {
		return nil, 0, ErrNoResults
	}
	table := findFirst(container, isTag("table"))
	if table == nil {
		return nil, 0, ErrNoResults
	}

	var tenders []Tender
	for _, body := range children(table, "tbody") {
		for _, row := range children(body, "tr") {
			tender, err := c.parseResultRow(row)
			if err != nil {
				c.logger().Warn(
					"skipping unreadable search result",
					"row", text(row),
					"error", err,
				)
				continue
			}
			tenders = append(tenders, tender)
		}
	}

	return tenders, searchPages(document), nil
}

// parseResultRow reads one row of search results. Its cells are the id,
// reference, title (a link to the tender), publication date, response
// deadline, process, buyers and countries; each date cell's title has the
// full time.
func (c *Client) parseResultRow(row *html.Node) (Tender, error) {
	cells := children(row, "td")
	if len(cells) < 8 {
		return Tender{}, fmt.Errorf(
			"eusupply: a result row has %d cells, not 8", len(cells),
		)
	}

	link := findFirst(cells[2], isTag("a"))
	if link == nil {
		return Tender{}, errors.New("eusupply: a result row has no link")
	}
	match := pidPattern.FindStringSubmatch(attr(link, "href"))
	if match == nil {
		return Tender{}, fmt.Errorf(
			"eusupply: no PID in the link %q", attr(link, "href"),
		)
	}

	published, err := cellTime(cells[3])
	if err != nil {
		return Tender{}, fmt.Errorf("publication date: %w", err)
	}

	tender := Tender{
		ID:        text(cells[0]),
		PID:       match[1],
		Reference: text(cells[1]),
		Title:     text(link),
		URL:       c.tenderURL(match[1]),
		Published: published,
		Process:   text(cells[5]),
		Buyer:     text(cells[6]),
		Countries: text(cells[7]),
	}
	if tender.ID == "" {
		return Tender{}, errors.New("eusupply: a result row has no id")
	}

	// A tender may have no deadline yet (e.g. market engagement).
	if deadline, err := cellTime(cells[4]); err == nil {
		tender.Deadline = &deadline
	}

	return tender, nil
}

// tenderURL returns the address of the public page of the tender with pid.
func (c *Client) tenderURL(pid string) string {
	return c.baseURL + tenderPath(pid)
}

// tenderPath returns the path of the public page of the tender with pid,
// which the search results' links redirect to.
func tenderPath(pid string) string {
	return "/ctm/Supplier/PublicPurchase/" + url.PathEscape(pid) +
		"/0/0?returnUrl=ctm/Supplier/PublicTenders&b=UK"
}

// cellTime reads the UK time in the title of a result cell's tooltip.
func cellTime(cell *html.Node) (time.Time, error) {
	tooltip := findFirst(cell, hasClass("div", "js-tooltip-content"))
	if tooltip == nil {
		return time.Time{}, errors.New("eusupply: no time in the cell")
	}
	return parseUKTime(attr(tooltip, "title"))
}

// parseUKTime reads a portal time ("21/10/2026 12:45:00"; the seconds may
// be left out) in UK time.
func parseUKTime(value string) (time.Time, error) {
	value = strings.TrimSpace(value)
	parsed, err := time.ParseInLocation(dateTimeLayout, value, ukTime)
	if err != nil {
		parsed, err = time.ParseInLocation("02/01/2006 15:04", value, ukTime)
	}
	if err != nil {
		return time.Time{}, fmt.Errorf("eusupply: time %q: %w", value, err)
	}
	return parsed, nil
}

// searchPages reads the number of pages from the pager ("Page 1 of 3"), or
// 1 if there is no pager.
func searchPages(document *html.Node) int {
	for _, pager := range findAll(document, hasClass("ul", "ctm-pager-info")) {
		for _, item := range children(pager, "li") {
			match := pagesPattern.FindStringSubmatch(text(item))
			if match == nil {
				continue
			}
			if pages, err := strconv.Atoi(match[1]); err == nil && pages > 0 {
				return pages
			}
		}
	}
	return 1
}