bwilliams
3a06132071
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
109 lines
2.9 KiB
Go
109 lines
2.9 KiB
Go
package eusupply
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
// ErrNoDetails means a tender's page had none of the labelled fields the
|
|
// public page always has: the page has changed, or isn't a tender's.
|
|
var ErrNoDetails = errors.New("eusupply: no tender details on the page")
|
|
|
|
// Details reads tender's own page and fills in its description, contract
|
|
// type, CPV codes, buyer address and number of lots, and sets HasDetails.
|
|
func (c *Client) Details(ctx context.Context, tender *Tender) error {
|
|
body, err := c.get(ctx, tenderPath(tender.PID))
|
|
if err != nil {
|
|
return fmt.Errorf("reading tender %s: %w", tender.ID, err)
|
|
}
|
|
|
|
if err := parseDetails(body, tender); err != nil {
|
|
return fmt.Errorf("reading tender %s: %w", tender.ID, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// parseDetails reads a tender's page into tender. Each field is a
|
|
// "ctm-content-label" div ("CPV codes:") followed by a <p> holding its
|
|
// value.
|
|
func parseDetails(body []byte, tender *Tender) error {
|
|
document, err := html.Parse(bytes.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
values := make(map[string]*html.Node)
|
|
labels := findAll(document, hasClass("div", "ctm-content-label"))
|
|
for _, label := range labels {
|
|
value := nextElement(label)
|
|
if value == nil || value.Data != "p" {
|
|
continue
|
|
}
|
|
name := strings.TrimSuffix(text(label), ":")
|
|
if _, found := values[name]; !found {
|
|
values[name] = value
|
|
}
|
|
}
|
|
if len(values) == 0 {
|
|
return ErrNoDetails
|
|
}
|
|
|
|
if value := values["Detailed description"]; value != nil {
|
|
tender.Description = strings.TrimSpace(rawText(value))
|
|
}
|
|
if value := values["Type of contract"]; value != nil {
|
|
tender.ContractType = text(value)
|
|
}
|
|
if value := values["CPV codes"]; value != nil {
|
|
tender.CPVCodes = parseCPVCodes(value)
|
|
}
|
|
if value := values["Buyer"]; value != nil {
|
|
tender.BuyerAddress = buyerAddress(value)
|
|
}
|
|
if deadline := values["Response deadline (UK time)"]; deadline != nil &&
|
|
tender.Deadline == nil {
|
|
if parsed, err := parseUKTime(text(deadline)); err == nil {
|
|
tender.Deadline = &parsed
|
|
}
|
|
}
|
|
tender.Lots = len(findAll(document, hasClass("div", "package-container")))
|
|
tender.HasDetails = true
|
|
|
|
return nil
|
|
}
|
|
|
|
// parseCPVCodes reads the CPV codes listed in value, one per <span>: the
|
|
// code, two non-breaking spaces, then its description.
|
|
func parseCPVCodes(value *html.Node) []CPVCode {
|
|
var codes []CPVCode
|
|
for _, span := range findAll(value, isTag("span")) {
|
|
code, description, _ := strings.Cut(text(span), " ")
|
|
if code == "" {
|
|
continue
|
|
}
|
|
codes = append(codes, CPVCode{
|
|
Code: code,
|
|
Description: description,
|
|
})
|
|
}
|
|
return codes
|
|
}
|
|
|
|
// buyerAddress reads the buyer's name and address, one line each, leaving
|
|
// out the "View profile" link.
|
|
func buyerAddress(value *html.Node) []string {
|
|
var address []string
|
|
for _, line := range lines(value) {
|
|
if line != "View profile" {
|
|
address = append(address, line)
|
|
}
|
|
}
|
|
return address
|
|
}
|