bwilliams
3a06132071
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
63 lines
2.0 KiB
Go
63 lines
2.0 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"reflect"
|
|
"time"
|
|
)
|
|
|
|
// JSCKSourceEUSupply is the EU Supply source's configuration key.
|
|
const JSCKSourceEUSupply string = "eu_supply"
|
|
|
|
// JSCSourceEUSupply configures scraping the public tenders on EU Supply's
|
|
// UK portal at BaseURL (see internal/eusupply). The client waits a random
|
|
// time from RequestMinGap to RequestMaxGap after each request; both must be
|
|
// greater than 0, as the site's own limit isn't published, and a
|
|
// RequestMaxGap less than RequestMinGap is read as RequestMinGap.
|
|
type JSCSourceEUSupply struct {
|
|
JSCSourcesSource
|
|
BaseURL string `cfg:"base_url" default:"https://uk.eu-supply.com"`
|
|
RequestMinGap time.Duration `cfg:"request_min_gap" default:"2s"`
|
|
RequestMaxGap time.Duration `cfg:"request_max_gap" default:"10s"`
|
|
}
|
|
|
|
// ---------- Section methods (see Section) ----------
|
|
|
|
// Map implements Section.
|
|
func (c JSCSourceEUSupply) Map() map[string]any { return mapOf(c) }
|
|
|
|
// Flatten implements Section.
|
|
func (c JSCSourceEUSupply) Flatten() map[string]any {
|
|
return flattenOf(JSCKSourceEUSupply, c)
|
|
}
|
|
|
|
// TypeOf implements Section.
|
|
func (c JSCSourceEUSupply) TypeOf(key string) (reflect.Kind, error) {
|
|
return typeOf(c, key)
|
|
}
|
|
|
|
// ValueOf implements Section.
|
|
func (c JSCSourceEUSupply) ValueOf(key string) (any, error) {
|
|
return valueOf(c, key)
|
|
}
|
|
|
|
// SetValue implements Section.
|
|
func (c *JSCSourceEUSupply) SetValue(key string, value any) error {
|
|
return setValue(c, key, value)
|
|
}
|
|
|
|
// ---------- Section specific methods ----------
|
|
|
|
// validate checks values before they are stored (see validator), as
|
|
// JSCSourcesSource's does, which this one hides. request_min_gap and
|
|
// request_max_gap must be greater than 0.
|
|
func (c *JSCSourceEUSupply) validate(key string, value any) error {
|
|
if (key == "request_min_gap" || key == "request_max_gap") &&
|
|
value.(time.Duration) <= 0 {
|
|
return errors.New(
|
|
"config: request_min_gap and request_max_gap must be greater than 0, as EU Supply publishes no rate limit",
|
|
)
|
|
}
|
|
return c.JSCSourcesSource.validate(key, value)
|
|
}
|