Added an EU Supply source, scraping the UK portal's public tenders
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
This commit is contained in:
1 parent
5c62376b6e
commit
3a06132071
25 files changed
+2520
-8
No files matched your search
@@ -288,6 +288,35 @@ func TestConcreteSourceSections(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEUSupplySection(t *testing.T) {
|
||||
conf := defaultConfig(t)
|
||||
got := conf.Sources.EUSupply
|
||||
if got.Enabled || got.Interval != 10*time.Minute ||
|
||||
got.BaseURL != "https://uk.eu-supply.com" ||
|
||||
got.RequestMinGap != 2*time.Second ||
|
||||
got.RequestMaxGap != 10*time.Second {
|
||||
t.Errorf("defaults = %+v", got)
|
||||
}
|
||||
|
||||
c := JSCSourceEUSupply{}
|
||||
if err := c.SetValue("request_min_gap", "5s"); err != nil ||
|
||||
c.RequestMinGap != 5*time.Second {
|
||||
t.Errorf("SetValue(request_min_gap, 5s): %v, %v", err, c.RequestMinGap)
|
||||
}
|
||||
for _, key := range []string{"request_min_gap", "request_max_gap"} {
|
||||
if err := c.SetValue(key, "0s"); err == nil {
|
||||
t.Errorf("SetValue(%s, 0s) succeeded", key)
|
||||
}
|
||||
}
|
||||
// Its own validate must still check interval, as JSCSourcesSource's does.
|
||||
if err := c.SetValue("interval", "0s"); err == nil {
|
||||
t.Error("SetValue(interval, 0s) succeeded")
|
||||
}
|
||||
if v, err := c.ValueOf("base_url"); err != nil || v != "" {
|
||||
t.Errorf("ValueOf(base_url) = %v, %v", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourcesSection(t *testing.T) {
|
||||
var c JSCSources
|
||||
if err := c.SetValue("fts_mock.api_root", "http://mock"); err != nil {
|
||||
@@ -317,8 +346,8 @@ func TestSourcesSection(t *testing.T) {
|
||||
if flat["sources.fts_mock.api_root"] != "http://mock" {
|
||||
t.Errorf("Flatten missing sources.fts_mock.api_root: %v", flat)
|
||||
}
|
||||
if len(flat) != 12 {
|
||||
t.Errorf("Flatten has %d keys; want 12: %v", len(flat), flat)
|
||||
if len(flat) != 17 {
|
||||
t.Errorf("Flatten has %d keys; want 17: %v", len(flat), flat)
|
||||
}
|
||||
|
||||
if err := c.SetValue("skip_rules_file", "rules.json"); err != nil ||
|
||||
|
||||
Reference in new issue
Block a user