Added an EU Supply source, scraping the UK portal's public tenders
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
This commit is contained in:
bwilliams committed
1 parent
5c62376b6e
commit
3a06132071
25 files changed
+2520
-8
No files matched your search
@@ -0,0 +1,208 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
func TestNormaliseTitle(t *testing.T) {
|
||||
tests := []struct {
|
||||
title string
|
||||
want string
|
||||
}{
|
||||
{"Pest Control Services", "pest control services"},
|
||||
{" RRE-01284 - For the provision ", "rre 01284 for the provision"},
|
||||
{"\u200b\u200bThe supply 3 kg of Li₂TiO₃", "the supply 3 kg of li₂tio₃"},
|
||||
{"Children & Young People's", "children young people s"},
|
||||
{"---", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := normaliseTitle(tt.title); got != tt.want {
|
||||
t.Errorf("normaliseTitle(%q) = %q; want %q", tt.title, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
first, second string
|
||||
want float64
|
||||
}{
|
||||
{"pest control services", "pest control services", 1},
|
||||
{"abcdefghij", "abcdefghix", 0.9},
|
||||
{"abcdefghij", "abcdefghxx", 0.8},
|
||||
{"kitten", "sitting", 1 - 3.0/7},
|
||||
{"", "", 0},
|
||||
{"abc", "", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := titleSimilarity(tt.first, tt.second)
|
||||
if got < tt.want-1e-9 || got > tt.want+1e-9 {
|
||||
t.Errorf(
|
||||
"titleSimilarity(%q, %q) = %v; want %v",
|
||||
tt.first, tt.second, got, tt.want,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimilarTitles(t *testing.T) {
|
||||
tests := []struct {
|
||||
first, second string
|
||||
want bool
|
||||
}{
|
||||
{"housing related support", "housing related support", true},
|
||||
// One edit in ten is exactly 90%.
|
||||
{"abcdefghij", "abcdefghix", true},
|
||||
{"abcdefghij", "abcdefghxx", false},
|
||||
{"pest control services", "pest control service", true},
|
||||
{"alternative provision", "alternative provision for schools", false},
|
||||
{"", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := similarTitles(tt.first, tt.second); got != tt.want {
|
||||
t.Errorf(
|
||||
"similarTitles(%q, %q) = %v; want %v",
|
||||
tt.first, tt.second, got, tt.want,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// similarTitles' banded, early-stopping distance agrees with the full one.
|
||||
func TestSimilarTitlesAgreesWithTitleSimilarity(t *testing.T) {
|
||||
//nolint:gosec // G404: seeded math/rand keeps the test deterministic
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
alphabet := []rune("abc d")
|
||||
random := func() string {
|
||||
runes := make([]rune, 5+rng.Intn(30))
|
||||
for i := range runes {
|
||||
runes[i] = alphabet[rng.Intn(len(alphabet))]
|
||||
}
|
||||
return string(runes)
|
||||
}
|
||||
mutate := func(title string) string {
|
||||
runes := []rune(title)
|
||||
for range rng.Intn(5) {
|
||||
i := rng.Intn(len(runes))
|
||||
switch rng.Intn(3) {
|
||||
case 0:
|
||||
runes[i] = alphabet[rng.Intn(len(alphabet))]
|
||||
case 1:
|
||||
runes = append(runes[:i], runes[i+1:]...)
|
||||
default:
|
||||
runes = append(runes[:i], append([]rune{'x'}, runes[i:]...)...)
|
||||
}
|
||||
if len(runes) == 0 {
|
||||
runes = []rune("a")
|
||||
}
|
||||
}
|
||||
return string(runes)
|
||||
}
|
||||
|
||||
for range 5000 {
|
||||
first := random()
|
||||
second := mutate(first)
|
||||
if rng.Intn(4) == 0 {
|
||||
second = random()
|
||||
}
|
||||
want := titleSimilarity(first, second) >= titleMatchThreshold-1e-9
|
||||
if got := similarTitles(first, second); got != want {
|
||||
t.Fatalf(
|
||||
"similarTitles(%q, %q) = %v; titleSimilarity = %v",
|
||||
first, second, got, titleSimilarity(first, second),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuyerSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
first, second string
|
||||
want float64
|
||||
known bool
|
||||
}{
|
||||
{"Jisc", "JISC SERVICES LIMITED", 1, true},
|
||||
{"The Borough Council of Calderdale", "Borough Council of Calderdale", 1, true},
|
||||
{"Wakefield Council", "The Council of the City of Wakefield", 1, true},
|
||||
{"City of York Council", "York and North Yorkshire Combined Authority", 1, true},
|
||||
{"UK Atomic Energy Authority", "UK Atomic Energy Authoritys", 1, true},
|
||||
{"Yorkshire Housing Limited", "Anchor Hanover Group", 0, true},
|
||||
// Only the shorter name's words count, so a name within another
|
||||
// matches it, rightly for York above, wrongly here: the title and
|
||||
// date must match too, so this is accepted.
|
||||
{"North East Lincolnshire", "North Lincolnshire Council", 1, true},
|
||||
{"Hull City Council", "East Riding of Yorkshire Council", 0, true},
|
||||
{"The Council", "Jisc", 0, false},
|
||||
{"", "Jisc", 0, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, known := buyerSimilarity(tt.first, tt.second)
|
||||
if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
|
||||
t.Errorf(
|
||||
"buyerSimilarity(%q, %q) = %v, %v; want %v, %v",
|
||||
tt.first, tt.second, got, known, tt.want, tt.known,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
first, second string
|
||||
want float64
|
||||
known bool
|
||||
}{
|
||||
{"Supply of pest control.", "Pest control: the supply", 1, true},
|
||||
{"Building materials for repairs", "Building works", 0.5, true},
|
||||
{"", "Anything at all", 0, false},
|
||||
{"a to be", "Anything at all", 0, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, known := descriptionSimilarity(tt.first, tt.second)
|
||||
if known != tt.known || got < tt.want-1e-9 || got > tt.want+1e-9 {
|
||||
t.Errorf(
|
||||
"descriptionSimilarity(%q, %q) = %v, %v; want %v, %v",
|
||||
tt.first, tt.second, got, known, tt.want, tt.known,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A title match is confirmed by the buyer or, failing that, the
|
||||
// description; with neither to compare, it isn't.
|
||||
func TestConfirmedRepeat(t *testing.T) {
|
||||
opportunity := func(buyer, description string) *types.Opportunity {
|
||||
return &types.Opportunity{
|
||||
BuyerName: buyer,
|
||||
Description: description,
|
||||
}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
first, last *types.Opportunity
|
||||
want bool
|
||||
}{
|
||||
{"buyer", opportunity("Jisc", ""), opportunity("JISC LIMITED", ""), true},
|
||||
{
|
||||
"description, not buyer",
|
||||
opportunity("Echelon", "building materials supply"),
|
||||
opportunity("Fairhive", "supply of building materials"),
|
||||
true,
|
||||
},
|
||||
{
|
||||
"neither",
|
||||
opportunity("Yorkshire Housing", "pest control in Leeds homes"),
|
||||
opportunity("Anchor Hanover", "pest control for retirement housing"),
|
||||
false,
|
||||
},
|
||||
{"nothing to compare", opportunity("", ""), opportunity("", ""), false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := confirmedRepeat(tt.first, tt.last); got != tt.want {
|
||||
t.Errorf("%s: confirmedRepeat = %v; want %v", tt.name, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user