Added an EU Supply source, scraping the UK portal's public tenders
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
This commit is contained in:
bwilliams committed
1 parent
5c62376b6e
commit
3a06132071
25 files changed
+2520
-8
No files matched your search
@@ -0,0 +1,324 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/config"
|
||||
"git.avsdev.uk/Go/job-search/internal/eusupply"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
var errTestStore = errors.New("test store failed")
|
||||
|
||||
// testTender is one row of a fake EU Supply search page, and its page.
|
||||
type testTender struct {
|
||||
id, pid, title string
|
||||
published string // "21/10/2026 12:45:00"
|
||||
buyer string
|
||||
description string
|
||||
}
|
||||
|
||||
// euSupplySite is a fake EU Supply portal: one page of search results with
|
||||
// tenders, and a page for each of them. It counts the tender pages read.
|
||||
type euSupplySite struct {
|
||||
mu sync.Mutex
|
||||
tenders []testTender
|
||||
detailed []string // the PIDs whose pages were read
|
||||
failPID string // a PID whose page fails
|
||||
}
|
||||
|
||||
func (s *euSupplySite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if r.Method == http.MethodPost {
|
||||
var rows strings.Builder
|
||||
for _, tender := range s.tenders {
|
||||
fmt.Fprintf(
|
||||
&rows,
|
||||
`<tr><td><div class="js-tooltip-content">%s</div></td>`+
|
||||
`<td><div class="js-tooltip-content"></div></td>`+
|
||||
`<td><a href="/app/rfq/rwlentrance_s.asp?PID=%s">%s</a></td>`+
|
||||
`<td><div class="js-tooltip-content" title="%s"></div></td>`+
|
||||
`<td></td><td></td><td>%s</td><td></td></tr>`,
|
||||
tender.id, tender.pid, tender.title, tender.published, tender.buyer,
|
||||
)
|
||||
}
|
||||
fmt.Fprintf(
|
||||
w,
|
||||
`<html><body><div id="searchResultContainer"><table><tbody>%s`+
|
||||
`</tbody></table></div></body></html>`,
|
||||
rows.String(),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
pid := strings.Split(strings.TrimPrefix(
|
||||
r.URL.Path, "/ctm/Supplier/PublicPurchase/",
|
||||
), "/")[0]
|
||||
s.detailed = append(s.detailed, pid)
|
||||
if pid == s.failPID {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
description := ""
|
||||
for _, tender := range s.tenders {
|
||||
if tender.pid == pid {
|
||||
description = tender.description
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(
|
||||
w,
|
||||
`<html><body><div class="ctm-content-label">Detailed description:</div>`+
|
||||
`<p>%s</p></body></html>`,
|
||||
description,
|
||||
)
|
||||
}
|
||||
|
||||
// fakeTenderStore is a tenderStore with some releases stored and other
|
||||
// sources' opportunities.
|
||||
type fakeTenderStore struct {
|
||||
stored map[string]bool // uuid "/" release id
|
||||
others []*types.Opportunity
|
||||
titlesErr error
|
||||
}
|
||||
|
||||
func (s *fakeTenderStore) ReleaseExists(
|
||||
ctx context.Context, uuid, releaseID string,
|
||||
) (bool, error) {
|
||||
return s.stored[uuid+"/"+releaseID], nil
|
||||
}
|
||||
|
||||
func (s *fakeTenderStore) ListOtherSourcesTitles(
|
||||
ctx context.Context, source string, from, before time.Time,
|
||||
) ([]store.OpportunityTitle, error) {
|
||||
if s.titlesErr != nil {
|
||||
return nil, s.titlesErr
|
||||
}
|
||||
var titles []store.OpportunityTitle
|
||||
for _, other := range s.others {
|
||||
if !other.LatestReleaseDate.Before(from) &&
|
||||
other.LatestReleaseDate.Before(before) {
|
||||
titles = append(titles, store.OpportunityTitle{
|
||||
UUID: other.UUID,
|
||||
Title: other.Title,
|
||||
LatestReleaseDate: other.LatestReleaseDate,
|
||||
})
|
||||
}
|
||||
}
|
||||
return titles, nil
|
||||
}
|
||||
|
||||
func (s *fakeTenderStore) GetOpportunity(
|
||||
ctx context.Context, uuid string,
|
||||
) (*types.Opportunity, error) {
|
||||
for _, other := range s.others {
|
||||
if other.UUID == uuid {
|
||||
return other, nil
|
||||
}
|
||||
}
|
||||
return nil, store.ErrNotFound
|
||||
}
|
||||
|
||||
// newEUSupplyClient returns a client for srv that doesn't wait.
|
||||
func newEUSupplyClient(srv *httptest.Server) *eusupply.Client {
|
||||
return eusupply.New(srv.URL, eusupply.WithRequestGap(0, 0))
|
||||
}
|
||||
|
||||
// Only tenders not stored are read and returned. The ones another source
|
||||
// has within the window, with a matching title and a matching buyer or
|
||||
// description, are marked Hidden and Skipped.
|
||||
func TestFetchEUSupply(t *testing.T) {
|
||||
const day = "20/10/2026 10:00:00"
|
||||
site := &euSupplySite{
|
||||
tenders: []testTender{
|
||||
{"1", "11", "Already stored", day, "", ""},
|
||||
// The same generic title from another buyer: not a repeat.
|
||||
{"2", "12", "Pest Control Services", day, "Yorkshire Housing Limited",
|
||||
"Pest control across our homes in Leeds"},
|
||||
// The same buyer, named differently: a repeat.
|
||||
{"3", "13", "Google Reseller Framework", day, "Jisc", ""},
|
||||
// An agent buying for the buyer, but the same description.
|
||||
{"4", "14", "Fairhive - Materials Procurement", day,
|
||||
"Echelon Consultancy",
|
||||
"Supply of building materials for Fairhive housing repairs"},
|
||||
// Alike, but too long before.
|
||||
{"5", "15", "Housing related support", day, "Hull City Council", ""},
|
||||
},
|
||||
}
|
||||
srv := httptest.NewServer(site)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
published := time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
|
||||
other := func(
|
||||
uuid, title, buyer, description string, released time.Time,
|
||||
) *types.Opportunity {
|
||||
return &types.Opportunity{
|
||||
UUID: uuid,
|
||||
Title: title,
|
||||
BuyerName: buyer,
|
||||
Description: description,
|
||||
LatestReleaseDate: released,
|
||||
}
|
||||
}
|
||||
lookup := &fakeTenderStore{
|
||||
stored: map[string]bool{
|
||||
"eusupply-uk-1/1": true,
|
||||
},
|
||||
others: []*types.Opportunity{
|
||||
other("ocds-pest", "PEST CONTROL SERVICE", "Anchor Hanover Group",
|
||||
"Pest control for retirement housing", published),
|
||||
other("ocds-jisc", "Google Reseller Framework",
|
||||
"JISC SERVICES LIMITED", "", published.Add(time.Hour)),
|
||||
other("ocds-fairhive", "Fairhive - Materials Procurement",
|
||||
"Fairhive Housing Limited",
|
||||
"Fairhive seeks supply of building materials for housing repairs",
|
||||
published),
|
||||
other("ocds-old", "Housing related support", "Hull City Council", "",
|
||||
published.Add(-40*24*time.Hour)),
|
||||
},
|
||||
}
|
||||
|
||||
opportunities, err := fetchEUSupply(
|
||||
context.Background(),
|
||||
slog.Default(),
|
||||
newEUSupplyClient(srv),
|
||||
lookup,
|
||||
published.Add(-24*time.Hour),
|
||||
time.Time{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := strings.Join(site.detailed, ","); got != "12,13,14,15" {
|
||||
t.Errorf("pages read = %s; want the new tenders' only", got)
|
||||
}
|
||||
marks := map[string]bool{}
|
||||
for _, opportunity := range opportunities {
|
||||
if opportunity.Source != "EU Supply - UK" {
|
||||
t.Errorf("%s: Source = %q", opportunity.UUID, opportunity.Source)
|
||||
}
|
||||
if opportunity.Marks.Hidden != opportunity.Marks.Skipped {
|
||||
t.Errorf("%s: marks = %+v", opportunity.UUID, opportunity.Marks)
|
||||
}
|
||||
marks[opportunity.UUID] = opportunity.Marks.Hidden
|
||||
}
|
||||
want := map[string]bool{
|
||||
"eusupply-uk-2": false,
|
||||
"eusupply-uk-3": true,
|
||||
"eusupply-uk-4": true,
|
||||
"eusupply-uk-5": false,
|
||||
}
|
||||
if fmt.Sprint(marks) != fmt.Sprint(want) {
|
||||
t.Errorf("hidden = %v; want %v", marks, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A tender page that fails stops the run, keeping the tenders read before
|
||||
// it; a failed title lookup returns nothing, so nothing is saved unmarked.
|
||||
func TestFetchEUSupplyErrors(t *testing.T) {
|
||||
site := &euSupplySite{
|
||||
tenders: []testTender{
|
||||
{"1", "11", "First", "20/10/2026 10:00:00", "", ""},
|
||||
{"2", "12", "Second", "20/10/2026 10:00:00", "", ""},
|
||||
{"3", "13", "Third", "20/10/2026 10:00:00", "", ""},
|
||||
},
|
||||
failPID: "12",
|
||||
}
|
||||
srv := httptest.NewServer(site)
|
||||
t.Cleanup(srv.Close)
|
||||
client := newEUSupplyClient(srv)
|
||||
since := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
opportunities, err := fetchEUSupply(
|
||||
context.Background(), slog.Default(), client, nil, since, time.Time{},
|
||||
)
|
||||
if !errors.Is(err, eusupply.ErrUnexpectedStatus) {
|
||||
t.Errorf("err = %v; want ErrUnexpectedStatus", err)
|
||||
}
|
||||
if len(opportunities) != 1 || opportunities[0].UUID != "eusupply-uk-1" {
|
||||
t.Errorf("opportunities = %v; want the first tender", opportunities)
|
||||
}
|
||||
|
||||
site.failPID = ""
|
||||
lookup := &fakeTenderStore{
|
||||
titlesErr: errTestStore,
|
||||
}
|
||||
opportunities, err = fetchEUSupply(
|
||||
context.Background(), slog.Default(), client, lookup, since, time.Time{},
|
||||
)
|
||||
if !errors.Is(err, errTestStore) || opportunities != nil {
|
||||
t.Errorf(
|
||||
"err = %v, %d opportunities; want the store's error and none",
|
||||
err, len(opportunities),
|
||||
)
|
||||
}
|
||||
|
||||
site.tenders = nil
|
||||
_, err = fetchEUSupply(
|
||||
context.Background(), slog.Default(), client, lookup, since, time.Time{},
|
||||
)
|
||||
if !errors.Is(err, ErrNoNewReleases) {
|
||||
t.Errorf("no tenders: err = %v; want ErrNoNewReleases", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Run starts the EU Supply source when it is enabled, and saves its
|
||||
// opportunities with their Source.
|
||||
func TestRunEUSupply(t *testing.T) {
|
||||
site := &euSupplySite{
|
||||
tenders: []testTender{
|
||||
{
|
||||
"1", "11", "A tender",
|
||||
time.Now().Format("02/01/2006 15:04:05"), "", "",
|
||||
},
|
||||
},
|
||||
}
|
||||
srv := httptest.NewServer(site)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
opportunityStore := openTestStore(t)
|
||||
sources := &config.JSCSources{}
|
||||
sources.EUSupply.Enabled = true
|
||||
sources.EUSupply.Interval = time.Hour
|
||||
sources.EUSupply.BaseURL = srv.URL
|
||||
sources.EUSupply.RequestMinGap = time.Millisecond
|
||||
sources.EUSupply.RequestMaxGap = time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- New(sources, WithStore(opportunityStore)).Run(ctx, time.Time{})
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var stored []string
|
||||
for time.Now().Before(deadline) {
|
||||
opportunity, err := opportunityStore.GetOpportunity(
|
||||
context.Background(), "eusupply-uk-1",
|
||||
)
|
||||
if err == nil {
|
||||
stored = append(stored, opportunity.Source)
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Run = %v", err)
|
||||
}
|
||||
if len(stored) != 1 || stored[0] != "EU Supply - UK" {
|
||||
t.Errorf("stored sources = %v; want EU Supply - UK", stored)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user