Added an EU Supply source, scraping the UK portal's public tenders
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
This commit is contained in:
bwilliams committed
1 parent
5c62376b6e
commit
3a06132071
25 files changed
+2520
-8
No files matched your search
@@ -0,0 +1,249 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/config"
|
||||
"git.avsdev.uk/Go/job-search/internal/eusupply"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
"git.avsdev.uk/Go/job-search/internal/util"
|
||||
)
|
||||
|
||||
// euSupplySource is the Source of the opportunities the EU Supply source
|
||||
// fetches.
|
||||
const euSupplySource = eusupply.Source
|
||||
|
||||
// repeatWindow is how far apart an EU Supply tender's publication and
|
||||
// another source's opportunity's latest release may be for the two to be
|
||||
// the same (see markRepeats).
|
||||
const repeatWindow = 30 * 24 * time.Hour
|
||||
|
||||
// tenderStore is what the EU Supply source reads from the store, to skip
|
||||
// tenders already stored and to find the ones another source has.
|
||||
type tenderStore interface {
|
||||
ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error)
|
||||
ListOtherSourcesTitles(
|
||||
ctx context.Context, source string, from, before time.Time,
|
||||
) ([]store.OpportunityTitle, error)
|
||||
GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error)
|
||||
}
|
||||
|
||||
// runEUSupply scrapes EU Supply's UK portal every configured interval until
|
||||
// ctx is cancelled, passing what each run fetches to save, even when the run
|
||||
// fails partway. A long window is backfilled first (see backfill). One
|
||||
// client is used throughout, so its gap between requests holds across runs.
|
||||
// lookup is nil without a store: every tender is then new, and none is
|
||||
// checked for repeats.
|
||||
func runEUSupply(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
conf *config.JSCSourceEUSupply,
|
||||
lookup tenderStore,
|
||||
save saveFunc,
|
||||
since time.Time,
|
||||
) error {
|
||||
client := eusupply.New(
|
||||
conf.BaseURL,
|
||||
eusupply.WithLogger(logger),
|
||||
eusupply.WithRequestGap(conf.RequestMinGap, conf.RequestMaxGap),
|
||||
)
|
||||
fetch := func(
|
||||
ctx context.Context, since, until time.Time,
|
||||
) ([]*types.Opportunity, error) {
|
||||
return fetchEUSupply(ctx, logger, client, lookup, since, until)
|
||||
}
|
||||
|
||||
since = backfill(
|
||||
ctx,
|
||||
logger,
|
||||
euSupplySource,
|
||||
conf.Interval,
|
||||
fetch,
|
||||
save,
|
||||
since,
|
||||
)
|
||||
|
||||
worker := util.Worker{
|
||||
Logger: tagged(logger),
|
||||
Name: euSupplySource,
|
||||
Interval: conf.Interval,
|
||||
Task: func(ctx context.Context, since time.Time) error {
|
||||
opportunities, err := fetch(ctx, since, time.Time{})
|
||||
return saveFetched(ctx, logger, save, opportunities, err)
|
||||
},
|
||||
}
|
||||
return worker.Run(ctx, since)
|
||||
}
|
||||
|
||||
// fetchEUSupply fetches the tenders published from since up to until (zero:
|
||||
// today) that aren't stored yet, with their own pages read, and marks the
|
||||
// ones another source already has Hidden and Skipped (see markRepeats). If
|
||||
// it fails partway, the tenders read in full before the error are returned
|
||||
// along with it. Finding nothing new returns ErrNoNewReleases.
|
||||
func fetchEUSupply(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
client *eusupply.Client,
|
||||
lookup tenderStore,
|
||||
since, until time.Time,
|
||||
) ([]*types.Opportunity, error) {
|
||||
tenders, fetchErr := client.Search(ctx, since, until)
|
||||
|
||||
fresh, err := newTenders(ctx, lookup, tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range fresh {
|
||||
if err := client.Details(ctx, &fresh[i]); err != nil {
|
||||
fresh = fresh[:i]
|
||||
fetchErr = errors.Join(fetchErr, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
opportunities := eusupply.Convert(fresh)
|
||||
if err := markRepeats(ctx, logger, lookup, opportunities); err != nil {
|
||||
// Saved unmarked, a repeat would stay visible, as marks are only
|
||||
// stored for a new opportunity: fetch them again next run instead.
|
||||
return nil, errors.Join(fetchErr, err)
|
||||
}
|
||||
|
||||
tagged(logger).Info(
|
||||
"fetched opportunities",
|
||||
"source", euSupplySource,
|
||||
"listed", len(tenders),
|
||||
"count", len(opportunities),
|
||||
)
|
||||
|
||||
if len(opportunities) == 0 && fetchErr == nil {
|
||||
return nil, ErrNoNewReleases
|
||||
}
|
||||
return opportunities, fetchErr
|
||||
}
|
||||
|
||||
// newTenders returns the tenders whose release isn't stored yet, so only
|
||||
// their pages are read. Saving a stored one again would replace its row
|
||||
// with the search's fields alone. With no lookup, every tender is new.
|
||||
func newTenders(
|
||||
ctx context.Context, lookup tenderStore, tenders []eusupply.Tender,
|
||||
) ([]eusupply.Tender, error) {
|
||||
if lookup == nil {
|
||||
return tenders, nil
|
||||
}
|
||||
|
||||
var fresh []eusupply.Tender
|
||||
for _, tender := range tenders {
|
||||
exists, err := lookup.ReleaseExists(
|
||||
ctx, eusupply.UUID(tender.ID), eusupply.ReleaseID(tender),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking tender %s: %w", tender.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
fresh = append(fresh, tender)
|
||||
}
|
||||
}
|
||||
|
||||
return fresh, nil
|
||||
}
|
||||
|
||||
// markRepeats marks Hidden and Skipped each opportunity that repeats one
|
||||
// from another source, most often its Find a Tender notice: its title is at
|
||||
// least titleMatchThreshold alike (see similarTitles), its latest release
|
||||
// is within repeatWindow, and its buyer or description matches too (see
|
||||
// confirmedRepeat). Only the opportunities whose titles match are read in
|
||||
// full. The store keeps the marks of an opportunity it already has, so only
|
||||
// new ones are marked. With no lookup it does nothing.
|
||||
func markRepeats(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
lookup tenderStore,
|
||||
opportunities []*types.Opportunity,
|
||||
) error {
|
||||
if lookup == nil || len(opportunities) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
earliest := opportunities[0].LatestReleaseDate
|
||||
latest := earliest
|
||||
for _, opportunity := range opportunities {
|
||||
earliest = minTime(earliest, opportunity.LatestReleaseDate)
|
||||
latest = maxTime(latest, opportunity.LatestReleaseDate)
|
||||
}
|
||||
others, err := lookup.ListOtherSourcesTitles(
|
||||
ctx,
|
||||
euSupplySource,
|
||||
earliest.Add(-repeatWindow),
|
||||
latest.Add(repeatWindow+time.Nanosecond),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
normalised := make([]string, len(others))
|
||||
for i, other := range others {
|
||||
normalised[i] = normaliseTitle(other.Title)
|
||||
}
|
||||
|
||||
repeats := 0
|
||||
for _, opportunity := range opportunities {
|
||||
title := normaliseTitle(opportunity.Title)
|
||||
for i, other := range others {
|
||||
gap := other.LatestReleaseDate.Sub(opportunity.LatestReleaseDate)
|
||||
if gap.Abs() > repeatWindow || !similarTitles(title, normalised[i]) {
|
||||
continue
|
||||
}
|
||||
repeated, err := lookup.GetOpportunity(ctx, other.UUID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading opportunity %s: %w", other.UUID, err)
|
||||
}
|
||||
if !confirmedRepeat(opportunity, repeated) {
|
||||
tagged(logger).Debug(
|
||||
"title matched, but not buyer or description",
|
||||
"uuid", opportunity.UUID,
|
||||
"title_matches", other.UUID,
|
||||
)
|
||||
continue
|
||||
}
|
||||
opportunity.Marks.Hidden = true
|
||||
opportunity.Marks.Skipped = true
|
||||
repeats++
|
||||
tagged(logger).Debug(
|
||||
"hid repeated opportunity",
|
||||
"uuid", opportunity.UUID,
|
||||
"repeats", other.UUID,
|
||||
)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
tagged(logger).Info(
|
||||
"hid repeated opportunities",
|
||||
"source", euSupplySource,
|
||||
"count", repeats,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// minTime returns the earlier of two times.
|
||||
func minTime(first, second time.Time) time.Time {
|
||||
if second.Before(first) {
|
||||
return second
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// maxTime returns the later of two times.
|
||||
func maxTime(first, second time.Time) time.Time {
|
||||
if second.After(first) {
|
||||
return second
|
||||
}
|
||||
return first
|
||||
}
|
||||
Reference in new issue
Block a user