Added an EU Supply source, scraping the UK portal's public tenders
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
This commit is contained in:
1 parent
5c62376b6e
commit
3a06132071
25 files changed
+2520
-8
No files matched your search
@@ -0,0 +1,101 @@
|
||||
package eusupply
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
// The opportunities Convert makes.
|
||||
const (
|
||||
// UUIDPrefix starts the UUID of every opportunity from the UK portal;
|
||||
// the tender's id follows it.
|
||||
UUIDPrefix = "eusupply-uk-"
|
||||
// uuidType is their UUIDType.
|
||||
uuidType = "eusupply"
|
||||
// releaseTag is their one release's tag: the portal lists open tenders.
|
||||
releaseTag = "tender"
|
||||
)
|
||||
|
||||
// mainCategories maps the portal's types of contract to OCDS's
|
||||
// mainProcurementCategory, as Find a Tender's opportunities have.
|
||||
var mainCategories = map[string]string{
|
||||
"Supplies": "goods",
|
||||
"Services": "services",
|
||||
"Works": "works",
|
||||
}
|
||||
|
||||
// UUID returns the UUID of the opportunity for the tender with id.
|
||||
func UUID(id string) string {
|
||||
return UUIDPrefix + id
|
||||
}
|
||||
|
||||
// ReleaseID returns the id of the one release of the opportunity for
|
||||
// tender: its id, as the portal gives each tender one page.
|
||||
func ReleaseID(tender Tender) string {
|
||||
return tender.ID
|
||||
}
|
||||
|
||||
// Convert returns the opportunity for each tender, with one release whose
|
||||
// JSON is the tender as read.
|
||||
func Convert(tenders []Tender) []*types.Opportunity {
|
||||
opportunities := make([]*types.Opportunity, 0, len(tenders))
|
||||
for _, tender := range tenders {
|
||||
opportunities = append(opportunities, convertOne(tender))
|
||||
}
|
||||
return opportunities
|
||||
}
|
||||
|
||||
// convertOne builds the opportunity for one tender.
|
||||
func convertOne(tender Tender) *types.Opportunity {
|
||||
published := tender.Published
|
||||
opportunity := &types.Opportunity{
|
||||
UUID: UUID(tender.ID),
|
||||
UUIDType: uuidType,
|
||||
Title: tender.Title,
|
||||
Status: "active",
|
||||
Description: tender.Description,
|
||||
BuyerName: tender.Buyer,
|
||||
Classifications: classifications(tender),
|
||||
MainCategory: mainCategories[tender.ContractType],
|
||||
MultiLot: tender.Lots > 1,
|
||||
StartDate: &published,
|
||||
EndDate: tender.Deadline,
|
||||
Source: Source,
|
||||
LatestReleaseDate: tender.Published,
|
||||
}
|
||||
|
||||
release := types.OpportunityRelease{
|
||||
UUID: opportunity.UUID,
|
||||
ReleaseID: ReleaseID(tender),
|
||||
ReleaseDate: tender.Published,
|
||||
Tags: []types.OpportunityReleaseTag{
|
||||
{
|
||||
Tag: releaseTag,
|
||||
},
|
||||
},
|
||||
}
|
||||
// A Tender is plain data, so marshalling it can't fail.
|
||||
if encoded, err := json.Marshal(tender); err == nil {
|
||||
release.JSON = string(encoded)
|
||||
}
|
||||
opportunity.Releases = []types.OpportunityRelease{release}
|
||||
|
||||
return opportunity
|
||||
}
|
||||
|
||||
// classifications returns the tender's CPV codes as Find a Tender's are
|
||||
// stored: without the check digit ("48000000", not "48000000-8").
|
||||
func classifications(tender Tender) []types.ClassificationCode {
|
||||
codes := make([]types.ClassificationCode, 0, len(tender.CPVCodes))
|
||||
for _, cpv := range tender.CPVCodes {
|
||||
code, _, _ := strings.Cut(cpv.Code, "-")
|
||||
codes = append(codes, types.ClassificationCode{
|
||||
IsCPV: true,
|
||||
Code: code,
|
||||
Classification: cpv.Description,
|
||||
})
|
||||
}
|
||||
return codes
|
||||
}
|
||||
Reference in new issue
Block a user