bwilliams
3a06132071
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
102 lines
2.9 KiB
Go
102 lines
2.9 KiB
Go
package eusupply
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
)
|
|
|
|
// The opportunities Convert makes.
|
|
const (
|
|
// UUIDPrefix starts the UUID of every opportunity from the UK portal;
|
|
// the tender's id follows it.
|
|
UUIDPrefix = "eusupply-uk-"
|
|
// uuidType is their UUIDType.
|
|
uuidType = "eusupply"
|
|
// releaseTag is their one release's tag: the portal lists open tenders.
|
|
releaseTag = "tender"
|
|
)
|
|
|
|
// mainCategories maps the portal's types of contract to OCDS's
|
|
// mainProcurementCategory, as Find a Tender's opportunities have.
|
|
var mainCategories = map[string]string{
|
|
"Supplies": "goods",
|
|
"Services": "services",
|
|
"Works": "works",
|
|
}
|
|
|
|
// UUID returns the UUID of the opportunity for the tender with id.
|
|
func UUID(id string) string {
|
|
return UUIDPrefix + id
|
|
}
|
|
|
|
// ReleaseID returns the id of the one release of the opportunity for
|
|
// tender: its id, as the portal gives each tender one page.
|
|
func ReleaseID(tender Tender) string {
|
|
return tender.ID
|
|
}
|
|
|
|
// Convert returns the opportunity for each tender, with one release whose
|
|
// JSON is the tender as read.
|
|
func Convert(tenders []Tender) []*types.Opportunity {
|
|
opportunities := make([]*types.Opportunity, 0, len(tenders))
|
|
for _, tender := range tenders {
|
|
opportunities = append(opportunities, convertOne(tender))
|
|
}
|
|
return opportunities
|
|
}
|
|
|
|
// convertOne builds the opportunity for one tender.
|
|
func convertOne(tender Tender) *types.Opportunity {
|
|
published := tender.Published
|
|
opportunity := &types.Opportunity{
|
|
UUID: UUID(tender.ID),
|
|
UUIDType: uuidType,
|
|
Title: tender.Title,
|
|
Status: "active",
|
|
Description: tender.Description,
|
|
BuyerName: tender.Buyer,
|
|
Classifications: classifications(tender),
|
|
MainCategory: mainCategories[tender.ContractType],
|
|
MultiLot: tender.Lots > 1,
|
|
StartDate: &published,
|
|
EndDate: tender.Deadline,
|
|
Source: Source,
|
|
LatestReleaseDate: tender.Published,
|
|
}
|
|
|
|
release := types.OpportunityRelease{
|
|
UUID: opportunity.UUID,
|
|
ReleaseID: ReleaseID(tender),
|
|
ReleaseDate: tender.Published,
|
|
Tags: []types.OpportunityReleaseTag{
|
|
{
|
|
Tag: releaseTag,
|
|
},
|
|
},
|
|
}
|
|
// A Tender is plain data, so marshalling it can't fail.
|
|
if encoded, err := json.Marshal(tender); err == nil {
|
|
release.JSON = string(encoded)
|
|
}
|
|
opportunity.Releases = []types.OpportunityRelease{release}
|
|
|
|
return opportunity
|
|
}
|
|
|
|
// classifications returns the tender's CPV codes as Find a Tender's are
|
|
// stored: without the check digit ("48000000", not "48000000-8").
|
|
func classifications(tender Tender) []types.ClassificationCode {
|
|
codes := make([]types.ClassificationCode, 0, len(tender.CPVCodes))
|
|
for _, cpv := range tender.CPVCodes {
|
|
code, _, _ := strings.Cut(cpv.Code, "-")
|
|
codes = append(codes, types.ClassificationCode{
|
|
IsCPV: true,
|
|
Code: code,
|
|
Classification: cpv.Description,
|
|
})
|
|
}
|
|
return codes
|
|
}
|