bwilliams
3a06132071
internal/eusupply replaces the old Node scraper (eu-supply.js) for uk.eu-supply.com. The site has no API, so it posts the public search form, which now needs the form's current defaults and reads the HTML with golang.org/x/net/html. Each tender not stored yet has its own page read too, for its description, contract type, CPV codes and buyer address. Tenders are stored with the source "EU Supply - UK" as eusupply-uk-<tender id>, with one release holding the tender as read. The site publishes no rate limit, so the client sets its own: one request at a time, with a random wait between sources.eu_supply.request_min_gap and request_max_gap (2s to 10s). The source has the same 7-day backfill as Find a Tender's. Most of its tenders are also on Find a Tender, so a new one is stored Hidden and Skipped when another source's opportunity released within 30 days has a title at least 90% alike, and its buyer or description matches too (80% of the shorter one's significant words).
115 lines
3.4 KiB
Go
115 lines
3.4 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"reflect"
|
|
"time"
|
|
)
|
|
|
|
// ========== Sources ==========
|
|
|
|
// JSCKSources is the sources section's configuration key.
|
|
const JSCKSources string = "sources"
|
|
|
|
// JSCSources configures each source of opportunities, one field
|
|
// per source. Each field's cfg tag must match the section's JSCK* constant.
|
|
// SkipRulesFile is the JSON file of rules that mark new opportunities
|
|
// Skipped (see internal/skiprules), read again when it changes; empty means
|
|
// none.
|
|
type JSCSources struct {
|
|
SkipRulesFile string `cfg:"skip_rules_file"`
|
|
FindATender JSCSourceFindATender `cfg:"find_a_tender"`
|
|
FTSMock JSCSourceFTSMock `cfg:"fts_mock"`
|
|
EUSupply JSCSourceEUSupply `cfg:"eu_supply"`
|
|
Sleeper JSCSourceSleeper `cfg:"sleeper"`
|
|
}
|
|
|
|
// ---------- Section methods (see Section) ----------
|
|
|
|
// Map implements Section.
|
|
func (c JSCSources) Map() map[string]any { return mapOf(c) }
|
|
|
|
// Flatten implements Section.
|
|
func (c JSCSources) Flatten() map[string]any {
|
|
return flattenOf(JSCKSources, c)
|
|
}
|
|
|
|
// TypeOf implements Section.
|
|
func (c JSCSources) TypeOf(key string) (reflect.Kind, error) {
|
|
return typeOf(c, key)
|
|
}
|
|
|
|
// ValueOf implements Section.
|
|
func (c JSCSources) ValueOf(key string) (any, error) {
|
|
return valueOf(c, key)
|
|
}
|
|
|
|
// SetValue implements Section.
|
|
func (c *JSCSources) SetValue(key string, value any) error {
|
|
return setValue(c, key, value)
|
|
}
|
|
|
|
// ========== Generic source ==========
|
|
|
|
// JSCSourcesSource holds the settings every source has, and is embedded
|
|
// in each source's section. Interval is how often an enabled source fetches;
|
|
// it must be greater than 0 (validate rejects 0 or less: a source is turned
|
|
// off with Enabled, not with its interval).
|
|
type JSCSourcesSource struct {
|
|
Enabled bool `cfg:"enabled"`
|
|
Interval time.Duration `cfg:"interval" default:"10m"`
|
|
}
|
|
|
|
// ---------- Section methods (see Section) ----------
|
|
|
|
// Map implements Section.
|
|
func (c JSCSourcesSource) Map() map[string]any { return mapOf(c) }
|
|
|
|
// Flatten implements Section.
|
|
func (c JSCSourcesSource) Flatten() map[string]any {
|
|
return flattenOf("", c)
|
|
}
|
|
|
|
// TypeOf implements Section.
|
|
func (c JSCSourcesSource) TypeOf(key string) (reflect.Kind, error) {
|
|
return typeOf(c, key)
|
|
}
|
|
|
|
// ValueOf implements Section.
|
|
func (c JSCSourcesSource) ValueOf(key string) (any, error) {
|
|
return valueOf(c, key)
|
|
}
|
|
|
|
// SetValue implements Section.
|
|
func (c *JSCSourcesSource) SetValue(key string, value any) error {
|
|
return setValue(c, key, value)
|
|
}
|
|
|
|
// Keys returns the keys directly in the section.
|
|
func (c *JSCSourcesSource) Keys() []string { return keysOf(c) }
|
|
|
|
// validate checks values before they are stored (see validator); it is
|
|
// promoted to each source's section. interval must be greater than 0.
|
|
func (c *JSCSourcesSource) validate(key string, value any) error {
|
|
if key == "interval" && value.(time.Duration) <= 0 {
|
|
return errors.New(
|
|
"config: interval must be greater than 0; set enabled to false to turn the source off",
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validateRateLimitRetries is the validate of a Find a Tender source's
|
|
// section: source's checks, then rate_limit_retries, which can't be
|
|
// negative.
|
|
func validateRateLimitRetries(
|
|
source *JSCSourcesSource, key string, value any,
|
|
) error {
|
|
if key == "rate_limit_retries" && value.(int) < 0 {
|
|
return errors.New(
|
|
"config: rate_limit_retries can't be negative; 0 fails at the first rate limit",
|
|
)
|
|
}
|
|
return source.validate(key, value)
|
|
}
|