bwilliams
c09c30ae71
Backfilling a long window (an early --since, or a long time offline) hit the API's rate limit, which could still be exceeded after waiting once, and then everything fetched so far was lost. Before each Find a Tender source's worker starts, a start point more than 7 days ago is now backfilled in 7-day windows bounded on both sides, oldest first, each saved before the next is fetched. A window that fails is saved as far as it got and fetched again from its start after the source's interval. The worker then carries on from the end of the last window. A 429 is now waited out up to sources.<source>.rate_limit_retries times (default 5) rather than once; fts-once --ignore-rate-limit uses 0. FetchAll returns what it fetched before an error along with it, and the workers save it, so a failed run no longer loses its data. findatender's FetchAll and FetchOnce take RequestOptions, which gains Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
52 lines
1.7 KiB
Go
52 lines
1.7 KiB
Go
package config
|
|
|
|
import "reflect"
|
|
|
|
// JSCKSourceFindATender is the Find a Tender source's configuration key.
|
|
const JSCKSourceFindATender string = "find_a_tender"
|
|
|
|
// JSCSourceFindATender configures fetching from the Find a Tender
|
|
// OCDS API at APIRoot. Empty: the source fails to start if it is enabled,
|
|
// and `fetcher fts-once` fails. RateLimitRetries is how many times a request
|
|
// refused for the API's rate limit is waited out and retried; 0 fails at the
|
|
// first, and it can't be negative.
|
|
type JSCSourceFindATender struct {
|
|
JSCSourcesSource
|
|
APIRoot string `cfg:"api_root" default:"https://www.find-tender.service.gov.uk/api/1.0"`
|
|
RateLimitRetries int `cfg:"rate_limit_retries" default:"5"`
|
|
}
|
|
|
|
// ---------- Section methods (see Section) ----------
|
|
|
|
// Map implements Section.
|
|
func (c JSCSourceFindATender) Map() map[string]any { return mapOf(c) }
|
|
|
|
// Flatten implements Section.
|
|
func (c JSCSourceFindATender) Flatten() map[string]any {
|
|
return flattenOf(JSCKSourceFindATender, c)
|
|
}
|
|
|
|
// TypeOf implements Section.
|
|
func (c JSCSourceFindATender) TypeOf(key string) (reflect.Kind, error) {
|
|
return typeOf(c, key)
|
|
}
|
|
|
|
// ValueOf implements Section.
|
|
func (c JSCSourceFindATender) ValueOf(key string) (any, error) {
|
|
return valueOf(c, key)
|
|
}
|
|
|
|
// SetValue implements Section.
|
|
func (c *JSCSourceFindATender) SetValue(key string, value any) error {
|
|
return setValue(c, key, value)
|
|
}
|
|
|
|
// ---------- Section specific methods ----------
|
|
|
|
// validate checks values before they are stored (see validator), as
|
|
// JSCSourcesSource's does, which this one hides. rate_limit_retries can't be
|
|
// negative.
|
|
func (c *JSCSourceFindATender) validate(key string, value any) error {
|
|
return validateRateLimitRetries(&c.JSCSourcesSource, key, value)
|
|
}
|