bwilliams
c09c30ae71
Backfilling a long window (an early --since, or a long time offline) hit the API's rate limit, which could still be exceeded after waiting once, and then everything fetched so far was lost. Before each Find a Tender source's worker starts, a start point more than 7 days ago is now backfilled in 7-day windows bounded on both sides, oldest first, each saved before the next is fetched. A window that fails is saved as far as it got and fetched again from its start after the source's interval. The worker then carries on from the end of the last window. A 429 is now waited out up to sources.<source>.rate_limit_retries times (default 5) rather than once; fts-once --ignore-rate-limit uses 0. FetchAll returns what it fetched before an error along with it, and the workers save it, so a failed run no longer loses its data. findatender's FetchAll and FetchOnce take RequestOptions, which gains Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
114 lines
3.3 KiB
Go
114 lines
3.3 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"reflect"
|
|
"time"
|
|
)
|
|
|
|
// ========== Sources ==========
|
|
|
|
// JSCKSources is the sources section's configuration key.
|
|
const JSCKSources string = "sources"
|
|
|
|
// JSCSources configures each source of opportunities, one field
|
|
// per source. Each field's cfg tag must match the section's JSCK* constant.
|
|
// SkipRulesFile is the JSON file of rules that mark new opportunities
|
|
// Skipped (see internal/skiprules), read again when it changes; empty means
|
|
// none.
|
|
type JSCSources struct {
|
|
SkipRulesFile string `cfg:"skip_rules_file"`
|
|
FindATender JSCSourceFindATender `cfg:"find_a_tender"`
|
|
FTSMock JSCSourceFTSMock `cfg:"fts_mock"`
|
|
Sleeper JSCSourceSleeper `cfg:"sleeper"`
|
|
}
|
|
|
|
// ---------- Section methods (see Section) ----------
|
|
|
|
// Map implements Section.
|
|
func (c JSCSources) Map() map[string]any { return mapOf(c) }
|
|
|
|
// Flatten implements Section.
|
|
func (c JSCSources) Flatten() map[string]any {
|
|
return flattenOf(JSCKSources, c)
|
|
}
|
|
|
|
// TypeOf implements Section.
|
|
func (c JSCSources) TypeOf(key string) (reflect.Kind, error) {
|
|
return typeOf(c, key)
|
|
}
|
|
|
|
// ValueOf implements Section.
|
|
func (c JSCSources) ValueOf(key string) (any, error) {
|
|
return valueOf(c, key)
|
|
}
|
|
|
|
// SetValue implements Section.
|
|
func (c *JSCSources) SetValue(key string, value any) error {
|
|
return setValue(c, key, value)
|
|
}
|
|
|
|
// ========== Generic source ==========
|
|
|
|
// JSCSourcesSource holds the settings every source has, and is embedded
|
|
// in each source's section. Interval is how often an enabled source fetches;
|
|
// it must be greater than 0 (validate rejects 0 or less: a source is turned
|
|
// off with Enabled, not with its interval).
|
|
type JSCSourcesSource struct {
|
|
Enabled bool `cfg:"enabled"`
|
|
Interval time.Duration `cfg:"interval" default:"10m"`
|
|
}
|
|
|
|
// ---------- Section methods (see Section) ----------
|
|
|
|
// Map implements Section.
|
|
func (c JSCSourcesSource) Map() map[string]any { return mapOf(c) }
|
|
|
|
// Flatten implements Section.
|
|
func (c JSCSourcesSource) Flatten() map[string]any {
|
|
return flattenOf("", c)
|
|
}
|
|
|
|
// TypeOf implements Section.
|
|
func (c JSCSourcesSource) TypeOf(key string) (reflect.Kind, error) {
|
|
return typeOf(c, key)
|
|
}
|
|
|
|
// ValueOf implements Section.
|
|
func (c JSCSourcesSource) ValueOf(key string) (any, error) {
|
|
return valueOf(c, key)
|
|
}
|
|
|
|
// SetValue implements Section.
|
|
func (c *JSCSourcesSource) SetValue(key string, value any) error {
|
|
return setValue(c, key, value)
|
|
}
|
|
|
|
// Keys returns the keys directly in the section.
|
|
func (c *JSCSourcesSource) Keys() []string { return keysOf(c) }
|
|
|
|
// validate checks values before they are stored (see validator); it is
|
|
// promoted to each source's section. interval must be greater than 0.
|
|
func (c *JSCSourcesSource) validate(key string, value any) error {
|
|
if key == "interval" && value.(time.Duration) <= 0 {
|
|
return errors.New(
|
|
"config: interval must be greater than 0; set enabled to false to turn the source off",
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validateRateLimitRetries is the validate of a Find a Tender source's
|
|
// section: source's checks, then rate_limit_retries, which can't be
|
|
// negative.
|
|
func validateRateLimitRetries(
|
|
source *JSCSourcesSource, key string, value any,
|
|
) error {
|
|
if key == "rate_limit_retries" && value.(int) < 0 {
|
|
return errors.New(
|
|
"config: rate_limit_retries can't be negative; 0 fails at the first rate limit",
|
|
)
|
|
}
|
|
return source.validate(key, value)
|
|
}
|