bwilliams
c09c30ae71
Backfilling a long window (an early --since, or a long time offline) hit the API's rate limit, which could still be exceeded after waiting once, and then everything fetched so far was lost. Before each Find a Tender source's worker starts, a start point more than 7 days ago is now backfilled in 7-day windows bounded on both sides, oldest first, each saved before the next is fetched. A window that fails is saved as far as it got and fetched again from its start after the source's interval. The worker then carries on from the end of the last window. A 429 is now waited out up to sources.<source>.rate_limit_retries times (default 5) rather than once; fts-once --ignore-rate-limit uses 0. FetchAll returns what it fetched before an error along with it, and the workers save it, so a failed run no longer loses its data. findatender's FetchAll and FetchOnce take RequestOptions, which gains Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
101 lines
2.6 KiB
Go
101 lines
2.6 KiB
Go
package findatender
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"time"
|
|
|
|
resty "resty.dev/v3"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/types"
|
|
)
|
|
|
|
// FindATender is a client for the Find a Tender OCDS API (or the mock
|
|
// server). Create it with New and call Shutdown when finished with it.
|
|
type FindATender struct {
|
|
httpClient *resty.Client
|
|
apiRoot string
|
|
ctx context.Context
|
|
log *slog.Logger
|
|
}
|
|
|
|
// Option configures a FindATender client.
|
|
type Option func(*FindATender)
|
|
|
|
// WithLogger makes the client log to logger instead of slog.Default().
|
|
func WithLogger(logger *slog.Logger) Option {
|
|
return func(ft *FindATender) {
|
|
ft.log = logger
|
|
}
|
|
}
|
|
|
|
// New returns a client for the OCDS API at apiRoot. Requests time out after
|
|
// 60 seconds and are retried once. Call Shutdown when finished with it. It
|
|
// logs to slog.Default() unless given WithLogger.
|
|
func New(apiRoot string, opts ...Option) *FindATender {
|
|
findATender := &FindATender{
|
|
apiRoot: apiRoot,
|
|
}
|
|
for _, opt := range opts {
|
|
opt(findATender)
|
|
}
|
|
|
|
findATender.httpClient = resty.New().
|
|
SetTimeout(60 * time.Second).
|
|
SetRetryCount(1)
|
|
|
|
return findATender
|
|
}
|
|
|
|
// logger returns the logger the client logs to, tagged with its component.
|
|
func (ft *FindATender) logger() *slog.Logger {
|
|
log := ft.log
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return log.With("component", "findatender")
|
|
}
|
|
|
|
// With returns a copy of the client whose requests use ctx, so they stop
|
|
// when it is cancelled. A nil ctx returns the client unchanged.
|
|
func (ft *FindATender) With(ctx context.Context) *FindATender {
|
|
if ctx == nil {
|
|
return ft
|
|
}
|
|
|
|
copied := *ft
|
|
copied.ctx = ctx
|
|
return &copied
|
|
}
|
|
|
|
// Shutdown closes the client's HTTP connections. A failure to close is
|
|
// logged, as there is nothing a caller could do about it.
|
|
func (ft *FindATender) Shutdown() {
|
|
if err := ft.httpClient.Close(); err != nil {
|
|
ft.logger().Warn("closing http client failed", "error", err)
|
|
}
|
|
}
|
|
|
|
// FetchOnce fetches the first page of the releases opts selects, and
|
|
// converts them to opportunities.
|
|
func (ft *FindATender) FetchOnce(
|
|
opts RequestOptions,
|
|
) ([]*types.Opportunity, error) {
|
|
releases, err := ft.fetchOnce(opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Convert(releases), nil
|
|
}
|
|
|
|
// FetchAll fetches every page of the releases opts selects, following each
|
|
// page's next link, and converts them to opportunities. If a page fails, the
|
|
// opportunities from the pages before it are returned with the error, so the
|
|
// caller can keep them.
|
|
func (ft *FindATender) FetchAll(
|
|
opts RequestOptions,
|
|
) ([]*types.Opportunity, error) {
|
|
releases, err := ft.fetchAll(opts)
|
|
return Convert(releases), err
|
|
}
|