Go/job-search
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added a backfill in 7-day windows, and fetches now keep their data on a failure

Browse Source 
Backfilling a long window (an early --since, or a long time offline)
hit the API's rate limit, which could still be exceeded after waiting
once, and then everything fetched so far was lost.

Before each Find a Tender source's worker starts, a start point more
than 7 days ago is now backfilled in 7-day windows bounded on both
sides, oldest first, each saved before the next is fetched. A window
that fails is saved as far as it got and fetched again from its start
after the source's interval. The worker then carries on from the end
of the last window.

A 429 is now waited out up to sources.<source>.rate_limit_retries
times (default 5) rather than once; fts-once --ignore-rate-limit uses
0. FetchAll returns what it fetched before an error along with it, and
the workers save it, so a failed run no longer loses its data.
findatender's FetchAll and FetchOnce take RequestOptions, which gains
Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
This commit is contained in:
bwilliams committed 2026-10-21 12:40:39 +01:00
1 parent ff2aa316b5
commit c09c30ae71
16 files changed
+683 -149
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/findatender/api_test.go
+82 -49
View File
@@ -129,6 +129,20 @@ func TestFetchWithOptsBuildsURI(t *testing.T) {
if !strings.HasSuffix(gotQuery, "&limit=100") {
t.Errorf("default limit not applied: %q", gotQuery)
}
res, err = ft.fetchWithOpts(RequestOptions{
Since: since,
Until: since.Add(7 * 24 * time.Hour),
})
if err != nil {
t.Fatal(err)
}
res.Body.Close()
want := "updatedFrom=2026-10-08T09:30:15&limit=100" +
"&updatedTo=2026-10-15T09:30:15"
if gotQuery != want {
t.Errorf("query = %q, want %q", gotQuery, want)
}
}
// A zero Since defaults to 24 hours ago.
@@ -158,31 +172,17 @@ func TestFetchWithOptsDefaultSince(t *testing.T) {
func TestFetchURIRateLimit(t *testing.T) {
tests := []struct {
name string
retryAfter string
allowRateLimit bool
wantErr error
wantErrText string
wantHits int32
name string
retryAfter string
retries int
wantErr error
wantErrText string
wantHits int32
}{
{"not allowed", "0", false, ErrRateLimitExceeded, "", 1},
{"allowed, retries", "0", true, nil, "", 2},
{
"allowed, bad retry-after",
"soon",
true,
nil,
"unknown retry-after value",
1,
},
{
"allowed, missing retry-after",
"",
true,
nil,
"unknown retry-after value",
1,
},
{"no retries", "0", 0, ErrRateLimitExceeded, "", 1},
{"retries", "0", 1, nil, "", 2},
{"bad retry-after", "soon", 1, nil, "unknown retry-after value", 1},
{"missing retry-after", "", 1, nil, "unknown retry-after value", 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -202,7 +202,7 @@ func TestFetchURIRateLimit(t *testing.T) {
},
)
res, err := ft.fetchURI(srv.URL+"/x", tt.allowRateLimit)
res, err := ft.fetchURI(srv.URL+"/x", tt.retries)
switch {
case tt.wantErr != nil:
if !errors.Is(err, tt.wantErr) {
@@ -241,7 +241,7 @@ func TestFetchURIRateLimitCancelled(t *testing.T) {
}()
start := time.Now()
_, err := withCtx.fetchURI(srv.URL, true)
_, err := withCtx.fetchURI(srv.URL, 1)
if !errors.Is(err, ErrFetchCancelled) {
t.Fatalf("err = %v, want ErrFetchCancelled", err)
}
@@ -250,20 +250,47 @@ func TestFetchURIRateLimitCancelled(t *testing.T) {
}
}
func TestFetchURIRateLimitTwice(t *testing.T) {
// A second 429 after waiting is reported as ErrRateLimitExceeded.
var hits atomic.Int32
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
})
_, err := ft.fetchURI(srv.URL, true)
if !errors.Is(err, ErrRateLimitExceeded) {
t.Fatalf("err = %v, want ErrRateLimitExceeded", err)
// Each 429 is waited out until the retries run out; the next one is
// reported as ErrRateLimitExceeded.
func TestFetchURIRateLimitRepeated(t *testing.T) {
tests := []struct {
name string
limited int32 // requests answered 429 before one succeeds
retries int
wantErr error
wantHits int32
}{
{"fewer than the retries", 3, 5, nil, 4},
{"as many as the retries", 3, 3, nil, 4},
{"more than the retries", 3, 2, ErrRateLimitExceeded, 3},
{"never succeeds", 1000, 4, ErrRateLimitExceeded, 5},
}
if n := hits.Load(); n != 2 {
t.Errorf("requests = %d, want 2", n)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var hits atomic.Int32
ft, srv := newTestFT(
t,
false,
func(w http.ResponseWriter, r *http.Request) {
if hits.Add(1) <= tt.limited {
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
return
}
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
},
)
res, err := ft.fetchURI(srv.URL, tt.retries)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("err = %v, want %v", err, tt.wantErr)
}
if err == nil {
res.Body.Close()
}
if n := hits.Load(); n != tt.wantHits {
t.Errorf("requests = %d, want %d", n, tt.wantHits)
}
})
}
}
@@ -281,7 +308,7 @@ func TestFetchURIUnexpectedStatus(t *testing.T) {
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
},
)
_, err := ft.fetchURI(srv.URL, true)
_, err := ft.fetchURI(srv.URL, 1)
if !errors.Is(err, ErrUnexpectedStatus) {
t.Fatalf("err = %v, want ErrUnexpectedStatus", err)
}
@@ -303,7 +330,7 @@ func TestFetchURIRateLimitThenError(t *testing.T) {
}
w.WriteHeader(http.StatusServiceUnavailable)
})
_, err := ft.fetchURI(srv.URL, true)
_, err := ft.fetchURI(srv.URL, 1)
if !errors.Is(err, ErrUnexpectedStatus) {
t.Fatalf("err = %v, want ErrUnexpectedStatus", err)
}
@@ -314,7 +341,7 @@ func TestFetchURIConnectionError(t *testing.T) {
})
srv.Close()
ft.httpClient.SetRetryCount(0)
if _, err := ft.fetchURI(srv.URL, true); err == nil {
if _, err := ft.fetchURI(srv.URL, 1); err == nil {
t.Fatal("expected an error from a closed server")
}
}
@@ -325,7 +352,7 @@ func TestFetchURICancelledContext(t *testing.T) {
})
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := ft.With(ctx).fetchURI(srv.URL, true); err == nil {
if _, err := ft.With(ctx).fetchURI(srv.URL, 1); err == nil {
t.Fatal("expected an error from a cancelled context")
}
}
@@ -404,7 +431,7 @@ func TestFetchNext(t *testing.T) {
Prev: srv.URL,
}},
} {
if _, err := ft.fetchNext(pkg, true); !errors.Is(err, ErrNoNext) {
if _, err := ft.fetchNext(pkg, 1); !errors.Is(err, ErrNoNext) {
t.Errorf("%s: err = %v, want ErrNoNext", name, err)
}
}
@@ -415,7 +442,7 @@ func TestFetchNext(t *testing.T) {
Next: srv.URL + "/page2",
},
},
true,
1,
)
if err != nil {
t.Fatal(err)
@@ -432,7 +459,7 @@ func responseFor(t *testing.T, body string) *resty.Response {
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, body)
})
res, err := ft.fetchURI(srv.URL, false)
res, err := ft.fetchURI(srv.URL, 0)
if err != nil {
t.Fatal(err)
}
@@ -676,7 +703,8 @@ func TestFetchAllErrors(t *testing.T) {
t.Error("first page bad body: expected an error")
}
// A later page fails, either to fetch or to decode.
// A later page fails, either to fetch or to decode: the error is returned
// with the releases fetched before it.
for name, fail := range map[string]http.HandlerFunc{
"rate limited": func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusTooManyRequests)
@@ -701,11 +729,16 @@ func TestFetchAllErrors(t *testing.T) {
},
)
srvURL = srv.URL
if _, err := ft.fetchAll(RequestOptions{
releases, err := ft.fetchAll(RequestOptions{
Since: time.Now(),
}); err == nil {
})
if err == nil {
t.Error("expected an error from the second page")
}
// The first page's releases are kept.
if len(releases) != 1 || releases[0].OCID != "ocds-1" {
t.Errorf("releases = %+v, want the first page's", releases)
}
})
}
}