bwilliams
c09c30ae71
Backfilling a long window (an early --since, or a long time offline) hit the API's rate limit, which could still be exceeded after waiting once, and then everything fetched so far was lost. Before each Find a Tender source's worker starts, a start point more than 7 days ago is now backfilled in 7-day windows bounded on both sides, oldest first, each saved before the next is fetched. A window that fails is saved as far as it got and fetched again from its start after the source's interval. The worker then carries on from the end of the last window. A 429 is now waited out up to sources.<source>.rate_limit_retries times (default 5) rather than once; fts-once --ignore-rate-limit uses 0. FetchAll returns what it fetched before an error along with it, and the workers save it, so a failed run no longer loses its data. findatender's FetchAll and FetchOnce take RequestOptions, which gains Until (updatedTo), and RateLimitRetries replaces AllowRateLimit.
1067 lines
27 KiB
Go
1067 lines
27 KiB
Go
package findatender
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"math"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
resty "resty.dev/v3"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/findatender/ocds"
|
|
"git.avsdev.uk/Go/job-search/internal/logging"
|
|
)
|
|
|
|
// quietLogs discards slog output for the duration of the test, or sends it
|
|
// through h when given.
|
|
func quietLogs(t *testing.T, h slog.Handler) {
|
|
t.Helper()
|
|
if h == nil {
|
|
h = slog.DiscardHandler
|
|
}
|
|
prev := slog.Default()
|
|
slog.SetDefault(slog.New(h))
|
|
t.Cleanup(func() {
|
|
slog.SetDefault(prev)
|
|
})
|
|
}
|
|
|
|
// newTestFT starts h on a test server and returns a FindATender pointed at it.
|
|
// With debug, the logging level is debug for the test, so requests are
|
|
// traced.
|
|
func newTestFT(
|
|
t *testing.T, debug bool, h http.HandlerFunc,
|
|
) (*FindATender, *httptest.Server) {
|
|
t.Helper()
|
|
quietLogs(t, nil)
|
|
if debug {
|
|
logging.SetLevel(slog.LevelDebug)
|
|
t.Cleanup(func() {
|
|
logging.SetLevel(slog.LevelInfo)
|
|
})
|
|
}
|
|
|
|
srv := httptest.NewServer(h)
|
|
t.Cleanup(srv.Close)
|
|
|
|
ft := New(srv.URL)
|
|
t.Cleanup(ft.Shutdown)
|
|
|
|
return ft, srv
|
|
}
|
|
|
|
func releaseJSON(ocid, id string) string {
|
|
return fmt.Sprintf(
|
|
`{"ocid":%q,"id":%q,"date":"2026-10-08T12:00:00Z","tag":["tender"],"initiationType":"tender"}`,
|
|
ocid,
|
|
id,
|
|
)
|
|
}
|
|
|
|
func packageJSON(next string, releases ...string) string {
|
|
links := ""
|
|
if next != "" {
|
|
links = fmt.Sprintf(`,"links":{"next":%q}`, next)
|
|
}
|
|
return fmt.Sprintf(
|
|
`{"uri":"u","version":"1.1","releases":[%s]%s}`,
|
|
strings.Join(releases, ","),
|
|
links,
|
|
)
|
|
}
|
|
|
|
func TestRateSleep(t *testing.T) {
|
|
if err := rateSleep(context.Background(), time.Millisecond); err != nil {
|
|
t.Errorf("elapsed sleep returned %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
start := time.Now()
|
|
if err := rateSleep(ctx, time.Hour); !errors.Is(err, context.Canceled) {
|
|
t.Errorf("cancelled sleep returned %v, want context.Canceled", err)
|
|
}
|
|
if time.Since(start) > time.Second {
|
|
t.Error("cancelled sleep did not return promptly")
|
|
}
|
|
}
|
|
|
|
func TestFetchWithOptsBuildsURI(t *testing.T) {
|
|
var gotPath, gotQuery string
|
|
ft, _ := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
gotPath, gotQuery = r.URL.Path, r.URL.RawQuery
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
|
|
since := time.Date(2026, 10, 8, 9, 30, 15, 0, time.UTC)
|
|
res, err := ft.fetchWithOpts(RequestOptions{
|
|
Since: since,
|
|
Limit: 7,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
if gotPath != "/ocdsReleasePackages" {
|
|
t.Errorf("path = %q", gotPath)
|
|
}
|
|
if gotQuery != "updatedFrom=2026-10-08T09:30:15&limit=7" {
|
|
t.Errorf("query = %q", gotQuery)
|
|
}
|
|
|
|
res, err = ft.fetchWithOpts(RequestOptions{
|
|
Since: since,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
if !strings.HasSuffix(gotQuery, "&limit=100") {
|
|
t.Errorf("default limit not applied: %q", gotQuery)
|
|
}
|
|
|
|
res, err = ft.fetchWithOpts(RequestOptions{
|
|
Since: since,
|
|
Until: since.Add(7 * 24 * time.Hour),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
want := "updatedFrom=2026-10-08T09:30:15&limit=100" +
|
|
"&updatedTo=2026-10-15T09:30:15"
|
|
if gotQuery != want {
|
|
t.Errorf("query = %q, want %q", gotQuery, want)
|
|
}
|
|
}
|
|
|
|
// A zero Since defaults to 24 hours ago.
|
|
func TestFetchWithOptsDefaultSince(t *testing.T) {
|
|
var gotQuery string
|
|
ft, _ := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
gotQuery = r.URL.Query().Get("updatedFrom")
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
// updatedFrom has whole seconds, so allow for the truncation.
|
|
before := time.Now().AddDate(0, 0, -1).Truncate(time.Second)
|
|
res, err := ft.fetchWithOpts(RequestOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
after := time.Now().AddDate(0, 0, -1)
|
|
|
|
from, err := time.ParseInLocation("2006-01-02T15:04:05", gotQuery, time.Local)
|
|
if err != nil {
|
|
t.Fatalf("updatedFrom = %q: %v", gotQuery, err)
|
|
}
|
|
if from.Before(before) || from.After(after) {
|
|
t.Errorf("updatedFrom = %v, want between %v and %v", from, before, after)
|
|
}
|
|
}
|
|
|
|
func TestFetchURIRateLimit(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
retryAfter string
|
|
retries int
|
|
wantErr error
|
|
wantErrText string
|
|
wantHits int32
|
|
}{
|
|
{"no retries", "0", 0, ErrRateLimitExceeded, "", 1},
|
|
{"retries", "0", 1, nil, "", 2},
|
|
{"bad retry-after", "soon", 1, nil, "unknown retry-after value", 1},
|
|
{"missing retry-after", "", 1, nil, "unknown retry-after value", 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
var hits atomic.Int32
|
|
ft, srv := newTestFT(
|
|
t,
|
|
false,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) == 1 {
|
|
if tt.retryAfter != "" {
|
|
w.Header().Set("Retry-After", tt.retryAfter)
|
|
}
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
},
|
|
)
|
|
|
|
res, err := ft.fetchURI(srv.URL+"/x", tt.retries)
|
|
switch {
|
|
case tt.wantErr != nil:
|
|
if !errors.Is(err, tt.wantErr) {
|
|
t.Fatalf("err = %v, want %v", err, tt.wantErr)
|
|
}
|
|
case tt.wantErrText != "":
|
|
if err == nil || !strings.Contains(err.Error(), tt.wantErrText) {
|
|
t.Fatalf("err = %v, want containing %q", err, tt.wantErrText)
|
|
}
|
|
default:
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode() != http.StatusOK {
|
|
t.Errorf("status = %d after retry", res.StatusCode())
|
|
}
|
|
}
|
|
if got := hits.Load(); got != tt.wantHits {
|
|
t.Errorf("server hit %d times, want %d", got, tt.wantHits)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchURIRateLimitCancelled(t *testing.T) {
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Retry-After", "3600")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
withCtx := ft.With(ctx)
|
|
go func() {
|
|
time.Sleep(20 * time.Millisecond)
|
|
cancel()
|
|
}()
|
|
|
|
start := time.Now()
|
|
_, err := withCtx.fetchURI(srv.URL, 1)
|
|
if !errors.Is(err, ErrFetchCancelled) {
|
|
t.Fatalf("err = %v, want ErrFetchCancelled", err)
|
|
}
|
|
if time.Since(start) > 5*time.Second {
|
|
t.Error("cancellation did not interrupt the rate limit wait")
|
|
}
|
|
}
|
|
|
|
// Each 429 is waited out until the retries run out; the next one is
|
|
// reported as ErrRateLimitExceeded.
|
|
func TestFetchURIRateLimitRepeated(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
limited int32 // requests answered 429 before one succeeds
|
|
retries int
|
|
wantErr error
|
|
wantHits int32
|
|
}{
|
|
{"fewer than the retries", 3, 5, nil, 4},
|
|
{"as many as the retries", 3, 3, nil, 4},
|
|
{"more than the retries", 3, 2, ErrRateLimitExceeded, 3},
|
|
{"never succeeds", 1000, 4, ErrRateLimitExceeded, 5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
var hits atomic.Int32
|
|
ft, srv := newTestFT(
|
|
t,
|
|
false,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) <= tt.limited {
|
|
w.Header().Set("Retry-After", "0")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
},
|
|
)
|
|
res, err := ft.fetchURI(srv.URL, tt.retries)
|
|
if !errors.Is(err, tt.wantErr) {
|
|
t.Fatalf("err = %v, want %v", err, tt.wantErr)
|
|
}
|
|
if err == nil {
|
|
res.Body.Close()
|
|
}
|
|
if n := hits.Load(); n != tt.wantHits {
|
|
t.Errorf("requests = %d, want %d", n, tt.wantHits)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchURIUnexpectedStatus(t *testing.T) {
|
|
for _, status := range []int{
|
|
http.StatusMovedPermanently, http.StatusBadRequest,
|
|
http.StatusNotFound, http.StatusInternalServerError,
|
|
} {
|
|
t.Run(http.StatusText(status), func(t *testing.T) {
|
|
ft, srv := newTestFT(
|
|
t,
|
|
false,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(status)
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
},
|
|
)
|
|
_, err := ft.fetchURI(srv.URL, 1)
|
|
if !errors.Is(err, ErrUnexpectedStatus) {
|
|
t.Fatalf("err = %v, want ErrUnexpectedStatus", err)
|
|
}
|
|
if !strings.Contains(err.Error(), fmt.Sprint(status)) {
|
|
t.Errorf("err = %q, want it to include the status %d", err, status)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A non-2xx after waiting out a 429 is reported as ErrUnexpectedStatus.
|
|
func TestFetchURIRateLimitThenError(t *testing.T) {
|
|
var hits atomic.Int32
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) == 1 {
|
|
w.Header().Set("Retry-After", "0")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
})
|
|
_, err := ft.fetchURI(srv.URL, 1)
|
|
if !errors.Is(err, ErrUnexpectedStatus) {
|
|
t.Fatalf("err = %v, want ErrUnexpectedStatus", err)
|
|
}
|
|
}
|
|
|
|
func TestFetchURIConnectionError(t *testing.T) {
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
})
|
|
srv.Close()
|
|
ft.httpClient.SetRetryCount(0)
|
|
if _, err := ft.fetchURI(srv.URL, 1); err == nil {
|
|
t.Fatal("expected an error from a closed server")
|
|
}
|
|
}
|
|
|
|
func TestFetchURICancelledContext(t *testing.T) {
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := ft.With(ctx).fetchURI(srv.URL, 1); err == nil {
|
|
t.Fatal("expected an error from a cancelled context")
|
|
}
|
|
}
|
|
|
|
func TestFetchURIDebugTrace(t *testing.T) {
|
|
handlers := map[string]func(io.Writer) slog.Handler{
|
|
"text": func(w io.Writer) slog.Handler {
|
|
return slog.NewTextHandler(w, &slog.HandlerOptions{
|
|
Level: slog.LevelDebug,
|
|
})
|
|
},
|
|
"json": func(w io.Writer) slog.Handler {
|
|
return slog.NewJSONHandler(w, &slog.HandlerOptions{
|
|
Level: slog.LevelDebug,
|
|
})
|
|
},
|
|
}
|
|
for name, mk := range handlers {
|
|
t.Run(name, func(t *testing.T) {
|
|
ft, _ := newTestFT(
|
|
t,
|
|
true,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
},
|
|
)
|
|
var logs strings.Builder
|
|
quietLogs(t, mk(&logs))
|
|
|
|
// The body must still be readable after the debug logging.
|
|
releases, err := ft.fetchOnce(RequestOptions{
|
|
Since: time.Now(),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(releases) != 1 {
|
|
t.Errorf("got %d releases, want 1", len(releases))
|
|
}
|
|
if !strings.Contains(logs.String(), "msg=fetched") &&
|
|
!strings.Contains(logs.String(), `"msg":"fetched"`) {
|
|
t.Errorf("no trace logged: %s", logs.String())
|
|
}
|
|
// Every field must be loggable (a *resty.Response isn't, as JSON).
|
|
if strings.Contains(logs.String(), "!ERROR") {
|
|
t.Errorf("a field could not be logged: %s", logs.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchURIDebugTraceConnectionError(t *testing.T) {
|
|
ft, srv := newTestFT(
|
|
t,
|
|
true,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
},
|
|
)
|
|
srv.Close()
|
|
ft.httpClient.SetRetryCount(0)
|
|
if _, err := fetchURI(ft, srv.URL); err == nil {
|
|
t.Fatal("expected an error from a closed server")
|
|
}
|
|
}
|
|
|
|
func TestFetchNext(t *testing.T) {
|
|
var gotPath string
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
gotPath = r.URL.Path
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
|
|
for name, pkg := range map[string]*ocds.ReleasePackage{
|
|
"nil links": {},
|
|
"empty next": {Links: &ocds.ReleasePackageLinks{
|
|
Prev: srv.URL,
|
|
}},
|
|
} {
|
|
if _, err := ft.fetchNext(pkg, 1); !errors.Is(err, ErrNoNext) {
|
|
t.Errorf("%s: err = %v, want ErrNoNext", name, err)
|
|
}
|
|
}
|
|
|
|
res, err := ft.fetchNext(
|
|
&ocds.ReleasePackage{
|
|
Links: &ocds.ReleasePackageLinks{
|
|
Next: srv.URL + "/page2",
|
|
},
|
|
},
|
|
1,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
if gotPath != "/page2" {
|
|
t.Errorf("fetched %q, want /page2", gotPath)
|
|
}
|
|
}
|
|
|
|
// responseFor fetches body from a test server so it can be unpacked.
|
|
func responseFor(t *testing.T, body string) *resty.Response {
|
|
t.Helper()
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, body)
|
|
})
|
|
res, err := ft.fetchURI(srv.URL, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
res.Body.Close()
|
|
})
|
|
return res
|
|
}
|
|
|
|
func TestSafeUnpackReleases(t *testing.T) {
|
|
body := `{
|
|
"uri": "https://example/pkg",
|
|
"version": "1.1",
|
|
"extensions": ["ext"],
|
|
"publishedDate": "2026-10-08T12:00:00Z",
|
|
"publisher": {"name": "FTS"},
|
|
"license": "OGL",
|
|
"publicationPolicy": "policy",
|
|
"links": {"next": "https://example/next"},
|
|
"releases": [
|
|
` + releaseJSON("ocds-1", "1") + `,
|
|
{"ocid": "ocds-bad", "id": "2", "date": 12345},
|
|
{"ocid": "ocds-3", "id": "3", "date": "2026-10-08", "tag": [], "initiationType": "tender"}
|
|
]
|
|
}`
|
|
|
|
pkg, err := safeUnpackReleases(slog.Default(), responseFor(t, body))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
copied := pkg.URI == "https://example/pkg" && pkg.Version == "1.1" &&
|
|
len(pkg.Extensions) == 1 && pkg.Publisher != nil &&
|
|
pkg.Publisher.Name == "FTS" && pkg.License == "OGL" &&
|
|
pkg.PublicationPolicy == "policy" && pkg.Links != nil &&
|
|
pkg.Links.Next == "https://example/next"
|
|
if !copied {
|
|
t.Errorf("package fields not copied: %+v", pkg)
|
|
}
|
|
if pkg.PublishedDate == nil ||
|
|
!pkg.PublishedDate.Equal(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("publishedDate = %v", pkg.PublishedDate)
|
|
}
|
|
if len(pkg.Releases) != 2 || pkg.Releases[0].OCID != "ocds-1" ||
|
|
pkg.Releases[1].OCID != "ocds-3" {
|
|
t.Errorf(
|
|
"releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases,
|
|
)
|
|
}
|
|
}
|
|
|
|
func TestSafeUnpackReleasesPublishedDate(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
date string // raw JSON value, empty to omit
|
|
want *time.Time
|
|
}{
|
|
{"omitted", "", nil},
|
|
{"empty string", `""`, nil},
|
|
{
|
|
"RFC3339",
|
|
`"2026-10-08T12:00:00Z"`,
|
|
ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)),
|
|
},
|
|
{
|
|
"date only",
|
|
`"2026-10-08"`,
|
|
ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC)),
|
|
},
|
|
{"unparseable", `"yesterday"`, nil},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
body := `{"releases":[]}`
|
|
if tt.date != "" {
|
|
body = `{"publishedDate":` + tt.date + `,"releases":[]}`
|
|
}
|
|
pkg, err := safeUnpackReleases(slog.Default(), responseFor(t, body))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
switch {
|
|
case tt.want == nil && pkg.PublishedDate != nil:
|
|
t.Errorf("publishedDate = %v, want nil", pkg.PublishedDate)
|
|
case tt.want != nil && (pkg.PublishedDate == nil ||
|
|
!pkg.PublishedDate.Equal(*tt.want)):
|
|
t.Errorf("publishedDate = %v, want %v", pkg.PublishedDate, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func ptrTime(t time.Time) *time.Time { return &t }
|
|
|
|
func TestSafeUnpackReleasesInvalidJSON(t *testing.T) {
|
|
for name, body := range map[string]string{
|
|
"not JSON": `<html>oops</html>`,
|
|
"wrong shape": `{"releases": {}}`,
|
|
"empty body": ``,
|
|
"publisher array": `{"publisher": []}`,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if _, err := safeUnpackReleases(
|
|
slog.Default(), responseFor(t, body),
|
|
); err == nil {
|
|
t.Error("expected a decode error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// pagedServer serves pages[i] for ?page=i (page 0 at the release package
|
|
// path), linking each page to the next.
|
|
func pagedServer(
|
|
t *testing.T, pages ...[]string,
|
|
) (*FindATender, *atomic.Int32) {
|
|
t.Helper()
|
|
var hits atomic.Int32
|
|
var srvURL string
|
|
ft, srv := newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
hits.Add(1)
|
|
|
|
i := 0
|
|
fmt.Sscan(r.URL.Query().Get("page"), &i)
|
|
|
|
next := ""
|
|
if i+1 < len(pages) {
|
|
next = fmt.Sprintf("%s/ocdsReleasePackages?page=%d", srvURL, i+1)
|
|
}
|
|
|
|
io.WriteString(w, packageJSON(next, pages[i]...))
|
|
})
|
|
srvURL = srv.URL
|
|
return ft, &hits
|
|
}
|
|
|
|
func TestFetchOnce(t *testing.T) {
|
|
ft, hits := pagedServer(
|
|
t,
|
|
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
|
[]string{releaseJSON("ocds-3", "1")},
|
|
)
|
|
|
|
releases, err := ft.fetchOnce(RequestOptions{
|
|
Since: time.Now(),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(releases) != 2 {
|
|
t.Errorf("got %d releases, want 2 (first page only)", len(releases))
|
|
}
|
|
if hits.Load() != 1 {
|
|
t.Errorf("server hit %d times, want 1", hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestFetchOnceErrors(t *testing.T) {
|
|
ft, _ := pagedServer(t, []string{})
|
|
if _, err := ft.fetchOnce(RequestOptions{
|
|
Since: time.Now(),
|
|
}); !errors.Is(err, ErrNoReleases) {
|
|
t.Errorf("empty page: err = %v, want ErrNoReleases", err)
|
|
}
|
|
|
|
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, "not json")
|
|
})
|
|
if _, err := ft.fetchOnce(RequestOptions{
|
|
Since: time.Now(),
|
|
}); err == nil {
|
|
t.Error("bad body: expected an error")
|
|
}
|
|
|
|
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
if _, err := ft.fetchOnce(
|
|
RequestOptions{
|
|
Since: time.Now(),
|
|
},
|
|
); !errors.Is(err, ErrRateLimitExceeded) {
|
|
t.Errorf("rate limited: err = %v, want ErrRateLimitExceeded", err)
|
|
}
|
|
}
|
|
|
|
func TestFetchAll(t *testing.T) {
|
|
ft, hits := pagedServer(
|
|
t,
|
|
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
|
[]string{},
|
|
[]string{releaseJSON("ocds-1", "2")},
|
|
)
|
|
|
|
releases, err := ft.fetchAll(RequestOptions{
|
|
Since: time.Now(),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var ids []string
|
|
for _, r := range releases {
|
|
ids = append(ids, r.OCID+"/"+r.ID)
|
|
}
|
|
if strings.Join(ids, ",") != "ocds-1/1,ocds-2/1,ocds-1/2" {
|
|
t.Errorf("releases = %v", ids)
|
|
}
|
|
if hits.Load() != 3 {
|
|
t.Errorf("server hit %d times, want 3", hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestFetchAllErrors(t *testing.T) {
|
|
ft, _ := pagedServer(t, []string{}, []string{})
|
|
if _, err := ft.fetchAll(RequestOptions{
|
|
Since: time.Now(),
|
|
}); !errors.Is(err, ErrNoReleases) {
|
|
t.Errorf("all pages empty: err = %v, want ErrNoReleases", err)
|
|
}
|
|
|
|
// First page fails.
|
|
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
if _, err := ft.fetchAll(
|
|
RequestOptions{
|
|
Since: time.Now(),
|
|
},
|
|
); !errors.Is(err, ErrRateLimitExceeded) {
|
|
t.Errorf("first page rate limited: err = %v", err)
|
|
}
|
|
|
|
ft, _ = newTestFT(t, false, func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, "not json")
|
|
})
|
|
if _, err := ft.fetchAll(RequestOptions{
|
|
Since: time.Now(),
|
|
}); err == nil {
|
|
t.Error("first page bad body: expected an error")
|
|
}
|
|
|
|
// A later page fails, either to fetch or to decode: the error is returned
|
|
// with the releases fetched before it.
|
|
for name, fail := range map[string]http.HandlerFunc{
|
|
"rate limited": func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
},
|
|
"bad body": func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, "not json")
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
var srvURL string
|
|
ft, srv := newTestFT(
|
|
t,
|
|
false,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Query().Get("page") == "" {
|
|
io.WriteString(
|
|
w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1")),
|
|
)
|
|
return
|
|
}
|
|
fail(w, r)
|
|
},
|
|
)
|
|
srvURL = srv.URL
|
|
releases, err := ft.fetchAll(RequestOptions{
|
|
Since: time.Now(),
|
|
})
|
|
if err == nil {
|
|
t.Error("expected an error from the second page")
|
|
}
|
|
// The first page's releases are kept.
|
|
if len(releases) != 1 || releases[0].OCID != "ocds-1" {
|
|
t.Errorf("releases = %+v, want the first page's", releases)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// fetchAll stops, keeping what it has, when a next link repeats an earlier one.
|
|
func TestFetchAllRepeatedNextLink(t *testing.T) {
|
|
tests := map[string]func(base string, page string) string{
|
|
// page 0 -> page 1 -> page 1 ...
|
|
"self link": func(base, page string) string {
|
|
return base + "/next?page=1"
|
|
},
|
|
// page 0 -> page 1 -> page 2 -> page 1 ...
|
|
"cycle": func(base, page string) string {
|
|
if page == "1" {
|
|
return base + "/next?page=2"
|
|
}
|
|
return base + "/next?page=1"
|
|
},
|
|
}
|
|
for name, nextFor := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
var hits atomic.Int32
|
|
var srvURL string
|
|
ft, srv := newTestFT(
|
|
t,
|
|
false,
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) > 10 {
|
|
t.Error("fetchAll did not stop following next links")
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
page := r.URL.Query().Get("page")
|
|
body := packageJSON(
|
|
nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"),
|
|
)
|
|
io.WriteString(w, body)
|
|
},
|
|
)
|
|
srvURL = srv.URL
|
|
|
|
releases, err := ft.fetchAll(RequestOptions{
|
|
Since: time.Now(),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(releases) != int(hits.Load()) {
|
|
t.Errorf(
|
|
"releases = %d, want one per request (%d)",
|
|
len(releases),
|
|
hits.Load(),
|
|
)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalLenient(t *testing.T) {
|
|
raw := json.RawMessage(`{
|
|
"date": "2006-10-02",
|
|
"publishedDate": "",
|
|
"tender": {
|
|
"tenderPeriod": {"startDate": "2006-10-02T15:33:02+07:00", "endDate": "2006-11-30"},
|
|
"description": "Bids close on 2006-11-30",
|
|
"expressionOfInterestDeadline": "2006-10-20"
|
|
},
|
|
"awards": [{"date": "2006-12-01", "id": "2006-12-01"}],
|
|
"value": 1234567.891
|
|
}`)
|
|
|
|
var got struct {
|
|
Date time.Time `json:"date"`
|
|
PublishedDate *time.Time `json:"publishedDate"`
|
|
Tender struct {
|
|
TenderPeriod struct {
|
|
StartDate *time.Time `json:"startDate"`
|
|
EndDate *time.Time `json:"endDate"`
|
|
} `json:"tenderPeriod"`
|
|
Description string `json:"description"`
|
|
ExpressionOfInterestDeadline *time.Time `json:"expressionOfInterestDeadline"`
|
|
} `json:"tender"`
|
|
Awards []struct {
|
|
Date *time.Time `json:"date"`
|
|
ID string `json:"id"`
|
|
} `json:"awards"`
|
|
Value json.Number `json:"value"`
|
|
}
|
|
if err := unmarshalLenient(raw, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
midnight := func(y int, m time.Month, d int) time.Time {
|
|
return time.Date(y, m, d, 0, 0, 0, 0, time.UTC)
|
|
}
|
|
if !got.Date.Equal(midnight(2006, 10, 2)) {
|
|
t.Errorf("date = %v", got.Date)
|
|
}
|
|
if got.PublishedDate != nil {
|
|
t.Errorf(
|
|
"empty publishedDate should decode as nil, got %v", got.PublishedDate,
|
|
)
|
|
}
|
|
wantStart := time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600))
|
|
if start := got.Tender.TenderPeriod.StartDate; start == nil ||
|
|
!start.Equal(wantStart) {
|
|
t.Errorf("full timestamp changed: %v", start)
|
|
}
|
|
if end := got.Tender.TenderPeriod.EndDate; end == nil ||
|
|
!end.Equal(midnight(2006, 11, 30)) {
|
|
t.Errorf("endDate = %v", end)
|
|
}
|
|
if d := got.Tender.ExpressionOfInterestDeadline; d == nil ||
|
|
!d.Equal(midnight(2006, 10, 20)) {
|
|
t.Errorf("expressionOfInterestDeadline = %v", d)
|
|
}
|
|
if got.Tender.Description != "Bids close on 2006-11-30" {
|
|
t.Errorf("free text changed: %q", got.Tender.Description)
|
|
}
|
|
if a := got.Awards[0]; a.Date == nil ||
|
|
!a.Date.Equal(midnight(2006, 12, 1)) || a.ID != "2006-12-01" {
|
|
t.Errorf("award = %+v (date should be repaired, id left alone)", a)
|
|
}
|
|
if got.Value != "1234567.891" {
|
|
t.Errorf("number changed: %s", got.Value)
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalLenientNumbers(t *testing.T) {
|
|
tests := []struct {
|
|
raw string
|
|
wantInt int64
|
|
wantFloat float64
|
|
}{
|
|
{`1e9999`, math.MaxInt64, math.MaxInt64},
|
|
{`-1e9999`, math.MinInt64, math.MinInt64},
|
|
{`1E+400`, math.MaxInt64, math.MaxInt64},
|
|
{`3`, 3, 3},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.raw, func(t *testing.T) {
|
|
var got struct {
|
|
Int *int64 `json:"int"`
|
|
Float *float64 `json:"float"`
|
|
}
|
|
raw := json.RawMessage(
|
|
`{"int": ` + tc.raw + `, "float": ` + tc.raw + `}`,
|
|
)
|
|
if err := unmarshalLenient(raw, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if *got.Int != tc.wantInt {
|
|
t.Errorf("int = %d; want %d", *got.Int, tc.wantInt)
|
|
}
|
|
if *got.Float != tc.wantFloat {
|
|
t.Errorf("float = %v; want %v", *got.Float, tc.wantFloat)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUnmarshalLenientNoLimit decodes OCDS's 1e9999 ("no limit") into the
|
|
// release fields Find a Tender sends it in, and checks the release can still
|
|
// be marshalled, as Convert does to store its JSON.
|
|
func TestUnmarshalLenientNoLimit(t *testing.T) {
|
|
raw := json.RawMessage(`{
|
|
"ocid": "ocds-1",
|
|
"id": "r1",
|
|
"date": "2026-10-16T15:52:30+01:00",
|
|
"tender": {
|
|
"lotDetails": {"maximumLotsBidPerSupplier": 1e9999},
|
|
"techniques": {
|
|
"frameworkAgreement": {"maximumParticipants": 1e9999}
|
|
}
|
|
}
|
|
}`)
|
|
|
|
var release ocds.Release
|
|
if err := unmarshalLenient(raw, &release); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lots := release.Tender.LotDetails.MaximumLotsBidPerSupplier
|
|
if lots == nil || *lots != math.MaxInt64 {
|
|
t.Errorf("maximumLotsBidPerSupplier = %v", lots)
|
|
}
|
|
framework := release.Tender.Techniques.FrameworkAgreement
|
|
if participants := framework.MaximumParticipants; participants == nil ||
|
|
*participants != math.MaxInt64 {
|
|
t.Errorf("maximumParticipants = %v", participants)
|
|
}
|
|
|
|
if _, err := json.Marshal(release); err != nil {
|
|
t.Errorf("marshalling the repaired release: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRepairJSONLeavesValidJSONAlone(t *testing.T) {
|
|
raw := json.RawMessage(`{"date": "2006-10-02T15:33:02+07:00", "title": "x"}`)
|
|
out, err := repairJSON(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(out, raw) {
|
|
t.Errorf("valid input was rewritten: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestRepairJSONRejectsBadJSON(t *testing.T) {
|
|
for _, raw := range []string{`{`, `{"date": "2006-10-02"} trailing`} {
|
|
if _, err := repairJSON(json.RawMessage(raw)); err == nil {
|
|
t.Errorf("expected an error for %q", raw)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalLenientErrors(t *testing.T) {
|
|
var v struct {
|
|
Date time.Time `json:"date"`
|
|
}
|
|
tests := map[string]string{
|
|
"bad JSON": `{`,
|
|
"wrong type": `{"date": 5}`,
|
|
"unparseable date": `{"date": "02/10/2006"}`,
|
|
"not a JSON object": `"2006-10-02"`,
|
|
}
|
|
for name, raw := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
if err := unmarshalLenient(json.RawMessage(raw), &v); err == nil {
|
|
t.Errorf("expected an error for %s", raw)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReleaseIDs(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
raw string
|
|
wantOCID string
|
|
wantID string
|
|
wantErr string // substring; empty means no error
|
|
}{
|
|
{"ocid then id", `{"ocid":"ocds-1","id":"r1"}`, "ocds-1", "r1", ""},
|
|
{"id then ocid", `{"id":"r1","ocid":"ocds-1"}`, "ocds-1", "r1", ""},
|
|
{
|
|
"numeric id keeps its digits",
|
|
`{"ocid":"ocds-1","id":12345678901234567890}`,
|
|
"ocds-1",
|
|
"12345678901234567890",
|
|
"",
|
|
},
|
|
{
|
|
"skips other fields",
|
|
`{"date":"2026-01-01","tender":{"items":[1,2]},"ocid":"ocds-1","id":"r1"}`,
|
|
"ocds-1",
|
|
"r1",
|
|
"",
|
|
},
|
|
{
|
|
"ignores errors after both ids",
|
|
`{"ocid":"ocds-1","id":"r1","tender":{"value":"oops"},` + `"bad":}`,
|
|
"ocds-1",
|
|
"r1",
|
|
"",
|
|
},
|
|
{"empty input", ``, "", "", "reading release"},
|
|
{"not an object", `["ocid","id"]`, "", "", "not a JSON object"},
|
|
{"empty object", `{}`, "", "", "no ocid or id"},
|
|
{"missing ocid", `{"id":"r1"}`, "", "r1", "no ocid"},
|
|
{"missing id", `{"ocid":"ocds-1"}`, "ocds-1", "", "no id"},
|
|
{"ocid wrong type", `{"ocid":5,"id":"r1"}`, "", "", "reading release ocid"},
|
|
{
|
|
"id wrong type",
|
|
`{"ocid":"ocds-1","id":true}`,
|
|
"ocds-1",
|
|
"",
|
|
"unexpected bool",
|
|
},
|
|
{
|
|
"id is an object",
|
|
`{"ocid":"ocds-1","id":{}}`,
|
|
"ocds-1",
|
|
"",
|
|
"unexpected map",
|
|
},
|
|
{
|
|
"invalid id JSON",
|
|
`{"ocid":"ocds-1","id":}`,
|
|
"ocds-1",
|
|
"",
|
|
"reading release id",
|
|
},
|
|
{
|
|
"invalid field before ids",
|
|
`{"date":nope,"ocid":"ocds-1","id":"r1"}`,
|
|
"",
|
|
"",
|
|
`reading release field "date"`,
|
|
},
|
|
{
|
|
"truncated after ocid",
|
|
`{"ocid":"ocds-1",`,
|
|
"ocds-1",
|
|
"",
|
|
"reading release",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ocid, id, err := releaseIDs(json.RawMessage(tt.raw))
|
|
if tt.wantErr == "" {
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
} else if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
|
t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
|
|
}
|
|
if ocid != tt.wantOCID || id != tt.wantID {
|
|
t.Errorf(
|
|
"got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
}
|