bwilliams
e19c8c2b69
gci is configured with the sections standard, default and localmodule, and goimports with the module path as its local prefix, so both formatters agree.
798 lines
24 KiB
Go
798 lines
24 KiB
Go
package findatender
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
resty "resty.dev/v3"
|
|
|
|
"git.avsdev.uk/Go/job-search/internal/config"
|
|
"git.avsdev.uk/Go/job-search/internal/findatender/ocds"
|
|
)
|
|
|
|
func testConfig(level string) *config.JobSearchConfig {
|
|
return &config.JobSearchConfig{Logging: config.JSCLogging{Level: level}}
|
|
}
|
|
|
|
// quietLogs discards slog output for the duration of the test, or sends it
|
|
// through h when given.
|
|
func quietLogs(t *testing.T, h slog.Handler) {
|
|
t.Helper()
|
|
if h == nil {
|
|
h = slog.NewTextHandler(io.Discard, nil)
|
|
}
|
|
prev := slog.Default()
|
|
slog.SetDefault(slog.New(h))
|
|
t.Cleanup(func() { slog.SetDefault(prev) })
|
|
}
|
|
|
|
// newTestFT starts h on a test server and returns a FindATender pointed at it.
|
|
func newTestFT(t *testing.T, level string, h http.HandlerFunc) (*FindATender, *httptest.Server) {
|
|
t.Helper()
|
|
quietLogs(t, nil)
|
|
srv := httptest.NewServer(h)
|
|
t.Cleanup(srv.Close)
|
|
ft := New(testConfig(level), srv.URL)
|
|
t.Cleanup(ft.Shutdown)
|
|
return ft, srv
|
|
}
|
|
|
|
func releaseJSON(ocid, id string) string {
|
|
return fmt.Sprintf(
|
|
`{"ocid":%q,"id":%q,"date":"2026-10-08T12:00:00Z","tag":["tender"],"initiationType":"tender"}`,
|
|
ocid,
|
|
id,
|
|
)
|
|
}
|
|
|
|
func packageJSON(next string, releases ...string) string {
|
|
links := ""
|
|
if next != "" {
|
|
links = fmt.Sprintf(`,"links":{"next":%q}`, next)
|
|
}
|
|
return fmt.Sprintf(
|
|
`{"uri":"u","version":"1.1","releases":[%s]%s}`,
|
|
strings.Join(releases, ","),
|
|
links,
|
|
)
|
|
}
|
|
|
|
func TestRateSleep(t *testing.T) {
|
|
if err := rateSleep(context.Background(), time.Millisecond); err != nil {
|
|
t.Errorf("elapsed sleep returned %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
start := time.Now()
|
|
if err := rateSleep(ctx, time.Hour); !errors.Is(err, context.Canceled) {
|
|
t.Errorf("cancelled sleep returned %v, want context.Canceled", err)
|
|
}
|
|
if time.Since(start) > time.Second {
|
|
t.Error("cancelled sleep did not return promptly")
|
|
}
|
|
}
|
|
|
|
func TestFetchWithOptsBuildsURI(t *testing.T) {
|
|
var gotPath, gotQuery string
|
|
ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
gotPath, gotQuery = r.URL.Path, r.URL.RawQuery
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
|
|
since := time.Date(2026, 10, 8, 9, 30, 15, 0, time.UTC)
|
|
res, err := ft.fetchWithOpts(RequestOptions{Since: since, Limit: 7})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
if gotPath != "/ocdsReleasePackages" {
|
|
t.Errorf("path = %q", gotPath)
|
|
}
|
|
if gotQuery != "updatedFrom=2026-10-08T09:30:15&limit=7" {
|
|
t.Errorf("query = %q", gotQuery)
|
|
}
|
|
|
|
res, err = ft.fetchWithOpts(RequestOptions{Since: since})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
if !strings.HasSuffix(gotQuery, "&limit=100") {
|
|
t.Errorf("default limit not applied: %q", gotQuery)
|
|
}
|
|
}
|
|
|
|
// A zero Since defaults to 24 hours ago.
|
|
func TestFetchWithOptsDefaultSince(t *testing.T) {
|
|
var gotQuery string
|
|
ft, _ := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
gotQuery = r.URL.Query().Get("updatedFrom")
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
// updatedFrom has whole seconds, so allow for the truncation.
|
|
before := time.Now().AddDate(0, 0, -1).Truncate(time.Second)
|
|
res, err := ft.fetchWithOpts(RequestOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
after := time.Now().AddDate(0, 0, -1)
|
|
|
|
from, err := time.ParseInLocation("2006-01-02T15:04:05", gotQuery, time.Local)
|
|
if err != nil {
|
|
t.Fatalf("updatedFrom = %q: %v", gotQuery, err)
|
|
}
|
|
if from.Before(before) || from.After(after) {
|
|
t.Errorf("updatedFrom = %v, want between %v and %v", from, before, after)
|
|
}
|
|
}
|
|
|
|
func TestFetchURIRateLimit(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
retryAfter string
|
|
allowRateLimit bool
|
|
wantErr error
|
|
wantErrText string
|
|
wantHits int32
|
|
}{
|
|
{"not allowed", "0", false, ErrRateLimitExceeded, "", 1},
|
|
{"allowed, retries", "0", true, nil, "", 2},
|
|
{"allowed, bad retry-after", "soon", true, nil, "unknown retry-after value", 1},
|
|
{"allowed, missing retry-after", "", true, nil, "unknown retry-after value", 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
var hits atomic.Int32
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) == 1 {
|
|
if tt.retryAfter != "" {
|
|
w.Header().Set("Retry-After", tt.retryAfter)
|
|
}
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
})
|
|
|
|
res, err := ft.fetchURI(srv.URL+"/x", tt.allowRateLimit)
|
|
switch {
|
|
case tt.wantErr != nil:
|
|
if !errors.Is(err, tt.wantErr) {
|
|
t.Fatalf("err = %v, want %v", err, tt.wantErr)
|
|
}
|
|
case tt.wantErrText != "":
|
|
if err == nil || !strings.Contains(err.Error(), tt.wantErrText) {
|
|
t.Fatalf("err = %v, want containing %q", err, tt.wantErrText)
|
|
}
|
|
default:
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode() != http.StatusOK {
|
|
t.Errorf("status = %d after retry", res.StatusCode())
|
|
}
|
|
}
|
|
if got := hits.Load(); got != tt.wantHits {
|
|
t.Errorf("server hit %d times, want %d", got, tt.wantHits)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchURIRateLimitCancelled(t *testing.T) {
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Retry-After", "3600")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
withCtx := ft.With(ctx)
|
|
go func() {
|
|
time.Sleep(20 * time.Millisecond)
|
|
cancel()
|
|
}()
|
|
|
|
start := time.Now()
|
|
if _, err := withCtx.fetchURI(srv.URL, true); !errors.Is(err, ErrFetchCancelled) {
|
|
t.Fatalf("err = %v, want ErrFetchCancelled", err)
|
|
}
|
|
if time.Since(start) > 5*time.Second {
|
|
t.Error("cancellation did not interrupt the rate limit wait")
|
|
}
|
|
}
|
|
|
|
func TestFetchURIRateLimitTwice(t *testing.T) {
|
|
// A second 429 after waiting is reported as ErrRateLimitExceeded.
|
|
var hits atomic.Int32
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
hits.Add(1)
|
|
w.Header().Set("Retry-After", "0")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrRateLimitExceeded) {
|
|
t.Fatalf("err = %v, want ErrRateLimitExceeded", err)
|
|
}
|
|
if n := hits.Load(); n != 2 {
|
|
t.Errorf("requests = %d, want 2", n)
|
|
}
|
|
}
|
|
|
|
func TestFetchURIUnexpectedStatus(t *testing.T) {
|
|
for _, status := range []int{
|
|
http.StatusMovedPermanently, http.StatusBadRequest,
|
|
http.StatusNotFound, http.StatusInternalServerError,
|
|
} {
|
|
t.Run(http.StatusText(status), func(t *testing.T) {
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(status)
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
})
|
|
_, err := ft.fetchURI(srv.URL, true)
|
|
if !errors.Is(err, ErrUnexpectedStatus) {
|
|
t.Fatalf("err = %v, want ErrUnexpectedStatus", err)
|
|
}
|
|
if !strings.Contains(err.Error(), fmt.Sprint(status)) {
|
|
t.Errorf("err = %q, want it to include the status %d", err, status)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A non-2xx after waiting out a 429 is reported as ErrUnexpectedStatus.
|
|
func TestFetchURIRateLimitThenError(t *testing.T) {
|
|
var hits atomic.Int32
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) == 1 {
|
|
w.Header().Set("Retry-After", "0")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
})
|
|
if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrUnexpectedStatus) {
|
|
t.Fatalf("err = %v, want ErrUnexpectedStatus", err)
|
|
}
|
|
}
|
|
|
|
func TestFetchURIConnectionError(t *testing.T) {
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {})
|
|
srv.Close()
|
|
ft.httpClient.SetRetryCount(0)
|
|
if _, err := ft.fetchURI(srv.URL, true); err == nil {
|
|
t.Fatal("expected an error from a closed server")
|
|
}
|
|
}
|
|
|
|
func TestFetchURICancelledContext(t *testing.T) {
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := ft.With(ctx).fetchURI(srv.URL, true); err == nil {
|
|
t.Fatal("expected an error from a cancelled context")
|
|
}
|
|
}
|
|
|
|
func TestFetchURIDebugTrace(t *testing.T) {
|
|
handlers := map[string]func(io.Writer) slog.Handler{
|
|
"text": func(w io.Writer) slog.Handler {
|
|
return slog.NewTextHandler(w, &slog.HandlerOptions{Level: slog.LevelDebug})
|
|
},
|
|
"json": func(w io.Writer) slog.Handler {
|
|
return slog.NewJSONHandler(w, &slog.HandlerOptions{Level: slog.LevelDebug})
|
|
},
|
|
}
|
|
for name, mk := range handlers {
|
|
t.Run(name, func(t *testing.T) {
|
|
ft, _ := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1")))
|
|
})
|
|
var logs strings.Builder
|
|
quietLogs(t, mk(&logs))
|
|
|
|
// The body must still be readable after the debug logging.
|
|
releases, err := ft.fetchOnce(RequestOptions{Since: time.Now()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(releases) != 1 {
|
|
t.Errorf("got %d releases, want 1", len(releases))
|
|
}
|
|
if !strings.Contains(logs.String(), "Resty") {
|
|
t.Errorf("no trace logged: %s", logs.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetchURIDebugTraceConnectionError(t *testing.T) {
|
|
ft, srv := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) {})
|
|
srv.Close()
|
|
ft.httpClient.SetRetryCount(0)
|
|
if _, err := fetchURI(ft, srv.URL); err == nil {
|
|
t.Fatal("expected an error from a closed server")
|
|
}
|
|
}
|
|
|
|
func TestFetchNext(t *testing.T) {
|
|
var gotPath string
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
gotPath = r.URL.Path
|
|
io.WriteString(w, packageJSON(""))
|
|
})
|
|
|
|
for name, pkg := range map[string]*ocds.ReleasePackage{
|
|
"nil links": {},
|
|
"empty next": {Links: &ocds.ReleasePackageLinks{Prev: srv.URL}},
|
|
} {
|
|
if _, err := ft.fetchNext(pkg, true); !errors.Is(err, ErrNoNext) {
|
|
t.Errorf("%s: err = %v, want ErrNoNext", name, err)
|
|
}
|
|
}
|
|
|
|
res, err := ft.fetchNext(
|
|
&ocds.ReleasePackage{Links: &ocds.ReleasePackageLinks{Next: srv.URL + "/page2"}},
|
|
true,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res.Body.Close()
|
|
if gotPath != "/page2" {
|
|
t.Errorf("fetched %q, want /page2", gotPath)
|
|
}
|
|
}
|
|
|
|
// responseFor fetches body from a test server so it can be unpacked.
|
|
func responseFor(t *testing.T, body string) *resty.Response {
|
|
t.Helper()
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, body)
|
|
})
|
|
res, err := ft.fetchURI(srv.URL, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { res.Body.Close() })
|
|
return res
|
|
}
|
|
|
|
func TestSafeUnpackReleases(t *testing.T) {
|
|
body := `{
|
|
"uri": "https://example/pkg",
|
|
"version": "1.1",
|
|
"extensions": ["ext"],
|
|
"publishedDate": "2026-10-08T12:00:00Z",
|
|
"publisher": {"name": "FTS"},
|
|
"license": "OGL",
|
|
"publicationPolicy": "policy",
|
|
"links": {"next": "https://example/next"},
|
|
"releases": [
|
|
` + releaseJSON("ocds-1", "1") + `,
|
|
{"ocid": "ocds-bad", "id": "2", "date": 12345},
|
|
{"ocid": "ocds-3", "id": "3", "date": "2026-10-08", "tag": [], "initiationType": "tender"}
|
|
]
|
|
}`
|
|
pkg, err := safeUnpackReleases(responseFor(t, body))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if pkg.URI != "https://example/pkg" || pkg.Version != "1.1" || len(pkg.Extensions) != 1 ||
|
|
pkg.Publisher == nil || pkg.Publisher.Name != "FTS" || pkg.License != "OGL" ||
|
|
pkg.PublicationPolicy != "policy" || pkg.Links == nil || pkg.Links.Next != "https://example/next" {
|
|
t.Errorf("package fields not copied: %+v", pkg)
|
|
}
|
|
if pkg.PublishedDate == nil ||
|
|
!pkg.PublishedDate.Equal(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("publishedDate = %v", pkg.PublishedDate)
|
|
}
|
|
if len(pkg.Releases) != 2 || pkg.Releases[0].OCID != "ocds-1" ||
|
|
pkg.Releases[1].OCID != "ocds-3" {
|
|
t.Errorf("releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases)
|
|
}
|
|
}
|
|
|
|
func TestSafeUnpackReleasesPublishedDate(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
date string // raw JSON value, empty to omit
|
|
want *time.Time
|
|
}{
|
|
{"omitted", "", nil},
|
|
{"empty string", `""`, nil},
|
|
{"RFC3339", `"2026-10-08T12:00:00Z"`, ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC))},
|
|
{"date only", `"2026-10-08"`, ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC))},
|
|
{"unparseable", `"yesterday"`, nil},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
body := `{"releases":[]}`
|
|
if tt.date != "" {
|
|
body = `{"publishedDate":` + tt.date + `,"releases":[]}`
|
|
}
|
|
pkg, err := safeUnpackReleases(responseFor(t, body))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
switch {
|
|
case tt.want == nil && pkg.PublishedDate != nil:
|
|
t.Errorf("publishedDate = %v, want nil", pkg.PublishedDate)
|
|
case tt.want != nil && (pkg.PublishedDate == nil || !pkg.PublishedDate.Equal(*tt.want)):
|
|
t.Errorf("publishedDate = %v, want %v", pkg.PublishedDate, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func ptrTime(t time.Time) *time.Time { return &t }
|
|
|
|
func TestSafeUnpackReleasesInvalidJSON(t *testing.T) {
|
|
for name, body := range map[string]string{
|
|
"not JSON": `<html>oops</html>`,
|
|
"wrong shape": `{"releases": {}}`,
|
|
"empty body": ``,
|
|
"publisher array": `{"publisher": []}`,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if _, err := safeUnpackReleases(responseFor(t, body)); err == nil {
|
|
t.Error("expected a decode error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// pagedServer serves pages[i] for ?page=i (page 0 at the release package
|
|
// path), linking each page to the next.
|
|
func pagedServer(t *testing.T, pages ...[]string) (*FindATender, *atomic.Int32) {
|
|
t.Helper()
|
|
var hits atomic.Int32
|
|
var srvURL string
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
hits.Add(1)
|
|
i := 0
|
|
fmt.Sscan(r.URL.Query().Get("page"), &i)
|
|
next := ""
|
|
if i+1 < len(pages) {
|
|
next = fmt.Sprintf("%s/ocdsReleasePackages?page=%d", srvURL, i+1)
|
|
}
|
|
io.WriteString(w, packageJSON(next, pages[i]...))
|
|
})
|
|
srvURL = srv.URL
|
|
return ft, &hits
|
|
}
|
|
|
|
func TestFetchOnce(t *testing.T) {
|
|
ft, hits := pagedServer(t,
|
|
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
|
[]string{releaseJSON("ocds-3", "1")},
|
|
)
|
|
releases, err := ft.fetchOnce(RequestOptions{Since: time.Now()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(releases) != 2 {
|
|
t.Errorf("got %d releases, want 2 (first page only)", len(releases))
|
|
}
|
|
if hits.Load() != 1 {
|
|
t.Errorf("server hit %d times, want 1", hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestFetchOnceErrors(t *testing.T) {
|
|
ft, _ := pagedServer(t, []string{})
|
|
if _, err := ft.fetchOnce(RequestOptions{Since: time.Now()}); !errors.Is(err, ErrNoReleases) {
|
|
t.Errorf("empty page: err = %v, want ErrNoReleases", err)
|
|
}
|
|
|
|
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, "not json")
|
|
})
|
|
if _, err := ft.fetchOnce(RequestOptions{Since: time.Now()}); err == nil {
|
|
t.Error("bad body: expected an error")
|
|
}
|
|
|
|
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
if _, err := ft.fetchOnce(
|
|
RequestOptions{Since: time.Now()},
|
|
); !errors.Is(
|
|
err,
|
|
ErrRateLimitExceeded,
|
|
) {
|
|
t.Errorf("rate limited: err = %v, want ErrRateLimitExceeded", err)
|
|
}
|
|
}
|
|
|
|
func TestFetchAll(t *testing.T) {
|
|
ft, hits := pagedServer(t,
|
|
[]string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")},
|
|
[]string{},
|
|
[]string{releaseJSON("ocds-1", "2")},
|
|
)
|
|
releases, err := ft.fetchAll(RequestOptions{Since: time.Now()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var ids []string
|
|
for _, r := range releases {
|
|
ids = append(ids, r.OCID+"/"+r.ID)
|
|
}
|
|
if strings.Join(ids, ",") != "ocds-1/1,ocds-2/1,ocds-1/2" {
|
|
t.Errorf("releases = %v", ids)
|
|
}
|
|
if hits.Load() != 3 {
|
|
t.Errorf("server hit %d times, want 3", hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestFetchAllErrors(t *testing.T) {
|
|
ft, _ := pagedServer(t, []string{}, []string{})
|
|
if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); !errors.Is(err, ErrNoReleases) {
|
|
t.Errorf("all pages empty: err = %v, want ErrNoReleases", err)
|
|
}
|
|
|
|
// First page fails.
|
|
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
})
|
|
if _, err := ft.fetchAll(
|
|
RequestOptions{Since: time.Now()},
|
|
); !errors.Is(
|
|
err,
|
|
ErrRateLimitExceeded,
|
|
) {
|
|
t.Errorf("first page rate limited: err = %v", err)
|
|
}
|
|
|
|
ft, _ = newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
io.WriteString(w, "not json")
|
|
})
|
|
if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); err == nil {
|
|
t.Error("first page bad body: expected an error")
|
|
}
|
|
|
|
// A later page fails, either to fetch or to decode.
|
|
for name, fail := range map[string]http.HandlerFunc{
|
|
"rate limited": func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTooManyRequests) },
|
|
"bad body": func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "not json") },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
var srvURL string
|
|
ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Query().Get("page") == "" {
|
|
io.WriteString(w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1")))
|
|
return
|
|
}
|
|
fail(w, r)
|
|
})
|
|
srvURL = srv.URL
|
|
if _, err := ft.fetchAll(RequestOptions{Since: time.Now()}); err == nil {
|
|
t.Error("expected an error from the second page")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// fetchAll stops, keeping what it has, when a next link repeats an earlier one.
|
|
func TestFetchAllRepeatedNextLink(t *testing.T) {
|
|
tests := map[string]func(base string, page string) string{
|
|
// page 0 -> page 1 -> page 1 ...
|
|
"self link": func(base, page string) string { return base + "/next?page=1" },
|
|
// page 0 -> page 1 -> page 2 -> page 1 ...
|
|
"cycle": func(base, page string) string {
|
|
if page == "1" {
|
|
return base + "/next?page=2"
|
|
}
|
|
return base + "/next?page=1"
|
|
},
|
|
}
|
|
for name, nextFor := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
var hits atomic.Int32
|
|
var srvURL string
|
|
ft, srv := newTestFT(t, "error", func(w http.ResponseWriter, r *http.Request) {
|
|
if hits.Add(1) > 10 {
|
|
t.Error("fetchAll did not stop following next links")
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
page := r.URL.Query().Get("page")
|
|
io.WriteString(w, packageJSON(
|
|
nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"),
|
|
))
|
|
})
|
|
srvURL = srv.URL
|
|
|
|
releases, err := ft.fetchAll(RequestOptions{Since: time.Now()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(releases) != int(hits.Load()) {
|
|
t.Errorf("releases = %d, want one per request (%d)", len(releases), hits.Load())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalLenientDates(t *testing.T) {
|
|
raw := json.RawMessage(`{
|
|
"date": "2006-10-02",
|
|
"publishedDate": "",
|
|
"tender": {
|
|
"tenderPeriod": {"startDate": "2006-10-02T15:33:02+07:00", "endDate": "2006-11-30"},
|
|
"description": "Bids close on 2006-11-30",
|
|
"expressionOfInterestDeadline": "2006-10-20"
|
|
},
|
|
"awards": [{"date": "2006-12-01", "id": "2006-12-01"}],
|
|
"value": 1234567.891
|
|
}`)
|
|
|
|
var got struct {
|
|
Date time.Time `json:"date"`
|
|
PublishedDate *time.Time `json:"publishedDate"`
|
|
Tender struct {
|
|
TenderPeriod struct {
|
|
StartDate *time.Time `json:"startDate"`
|
|
EndDate *time.Time `json:"endDate"`
|
|
} `json:"tenderPeriod"`
|
|
Description string `json:"description"`
|
|
ExpressionOfInterestDeadline *time.Time `json:"expressionOfInterestDeadline"`
|
|
} `json:"tender"`
|
|
Awards []struct {
|
|
Date *time.Time `json:"date"`
|
|
ID string `json:"id"`
|
|
} `json:"awards"`
|
|
Value json.Number `json:"value"`
|
|
}
|
|
if err := unmarshalLenientDates(raw, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
midnight := func(y int, m time.Month, d int) time.Time { return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) }
|
|
if !got.Date.Equal(midnight(2006, 10, 2)) {
|
|
t.Errorf("date = %v", got.Date)
|
|
}
|
|
if got.PublishedDate != nil {
|
|
t.Errorf("empty publishedDate should decode as nil, got %v", got.PublishedDate)
|
|
}
|
|
if start := got.Tender.TenderPeriod.StartDate; start == nil ||
|
|
!start.Equal(time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600))) {
|
|
t.Errorf("full timestamp changed: %v", start)
|
|
}
|
|
if end := got.Tender.TenderPeriod.EndDate; end == nil || !end.Equal(midnight(2006, 11, 30)) {
|
|
t.Errorf("endDate = %v", end)
|
|
}
|
|
if d := got.Tender.ExpressionOfInterestDeadline; d == nil || !d.Equal(midnight(2006, 10, 20)) {
|
|
t.Errorf("expressionOfInterestDeadline = %v", d)
|
|
}
|
|
if got.Tender.Description != "Bids close on 2006-11-30" {
|
|
t.Errorf("free text changed: %q", got.Tender.Description)
|
|
}
|
|
if a := got.Awards[0]; a.Date == nil || !a.Date.Equal(midnight(2006, 12, 1)) ||
|
|
a.ID != "2006-12-01" {
|
|
t.Errorf("award = %+v (date should be repaired, id left alone)", a)
|
|
}
|
|
if got.Value != "1234567.891" {
|
|
t.Errorf("number changed: %s", got.Value)
|
|
}
|
|
}
|
|
|
|
func TestRepairDatesLeavesValidJSONAlone(t *testing.T) {
|
|
raw := json.RawMessage(`{"date": "2006-10-02T15:33:02+07:00", "title": "x"}`)
|
|
out, err := repairDates(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(out, raw) {
|
|
t.Errorf("valid input was rewritten: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestRepairDatesRejectsBadJSON(t *testing.T) {
|
|
for _, raw := range []string{`{`, `{"date": "2006-10-02"} trailing`} {
|
|
if _, err := repairDates(json.RawMessage(raw)); err == nil {
|
|
t.Errorf("expected an error for %q", raw)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalLenientDatesErrors(t *testing.T) {
|
|
var v struct {
|
|
Date time.Time `json:"date"`
|
|
}
|
|
tests := map[string]string{
|
|
"bad JSON": `{`,
|
|
"wrong type": `{"date": 5}`,
|
|
"unparseable date": `{"date": "02/10/2006"}`,
|
|
"not a JSON object": `"2006-10-02"`,
|
|
}
|
|
for name, raw := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
if err := unmarshalLenientDates(json.RawMessage(raw), &v); err == nil {
|
|
t.Errorf("expected an error for %s", raw)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReleaseIDs(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
raw string
|
|
wantOCID string
|
|
wantID string
|
|
wantErr string // substring; empty means no error
|
|
}{
|
|
{"ocid then id", `{"ocid":"ocds-1","id":"r1"}`, "ocds-1", "r1", ""},
|
|
{"id then ocid", `{"id":"r1","ocid":"ocds-1"}`, "ocds-1", "r1", ""},
|
|
{
|
|
"numeric id keeps its digits",
|
|
`{"ocid":"ocds-1","id":12345678901234567890}`,
|
|
"ocds-1",
|
|
"12345678901234567890",
|
|
"",
|
|
},
|
|
{
|
|
"skips other fields",
|
|
`{"date":"2026-01-01","tender":{"items":[1,2]},"ocid":"ocds-1","id":"r1"}`,
|
|
"ocds-1",
|
|
"r1",
|
|
"",
|
|
},
|
|
{
|
|
"ignores errors after both ids",
|
|
`{"ocid":"ocds-1","id":"r1","tender":{"value":"oops"},` + `"bad":}`,
|
|
"ocds-1",
|
|
"r1",
|
|
"",
|
|
},
|
|
{"empty input", ``, "", "", "reading release"},
|
|
{"not an object", `["ocid","id"]`, "", "", "not a JSON object"},
|
|
{"empty object", `{}`, "", "", "no ocid or id"},
|
|
{"missing ocid", `{"id":"r1"}`, "", "r1", "no ocid"},
|
|
{"missing id", `{"ocid":"ocds-1"}`, "ocds-1", "", "no id"},
|
|
{"ocid wrong type", `{"ocid":5,"id":"r1"}`, "", "", "reading release ocid"},
|
|
{"id wrong type", `{"ocid":"ocds-1","id":true}`, "ocds-1", "", "unexpected bool"},
|
|
{"id is an object", `{"ocid":"ocds-1","id":{}}`, "ocds-1", "", "unexpected map"},
|
|
{"invalid id JSON", `{"ocid":"ocds-1","id":}`, "ocds-1", "", "reading release id"},
|
|
{
|
|
"invalid field before ids",
|
|
`{"date":nope,"ocid":"ocds-1","id":"r1"}`,
|
|
"",
|
|
"",
|
|
`reading release field "date"`,
|
|
},
|
|
{"truncated after ocid", `{"ocid":"ocds-1",`, "ocds-1", "", "reading release"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ocid, id, err := releaseIDs(json.RawMessage(tt.raw))
|
|
if tt.wantErr == "" {
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
} else if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
|
t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
|
|
}
|
|
if ocid != tt.wantOCID || id != tt.wantID {
|
|
t.Errorf("got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID)
|
|
}
|
|
})
|
|
}
|
|
}
|