Added a "Published in the last X days" filter, applied by the backend

GET /api/opportunities takes released_from, an RFC 3339 time, and lists
only the opportunities whose latest release is at or after it. It works
with paging and with changed_after, which keeps the same batch either
way; an invalid value is 400.

The frontend's filter defaults to 7 days, from midnight UK time, and
sends it with every load and catch-up, so a year of opportunities isn't
downloaded to show a week's. Changing the days loads the list again;
Clear filters goes back to 7.
This commit is contained in:
bwilliams committed 2026-10-19 10:05:45 +01:00
1 parent b8244da2e5
commit 746fa0a9c1
19 files changed
+333 -64

No files matched your search

+19
View File
@@ -3,6 +3,7 @@ package api
import (
"fmt"
"strconv"
"time"
)
// intParam parses the query parameter text as an integer from low to high,
@@ -21,3 +22,21 @@ func intParam(text string, fallback, low, high int) (int, error) {
return number, nil
}
// timeParam parses the query parameter name's text as an RFC 3339 time, or
// returns the zero time if it is empty. Its error is meant for the client.
func timeParam(name, text string) (time.Time, error) {
if text == "" {
return time.Time{}, nil
}
when, err := time.Parse(time.RFC3339, text)
if err != nil {
return time.Time{}, fmt.Errorf(
"%s must be an RFC 3339 date and time, e.g. 2026-10-12T00:00:00+01:00",
name,
)
}
return when, nil
}
+26 -11
View File
@@ -7,6 +7,7 @@ import (
"reflect"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
@@ -71,7 +72,8 @@ func (h *opportunityRoutes) Routes(router chi.Router) {
// list serves GET /opportunities: a page of opportunities, newest release
// first, with their classifications and marks but not their releases. The
// page is set by the offset and limit (default 50, at most 100) query
// parameters. With changed_after, it serves the opportunities changed since
// parameters, and released_from leaves out opportunities whose latest release
// is before it. With changed_after, it serves the opportunities changed since
// that batch instead (see listChanged).
//
// @Summary List opportunities
@@ -85,27 +87,38 @@ func (h *opportunityRoutes) Routes(router chi.Router) {
// @Description releases or with changed marks), and batch is the newest
// @Description batch they include. offset and limit can't be given with
// @Description changed_after. An unknown batch is 410 Gone: reload the
// @Description whole list.
// @Description whole list. released_from (RFC 3339, inclusive) leaves out
// @Description opportunities whose latest release is before it, in a page
// @Description or with changed_after.
// @Tags opportunities
// @Produce json
// @Param offset query int false "Opportunities to skip" minimum(0) default(0)
// @Param limit query int false "Page size" minimum(1) maximum(100) default(50)
// @Param changed_after query string false "List the opportunities changed after this batch instead"
// @Param released_from query string false "Only opportunities whose latest release is at or after this time" format(date-time)
// @Success 200 {object} OpportunityPage
// @Failure 400 {object} httpx.Problem "Invalid offset or limit, or either with changed_after"
// @Failure 400 {object} httpx.Problem "Invalid offset, limit or released_from, or offset or limit with changed_after"
// @Failure 410 {object} httpx.Problem "Unknown batch"
// @Failure 500 {object} httpx.Problem
// @Router /opportunities [get]
func (h *opportunityRoutes) list(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
releasedFrom, err := timeParam(
"released_from", query.Get("released_from"),
)
if err != nil {
httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error())
return
}
if query.Has("changed_after") {
h.listChanged(w, r)
h.listChanged(w, r, releasedFrom)
return
}
offset, offsetErr := intParam(query.Get("offset"), 0, 0, 1<<30)
limit, limitErr := intParam(query.Get("limit"), 50, 1, 100)
err := errors.Join(offsetErr, limitErr)
err = errors.Join(offsetErr, limitErr)
if err != nil {
httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error())
return
@@ -123,8 +136,9 @@ func (h *opportunityRoutes) list(w http.ResponseWriter, r *http.Request) {
opportunities, err := h.store.ListOpportunities(
r.Context(),
store.OpportunityFilter{
Limit: limit + 1,
Offset: offset,
Limit: limit + 1,
Offset: offset,
ReleasedFrom: releasedFrom,
},
)
if err != nil {
@@ -148,10 +162,11 @@ func (h *opportunityRoutes) list(w http.ResponseWriter, r *http.Request) {
// listChanged serves GET /opportunities?changed_after=batch: every
// opportunity changed after the change batch, as one page, with the newest
// batch it includes. An unknown batch, such as one from another database, is
// 410 Gone, as the client can't catch up from it.
// batch it includes, leaving out those whose latest release is before
// releasedFrom unless it is zero. An unknown batch, such as one from another
// database, is 410 Gone, as the client can't catch up from it.
func (h *opportunityRoutes) listChanged(
w http.ResponseWriter, r *http.Request,
w http.ResponseWriter, r *http.Request, releasedFrom time.Time,
) {
query := r.URL.Query()
if query.Has("offset") || query.Has("limit") {
@@ -165,7 +180,7 @@ func (h *opportunityRoutes) listChanged(
}
opportunities, batch, err := h.store.ListChangedOpportunities(
r.Context(), query.Get("changed_after"),
r.Context(), query.Get("changed_after"), releasedFrom,
)
if errors.Is(err, store.ErrUnknownBatch) {
httpx.WriteProblem(
+52 -3
View File
@@ -104,7 +104,8 @@ func uuids(page OpportunityPage) []string {
return ids
}
// Opportunities are listed newest first, a page at a time.
// Opportunities are listed newest first, a page at a time, from
// released_from if given (ocds-N is released 2026-10-0N at 12:00 UTC).
func TestListOpportunitiesPaging(t *testing.T) {
url := opportunitiesServer(t, 5)
all := []string{"ocds-5", "ocds-4", "ocds-3", "ocds-2", "ocds-1"}
@@ -120,6 +121,12 @@ func TestListOpportunitiesPaging(t *testing.T) {
{"?limit=5", all, false},
{"?offset=10", nil, false},
{"", all, false},
{"?released_from=", all, false},
{"?released_from=2026-10-03T12:00:00Z", []string{"ocds-5", "ocds-4", "ocds-3"}, false},
{"?released_from=2026-10-03T13:00:00%2B01:00", []string{"ocds-5", "ocds-4", "ocds-3"}, false},
{"?released_from=2026-10-03T12:00:01Z", []string{"ocds-5", "ocds-4"}, false},
{"?released_from=2026-10-02T00:00:00Z&limit=1", []string{"ocds-5"}, true},
{"?released_from=2026-10-02T00:00:00Z&limit=2&offset=2", []string{"ocds-3", "ocds-2"}, false},
}
for _, tt := range tests {
var page OpportunityPage
@@ -161,9 +168,19 @@ func TestListOpportunitiesEmpty(t *testing.T) {
}
}
// Bad paging or released_from is 400. An unencoded "+" in released_from
// arrives as a space, so it is refused too.
func TestListOpportunitiesBadPaging(t *testing.T) {
url := opportunitiesServer(t, 1)
queries := []string{"?limit=0", "?limit=101", "?limit=x", "?offset=-1"}
queries := []string{
"?limit=0",
"?limit=101",
"?limit=x",
"?offset=-1",
"?released_from=x",
"?released_from=2026-10-03",
"?released_from=2026-10-03T13:00:00+01:00",
}
for _, query := range queries {
status := request(t, "GET", url+"/opportunities"+query, nil)
if status != 400 {
@@ -211,6 +228,36 @@ func TestListChangedOpportunities(t *testing.T) {
)
}
// released_from leaves out ocds-2 (released on the 2nd), not the batch.
var bounded OpportunityPage
status = request(
t,
"GET",
url+"/opportunities?released_from=2026-10-03T00:00:00Z"+
"&changed_after="+page.Batch,
&bounded,
)
if status != 200 || len(bounded.Opportunities) != 0 ||
bounded.Batch != changed.Batch {
t.Errorf(
"released_from the 3rd: status %d, %+v; want [] and batch %q",
status, bounded, changed.Batch,
)
}
status = request(
t,
"GET",
url+"/opportunities?changed_after="+page.Batch+
"&released_from=2026-10-02T12:00:00Z",
&bounded,
)
if status != 200 || strings.Join(uuids(bounded), ",") != "ocds-2" {
t.Errorf(
"released_from the 2nd: status %d, %+v; want ocds-2",
status, bounded,
)
}
var latest OpportunityPage
request(t, "GET", url+"/opportunities?limit=1", &latest)
if latest.Batch != changed.Batch {
@@ -218,7 +265,8 @@ func TestListChangedOpportunities(t *testing.T) {
}
}
// An unknown batch is 410 Gone, and paging can't be used with changed_after.
// An unknown batch is 410 Gone, and paging can't be used with changed_after;
// a bad released_from is 400 there too.
func TestListChangedOpportunitiesRefused(t *testing.T) {
url := opportunitiesServer(t, 1)
var page OpportunityPage
@@ -232,6 +280,7 @@ func TestListChangedOpportunitiesRefused(t *testing.T) {
{"?changed_after=", http.StatusGone},
{"?changed_after=" + page.Batch + "&limit=5", http.StatusBadRequest},
{"?offset=0&changed_after=" + page.Batch, http.StatusBadRequest},
{"?changed_after=" + page.Batch + "&released_from=x", http.StatusBadRequest},
}
for _, tt := range tests {
status := request(t, "GET", url+"/opportunities"+tt.query, nil)
+2 -2
View File
@@ -110,7 +110,7 @@ ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
})
rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags
batch, err := s.LatestChangeBatch(ctx) // hash of the newest change batch, or ""
ts, batch, err = s.ListChangedOpportunities(ctx, batch) // changed since; ErrUnknownBatch
ts, batch, err = s.ListChangedOpportunities(ctx, batch, time.Time{}) // changed since; ErrUnknownBatch
ok, err := s.ReleaseExists(ctx, uuid, releaseID)
```
@@ -127,7 +127,7 @@ ok, err := s.ReleaseExists(ctx, uuid, releaseID)
**Listing.** `ListOpportunities` filters by status, classification code, title text, a range on `LatestReleaseDate` and each mark (`MarkSet` or `MarkUnset`; an opportunity with no marks stored has every mark unset), newest first, 50 per page by default. It loads classifications and marks but not releases (which can be large); those are loaded by `ListReleases` or `GetOpportunity`.
`LatestChangeBatch` returns the hash of the newest change batch ("" if there is none). Read before listing, it is the batch to give `ListChangedOpportunities`, which returns the opportunities in every later batch, once each, unpaged and ordered as `ListOpportunities` orders them, with the hash of the newest of those batches to give it next time. Batches recorded while it runs are left for next time rather than missed. A hash it doesn't know, such as a deleted batch's, is `ErrUnknownBatch`.
`LatestChangeBatch` returns the hash of the newest change batch ("" if there is none). Read before listing, it is the batch to give `ListChangedOpportunities`, which returns the opportunities in every later batch, once each, unpaged and ordered as `ListOpportunities` orders them, with the hash of the newest of those batches to give it next time. A non-zero `releasedFrom` leaves out opportunities whose latest release is before it, as `OpportunityFilter.ReleasedFrom` does; the hash is the same either way. Batches recorded while it runs are left for next time rather than missed. A hash it doesn't know, such as a deleted batch's, is `ErrUnknownBatch`.
**Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted.
+2 -2
View File
@@ -46,7 +46,7 @@ func TestChangeBatchAfterClockWentBack(t *testing.T) {
t.Errorf("batch created at %v, not after %v", latest.CreatedAt, future)
}
got, hash, err := s.ListChangedOpportunities(ctx, "future")
got, hash, err := s.ListChangedOpportunities(ctx, "future", time.Time{})
if err != nil {
t.Fatal(err)
}
@@ -101,7 +101,7 @@ func TestChangeBatchRetention(t *testing.T) {
&records,
`SELECT COUNT(*) FROM opportunity_change_batch_opportunities`,
)
_, _, err = s.ListChangedOpportunities(ctx, old)
_, _, err = s.ListChangedOpportunities(ctx, old, time.Time{})
if retention > 0 {
if batches != 1 || records != 1 ||
+7 -2
View File
@@ -313,7 +313,8 @@ ON CONFLICT DO NOTHING`
// selectOpportunitiesChangedBetween reads, into opportunityRows, the
// opportunities in the batches created after the first argument and at or
// before the second, once each, ordered as ListOpportunities orders them.
// before the second, once each. ListChangedOpportunities may add AND
// clauses, then orderOpportunitiesChanged.
const selectOpportunitiesChangedBetween = `
SELECT t.*
FROM opportunities t
@@ -322,5 +323,9 @@ WHERE t.uuid IN (
FROM opportunity_change_batch_opportunities bo
JOIN opportunity_change_batches b ON b.id = bo.batch_id
WHERE b.created_at > ? AND b.created_at <= ?
)
)`
// orderOpportunitiesChanged ends selectOpportunitiesChangedBetween, ordering
// as ListOpportunities orders.
const orderOpportunitiesChanged = `
ORDER BY t.latest_release_date DESC, t.uuid`
+5 -1
View File
@@ -124,7 +124,11 @@ var checks = map[string]queryCheck{
// opportunity_change_batch_opportunities
"insertChangeBatchOpportunity": {sql: insertChangeBatchOpportunity},
"selectOpportunitiesChangedBetween": {
sql: selectOpportunitiesChangedBetween,
sql: selectOpportunitiesChangedBetween + orderOpportunitiesChanged,
},
"orderOpportunitiesChanged": {
sql: selectOpportunitiesChangedBetween + "\nAND " +
filterOpportunityReleasedFrom + orderOpportunitiesChanged,
},
}
+14 -8
View File
@@ -776,10 +776,14 @@ func (s *Store) LatestChangeBatch(ctx context.Context) (string, error) {
// those batches, to give it next time. With no newer batches, it returns no
// opportunities and after. An unknown hash is ErrUnknownBatch.
//
// Unless releasedFrom is zero, only opportunities whose latest release is on
// or after it are returned (as OpportunityFilter.ReleasedFrom); the hash
// returned is the same either way.
//
// Batches recorded while it runs are left for next time, so none is missed:
// only those up to the newest batch when it starts are read.
func (s *Store) ListChangedOpportunities(
ctx context.Context, after string,
ctx context.Context, after string, releasedFrom time.Time,
) ([]types.Opportunity, string, error) {
var since changeBatch
err := s.db.GetContext(ctx, &since, selectChangeBatchByHash, after)
@@ -796,14 +800,16 @@ func (s *Store) ListChangedOpportunities(
return nil, "", fmt.Errorf("reading the latest change batch: %w", err)
}
query := selectOpportunitiesChangedBetween
args := []any{since.CreatedAt, latest.CreatedAt}
if !releasedFrom.IsZero() {
query += "\nAND " + filterOpportunityReleasedFrom
args = append(args, releasedFrom.UTC())
}
query += orderOpportunitiesChanged
var rows []opportunityRow
err = s.db.SelectContext(
ctx,
&rows,
selectOpportunitiesChangedBetween,
since.CreatedAt,
latest.CreatedAt,
)
err = s.db.SelectContext(ctx, &rows, query, args...)
if err != nil {
return nil, "", fmt.Errorf("listing changed opportunities: %w", err)
}
+16 -2
View File
@@ -1020,7 +1020,9 @@ func TestChangeBatches(t *testing.T) {
}
changed := func(after string) ([]types.Opportunity, string) {
t.Helper()
opportunities, latest, err := s.ListChangedOpportunities(ctx, after)
opportunities, latest, err := s.ListChangedOpportunities(
ctx, after, time.Time{},
)
if err != nil {
t.Fatal(err)
}
@@ -1074,6 +1076,18 @@ func TestChangeBatches(t *testing.T) {
t.Errorf("changed opportunity without classifications: %+v", got[1])
}
// releasedFrom leaves out ocds-2 (released day 2), not the batch.
got, hash, err := s.ListChangedOpportunities(ctx, first, day(3))
if err != nil {
t.Fatal(err)
}
if !equal(uuids(got), []string{"ocds-3"}) || hash != second {
t.Errorf(
"changed from day 3: %v, %q; want [ocds-3], %q",
uuids(got), hash, second,
)
}
mustSetMarks(t, s, types.OpportunityMarks{
UUID: "ocds-1",
Skipped: true,
@@ -1091,7 +1105,7 @@ func TestChangeBatches(t *testing.T) {
t.Errorf("changed after latest: %v, %q; want [], %q", got, again, hash)
}
_, _, err := s.ListChangedOpportunities(ctx, "unknown")
_, _, err = s.ListChangedOpportunities(ctx, "unknown", time.Time{})
if !errors.Is(err, store.ErrUnknownBatch) {
t.Errorf("unknown batch: err %v, want ErrUnknownBatch", err)
}