Added change batches, so clients can catch up on changed opportunities

Every save that stores new opportunities or new releases records them in a
change batch (migration 00008: opportunity_change_batches and
opportunity_change_batch_opportunities; the opportunities table is
unchanged), as does every change of marks. A batch is named by a random
SHA-256 hash, so it can't be guessed, and ordered by its created_at, which
is kept strictly increasing even if the clock goes back.

GET /api/opportunities pages now carry the newest batch, read before the
page, and GET /api/opportunities?changed_after=<batch> returns every
opportunity changed since that batch, unpaged, with the newest batch it
includes. Offset and limit can't be combined with it, and an unknown batch
is 410 Gone, telling the client to reload the whole list.

OpportunitiesUpdated now carries its save's batch, and a "large" flag: over
50 opportunities, new and updated are left empty and only the summary
counts them, so a big catch-up no longer sends tens of thousands of UUIDs.

internal/fetcher/fetcher.go

@@ -238,7 +238,7 @@ func (f *Fetcher) save(
 	)
 
 	f.publishClassificationChanges(result)
-	f.publishUpdates(opportunities, result.OpportunityResults)
+	f.publishUpdates(opportunities, result)
 
 	return nil
 }
@@ -277,12 +277,14 @@ func (f *Fetcher) publishClassificationChanges(
 }
 
 // publishUpdates publishes an events.OpportunitiesUpdated event for the
-// opportunities that results, from saving them, show were new or gained new
+// opportunities that result, from saving them, shows were new or gained new
 // releases, if there were any and a bus was given WithEvents. An opportunity
 // whose row was only rewritten, such as one fetched again from the newest
-// stored release, isn't counted as updated.
+// stored release, isn't counted as updated. Over
+// events.MaxListedOpportunities, they are only counted (see
+// events.OpportunitiesUpdatedPayload).
 func (f *Fetcher) publishUpdates(
-	opportunities []*types.Opportunity, results []store.SaveResult,
+	opportunities []*types.Opportunity, result store.SaveMultipleResult,
 ) {
 	if f.events == nil {
 		return
@@ -291,12 +293,13 @@ func (f *Fetcher) publishUpdates(
 	payload := events.OpportunitiesUpdatedPayload{
 		New:     []string{},
 		Updated: []string{},
+		Batch:   result.ChangeBatch,
 	}
-	for i, result := range results {
+	for i, saved := range result.OpportunityResults {
 		switch {
-		case result.NewOpportunity:
+		case saved.NewOpportunity:
 			payload.New = append(payload.New, opportunities[i].UUID)
-		case result.NewReleases > 0:
+		case saved.NewReleases > 0:
 			payload.Updated = append(payload.Updated, opportunities[i].UUID)
 		}
 	}
@@ -307,6 +310,11 @@ func (f *Fetcher) publishUpdates(
 	if payload.Summary.New == 0 && payload.Summary.Updated == 0 {
 		return
 	}
+	if len(payload.New)+len(payload.Updated) > events.MaxListedOpportunities {
+		payload.New = []string{}
+		payload.Updated = []string{}
+		payload.Large = true
+	}
 
 	f.events.Publish(events.Event{
 		Type:    events.OpportunitiesUpdated,

internal/fetcher/fetcher_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
@@ -708,6 +709,18 @@ func TestSavePublishesUpdates(t *testing.T) {
 	save(onePackage) // fetched again: nothing published
 	save(newRelease) // a new release of a stored opportunity
 
+	// Each event names its own batch; the hashes are random.
+	if len(published) == 2 {
+		if published[0].Batch == "" ||
+			published[0].Batch == published[1].Batch {
+			t.Errorf(
+				"batches %q and %q; want two different ones",
+				published[0].Batch, published[1].Batch,
+			)
+		}
+		published[0].Batch, published[1].Batch = "", ""
+	}
+
 	want := []events.OpportunitiesUpdatedPayload{
 		{
 			New:     []string{"ocds-1"},
@@ -729,6 +742,61 @@ func TestSavePublishesUpdates(t *testing.T) {
 	}
 }
 
+// More than events.MaxListedOpportunities are only counted.
+func TestPublishUpdatesLarge(t *testing.T) {
+	var bus events.Bus
+	var published []events.OpportunitiesUpdatedPayload
+	bus.Subscribe(func(event events.Event) {
+		payload := event.Payload.(events.OpportunitiesUpdatedPayload)
+		published = append(published, payload)
+	})
+	f := New(&config.JSCSources{}, WithEvents(&bus))
+
+	publish := func(count int) {
+		t.Helper()
+		var opportunities []*types.Opportunity
+		result := store.SaveMultipleResult{
+			ChangeBatch: "batch",
+		}
+		for i := range count {
+			opportunities = append(opportunities, &types.Opportunity{
+				UUID: fmt.Sprintf("ocds-%d", i),
+			})
+			result.OpportunityResults = append(
+				result.OpportunityResults,
+				store.SaveResult{
+					NewOpportunity: i%2 == 0,
+					NewReleases:    1,
+				},
+			)
+		}
+		f.publishUpdates(opportunities, result)
+	}
+
+	publish(events.MaxListedOpportunities)
+	publish(events.MaxListedOpportunities + 1)
+	if len(published) != 2 {
+		t.Fatalf("published %d events; want 2", len(published))
+	}
+
+	listed := published[0]
+	if listed.Large ||
+		len(listed.New)+len(listed.Updated) != events.MaxListedOpportunities {
+		t.Errorf("at the limit: %+v; want every UUID listed", listed)
+	}
+
+	large := published[1]
+	want := events.OpportunitiesSummary{
+		New:     26,
+		Updated: 25,
+	}
+	if !large.Large || len(large.New) != 0 || len(large.Updated) != 0 ||
+		large.New == nil || large.Updated == nil ||
+		large.Summary != want || large.Batch != "batch" {
+		t.Errorf("over the limit: %+v; want only counted", large)
+	}
+}
+
 // Without WithEvents, a save publishes nothing and still succeeds.
 func TestSaveWithoutEvents(t *testing.T) {
 	f := New(&config.JSCSources{}, WithStore(openTestStore(t)))