Added change batches, so clients can catch up on changed opportunities

Every save that stores new opportunities or new releases records them in a
change batch (migration 00008: opportunity_change_batches and
opportunity_change_batch_opportunities; the opportunities table is
unchanged), as does every change of marks. A batch is named by a random
SHA-256 hash, so it can't be guessed, and ordered by its created_at, which
is kept strictly increasing even if the clock goes back.

GET /api/opportunities pages now carry the newest batch, read before the
page, and GET /api/opportunities?changed_after=<batch> returns every
opportunity changed since that batch, unpaged, with the newest batch it
includes. Offset and limit can't be combined with it, and an unknown batch
is 410 Gone, telling the client to reload the whole list.

OpportunitiesUpdated now carries its save's batch, and a "large" flag: over
50 opportunities, new and updated are left empty and only the summary
counts them, so a big catch-up no longer sends tens of thousands of UUIDs.

internal/store/store.go

@@ -5,7 +5,10 @@ package store
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"strings"
@@ -22,6 +25,9 @@ import (
 var (
 	// ErrNotFound is returned when a requested opportunity does not exist.
 	ErrNotFound = errors.New("store: not found")
+	// ErrUnknownBatch is returned by ListChangedOpportunities for a hash
+	// that names no change batch.
+	ErrUnknownBatch = errors.New("store: unknown change batch")
 	// ErrNeedsMigration is returned by Open when the database has pending
 	// migrations. Apply them with the migrate command, or pass
 	// WithAutoMigrate.
@@ -69,6 +75,10 @@ type SaveMultipleResult struct {
 	// a stored opportunity had before the save and none has after it, in
 	// code order; nil if there are none.
 	RemovedClassificationIDs []int64
+	// ChangeBatch is the hash of the change batch recording the
+	// opportunities the save stored for the first time or gave new releases
+	// (see ListChangedOpportunities); "" if there are none.
+	ChangeBatch string
 }
 
 // SaveResult reports what SaveOpportunity changed.
@@ -254,6 +264,15 @@ type opportunityRow struct {
 	types.Opportunity
 }
 
+// changeBatch is a row of opportunity_change_batches: a batch of changed
+// opportunities, named to clients by its Hash. ID and CreatedAt are internal
+// to the store.
+type changeBatch struct {
+	ID        int64     `db:"id"`
+	Hash      string    `db:"hash"`
+	CreatedAt time.Time `db:"created_at"`
+}
+
 // SaveOpportunity stores opportunity and its releases in one transaction.
 //
 // Releases (and their tags) are write-once: ones already stored are skipped.
@@ -265,6 +284,9 @@ type opportunityRow struct {
 // opportunity.Marks are stored only for a new opportunity; existing marks are
 // kept (use SetMarks to change them).
 //
+// A new opportunity, or one given new releases, is recorded in a change
+// batch of its own (see ListChangedOpportunities).
+//
 // opportunity is normalised in place (see prepareOpportunity), and when it is
 // written its classifications' IDs are filled in.
 func (s *Store) SaveOpportunity(
@@ -285,6 +307,15 @@ func (s *Store) SaveOpportunity(
 		return SaveResult{}, err
 	}
 
+	_, err = recordChangeBatch(
+		ctx,
+		tx,
+		changedUUIDs([]*types.Opportunity{opportunity}, []SaveResult{result}),
+	)
+	if err != nil {
+		return SaveResult{}, err
+	}
+
 	if err := tx.Commit(); err != nil {
 		return SaveResult{}, err
 	}
@@ -307,7 +338,9 @@ func (s *Store) SaveOpportunity(
 // Each opportunity is normalised in place (see prepareOpportunity), and when
 // it is written its classifications' IDs are filled in. The result reports
 // the classification codes the save brought into or out of use (see
-// ListClassifications).
+// ListClassifications), and the hash of the change batch recording the
+// opportunities that were new or given new releases (see
+// ListChangedOpportunities).
 func (s *Store) SaveOpportunities(
 	ctx context.Context, opportunities []*types.Opportunity,
 ) (SaveMultipleResult, error) {
@@ -354,6 +387,13 @@ func (s *Store) SaveOpportunities(
 	total.AddedClassifications = added
 	total.RemovedClassificationIDs = removed
 
+	total.ChangeBatch, err = recordChangeBatch(
+		ctx, tx, changedUUIDs(opportunities, total.OpportunityResults),
+	)
+	if err != nil {
+		return SaveMultipleResult{}, err
+	}
+
 	if err := tx.Commit(); err != nil {
 		return SaveMultipleResult{}, err
 	}
@@ -406,6 +446,82 @@ func classificationChanges(
 	return added, removed
 }
 
+// changedUUIDs returns the UUIDs of the opportunities that results, from
+// saving them, show were new or given new releases. An opportunity whose row
+// was only rewritten, such as one fetched again from its newest stored
+// release, hasn't changed.
+func changedUUIDs(
+	opportunities []*types.Opportunity, results []SaveResult,
+) []string {
+	var uuids []string
+	for i, result := range results {
+		if result.NewOpportunity || result.NewReleases > 0 {
+			uuids = append(uuids, opportunities[i].UUID)
+		}
+	}
+
+	return uuids
+}
+
+// recordChangeBatch records a change batch of the opportunities uuids inside
+// tx and returns its hash, or "" without recording anything if uuids is
+// empty.
+//
+// The batch's created_at is the current time, or just after the newest
+// batch's if that isn't later (the clock went back, or two batches came
+// within its resolution), so batches are ordered by created_at. tx holds the
+// write lock from its start (see dsn), so no other batch can be recorded in
+// between.
+func recordChangeBatch(
+	ctx context.Context, tx *sqlx.Tx, uuids []string,
+) (string, error) {
+	if len(uuids) == 0 {
+		return "", nil
+	}
+
+	var latest changeBatch
+	err := tx.GetContext(ctx, &latest, selectLatestChangeBatch)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return "", fmt.Errorf("reading the latest change batch: %w", err)
+	}
+
+	createdAt := time.Now().UTC()
+	if !createdAt.After(latest.CreatedAt) {
+		createdAt = latest.CreatedAt.Add(time.Nanosecond)
+	}
+	hash := newBatchHash(createdAt)
+
+	var batchID int64
+	err = tx.GetContext(ctx, &batchID, insertChangeBatch, hash, createdAt)
+	if err != nil {
+		return "", fmt.Errorf("recording a change batch: %w", err)
+	}
+
+	for _, uuid := range uuids {
+		_, err = tx.ExecContext(ctx, insertChangeBatchOpportunity, batchID, uuid)
+		if err != nil {
+			return "", fmt.Errorf(
+				"recording %s in a change batch: %w", uuid, err,
+			)
+		}
+	}
+
+	return hash, nil
+}
+
+// newBatchHash returns the hash of a change batch created at createdAt: the
+// hex SHA-256 hash of random bytes and the time. The random bytes make it
+// unguessable, so a client can only name a batch it was given.
+func newBatchHash(createdAt time.Time) string {
+	random := make([]byte, 32)
+	rand.Read(random) // crypto/rand.Read never returns an error
+
+	hash := sha256.Sum256(
+		append(random, createdAt.Format(time.RFC3339Nano)...),
+	)
+	return hex.EncodeToString(hash[:])
+}
+
 // saveOpportunity does the work of SaveOpportunity and SaveOpportunities for
 // one prepared opportunity inside tx. isNew reports that the opportunity was
 // not already stored.
@@ -539,7 +655,8 @@ func writeOpportunity(
 
 // SetMarks replaces the marks of the opportunity marks.UUID, or returns
 // ErrNotFound. Marks that fail marks.Validate (e.g. types.ErrWonAndLost) are
-// not stored.
+// not stored. The opportunity is recorded in a change batch of its own (see
+// ListChangedOpportunities), so a client catching up sees its new marks.
 func (s *Store) SetMarks(
 	ctx context.Context, marks types.OpportunityMarks,
 ) error {
@@ -550,11 +667,26 @@ func (s *Store) SetMarks(
 		return fmt.Errorf("opportunity %s: %w", marks.UUID, err)
 	}
 
-	_, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, marks)
+	tx, err := s.db.BeginTxx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback() //nolint:errcheck // no-op after Commit
+
+	_, err = tx.NamedExecContext(ctx, upsertOpportunityMarks, marks)
 	if isForeignKeyError(err) {
 		return ErrNotFound
 	}
-	return err
+	if err != nil {
+		return err
+	}
+
+	_, err = recordChangeBatch(ctx, tx, []string{marks.UUID})
+	if err != nil {
+		return err
+	}
+
+	return tx.Commit()
 }
 
 // isForeignKeyError reports whether err is SQLite rejecting a write that
@@ -606,6 +738,76 @@ func (s *Store) ListOpportunities(
 		return nil, err
 	}
 
+	return s.opportunitiesFromRows(ctx, rows)
+}
+
+// LatestChangeBatch returns the hash of the newest change batch, or "" if
+// none has been recorded. Read before listing opportunities, it is the batch
+// to give ListChangedOpportunities later to get what changed since.
+func (s *Store) LatestChangeBatch(ctx context.Context) (string, error) {
+	var latest changeBatch
+	err := s.db.GetContext(ctx, &latest, selectLatestChangeBatch)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("reading the latest change batch: %w", err)
+	}
+
+	return latest.Hash, nil
+}
+
+// ListChangedOpportunities returns the opportunities recorded in the change
+// batches created after the batch with the hash after, once each, as
+// ListOpportunities returns them but unpaged, and the hash of the newest of
+// those batches, to give it next time. With no newer batches, it returns no
+// opportunities and after. An unknown hash is ErrUnknownBatch.
+//
+// Batches recorded while it runs are left for next time, so none is missed:
+// only those up to the newest batch when it starts are read.
+func (s *Store) ListChangedOpportunities(
+	ctx context.Context, after string,
+) ([]types.Opportunity, string, error) {
+	var since changeBatch
+	err := s.db.GetContext(ctx, &since, selectChangeBatchByHash, after)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, "", ErrUnknownBatch
+	}
+	if err != nil {
+		return nil, "", fmt.Errorf("reading change batch %s: %w", after, err)
+	}
+
+	var latest changeBatch
+	err = s.db.GetContext(ctx, &latest, selectLatestChangeBatch)
+	if err != nil {
+		return nil, "", fmt.Errorf("reading the latest change batch: %w", err)
+	}
+
+	var rows []opportunityRow
+	err = s.db.SelectContext(
+		ctx,
+		&rows,
+		selectOpportunitiesChangedBetween,
+		since.CreatedAt,
+		latest.CreatedAt,
+	)
+	if err != nil {
+		return nil, "", fmt.Errorf("listing changed opportunities: %w", err)
+	}
+
+	opportunities, err := s.opportunitiesFromRows(ctx, rows)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return opportunities, latest.Hash, nil
+}
+
+// opportunitiesFromRows returns the opportunities in rows, in order, with
+// their classifications and marks. The result is never nil.
+func (s *Store) opportunitiesFromRows(
+	ctx context.Context, rows []opportunityRow,
+) ([]types.Opportunity, error) {
 	opportunities := make([]types.Opportunity, len(rows))
 	for i, row := range rows {
 		opportunities[i] = row.Opportunity