First pass implementation of an SQLite backing store for fetched opportunities

This commit is contained in:
bwilliams committed 2026-10-13 12:52:17 +01:00
1 parent 9b0e6dccfe
commit 045492a148
29 files changed
+3132 -9

No files matched your search

+596
View File
@@ -0,0 +1,596 @@
// Package store persists opportunities and their OCDS releases in SQLite using
// sqlx and hand-written SQL. The SQL is in queries.go; this file holds the
// logic. Requires building with -tags sqlite_fts5.
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"unicode"
"git.avsdev.uk/Go/job-search/internal/types"
"github.com/jmoiron/sqlx"
"github.com/mattn/go-sqlite3"
)
// ErrNotFound is returned when a requested opportunity does not exist.
var ErrNotFound = errors.New("store: not found")
// DefaultListLimit is used when OpportunityFilter.Limit is zero or negative.
const DefaultListLimit = 50
// Store is the opportunity database. Open it with Open; it is safe for
// concurrent use.
type Store struct {
db *sqlx.DB
}
// SaveMultipleResult reports what SaveOpportunities changed.
type SaveMultipleResult struct {
// NewOpportunities is the number of opportunities that were not already stored.
NewOpportunities int
// NewReleases is the number of releases that were not already stored.
NewReleases int
// UpdatedOpportunities is incremented for each opportunity row and its classifications
// that are updated (when every release in the save was older than the
// one the stored opportunity was last updated from). It does not include new
// opportunities.
UpdatedOpportunities int
// OpportunityResults is a slice of the results of each opportunity saved.
OpportunityResults []SaveResult
}
// SaveResult reports what SaveOpportunity changed.
type SaveResult struct {
// OpportunityUpdated is true if the opportunity row and its classifications were
// written. It is false when every release in the save was older than the
// one the stored opportunity was last updated from.
OpportunityUpdated bool
// NewReleases is the number of releases that were not already stored.
NewReleases int
}
// MarkFilter filters ListOpportunities on one of an opportunity's marks.
type MarkFilter int
const (
// MarkAny does not filter. It is the zero value.
MarkAny MarkFilter = iota
// MarkSet matches opportunities with the mark set.
MarkSet
// MarkUnset matches opportunities with the mark unset, including those
// with no marks stored.
MarkUnset
)
// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued fields do not
// filter.
type OpportunityFilter struct {
Status string
// ClassificationCode matches opportunities with a classification of this code
// in any scheme.
ClassificationCode string
// Title is free text matched against opportunity titles: every word must
// appear, and each word also matches as a prefix ("cloud migr" matches
// "Cloud migration services").
Title string
// ReleasedFrom (inclusive) and ReleasedBefore (exclusive) bound
// Opportunity.LatestReleaseDate.
ReleasedFrom time.Time
ReleasedBefore time.Time
// Each mark filter matches Opportunity.Marks' field of the same name.
Hidden MarkFilter
Skipped MarkFilter
Interested MarkFilter
BidSubmitted MarkFilter
Won MarkFilter
Lost MarkFilter
// Limit defaults to DefaultListLimit. Results are ordered newest
// LatestReleaseDate first.
Limit int
Offset int
}
func (f OpportunityFilter) limit() int {
if f.Limit <= 0 {
return DefaultListLimit
}
return f.Limit
}
// prepareOpportunity validates t and normalises it in place before saving: child
// rows get the opportunity's UUID and their release id, times are converted to
// UTC (so stored timestamps compare correctly as text), and
// LatestReleaseDate becomes the newest release date in the save.
func prepareOpportunity(t *types.Opportunity) error {
if t == nil {
return errors.New("store: nil opportunity")
}
if t.UUID == "" {
return errors.New("store: opportunity has no UUID")
}
t.Marks.UUID = t.UUID
if err := t.Marks.Validate(); err != nil {
return fmt.Errorf("store: opportunity %s: %w", t.UUID, err)
}
latest := t.LatestReleaseDate
for i := range t.Releases {
r := &t.Releases[i]
if r.ReleaseID == "" {
return fmt.Errorf("store: opportunity %s: release %d has no release id", t.UUID, i)
}
r.UUID = t.UUID
r.ReleaseDate = r.ReleaseDate.UTC()
if r.ReleaseDate.After(latest) {
latest = r.ReleaseDate
}
for j := range r.Tags {
r.Tags[j].UUID = t.UUID
r.Tags[j].ReleaseID = r.ReleaseID
}
}
if latest.IsZero() {
return fmt.Errorf("store: opportunity %s has no release date", t.UUID)
}
t.LatestReleaseDate = latest.UTC()
t.StartDate = utcPtr(t.StartDate)
t.EndDate = utcPtr(t.EndDate)
return nil
}
func utcPtr(t *time.Time) *time.Time {
if t == nil {
return nil
}
u := t.UTC()
return &u
}
// ftsQuery turns free text into an FTS5 query that requires every word, each
// as a prefix match. Words are split the same way the unicode61 tokenizer
// splits them and each is quoted, so user input can never be interpreted as
// FTS5 syntax (operators, column filters, unbalanced quotes). It returns ""
// when the text contains no words.
func ftsQuery(text string) string {
words := strings.FieldsFunc(text, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
})
for i, w := range words {
words[i] = `"` + w + `"*`
}
return strings.Join(words, " ")
}
// Open opens the database at path. The schema must be exactly at this
// build's latest migration: Open returns ErrNeedsMigration if migrations are
// pending (run the migrate command first), or ErrSchemaTooNew if the database
// was migrated by a newer build. WithAutoMigrate applies pending migrations
// instead.
//
// A database file that doesn't exist is created, and then needs migrating.
func Open(ctx context.Context, path string, opts ...Option) (*Store, error) {
var o options
for _, opt := range opts {
opt(&o)
}
sqldb, err := OpenDB(path)
if err != nil {
return nil, err
}
if err := sqldb.PingContext(ctx); err != nil {
return nil, errors.Join(err, sqldb.Close())
}
if err := prepareSchema(ctx, sqldb, o); err != nil {
return nil, errors.Join(err, sqldb.Close())
}
return &Store{db: sqlx.NewDb(sqldb, "sqlite3")}, nil
}
// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }
// Reads select whole rows ("SELECT *", "t.*", "c.*"). sqlx rejects a result
// column with no matching field, so a column added by a migration makes the
// reads fail loudly ("missing destination name ...") until it is mapped,
// instead of being silently ignored.
//
// opportunityRow gives the opportunities table's internal id column a home, so it stays
// out of types.Opportunity. sqlx maps the embedded Opportunity's fields as if declared
// here. The other tables have no internal columns and read straight into
// internal/types.
type opportunityRow struct {
ID int64 `db:"id"`
types.Opportunity
}
// SaveOpportunity stores t and its releases in one transaction.
//
// Releases (and their tags) are write-once: ones already stored are skipped.
// The opportunity row and its classifications are replaced only if t's newest
// release is at least as new as the release the stored opportunity came from, so
// a late-arriving older release never undoes a newer one.
//
// t.Marks are stored only for a new opportunity; existing marks are kept (use
// SetMarks to change them).
//
// t is normalised in place (see prepareOpportunity), and when the opportunity is
// written its classifications' IDs are filled in.
func (s *Store) SaveOpportunity(ctx context.Context, t *types.Opportunity) (SaveResult, error) {
if err := prepareOpportunity(t); err != nil {
return SaveResult{}, err
}
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return SaveResult{}, err
}
defer tx.Rollback() //nolint:errcheck // no-op after Commit
res, _, err := saveOpportunity(ctx, tx, t)
if err != nil {
return SaveResult{}, err
}
if err := tx.Commit(); err != nil {
return SaveResult{}, err
}
return res, nil
}
// SaveOpportunities stores t and its releases in one transaction.
//
// Releases (and their tags) are write-once: ones already stored are skipped.
// The opportunity row and its classifications are replaced only if t's newest
// release is at least as new as the release the stored opportunity came from, so
// a late-arriving older release never undoes a newer one.
//
// t.Marks are stored only for a new opportunity; existing marks are kept (use
// SetMarks to change them).
//
// t is normalised in place (see prepareOpportunity), and when the opportunity is
// written its classifications' IDs are filled in.
func (s *Store) SaveOpportunities(
ctx context.Context, opportunities []*types.Opportunity,
) (SaveMultipleResult, error) {
for _, t := range opportunities {
if err := prepareOpportunity(t); err != nil {
return SaveMultipleResult{}, err
}
}
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return SaveMultipleResult{}, err
}
defer tx.Rollback() //nolint:errcheck // no-op after Commit
allRes := SaveMultipleResult{OpportunityResults: make([]SaveResult, len(opportunities))}
for i, t := range opportunities {
res, isNew, err := saveOpportunity(ctx, tx, t)
if err != nil {
return SaveMultipleResult{}, err
}
allRes.OpportunityResults[i] = res
allRes.NewReleases += res.NewReleases
switch {
case isNew:
allRes.NewOpportunities++
case res.OpportunityUpdated:
allRes.UpdatedOpportunities++
}
}
if err := tx.Commit(); err != nil {
return SaveMultipleResult{}, err
}
return allRes, nil
}
// saveOpportunity does the work of SaveOpportunity and SaveOpportunities for
// one prepared opportunity inside tx. isNew reports that the opportunity was
// not already stored.
func saveOpportunity(
ctx context.Context, tx *sqlx.Tx, t *types.Opportunity,
) (res SaveResult, isNew bool, err error) {
var stored time.Time
err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, t.UUID)
switch {
case errors.Is(err, sql.ErrNoRows):
isNew = true
res.OpportunityUpdated = true
case err != nil:
return SaveResult{}, false, fmt.Errorf("store: reading opportunity %s: %w", t.UUID, err)
default:
res.OpportunityUpdated = !t.LatestReleaseDate.Before(stored)
}
if res.OpportunityUpdated {
if err := writeOpportunity(ctx, tx, t); err != nil {
return SaveResult{}, false, fmt.Errorf("store: writing opportunity %s: %w", t.UUID, err)
}
}
for _, r := range t.Releases {
inserted, err := insertReleaseWithTags(ctx, tx, r)
if err != nil {
return SaveResult{}, false, err
}
if inserted {
res.NewReleases++
}
}
return res, isNew, nil
}
// insertReleaseWithTags stores r and its tags unless r is already stored, and
// reports whether it was inserted.
func insertReleaseWithTags(
ctx context.Context, tx *sqlx.Tx, r types.OpportunityRelease,
) (bool, error) {
result, err := tx.NamedExecContext(ctx, insertRelease, r)
if err != nil {
return false, fmt.Errorf("store: inserting release %s/%s: %w", r.UUID, r.ReleaseID, err)
}
if n, err := result.RowsAffected(); err != nil || n == 0 {
return false, err // n == 0: already stored
}
for _, tag := range r.Tags {
if _, err := tx.NamedExecContext(ctx, insertReleaseTag, tag); err != nil {
return false, fmt.Errorf(
"store: inserting tag for %s/%s: %w", r.UUID, r.ReleaseID, err,
)
}
}
return true, nil
}
// writeOpportunity upserts t's row, gives it a marks row if it has none
// (existing marks are kept), and replaces its classifications, filling in
// their IDs.
func writeOpportunity(ctx context.Context, tx *sqlx.Tx, t *types.Opportunity) (err error) {
if _, err := tx.NamedExecContext(ctx, upsertOpportunity, t); err != nil {
return err
}
if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, t.Marks); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, t.UUID); err != nil {
return err
}
stmt, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID)
if err != nil {
return err
}
defer func() { err = errors.Join(err, stmt.Close()) }()
for i := range t.Classifications {
c := &t.Classifications[i]
if err := stmt.GetContext(ctx, &c.ID, c); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, insertOpportunityClassification, t.UUID, c.ID); err != nil {
return err
}
}
return nil
}
// SetMarks replaces the marks of the opportunity m.UUID, or returns ErrNotFound.
// Marks that fail m.Validate (e.g. types.ErrWonAndLost) are not stored.
func (s *Store) SetMarks(ctx context.Context, m types.OpportunityMarks) error {
if m.UUID == "" {
return errors.New("store: marks have no UUID")
}
if err := m.Validate(); err != nil {
return fmt.Errorf("store: opportunity %s: %w", m.UUID, err)
}
_, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, m)
if isForeignKeyError(err) {
return ErrNotFound
}
return err
}
// isForeignKeyError reports whether err is SQLite rejecting a write that
// breaks a foreign key, e.g. one referring to a missing opportunity.
func isForeignKeyError(err error) bool {
var se sqlite3.Error
return errors.As(err, &se) && se.ExtendedCode == sqlite3.ErrConstraintForeignKey
}
// GetOpportunity returns the opportunity with its classifications, marks and
// all of its releases (with tags), or ErrNotFound.
func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) {
var row opportunityRow
err := s.db.GetContext(ctx, &row, selectOpportunityByUUID, uuid)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
opportunities := []types.Opportunity{row.Opportunity}
if err := s.loadClassifications(ctx, opportunities); err != nil {
return nil, err
}
if err := s.loadMarks(ctx, opportunities); err != nil {
return nil, err
}
if opportunities[0].Releases, err = s.ListReleases(ctx, uuid); err != nil {
return nil, err
}
return &opportunities[0], nil
}
// ListOpportunities returns opportunities matching f, with their classifications
// and marks but without releases (use ListReleases or GetOpportunity for those).
func (s *Store) ListOpportunities(
ctx context.Context,
f OpportunityFilter,
) ([]types.Opportunity, error) {
query, args := listOpportunitiesQuery(f)
var rows []opportunityRow
if err := s.db.SelectContext(ctx, &rows, query, args...); err != nil {
return nil, err
}
opportunities := make([]types.Opportunity, len(rows))
for i, row := range rows {
opportunities[i] = row.Opportunity
}
if err := s.loadClassifications(ctx, opportunities); err != nil {
return nil, err
}
if err := s.loadMarks(ctx, opportunities); err != nil {
return nil, err
}
return opportunities, nil
}
// listOpportunitiesQuery assembles the ListOpportunities query from the fragments in
// queries.go, with one argument per filter that is set (none for mark filters).
func listOpportunitiesQuery(f OpportunityFilter) (string, []any) {
var where []string
var args []any
add := func(clause string, arg any) {
where = append(where, clause)
args = append(args, arg)
}
if f.Status != "" {
add(filterOpportunityStatus, f.Status)
}
if f.ClassificationCode != "" {
add(filterOpportunityClassificationCode, f.ClassificationCode)
}
if q := ftsQuery(f.Title); q != "" {
add(filterOpportunityTitleMatch, q)
}
if !f.ReleasedFrom.IsZero() {
add(filterOpportunityReleasedFrom, f.ReleasedFrom.UTC())
}
if !f.ReleasedBefore.IsZero() {
add(filterOpportunityReleasedBefore, f.ReleasedBefore.UTC())
}
for _, m := range []struct {
filter MarkFilter
clause string
}{
{f.Hidden, filterOpportunityMarkHidden},
{f.Skipped, filterOpportunityMarkSkipped},
{f.Interested, filterOpportunityMarkInterested},
{f.BidSubmitted, filterOpportunityMarkBidSubmitted},
{f.Won, filterOpportunityMarkWon},
{f.Lost, filterOpportunityMarkLost},
} {
switch m.filter {
case MarkSet:
where = append(where, m.clause)
case MarkUnset:
where = append(where, "NOT "+m.clause)
}
}
query := listOpportunitiesSelect
if len(where) > 0 {
query += "\nWHERE " + strings.Join(where, "\n\tAND ")
}
query += listOpportunitiesOrderAndPage
return query, append(args, f.limit(), f.Offset)
}
// loadClassifications fills in Classifications for all opportunities with one query.
func (s *Store) loadClassifications(ctx context.Context, opportunities []types.Opportunity) error {
if len(opportunities) == 0 {
return nil
}
uuids := make([]string, len(opportunities))
byUUID := make(map[string]*types.Opportunity, len(opportunities))
for i := range opportunities {
uuids[i] = opportunities[i].UUID
byUUID[opportunities[i].UUID] = &opportunities[i]
}
query, args, err := sqlx.In(selectClassificationsForOpportunities, uuids)
if err != nil {
return err
}
// opportunity_uuid says which opportunity each row belongs to.
var rows []struct {
OpportunityUUID string `db:"opportunity_uuid"`
types.ClassificationCode
}
if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil {
return err
}
for _, r := range rows {
t := byUUID[r.OpportunityUUID]
t.Classifications = append(t.Classifications, r.ClassificationCode)
}
return nil
}
// loadMarks fills in Marks for all opportunities with one query. An
// opportunity with no marks row gets zero marks carrying its UUID.
func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity) error {
if len(opportunities) == 0 {
return nil
}
uuids := make([]string, len(opportunities))
byUUID := make(map[string]*types.Opportunity, len(opportunities))
for i := range opportunities {
uuids[i] = opportunities[i].UUID
byUUID[opportunities[i].UUID] = &opportunities[i]
opportunities[i].Marks = types.OpportunityMarks{UUID: opportunities[i].UUID}
}
query, args, err := sqlx.In(selectMarksForOpportunities, uuids)
if err != nil {
return err
}
var rows []types.OpportunityMarks
if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil {
return err
}
for _, m := range rows {
byUUID[m.UUID].Marks = m
}
return nil
}
// ListReleases returns a opportunity's releases with their tags, oldest first.
func (s *Store) ListReleases(ctx context.Context, uuid string) ([]types.OpportunityRelease, error) {
releases := []types.OpportunityRelease{}
if err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid); err != nil {
return nil, err
}
byID := make(map[string]*types.OpportunityRelease, len(releases))
for i := range releases {
byID[releases[i].ReleaseID] = &releases[i]
}
var tags []types.OpportunityReleaseTag
if err := s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid); err != nil {
return nil, err
}
for _, tag := range tags {
if r := byID[tag.ReleaseID]; r != nil {
r.Tags = append(r.Tags, tag)
}
}
return releases, nil
}
// ReleaseExists reports whether a release is already stored, so a fetcher
// can skip converting it.
func (s *Store) ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) {
var exists bool
err := s.db.GetContext(ctx, &exists, selectReleaseExists, uuid, releaseID)
return exists, err
}