Files
job-search/internal/store/queries.go
T
bwilliams b3d20c1db0 Made the fetcher save what it fetches, with fetcher.WithStore
With WithStore, the Find a Tender and mock workers save each run's
opportunities with SaveOpportunities and log what changed; a failed save
fails the run, so the checkpoint stays put and the next run retries.
Without a store nothing is persisted. The mock's opportunities are saved
with Source "FTSMock" rather than the converter's "FindATender".

fetcher service and job-search serve open the store at database.file
(cli.OpenConfiguredStore), or warn and run without one. A missing file
isn't created, and a database needing migration says to run job-search
migrate.

Fixed upsertOpportunity, which never wrote the source column, so every
saved opportunity had an empty Source.
2026-10-16 11:26:16 +01:00

257 lines
9.4 KiB
Go

package store
// All SQL used by the store, grouped by table in migration order.
//
// Conventions:
// - Statements are named <verb><Subject>, e.g. insertRelease. Fragments
// combined at run time are named after what they build (listOpportunities*,
// filterOpportunity*).
// - Named-parameter statements (":column", bound from a struct's db tags)
// say so in their comment. Everything else uses "?" placeholders.
// - Reads select whole rows (*, t.*, c.*) so a column added by a migration
// but not mapped to a struct field fails loudly; see opportunityRow.
// - Keep comments out of the SQL strings: sqlx reads a colon followed by a
// name as a parameter, even inside an SQL comment.
// - Every constant here is prepared against the migrated schema by
// TestQueriesPrepare (queries_test.go), which also fails if a constant is
// added without being listed there.
// ---------- opportunities ----------
// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares
// against.
const selectOpportunityLatestReleaseDate = `
SELECT latest_release_date FROM opportunities WHERE uuid = ?`
// upsertOpportunity inserts or updates a opportunity. Named parameters
// (types.Opportunity).
//
// An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first,
// cascading to the opportunity's releases and changing its id.
const upsertOpportunity = `
INSERT INTO opportunities (
uuid, uuid_type, title, status, description, buyer_name, buyer_id,
main_category, procurement_method, multi_lot, sme_suitable,
vcse_suitable, start_date, end_date, gross_value, has_documents, source,
latest_release_date
) VALUES (
:uuid, :uuid_type, :title, :status, :description, :buyer_name, :buyer_id,
:main_category, :procurement_method, :multi_lot, :sme_suitable,
:vcse_suitable, :start_date, :end_date, :gross_value, :has_documents,
:source, :latest_release_date
)
ON CONFLICT (uuid) DO UPDATE SET
uuid_type = excluded.uuid_type,
title = excluded.title,
status = excluded.status,
description = excluded.description,
buyer_name = excluded.buyer_name,
buyer_id = excluded.buyer_id,
main_category = excluded.main_category,
procurement_method = excluded.procurement_method,
multi_lot = excluded.multi_lot,
sme_suitable = excluded.sme_suitable,
vcse_suitable = excluded.vcse_suitable,
start_date = excluded.start_date,
end_date = excluded.end_date,
gross_value = excluded.gross_value,
has_documents = excluded.has_documents,
source = excluded.source,
latest_release_date = excluded.latest_release_date`
// selectOpportunityByUUID reads one opportunity into a opportunityRow.
const selectOpportunityByUUID = `
SELECT t.* FROM opportunities t WHERE t.uuid = ?`
// ListOpportunities is assembled by listOpportunitiesQuery as:
//
// listOpportunitiesSelect [WHERE filter AND filter ...] listOpportunitiesOrderAndPage
//
// Each filterOpportunity* fragment takes exactly one "?" argument, except
// filterOpportunityMark* fragments, which take none. A mark fragment is true
// when the mark is set and is negated with NOT to match it unset; an
// opportunity with no marks row has every mark unset.
const (
// listOpportunitiesSelect selects every opportunity, as t.
listOpportunitiesSelect = `
SELECT t.* FROM opportunities t`
// listOpportunitiesOrderAndPage orders by latest release, newest first,
// and takes a page: the limit and offset arguments.
listOpportunitiesOrderAndPage = `
ORDER BY t.latest_release_date DESC, t.uuid
LIMIT ? OFFSET ?`
// filterOpportunityStatus matches a status.
filterOpportunityStatus = `t.status = ?`
// filterOpportunityClassificationCode matches a classification code.
filterOpportunityClassificationCode = `EXISTS (
SELECT 1
FROM opportunity_classifications tc
JOIN classification_codes c ON c.id = tc.classification_id
WHERE tc.opportunity_uuid = t.uuid AND c.code = ?
)`
// filterOpportunityTitleMatch matches a full-text search of the title. The
// argument must come from ftsQuery, never raw user input.
filterOpportunityTitleMatch = `t.id IN (
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
)`
// filterOpportunityReleasedFrom matches a latest release on or after a
// time.
filterOpportunityReleasedFrom = `t.latest_release_date >= ?`
// filterOpportunityReleasedBefore matches a latest release before a time.
filterOpportunityReleasedBefore = `t.latest_release_date < ?`
// filterOpportunityMarkHidden matches the hidden mark set.
filterOpportunityMarkHidden = `EXISTS (
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.hidden
)`
// filterOpportunityMarkSkipped matches the skipped mark set.
filterOpportunityMarkSkipped = `EXISTS (
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.skipped
)`
// filterOpportunityMarkInterested matches the interested mark set.
filterOpportunityMarkInterested = `EXISTS (
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.interested
)`
// filterOpportunityMarkBidSubmitted matches the bid submitted mark set.
filterOpportunityMarkBidSubmitted = `EXISTS (
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.bid_submitted
)`
// filterOpportunityMarkWon matches the won mark set.
filterOpportunityMarkWon = `EXISTS (
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.won
)`
// filterOpportunityMarkLost matches the lost mark set.
filterOpportunityMarkLost = `EXISTS (
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.lost
)`
)
// ---------- opportunity_releases ----------
// insertRelease stores a release; 0 rows affected means it was already
// stored. Named parameters (types.OCDSReleaseJSON).
const insertRelease = `
INSERT INTO opportunity_releases (uuid, release_id, release_date, data)
VALUES (:uuid, :release_id, :release_date, :data)
ON CONFLICT DO NOTHING`
// selectReleasesByOpportunity reads a opportunity's releases, oldest first.
const selectReleasesByOpportunity = `
SELECT *
FROM opportunity_releases
WHERE uuid = ?
ORDER BY release_date, release_id`
// selectReleaseExists returns a single boolean.
const selectReleaseExists = `
SELECT EXISTS (
SELECT 1 FROM opportunity_releases WHERE uuid = ? AND release_id = ?
)`
// ---------- opportunity_release_tags ----------
// insertReleaseTag stores a tag. Named parameters (types.OCDSReleaseTags).
const insertReleaseTag = `
INSERT INTO opportunity_release_tags (uuid, release_id, tag)
VALUES (:uuid, :release_id, :tag)
ON CONFLICT DO NOTHING`
// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's
// releases.
const selectReleaseTagsByOpportunity = `
SELECT *
FROM opportunity_release_tags
WHERE uuid = ?
ORDER BY release_id, tag`
// ---------- classification_codes ----------
// upsertClassificationCodeReturningID returns the id of the row matching
// every content column, inserting it first if needed. Named parameters
// (types.ClassificationCode).
//
// The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING yield the
// existing row's id on conflict.
const upsertClassificationCodeReturningID = `
INSERT INTO classification_codes (is_cpv, code, classification)
VALUES (:is_cpv, :code, :classification)
ON CONFLICT (is_cpv, code, classification)
DO UPDATE SET is_cpv = excluded.is_cpv
RETURNING id`
// ---------- opportunity_classifications ----------
// deleteOpportunityClassifications clears a opportunity's mappings before they
// are rewritten.
const deleteOpportunityClassifications = `
DELETE FROM opportunity_classifications WHERE opportunity_uuid = ?`
// insertOpportunityClassification maps a opportunity to a classification code.
const insertOpportunityClassification = `
INSERT INTO opportunity_classifications (opportunity_uuid, classification_id)
VALUES (?, ?)
ON CONFLICT DO NOTHING`
// selectClassificationsForOpportunities reads the classifications of several
// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by
// Rebind. c.* must map entirely onto types.ClassificationCode.
const selectClassificationsForOpportunities = `
SELECT tc.opportunity_uuid, c.*
FROM opportunity_classifications tc
JOIN classification_codes c ON c.id = tc.classification_id
WHERE tc.opportunity_uuid IN (?)
ORDER BY tc.opportunity_uuid, c.code, c.id`
// ---------- opportunity_marks ----------
// insertOpportunityMarks gives a new opportunity its marks row; an existing
// row is left alone, so saving fetched data never resets marks set by the
// user. Named parameters (types.OpportunityMarks).
const insertOpportunityMarks = `
INSERT INTO opportunity_marks (
uuid, hidden, interested, skipped, bid_submitted, won, lost
)
VALUES (
:uuid, :hidden, :interested, :skipped, :bid_submitted, :won, :lost
)
ON CONFLICT (uuid) DO NOTHING`
// upsertOpportunityMarks inserts or updates an opportunity's marks. Named
// parameters (types.OpportunityMarks).
//
// An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first
// which is unnecessary.
const upsertOpportunityMarks = `
INSERT INTO opportunity_marks (
uuid, hidden, interested, skipped, bid_submitted, won, lost
)
VALUES (
:uuid, :hidden, :interested, :skipped, :bid_submitted, :won, :lost
)
ON CONFLICT (uuid) DO UPDATE SET
hidden = excluded.hidden,
interested = excluded.interested,
skipped = excluded.skipped,
bid_submitted = excluded.bid_submitted,
won = excluded.won,
lost = excluded.lost`
// selectMarksForOpportunities reads the marks of several opportunities.
// "IN (?)" is expanded by sqlx.In, which must be followed by Rebind.
const selectMarksForOpportunities = `
SELECT *
FROM opportunity_marks
WHERE uuid IN (?)
ORDER BY uuid`