Replaced the Title filter with a general search, run by the backend

GET /api/opportunities/search?q= returns the UUIDs of the matching
opportunities, and the frontend filters its list by them. Words without
a prompt search the title and description; title:, buyer:, buyer_id:
and cpv: search only their own field, and a leading ! leaves out what a
term matches. Repeats of one prompt are OR, everything else AND. CPV
terms are checked per code: cpv:72 !cpv:7221 needs a code starting 72
that doesn't start 7221.

Unknown prompts are reported, and every response carries the help that
the search box's "?" panel shows. Migration 00011 adds the description
and buyer name to the full-text index, and indexes CPV descriptions.
This commit is contained in:
bwilliams committed 2026-10-22 13:10:25 +01:00
1 parent 0a777792d8
commit b0230e6847
29 files changed
+1913 -43

No files matched your search

+72
View File
@@ -13,6 +13,7 @@ import (
"git.avsdev.uk/Go/job-search/internal/events"
"git.avsdev.uk/Go/job-search/internal/httpx"
"git.avsdev.uk/Go/job-search/internal/search"
"git.avsdev.uk/Go/job-search/internal/store"
"git.avsdev.uk/Go/job-search/internal/types"
)
@@ -38,6 +39,21 @@ type OpportunityPage struct {
Batch string `json:"batch"`
}
// OpportunitySearch is the result of a general search, as GET
// /opportunities/search returns it: the UUIDs of the opportunities that
// match, in no particular order. MatchesAll reports a search with nothing to
// search for (blank, or only unknown prompts), which matches every
// opportunity; UUIDs is then empty. UnknownPrompts are the prompts in the
// search that aren't known, as typed, without their colon. Prompts and
// Syntax are the help for the search box.
type OpportunitySearch struct {
UUIDs []string `json:"uuids"`
MatchesAll bool `json:"matches_all"`
UnknownPrompts []string `json:"unknown_prompts"`
Prompts []search.Prompt `json:"prompts"`
Syntax []search.Example `json:"syntax"`
}
// opportunityRoutes wires HTTP handlers to the opportunity store.
// Opportunities are read-only; only their marks can be changed.
type opportunityRoutes struct {
@@ -64,6 +80,7 @@ func newOpportunityRoutes(
// Routes registers the opportunity routes on router (mounted under /api).
func (h *opportunityRoutes) Routes(router chi.Router) {
router.Get("/opportunities", h.list)
router.Get("/opportunities/search", h.search)
router.Get("/opportunities/{uuid}", h.get)
router.Put("/opportunities/{uuid}/marks/{mark}", h.setMark)
router.Delete("/opportunities/{uuid}/marks/{mark}", h.clearMark)
@@ -203,6 +220,61 @@ func (h *opportunityRoutes) listChanged(
})
}
// search serves GET /opportunities/search: the UUIDs of the opportunities
// matching the general search in the q query parameter (see package
// search), with the help for the search box. released_from bounds it as it
// bounds list.
//
// @Summary Search opportunities
// @Description The UUIDs of the opportunities matching a general search,
// @Description in no particular order. q is words and "quoted phrases",
// @Description each optionally with a prompt (title:, buyer:, buyer_id:,
// @Description cpv:) and a leading ! to leave out what it matches; prompts
// @Description and syntax explain it. A search with nothing to search for
// @Description (blank, or only unknown prompts) has matches_all set and no
// @Description uuids. unknown_prompts lists the prompts that aren't known,
// @Description as typed; their terms are ignored. released_from (RFC 3339,
// @Description inclusive) leaves out opportunities whose latest release is
// @Description before it.
// @Tags opportunities
// @Produce json
// @Param q query string false "The search"
// @Param released_from query string false "Only opportunities whose latest release is at or after this time" format(date-time)
// @Success 200 {object} OpportunitySearch
// @Failure 400 {object} httpx.Problem "Invalid released_from"
// @Failure 500 {object} httpx.Problem
// @Router /opportunities/search [get]
func (h *opportunityRoutes) search(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
releasedFrom, err := timeParam(
"released_from", query.Get("released_from"),
)
if err != nil {
httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error())
return
}
parsed := search.Parse(query.Get("q"))
result := OpportunitySearch{
UUIDs: []string{},
MatchesAll: parsed.MatchesAll(),
UnknownPrompts: parsed.UnknownPrompts,
Prompts: search.Prompts,
Syntax: search.Syntax,
}
if !result.MatchesAll {
result.UUIDs, err = h.store.SearchOpportunities(
r.Context(), parsed, releasedFrom,
)
if err != nil {
httpx.WriteProblem(w, r, http.StatusInternalServerError, "")
return
}
}
httpx.WriteJSON(w, http.StatusOK, result)
}
// get serves GET /opportunities/{uuid}: one opportunity with its
// classifications, marks and releases.
//
+52
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
@@ -483,3 +484,54 @@ func TestChangeMarkPublishes(t *testing.T) {
}
}
}
// GET /opportunities/search returns the matching UUIDs, bounded by
// released_from, with the help; a search with nothing to search for matches
// all, and unknown prompts are listed.
func TestSearchOpportunities(t *testing.T) {
url := opportunitiesServer(t, 5) + "/opportunities/search"
from3 := "&released_from=2026-10-03T00:00:00Z"
cases := []struct {
query string
uuids []string
matchesAll bool
unknown []string
}{
{"", []string{}, true, []string{}},
{"?q=", []string{}, true, []string{}},
{"?q=cpv:72", []string{"ocds-1", "ocds-3", "ocds-5"}, false, []string{}},
{"?q=cpv:72" + from3, []string{"ocds-3", "ocds-5"}, false, []string{}},
{"?q=opportunity%20!cpv:72", []string{"ocds-2", "ocds-4"}, false, []string{}},
{"?q=nothing", []string{}, false, []string{}},
{"?q=foo:x%20Bar:y", []string{}, true, []string{"foo", "Bar"}},
{"?q=foo:x%20title:3", []string{"ocds-3"}, false, []string{"foo"}},
}
for _, tc := range cases {
var got OpportunitySearch
status := request(t, http.MethodGet, url+tc.query, &got)
slices.Sort(got.UUIDs)
if status != http.StatusOK || !slices.Equal(got.UUIDs, tc.uuids) ||
got.MatchesAll != tc.matchesAll ||
!slices.Equal(got.UnknownPrompts, tc.unknown) {
t.Errorf("%s: status %d, %+v; want %v, matches_all %v, unknown %v",
tc.query, status, got, tc.uuids, tc.matchesAll, tc.unknown)
}
if len(got.Prompts) == 0 || len(got.Syntax) == 0 {
t.Errorf("%s: no help: %+v", tc.query, got)
}
}
// Empty lists are [], never null.
var raw map[string]json.RawMessage
request(t, http.MethodGet, url+"?q=nothing", &raw)
for _, field := range []string{"uuids", "unknown_prompts"} {
if string(raw[field]) != "[]" {
t.Errorf("%s = %s; want []", field, raw[field])
}
}
status := request(t, http.MethodGet, url+"?q=x&released_from=x", nil)
if status != http.StatusBadRequest {
t.Errorf("bad released_from: status %d; want 400", status)
}
}
+222
View File
@@ -0,0 +1,222 @@
// Package search parses the general opportunity search: words and quoted
// phrases, each optionally with a prompt naming the field it searches
// (title:, buyer:, buyer_id:, cpv:) and a leading "!" to negate it. The
// store runs a parsed Query (store.SearchOpportunities).
package search
import (
"regexp"
"strings"
"unicode"
)
// Field is what a term searches.
type Field int
// Values of Field.
const (
// FieldText is a term with no prompt: the title or the description.
FieldText Field = iota
// FieldTitle is title:, the title only.
FieldTitle
// FieldBuyer is buyer:, the buyer's name.
FieldBuyer
// FieldBuyerID is buyer_id:, the buyer's ID, matched whole.
FieldBuyerID
// FieldCPV is cpv:, a CPV code prefix or words of a CPV description.
FieldCPV
)
// Prompt is a known prompt, with a line of help for the user.
type Prompt struct {
Name string `json:"name"`
Help string `json:"help"`
}
// Example is an example of the search syntax, with a line of help for the
// user.
type Example struct {
Example string `json:"example"`
Help string `json:"help"`
}
// Prompts are the known prompts, in the order to show them.
var Prompts = []Prompt{
{Name: "title", Help: "Words in the title only."},
{Name: "buyer", Help: "Words in the buyer's name."},
{
Name: "buyer_id",
Help: "The buyer's ID, whole and in any case (e.g. GB-FTS-148803).",
},
{
Name: "cpv",
Help: "The start of a CPV code (cpv:72), or words in a CPV code's description (cpv:food).",
},
}
// Syntax explains the search syntax, in the order to show it.
var Syntax = []Example{
{
Example: "cloud hosting",
Help: "Words without a prompt are searched for in the title and the description. Every word must be found, though not necessarily in the same one.",
},
{
Example: "migr",
Help: "A word also matches the start of a longer word, and other forms of it: migr finds migration, and migrating finds migration.",
},
{
Example: `"road repairs"`,
Help: "Quotes search for a phrase: its words together, in that order.",
},
{
Example: `title:"road repairs"`,
Help: "A prompt searches only its own field, for one word or one quoted phrase.",
},
{
Example: "buyer:leeds buyer:bradford",
Help: "Repeats of the same prompt match if any of them does: Leeds or Bradford.",
},
{
Example: "cloud buyer:leeds cpv:72",
Help: "Different prompts, and words without one, must all match.",
},
{
Example: "!cloud !buyer:leeds",
Help: "! in front of a word, a phrase or a prompt leaves out what it matches.",
},
{
Example: "cpv:72 !cpv:7221",
Help: "CPV terms are checked against each of an opportunity's codes: this needs a code starting 72 that doesn't start 7221. !cpv: without cpv: leaves out every opportunity with a matching code.",
},
}
// prompts maps each known prompt's name to its field.
var prompts = map[string]Field{
"title": FieldTitle,
"buyer": FieldBuyer,
"buyer_id": FieldBuyerID,
"cpv": FieldCPV,
}
// promptPattern matches a prompt at the start of a term: a name of letters
// and underscores, then a colon. A colon after anything else (12:00) is
// part of the term.
var promptPattern = regexp.MustCompile(`^([A-Za-z_]+):`)
// cpvCodePattern matches a CPV code or the start of one, with an optional
// check digit.
var cpvCodePattern = regexp.MustCompile(`^([0-9]+)(?:-[0-9])?$`)
// Term is one search term: a word or a quoted phrase, as typed.
type Term struct {
Field Field
Negated bool
Value string
}
// Words returns the term's words, split as the store's full-text index
// splits them: on anything but letters and numbers.
func (t Term) Words() []string {
return strings.FieldsFunc(t.Value, func(char rune) bool {
return !unicode.IsLetter(char) && !unicode.IsNumber(char)
})
}
// CPVCode returns the digits of a cpv: term that is a code or the start of
// one, without its check digit, and reports whether it is one. Any other
// cpv: term searches CPV descriptions.
func (t Term) CPVCode() (string, bool) {
match := cpvCodePattern.FindStringSubmatch(t.Value)
if match == nil {
return "", false
}
return match[1], true
}
// usable reports whether the term can be searched for: a buyer ID that
// isn't blank, or any other term with at least one word.
func (t Term) usable() bool {
if t.Field == FieldBuyerID {
return strings.TrimSpace(t.Value) != ""
}
return len(t.Words()) > 0
}
// Query is a parsed search.
type Query struct {
// Terms are the terms to search for, all usable.
Terms []Term
// UnknownPrompts are the prompts that aren't known, as typed, without
// their colon, once each. Their terms are left out of Terms.
UnknownPrompts []string
}
// MatchesAll reports whether the query has no terms, so it matches every
// opportunity.
func (q Query) MatchesAll() bool {
return len(q.Terms) == 0
}
// Parse parses text as a search. It never fails: text that isn't valid
// syntax, such as an unclosed quote, is read as plain words, and a term
// with no words (title: or !!!) is left out.
func Parse(text string) Query {
query := Query{
UnknownPrompts: []string{},
}
unknown := map[string]bool{}
for rest := strings.TrimSpace(text); rest != ""; {
var term Term
var prompt string
var known bool
term, prompt, known, rest = nextTerm(rest)
rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
switch {
case !known:
if !unknown[prompt] {
unknown[prompt] = true
query.UnknownPrompts = append(query.UnknownPrompts, prompt)
}
case term.usable():
query.Terms = append(query.Terms, term)
}
}
return query
}
// nextTerm reads the term at the start of text, which mustn't start with
// space. It returns the term, its prompt as typed ("" if none), whether the
// prompt is known, and the text after the term.
func nextTerm(
text string,
) (term Term, prompt string, known bool, rest string) {
if strings.HasPrefix(text, "!") {
term.Negated = true
text = text[1:]
}
if match := promptPattern.FindStringSubmatch(text); match != nil {
prompt = match[1]
text = text[len(match[0]):]
}
term.Field, known = prompts[strings.ToLower(prompt)]
if prompt == "" {
term.Field, known = FieldText, true
}
if strings.HasPrefix(text, `"`) {
term.Value, rest, _ = strings.Cut(text[1:], `"`)
return term, prompt, known, rest
}
end := strings.IndexFunc(text, unicode.IsSpace)
if end < 0 {
end = len(text)
}
term.Value = text[:end]
return term, prompt, known, text[end:]
}
+106
View File
@@ -0,0 +1,106 @@
package search
import (
"reflect"
"slices"
"testing"
)
func TestParse(t *testing.T) {
term := func(field Field, value string) Term {
return Term{
Field: field,
Value: value,
}
}
text := func(value string) Term {
return term(FieldText, value)
}
not := func(term Term) Term {
term.Negated = true
return term
}
none := []string{}
cases := []struct {
input string
terms []Term
unknown []string
}{
{"", nil, none},
{" ", nil, none},
{"cloud hosting", []Term{text("cloud"), text("hosting")}, none},
{`"road repairs" leeds`, []Term{text("road repairs"), text("leeds")}, none},
{`title:"road repairs"`, []Term{term(FieldTitle, "road repairs")}, none},
{"TITLE:road Buyer:leeds", []Term{term(FieldTitle, "road"), term(FieldBuyer, "leeds")}, none},
{"buyer_id:GB-1 cpv:72", []Term{term(FieldBuyerID, "GB-1"), term(FieldCPV, "72")}, none},
{"!cloud !cpv:food", []Term{not(text("cloud")), not(term(FieldCPV, "food"))}, none},
{`!title:"a b"`, []Term{not(term(FieldTitle, "a b"))}, none},
{"12:00", []Term{text("12:00")}, none},
{`"unclosed phrase`, []Term{text("unclosed phrase")}, none},
{`title: !!! "" ! cloud`, []Term{text("cloud")}, none},
{"foo:bar cloud Foo:baz foo:x baz:", []Term{text("cloud")}, []string{"foo", "Foo", "baz"}},
{"buyer_id:", nil, none},
}
for _, tc := range cases {
got := Parse(tc.input)
if !reflect.DeepEqual(got.Terms, tc.terms) {
t.Errorf("%q: terms %+v, want %+v", tc.input, got.Terms, tc.terms)
}
if !slices.Equal(got.UnknownPrompts, tc.unknown) {
t.Errorf(
"%q: unknown %q, want %q",
tc.input, got.UnknownPrompts, tc.unknown,
)
}
if got.UnknownPrompts == nil {
t.Errorf("%q: UnknownPrompts is nil", tc.input)
}
if got.MatchesAll() != (len(tc.terms) == 0) {
t.Errorf("%q: MatchesAll %v", tc.input, got.MatchesAll())
}
}
}
func TestTermWordsAndCPVCode(t *testing.T) {
cases := []struct {
value string
words []string
code string
isCPV bool
}{
{"road-repairs", []string{"road", "repairs"}, "", false},
{"Café's", []string{"Café", "s"}, "", false},
{"72", []string{"72"}, "72", true},
{"72210000-5", []string{"72210000", "5"}, "72210000", true},
{"72-55", []string{"72", "55"}, "", false},
{"72a", []string{"72a"}, "", false},
}
for _, tc := range cases {
term := Term{
Field: FieldCPV,
Value: tc.value,
}
if got := term.Words(); !slices.Equal(got, tc.words) {
t.Errorf("%q: words %q, want %q", tc.value, got, tc.words)
}
code, ok := term.CPVCode()
if code != tc.code || ok != tc.isCPV {
t.Errorf(
"%q: CPVCode %q %v, want %q %v",
tc.value, code, ok, tc.code, tc.isCPV,
)
}
}
}
// Every known prompt has help, and every prompt with help is known.
func TestPromptsHaveHelp(t *testing.T) {
if len(Prompts) != len(prompts) {
t.Errorf("%d prompts with help, %d known", len(Prompts), len(prompts))
}
for _, prompt := range Prompts {
if _, ok := prompts[prompt.Name]; !ok || prompt.Help == "" {
t.Errorf("prompt %q: known %v, help %q", prompt.Name, ok, prompt.Help)
}
}
}
+13 -2
View File
@@ -4,7 +4,7 @@ A SQLite store for opportunities and their OCDS releases, built on sqlx + goose
## Build and test
The title search uses SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
The title and general searches use SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
```bash
go test -tags sqlite_fts5 ./...
@@ -24,7 +24,9 @@ internal/store/migrate_test.go version checks, stepping up and down, failed-
internal/store/queries.go all SQL, as named constants grouped by table
internal/store/queries_test.go prepares every query against the migrated schema
internal/store/store.go Store: logic, using the queries via sqlx
internal/store/search.go SearchOpportunities: the general search's query
internal/store/store_test.go behavioural tests
internal/store/search_test.go behavioural tests of the general search
internal/store/common.go common for migrate, store and direct access
```
@@ -38,7 +40,8 @@ internal/store/common.go common for migrate, store and direct access
| `opportunity_release_tags` | (`uuid`, `release_id`, `tag`) | Write-once. (`uuid`, `release_id`) references `opportunity_releases`. |
| `classification_codes` | `id` | Shared between opportunities. A CPV code has one row (unique on `code` where `is_cpv`), keeping the longest description it has been given: EU Supply cuts descriptions to 64 characters. A code of any other scheme gets a row per distinct content. The table grows over time to include codes not already seen in fetched opportunities. |
| `opportunity_classifications` | (`opportunity_uuid`, `classification_id`) | Mapping table; replaced as a set when the opportunity is updated. |
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, kept in sync by triggers. |
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, `description` and `buyer_name`, kept in sync by triggers. |
| `classification_codes_fts` | rowid = `classification_codes.id` | FTS5 index on `classification_codes.classification` (the description), kept in sync by triggers. |
| `opportunity_change_batches` | `id`; `hash` (unique) | Write-once. One row per save that adds opportunities or gives them new releases, and per change of marks. `hash` names the batch to clients (random, so unguessable); `created_at` orders the batches. |
| `opportunity_change_batch_opportunities` | (`batch_id`, `opportunity_uuid`) | The opportunities in each batch. References both tables, cascading on delete. |
@@ -71,6 +74,7 @@ All SQL is in `queries.go` as constants, grouped by table in migration order; `s
- **TestQueriesPrepare** prepares every query constant against a freshly migrated database, so a syntax error, an unknown table or unknown column fails the test and names the failing statement.
- **TestQueriesAllListed** parses `queries.go` and fails if a constant isn't listed in that test's `checks` map. Adding a query therefore means adding one line there, and it can't be skipped by accident.
- **TestSearchOpportunitiesQueryAllTerms** does the same for `SearchOpportunities`, with a term of every kind, plain and negated.
- **TestListOpportunitiesQueryAllFilters** builds the `ListOpportunities` query with every filter set and runs it, checking that the fragments combine and that the arguments line up with the placeholders.
- **TestMarksCheckConstraint** checks the database itself rejects `won` and `lost` set together, for writes that bypass the Go validation.
@@ -111,6 +115,7 @@ ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags
batch, err := s.LatestChangeBatch(ctx) // hash of the newest change batch, or ""
ts, batch, err = s.ListChangedOpportunities(ctx, batch, time.Time{}) // changed since; ErrUnknownBatch
uuids, err := s.SearchOpportunities(ctx, search.Parse("cloud buyer:leeds"), releasedFrom) // never nil
ok, err := s.ReleaseExists(ctx, uuid, releaseID)
```
@@ -132,3 +137,9 @@ ok, err := s.ReleaseExists(ctx, uuid, releaseID)
**Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted.
**General search.** `SearchOpportunities` runs a `search.Query` (parsed by `internal/search`, which documents the syntax) and returns only the matching UUIDs, for the frontend to filter its own list by. It is assembled by `searchOpportunitiesQuery` from the `searchOpportunity*` and `searchCPV*` fragments:
- Words without a prompt search `{title description}` in `opportunities_fts`, and must all match. `title:` and `buyer:` search the `title` and `buyer_name` columns; repeats of one prompt are joined by OR into one clause. Every negated full-text term goes into one `NOT IN` clause, joined by OR. Each term is one FTS5 phrase built from `Term.Words`, its last word a prefix, so typed input is never FTS5 syntax.
- `buyer_id:` matches the whole ID, ignoring case (`IN`, expanded by `sqlx.In`).
- CPV terms are checked against each code, inside one `EXISTS` on the opportunity's CPV codes: a code must match a `cpv:` term (a code prefix by `LIKE`, or words in `classification_codes_fts`) and no `!cpv:` term. With only `!cpv:` terms the clause is `NOT EXISTS` a code matching any of them, so an opportunity with no CPV codes passes it.
- `OpportunityFilter.Title` searches only the `title` column of the same index.
+65
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
@@ -402,3 +403,67 @@ func TestMigration10MergesCPVDuplicates(t *testing.T) {
t.Error("a second row for a CPV code was accepted")
}
}
// Migration 11 indexes the description, buyer name and classification
// descriptions of rows already stored, and its Down leaves the title
// index working.
func TestMigration11IndexesStoredRows(t *testing.T) {
ctx := context.Background()
m, db := newMigrator(t)
if _, err := m.UpTo(ctx, 10); err != nil {
t.Fatal(err)
}
setup := []string{
`INSERT INTO opportunities (uuid, uuid_type, title, status,
description, buyer_name, latest_release_date)
VALUES ('a', 'ocid', 'Cloud', 'active', 'Servers', 'Leeds',
'2026-10-01')`,
`INSERT INTO classification_codes (id, is_cpv, code, classification)
VALUES (1, 1, '15800000', 'Miscellaneous food products')`,
}
for _, statement := range setup {
if _, err := db.ExecContext(ctx, statement); err != nil {
t.Fatal(err)
}
}
if _, err := m.UpTo(ctx, 11); err != nil {
t.Fatal(err)
}
count := func(query string) int {
t.Helper()
var n int
if err := db.QueryRowContext(ctx, query).Scan(&n); err != nil {
t.Fatalf("%s: %v", query, err)
}
return n
}
matches := []struct {
table, match string
want int
}{
{"opportunities_fts", "description : servers", 1},
{"opportunities_fts", "buyer_name : leeds", 1},
{"opportunities_fts", "title : servers", 0},
{"classification_codes_fts", "food", 1},
}
for _, tc := range matches {
got := count(fmt.Sprintf(
"SELECT count(*) FROM %s WHERE %[1]s MATCH '%s'", tc.table, tc.match,
))
if got != tc.want {
t.Errorf("%s MATCH %q: %d; want %d", tc.table, tc.match, got, tc.want)
}
}
if _, err := m.DownTo(ctx, 10); err != nil {
t.Fatal(err)
}
got := count(`SELECT count(*) FROM opportunities_fts
WHERE opportunities_fts MATCH 'cloud'`)
if got != 1 {
t.Errorf("title search after Down: %d; want 1", got)
}
}
@@ -0,0 +1,119 @@
-- +goose Up
-- +goose StatementBegin
-- The general search: opportunities_fts (migration 00006) now also indexes
-- the description and the buyer's name, and classification_codes_fts
-- indexes classification descriptions. Both are kept in sync by triggers.
DROP TRIGGER IF EXISTS "opportunities_fts_update";
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
DROP TABLE IF EXISTS "opportunities_fts";
CREATE VIRTUAL TABLE "opportunities_fts" USING fts5(
"title",
"description",
"buyer_name",
content = 'opportunities',
content_rowid = 'id',
tokenize = 'porter unicode61 remove_diacritics 2'
);
CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name")
VALUES (new."id", new."title", new."description", new."buyer_name");
END;
CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts"
("opportunities_fts", "rowid", "title", "description", "buyer_name")
VALUES ('delete', old."id", old."title", old."description", old."buyer_name");
END;
CREATE TRIGGER "opportunities_fts_update"
AFTER UPDATE OF "title", "description", "buyer_name" ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts"
("opportunities_fts", "rowid", "title", "description", "buyer_name")
VALUES ('delete', old."id", old."title", old."description", old."buyer_name");
INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name")
VALUES (new."id", new."title", new."description", new."buyer_name");
END;
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
CREATE VIRTUAL TABLE "classification_codes_fts" USING fts5(
"classification",
content = 'classification_codes',
content_rowid = 'id',
tokenize = 'porter unicode61 remove_diacritics 2'
);
CREATE TRIGGER "classification_codes_fts_insert"
AFTER INSERT ON "classification_codes"
BEGIN
INSERT INTO "classification_codes_fts" ("rowid", "classification")
VALUES (new."id", new."classification");
END;
CREATE TRIGGER "classification_codes_fts_delete"
AFTER DELETE ON "classification_codes"
BEGIN
INSERT INTO "classification_codes_fts"
("classification_codes_fts", "rowid", "classification")
VALUES ('delete', old."id", old."classification");
END;
CREATE TRIGGER "classification_codes_fts_update"
AFTER UPDATE OF "classification" ON "classification_codes"
BEGIN
INSERT INTO "classification_codes_fts"
("classification_codes_fts", "rowid", "classification")
VALUES ('delete', old."id", old."classification");
INSERT INTO "classification_codes_fts" ("rowid", "classification")
VALUES (new."id", new."classification");
END;
INSERT INTO "classification_codes_fts" ("classification_codes_fts")
VALUES ('rebuild');
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TRIGGER IF EXISTS "classification_codes_fts_update";
DROP TRIGGER IF EXISTS "classification_codes_fts_delete";
DROP TRIGGER IF EXISTS "classification_codes_fts_insert";
DROP TABLE IF EXISTS "classification_codes_fts";
DROP TRIGGER IF EXISTS "opportunities_fts_update";
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
DROP TABLE IF EXISTS "opportunities_fts";
CREATE VIRTUAL TABLE "opportunities_fts" USING fts5(
"title",
content = 'opportunities',
content_rowid = 'id',
tokenize = 'porter unicode61 remove_diacritics 2'
);
CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
END;
CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
VALUES ('delete', old."id", old."title");
END;
CREATE TRIGGER "opportunities_fts_update" AFTER UPDATE OF "title" ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
VALUES ('delete', old."id", old."title");
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
END;
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
-- +goose StatementEnd
+48 -1
View File
@@ -107,7 +107,8 @@ LIMIT ? OFFSET ?`
)`
// filterOpportunityTitleMatch matches a full-text search of the title. The
// argument must come from ftsQuery, never raw user input.
// argument must be ftsQuery's, limited to the title column (see
// listOpportunitiesQuery), never raw user input.
filterOpportunityTitleMatch = `t.id IN (
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
)`
@@ -150,6 +151,52 @@ LIMIT ? OFFSET ?`
)`
)
// SearchOpportunities is assembled by searchOpportunitiesQuery as:
//
// searchOpportunitiesSelect
// [WHERE clause AND clause ...]
//
// Each clause is a searchOpportunity* fragment, perhaps with NOT in front,
// or filterOpportunityReleasedFrom. searchOpportunityCPV's %s is filled
// with searchCPV* fragments joined by OR, AND and NOT. A full-text argument
// must be built from search.Term.Words, never raw user input.
const (
// searchOpportunitiesSelect selects every opportunity's UUID, as t.
searchOpportunitiesSelect = `
SELECT t.uuid FROM opportunities t`
// searchOpportunityText matches a full-text search of the title,
// description and buyer_name columns.
searchOpportunityText = `t.id IN (
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
)`
// searchOpportunityBuyerID matches any of several buyer IDs, ignoring
// case. "IN (?)" is expanded by sqlx.In, which must be followed by
// Rebind.
searchOpportunityBuyerID = `t.buyer_id COLLATE NOCASE IN (?)`
// searchOpportunityCPV matches an opportunity with a CPV code meeting
// the condition put in place of %s, which reads the code as c.
searchOpportunityCPV = `EXISTS (
SELECT 1
FROM opportunity_classifications tc
JOIN classification_codes c ON c.id = tc.classification_id
WHERE tc.opportunity_uuid = t.uuid AND c.is_cpv AND %s
)`
// searchCPVCodePrefix matches a code by a LIKE pattern: its first
// digits followed by %.
searchCPVCodePrefix = `c.code LIKE ?`
// searchCPVDescription matches a full-text search of the code's
// description.
searchCPVDescription = `c.id IN (
SELECT rowid FROM classification_codes_fts
WHERE classification_codes_fts MATCH ?
)`
)
// selectOtherSourcesTitles lists the uuid, title and latest_release_date of
// the opportunities not from one source whose latest_release_date is in
// [from, before); it walks the latest_release_date index.
+47
View File
@@ -2,6 +2,7 @@ package store
import (
"context"
"fmt"
"go/ast"
"go/parser"
"go/token"
@@ -11,6 +12,8 @@ import (
"time"
"github.com/jmoiron/sqlx"
"git.avsdev.uk/Go/job-search/internal/search"
)
// Run with: go test -tags sqlite_fts5 ./...
@@ -81,6 +84,24 @@ var checks = map[string]queryCheck{
sql: withFilter(filterOpportunityMarkLost),
},
"searchOpportunitiesSelect": {sql: searchOpportunitiesSelect},
"searchOpportunityText": {
sql: withSearch(searchOpportunityText),
},
"searchOpportunityBuyerID": {
sql: withSearch(searchOpportunityBuyerID),
in: true,
},
"searchOpportunityCPV": {
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, "1")),
},
"searchCPVCodePrefix": {
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVCodePrefix)),
},
"searchCPVDescription": {
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVDescription)),
},
// opportunity_ocds_releases
"insertRelease": {sql: insertRelease, named: true},
"selectReleasesByOpportunity": {sql: selectReleasesByOpportunity},
@@ -140,6 +161,10 @@ func withFilter(fragment string) string {
listOpportunitiesOrderAndPage
}
func withSearch(fragment string) string {
return searchOpportunitiesSelect + "\nWHERE " + fragment
}
func openTestDB(t *testing.T) *sqlx.DB {
t.Helper()
s, err := Open(
@@ -223,6 +248,28 @@ func TestListOpportunitiesQueryAllFilters(t *testing.T) {
}
}
// TestSearchOpportunitiesQueryAllTerms runs the assembled search query with
// a term of every kind, both plain and negated, checking the fragments
// combine and the arguments line up with the placeholders.
func TestSearchOpportunitiesQueryAllTerms(t *testing.T) {
db := openTestDB(t)
query := search.Parse(`cloud "road repairs" title:a title:b buyer:c ` +
`buyer_id:GB-1 buyer_id:GB-2 cpv:72 cpv:food !cloud !title:d ` +
`!buyer:e !buyer_id:GB-3 !cpv:7221 !cpv:fuel`)
statement, args := searchOpportunitiesQuery(
query, time.Now().Add(-time.Hour),
)
statement, args, err := sqlx.In(statement, args...)
if err != nil {
t.Fatal(err)
}
var uuids []string
if err := db.Select(&uuids, db.Rebind(statement), args...); err != nil {
t.Errorf("run with %d args:\n%s\n%v", len(args), statement, err)
}
}
// TestQueriesAllListed fails if queries.go declares a constant that checks
// doesn't list, so every new statement is prepared by TestQueriesPrepare.
func TestQueriesAllListed(t *testing.T) {
+202
View File
@@ -0,0 +1,202 @@
package store
import (
"context"
"fmt"
"strings"
"time"
"github.com/jmoiron/sqlx"
"git.avsdev.uk/Go/job-search/internal/search"
)
// ftsColumns are the opportunities_fts columns each field's terms search,
// as an FTS5 column filter.
var ftsColumns = map[search.Field]string{
search.FieldText: "{title description}",
search.FieldTitle: "title",
search.FieldBuyer: "buyer_name",
}
// anyOfFields are the fields whose terms match if any of them does, in the
// order their clauses are added.
var anyOfFields = []search.Field{search.FieldTitle, search.FieldBuyer}
// searchClauses holds a search's terms, sorted by how they combine.
type searchClauses struct {
// required are full-text matches (words without a prompt) that must
// all match.
required []string
// anyOf are full-text matches by field, of which one must match.
anyOf map[search.Field][]string
// excluded are full-text matches of which none may match.
excluded []string
buyerIDs []string
excludedBuyerIDs []string
// cpv and excludedCPV are conditions on one CPV code (searchCPV*
// fragments), with their arguments.
cpv []string
cpvArgs []any
excludedCPV []string
excludedCPVArgs []any
}
// SearchOpportunities returns the UUIDs of the opportunities matching
// query, in no particular order and never nil. Unless releasedFrom is zero,
// only opportunities whose latest release is on or after it are returned
// (as OpportunityFilter.ReleasedFrom). A query with no terms matches every
// opportunity.
func (s *Store) SearchOpportunities(
ctx context.Context, query search.Query, releasedFrom time.Time,
) ([]string, error) {
statement, args := searchOpportunitiesQuery(query, releasedFrom)
statement, args, err := sqlx.In(statement, args...)
if err != nil {
return nil, fmt.Errorf("expanding the search query: %w", err)
}
uuids := []string{}
err = s.db.SelectContext(ctx, &uuids, s.db.Rebind(statement), args...)
if err != nil {
return nil, fmt.Errorf("searching opportunities: %w", err)
}
return uuids, nil
}
// searchOpportunitiesQuery assembles the SearchOpportunities query from the
// fragments in queries.go, with its arguments. It must be expanded by
// sqlx.In.
func searchOpportunitiesQuery(
query search.Query, releasedFrom time.Time,
) (string, []any) {
clauses := sortTerms(query.Terms)
var where []string
var args []any
add := func(clause string, arguments ...any) {
where = append(where, clause)
args = append(args, arguments...)
}
if len(clauses.required) > 0 {
add(searchOpportunityText, strings.Join(clauses.required, " AND "))
}
for _, field := range anyOfFields {
if matches := clauses.anyOf[field]; len(matches) > 0 {
add(searchOpportunityText, strings.Join(matches, " OR "))
}
}
if len(clauses.excluded) > 0 {
add(
"NOT "+searchOpportunityText,
strings.Join(clauses.excluded, " OR "),
)
}
if len(clauses.buyerIDs) > 0 {
add(searchOpportunityBuyerID, clauses.buyerIDs)
}
if len(clauses.excludedBuyerIDs) > 0 {
add("NOT "+searchOpportunityBuyerID, clauses.excludedBuyerIDs)
}
if clause, arguments := clauses.cpvClause(); clause != "" {
add(clause, arguments...)
}
if !releasedFrom.IsZero() {
add(filterOpportunityReleasedFrom, releasedFrom.UTC())
}
statement := searchOpportunitiesSelect
if len(where) > 0 {
statement += "\nWHERE " + strings.Join(where, "\n\tAND ")
}
return statement, args
}
// sortTerms sorts terms by how they combine into the query.
func sortTerms(terms []search.Term) searchClauses {
clauses := searchClauses{
anyOf: map[search.Field][]string{},
}
for _, term := range terms {
switch {
case term.Field == search.FieldBuyerID && term.Negated:
clauses.excludedBuyerIDs = append(
clauses.excludedBuyerIDs, term.Value,
)
case term.Field == search.FieldBuyerID:
clauses.buyerIDs = append(clauses.buyerIDs, term.Value)
case term.Field == search.FieldCPV && term.Negated:
condition, argument := cpvCondition(term)
clauses.excludedCPV = append(clauses.excludedCPV, condition)
clauses.excludedCPVArgs = append(
clauses.excludedCPVArgs, argument,
)
case term.Field == search.FieldCPV:
condition, argument := cpvCondition(term)
clauses.cpv = append(clauses.cpv, condition)
clauses.cpvArgs = append(clauses.cpvArgs, argument)
case term.Negated:
clauses.excluded = append(clauses.excluded, textMatch(term))
case term.Field == search.FieldText:
clauses.required = append(clauses.required, textMatch(term))
default:
clauses.anyOf[term.Field] = append(
clauses.anyOf[term.Field], textMatch(term),
)
}
}
return clauses
}
// cpvClause returns the clause for the CPV terms, with its arguments, or ""
// if there are none. With cpv: terms, an opportunity needs a code that one
// of them matches and no !cpv: term does; with only !cpv: terms, it needs
// no code that any of them matches.
func (c searchClauses) cpvClause() (string, []any) {
excluded := "(" + strings.Join(c.excludedCPV, " OR ") + ")"
switch {
case len(c.cpv) > 0 && len(c.excludedCPV) > 0:
condition := "(" + strings.Join(c.cpv, " OR ") + ")" +
" AND NOT " + excluded
return fmt.Sprintf(searchOpportunityCPV, condition),
append(append([]any{}, c.cpvArgs...), c.excludedCPVArgs...)
case len(c.cpv) > 0:
condition := "(" + strings.Join(c.cpv, " OR ") + ")"
return fmt.Sprintf(searchOpportunityCPV, condition), c.cpvArgs
case len(c.excludedCPV) > 0:
return "NOT " + fmt.Sprintf(searchOpportunityCPV, excluded),
c.excludedCPVArgs
}
return "", nil
}
// cpvCondition returns the condition on one CPV code for a cpv: term, with
// its argument: a code prefix, or a full-text search of its description.
func cpvCondition(term search.Term) (string, any) {
if code, ok := term.CPVCode(); ok {
return searchCPVCodePrefix, code + "%"
}
return searchCPVDescription, ftsPhrase(term.Words())
}
// textMatch returns the FTS5 query of opportunities_fts for a full-text
// term: its words as a phrase, in the columns its field searches.
func textMatch(term search.Term) string {
return ftsColumns[term.Field] + " : " + ftsPhrase(term.Words())
}
// ftsPhrase returns an FTS5 phrase of words, the last also matching as a
// prefix. The words must be split as search.Term.Words splits them, so
// they hold no quotes or FTS5 syntax.
func ftsPhrase(words []string) string {
return `"` + strings.Join(words, " ") + `"*`
}
+180
View File
@@ -0,0 +1,180 @@
package store_test
import (
"context"
"fmt"
"slices"
"testing"
"time"
"git.avsdev.uk/Go/job-search/internal/search"
"git.avsdev.uk/Go/job-search/internal/store"
"git.avsdev.uk/Go/job-search/internal/types"
)
// searchStore returns a store holding five opportunities, s1 to s5, released
// on days 1 to 5, for the search tests.
func searchStore(t *testing.T) *store.Store {
t.Helper()
s := openStore(t)
const software = "Programming services of packaged software products"
cpv := func(code, description string) types.ClassificationCode {
return types.ClassificationCode{
IsCPV: true,
Code: code,
Classification: description,
}
}
rows := []struct {
title, description, buyer, buyerID string
classifications []types.ClassificationCode
}{
{
title: "Cloud migration services",
description: "Moving servers to the cloud",
buyer: "Leeds City Council",
buyerID: "GB-LEEDS",
classifications: []types.ClassificationCode{
cpv("72100000", "Hardware consultancy services"),
cpv("72210000", software),
},
},
{
title: "Road repairs",
description: "Pothole repairs in Bradford",
buyer: "Bradford Council",
buyerID: "GB-BRAD",
classifications: []types.ClassificationCode{
cpv("45233141", "Road-maintenance works"),
},
},
{
title: "School meals",
description: "Catering and cloud-based ordering",
buyer: "Leeds Schools Trust",
buyerID: "GB-LST",
classifications: []types.ClassificationCode{
cpv("55524000", "School catering services"),
cpv("15800000", "Miscellaneous food products"),
},
},
{
title: "Software licences",
description: "Licences",
buyer: "York Council",
buyerID: "GB-YORK",
classifications: []types.ClassificationCode{
cpv("72210000", software),
},
},
{
title: "Office cleaning",
description: "Cleaning",
buyer: "Hull Council",
buyerID: "GB-HULL",
},
}
for i, row := range rows {
o := opportunity(fmt.Sprintf("s%d", i+1), row.title, i+1, "r1")
o.Description = row.description
o.BuyerName = row.buyer
o.BuyerID = row.buyerID
o.Classifications = row.classifications
mustSave(t, s, o)
}
return s
}
func TestSearchOpportunities(t *testing.T) {
s := searchStore(t)
all := []string{"s1", "s2", "s3", "s4", "s5"}
cases := []struct {
query string
want []string
}{
{"", all},
{"title: !!! \"\"", all},
{"cloud", []string{"s1", "s3"}},
{"CLOUD", []string{"s1", "s3"}},
{"migr", []string{"s1"}},
{"migrating", []string{"s1"}},
{"cloud servers", []string{"s1"}},
{"leeds", []string{}},
{"bradford", []string{"s2"}},
{`"road repairs"`, []string{"s2"}},
{`"repairs road"`, []string{}},
{"title:cloud", []string{"s1"}},
{"title:pothole", []string{}},
{"title:cloud title:road", []string{"s1", "s2"}},
{"buyer:leeds", []string{"s1", "s3"}},
{"buyer:leeds buyer:bradford", []string{"s1", "s2", "s3"}},
{`buyer:"leeds city"`, []string{"s1"}},
{"cloud buyer:leeds", []string{"s1", "s3"}},
{"title:cloud buyer:leeds", []string{"s1"}},
{`cloud !buyer:"leeds city"`, []string{"s3"}},
{"!cloud", []string{"s2", "s4", "s5"}},
{"!cloud !road", []string{"s4", "s5"}},
{"buyer_id:gb-leeds", []string{"s1"}},
{"buyer_id:GB-LEEDS buyer_id:gb-york", []string{"s1", "s4"}},
{"buyer_id:GB", []string{}},
{"!buyer_id:GB-LEEDS", []string{"s2", "s3", "s4", "s5"}},
{"cpv:72", []string{"s1", "s4"}},
{"cpv:72210000-5", []string{"s1", "s4"}},
{"cpv:72 !cpv:7221", []string{"s1"}},
{"!cpv:7221", []string{"s2", "s3", "s5"}},
{"cpv:food", []string{"s3"}},
{"!cpv:food", []string{"s1", "s2", "s4", "s5"}},
{"cpv:catering cpv:road", []string{"s2", "s3"}},
{"cpv:software !cpv:hardware", []string{"s1", "s4"}},
{"cpv:72 buyer:york", []string{"s4"}},
{"foo:cloud", all},
{`cloud" OR title:road*`, []string{}},
}
for _, tc := range cases {
got, err := s.SearchOpportunities(
context.Background(), search.Parse(tc.query), time.Time{},
)
if err != nil {
t.Fatalf("%q: %v", tc.query, err)
}
slices.Sort(got)
if !slices.Equal(got, tc.want) {
t.Errorf("%q: got %v, want %v", tc.query, got, tc.want)
}
}
}
// released_from bounds the search as it bounds ListOpportunities, and
// changing a searched field updates the index.
func TestSearchOpportunitiesReleasedFromAndUpdates(t *testing.T) {
ctx := context.Background()
s := searchStore(t)
got, err := s.SearchOpportunities(ctx, search.Parse("buyer:leeds"), day(3))
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, []string{"s3"}) {
t.Errorf("released from day 3: got %v, want [s3]", got)
}
o := opportunity("s5", "Office cleaning", 6, "r1", "r2")
o.Description = "Window washing"
o.BuyerName = "Wakefield Council"
mustSave(t, s, o)
for query, want := range map[string][]string{
"washing": {"s5"},
"buyer:wakefield": {"s5"},
"cleaning": {"s5"},
"buyer:hull": {},
"cpv:agricultural": {"s5"},
} {
got, err := s.SearchOpportunities(ctx, search.Parse(query), time.Time{})
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, want) {
t.Errorf("%q after update: got %v, want %v", query, got, want)
}
}
}
+1 -1
View File
@@ -861,7 +861,7 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) {
add(filterOpportunityClassificationCode, filter.ClassificationCode)
}
if match := ftsQuery(filter.Title); match != "" {
add(filterOpportunityTitleMatch, match)
add(filterOpportunityTitleMatch, "title : ("+match+")")
}
if !filter.ReleasedFrom.IsZero() {
add(filterOpportunityReleasedFrom, filter.ReleasedFrom.UTC())
+3
View File
@@ -453,6 +453,9 @@ func TestListOpportunitiesFilters(t *testing.T) {
{"title case", store.OpportunityFilter{
Title: "ROAD",
}, []string{"ocds-2"}},
{"title only, not description", store.OpportunityFilter{
Title: "description",
}, []string{}},
{
"title syntax is not FTS syntax",
store.OpportunityFilter{