diff --git a/README.md b/README.md index 90ff21d..e2bf6a0 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ second one fails with "address already in use". | `internal/cli` | Cobra commands and shared CLI setup | | `internal/config` | Configuration loading, validation and editing | | `internal/fetcher` | Runs a worker per enabled source | +| `internal/search` | Parsing of the general opportunity search (`title:`, `buyer:`, `buyer_id:`, `cpv:`, `!`) and its help | | `internal/skiprules` | The rules that mark new opportunities Skipped (see [internal/skiprules](internal/skiprules/README.md)) | | `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) | | `internal/findatender` | FTS API client and conversion from OCDS to JobSearch types | diff --git a/internal/api/opportunities.go b/internal/api/opportunities.go index 7a317fa..a0f1ed0 100644 --- a/internal/api/opportunities.go +++ b/internal/api/opportunities.go @@ -13,6 +13,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/events" "git.avsdev.uk/Go/job-search/internal/httpx" + "git.avsdev.uk/Go/job-search/internal/search" "git.avsdev.uk/Go/job-search/internal/store" "git.avsdev.uk/Go/job-search/internal/types" ) @@ -38,6 +39,21 @@ type OpportunityPage struct { Batch string `json:"batch"` } +// OpportunitySearch is the result of a general search, as GET +// /opportunities/search returns it: the UUIDs of the opportunities that +// match, in no particular order. MatchesAll reports a search with nothing to +// search for (blank, or only unknown prompts), which matches every +// opportunity; UUIDs is then empty. UnknownPrompts are the prompts in the +// search that aren't known, as typed, without their colon. Prompts and +// Syntax are the help for the search box. +type OpportunitySearch struct { + UUIDs []string `json:"uuids"` + MatchesAll bool `json:"matches_all"` + UnknownPrompts []string `json:"unknown_prompts"` + Prompts []search.Prompt `json:"prompts"` + Syntax []search.Example `json:"syntax"` +} + // opportunityRoutes wires HTTP handlers to the opportunity store. // Opportunities are read-only; only their marks can be changed. type opportunityRoutes struct { @@ -64,6 +80,7 @@ func newOpportunityRoutes( // Routes registers the opportunity routes on router (mounted under /api). func (h *opportunityRoutes) Routes(router chi.Router) { router.Get("/opportunities", h.list) + router.Get("/opportunities/search", h.search) router.Get("/opportunities/{uuid}", h.get) router.Put("/opportunities/{uuid}/marks/{mark}", h.setMark) router.Delete("/opportunities/{uuid}/marks/{mark}", h.clearMark) @@ -203,6 +220,61 @@ func (h *opportunityRoutes) listChanged( }) } +// search serves GET /opportunities/search: the UUIDs of the opportunities +// matching the general search in the q query parameter (see package +// search), with the help for the search box. released_from bounds it as it +// bounds list. +// +// @Summary Search opportunities +// @Description The UUIDs of the opportunities matching a general search, +// @Description in no particular order. q is words and "quoted phrases", +// @Description each optionally with a prompt (title:, buyer:, buyer_id:, +// @Description cpv:) and a leading ! to leave out what it matches; prompts +// @Description and syntax explain it. A search with nothing to search for +// @Description (blank, or only unknown prompts) has matches_all set and no +// @Description uuids. unknown_prompts lists the prompts that aren't known, +// @Description as typed; their terms are ignored. released_from (RFC 3339, +// @Description inclusive) leaves out opportunities whose latest release is +// @Description before it. +// @Tags opportunities +// @Produce json +// @Param q query string false "The search" +// @Param released_from query string false "Only opportunities whose latest release is at or after this time" format(date-time) +// @Success 200 {object} OpportunitySearch +// @Failure 400 {object} httpx.Problem "Invalid released_from" +// @Failure 500 {object} httpx.Problem +// @Router /opportunities/search [get] +func (h *opportunityRoutes) search(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + releasedFrom, err := timeParam( + "released_from", query.Get("released_from"), + ) + if err != nil { + httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error()) + return + } + + parsed := search.Parse(query.Get("q")) + result := OpportunitySearch{ + UUIDs: []string{}, + MatchesAll: parsed.MatchesAll(), + UnknownPrompts: parsed.UnknownPrompts, + Prompts: search.Prompts, + Syntax: search.Syntax, + } + if !result.MatchesAll { + result.UUIDs, err = h.store.SearchOpportunities( + r.Context(), parsed, releasedFrom, + ) + if err != nil { + httpx.WriteProblem(w, r, http.StatusInternalServerError, "") + return + } + } + + httpx.WriteJSON(w, http.StatusOK, result) +} + // get serves GET /opportunities/{uuid}: one opportunity with its // classifications, marks and releases. // diff --git a/internal/api/opportunities_test.go b/internal/api/opportunities_test.go index f780912..d44df41 100644 --- a/internal/api/opportunities_test.go +++ b/internal/api/opportunities_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "slices" "strings" "sync" "testing" @@ -483,3 +484,54 @@ func TestChangeMarkPublishes(t *testing.T) { } } } + +// GET /opportunities/search returns the matching UUIDs, bounded by +// released_from, with the help; a search with nothing to search for matches +// all, and unknown prompts are listed. +func TestSearchOpportunities(t *testing.T) { + url := opportunitiesServer(t, 5) + "/opportunities/search" + from3 := "&released_from=2026-10-03T00:00:00Z" + cases := []struct { + query string + uuids []string + matchesAll bool + unknown []string + }{ + {"", []string{}, true, []string{}}, + {"?q=", []string{}, true, []string{}}, + {"?q=cpv:72", []string{"ocds-1", "ocds-3", "ocds-5"}, false, []string{}}, + {"?q=cpv:72" + from3, []string{"ocds-3", "ocds-5"}, false, []string{}}, + {"?q=opportunity%20!cpv:72", []string{"ocds-2", "ocds-4"}, false, []string{}}, + {"?q=nothing", []string{}, false, []string{}}, + {"?q=foo:x%20Bar:y", []string{}, true, []string{"foo", "Bar"}}, + {"?q=foo:x%20title:3", []string{"ocds-3"}, false, []string{"foo"}}, + } + for _, tc := range cases { + var got OpportunitySearch + status := request(t, http.MethodGet, url+tc.query, &got) + slices.Sort(got.UUIDs) + if status != http.StatusOK || !slices.Equal(got.UUIDs, tc.uuids) || + got.MatchesAll != tc.matchesAll || + !slices.Equal(got.UnknownPrompts, tc.unknown) { + t.Errorf("%s: status %d, %+v; want %v, matches_all %v, unknown %v", + tc.query, status, got, tc.uuids, tc.matchesAll, tc.unknown) + } + if len(got.Prompts) == 0 || len(got.Syntax) == 0 { + t.Errorf("%s: no help: %+v", tc.query, got) + } + } + + // Empty lists are [], never null. + var raw map[string]json.RawMessage + request(t, http.MethodGet, url+"?q=nothing", &raw) + for _, field := range []string{"uuids", "unknown_prompts"} { + if string(raw[field]) != "[]" { + t.Errorf("%s = %s; want []", field, raw[field]) + } + } + + status := request(t, http.MethodGet, url+"?q=x&released_from=x", nil) + if status != http.StatusBadRequest { + t.Errorf("bad released_from: status %d; want 400", status) + } +} diff --git a/internal/search/search.go b/internal/search/search.go new file mode 100644 index 0000000..746d64c --- /dev/null +++ b/internal/search/search.go @@ -0,0 +1,222 @@ +// Package search parses the general opportunity search: words and quoted +// phrases, each optionally with a prompt naming the field it searches +// (title:, buyer:, buyer_id:, cpv:) and a leading "!" to negate it. The +// store runs a parsed Query (store.SearchOpportunities). +package search + +import ( + "regexp" + "strings" + "unicode" +) + +// Field is what a term searches. +type Field int + +// Values of Field. +const ( + // FieldText is a term with no prompt: the title or the description. + FieldText Field = iota + // FieldTitle is title:, the title only. + FieldTitle + // FieldBuyer is buyer:, the buyer's name. + FieldBuyer + // FieldBuyerID is buyer_id:, the buyer's ID, matched whole. + FieldBuyerID + // FieldCPV is cpv:, a CPV code prefix or words of a CPV description. + FieldCPV +) + +// Prompt is a known prompt, with a line of help for the user. +type Prompt struct { + Name string `json:"name"` + Help string `json:"help"` +} + +// Example is an example of the search syntax, with a line of help for the +// user. +type Example struct { + Example string `json:"example"` + Help string `json:"help"` +} + +// Prompts are the known prompts, in the order to show them. +var Prompts = []Prompt{ + {Name: "title", Help: "Words in the title only."}, + {Name: "buyer", Help: "Words in the buyer's name."}, + { + Name: "buyer_id", + Help: "The buyer's ID, whole and in any case (e.g. GB-FTS-148803).", + }, + { + Name: "cpv", + Help: "The start of a CPV code (cpv:72), or words in a CPV code's description (cpv:food).", + }, +} + +// Syntax explains the search syntax, in the order to show it. +var Syntax = []Example{ + { + Example: "cloud hosting", + Help: "Words without a prompt are searched for in the title and the description. Every word must be found, though not necessarily in the same one.", + }, + { + Example: "migr", + Help: "A word also matches the start of a longer word, and other forms of it: migr finds migration, and migrating finds migration.", + }, + { + Example: `"road repairs"`, + Help: "Quotes search for a phrase: its words together, in that order.", + }, + { + Example: `title:"road repairs"`, + Help: "A prompt searches only its own field, for one word or one quoted phrase.", + }, + { + Example: "buyer:leeds buyer:bradford", + Help: "Repeats of the same prompt match if any of them does: Leeds or Bradford.", + }, + { + Example: "cloud buyer:leeds cpv:72", + Help: "Different prompts, and words without one, must all match.", + }, + { + Example: "!cloud !buyer:leeds", + Help: "! in front of a word, a phrase or a prompt leaves out what it matches.", + }, + { + Example: "cpv:72 !cpv:7221", + Help: "CPV terms are checked against each of an opportunity's codes: this needs a code starting 72 that doesn't start 7221. !cpv: without cpv: leaves out every opportunity with a matching code.", + }, +} + +// prompts maps each known prompt's name to its field. +var prompts = map[string]Field{ + "title": FieldTitle, + "buyer": FieldBuyer, + "buyer_id": FieldBuyerID, + "cpv": FieldCPV, +} + +// promptPattern matches a prompt at the start of a term: a name of letters +// and underscores, then a colon. A colon after anything else (12:00) is +// part of the term. +var promptPattern = regexp.MustCompile(`^([A-Za-z_]+):`) + +// cpvCodePattern matches a CPV code or the start of one, with an optional +// check digit. +var cpvCodePattern = regexp.MustCompile(`^([0-9]+)(?:-[0-9])?$`) + +// Term is one search term: a word or a quoted phrase, as typed. +type Term struct { + Field Field + Negated bool + Value string +} + +// Words returns the term's words, split as the store's full-text index +// splits them: on anything but letters and numbers. +func (t Term) Words() []string { + return strings.FieldsFunc(t.Value, func(char rune) bool { + return !unicode.IsLetter(char) && !unicode.IsNumber(char) + }) +} + +// CPVCode returns the digits of a cpv: term that is a code or the start of +// one, without its check digit, and reports whether it is one. Any other +// cpv: term searches CPV descriptions. +func (t Term) CPVCode() (string, bool) { + match := cpvCodePattern.FindStringSubmatch(t.Value) + if match == nil { + return "", false + } + return match[1], true +} + +// usable reports whether the term can be searched for: a buyer ID that +// isn't blank, or any other term with at least one word. +func (t Term) usable() bool { + if t.Field == FieldBuyerID { + return strings.TrimSpace(t.Value) != "" + } + return len(t.Words()) > 0 +} + +// Query is a parsed search. +type Query struct { + // Terms are the terms to search for, all usable. + Terms []Term + // UnknownPrompts are the prompts that aren't known, as typed, without + // their colon, once each. Their terms are left out of Terms. + UnknownPrompts []string +} + +// MatchesAll reports whether the query has no terms, so it matches every +// opportunity. +func (q Query) MatchesAll() bool { + return len(q.Terms) == 0 +} + +// Parse parses text as a search. It never fails: text that isn't valid +// syntax, such as an unclosed quote, is read as plain words, and a term +// with no words (title: or !!!) is left out. +func Parse(text string) Query { + query := Query{ + UnknownPrompts: []string{}, + } + unknown := map[string]bool{} + + for rest := strings.TrimSpace(text); rest != ""; { + var term Term + var prompt string + var known bool + term, prompt, known, rest = nextTerm(rest) + rest = strings.TrimLeftFunc(rest, unicode.IsSpace) + + switch { + case !known: + if !unknown[prompt] { + unknown[prompt] = true + query.UnknownPrompts = append(query.UnknownPrompts, prompt) + } + case term.usable(): + query.Terms = append(query.Terms, term) + } + } + + return query +} + +// nextTerm reads the term at the start of text, which mustn't start with +// space. It returns the term, its prompt as typed ("" if none), whether the +// prompt is known, and the text after the term. +func nextTerm( + text string, +) (term Term, prompt string, known bool, rest string) { + if strings.HasPrefix(text, "!") { + term.Negated = true + text = text[1:] + } + + if match := promptPattern.FindStringSubmatch(text); match != nil { + prompt = match[1] + text = text[len(match[0]):] + } + term.Field, known = prompts[strings.ToLower(prompt)] + if prompt == "" { + term.Field, known = FieldText, true + } + + if strings.HasPrefix(text, `"`) { + term.Value, rest, _ = strings.Cut(text[1:], `"`) + return term, prompt, known, rest + } + + end := strings.IndexFunc(text, unicode.IsSpace) + if end < 0 { + end = len(text) + } + term.Value = text[:end] + + return term, prompt, known, text[end:] +} diff --git a/internal/search/search_test.go b/internal/search/search_test.go new file mode 100644 index 0000000..841670c --- /dev/null +++ b/internal/search/search_test.go @@ -0,0 +1,106 @@ +package search + +import ( + "reflect" + "slices" + "testing" +) + +func TestParse(t *testing.T) { + term := func(field Field, value string) Term { + return Term{ + Field: field, + Value: value, + } + } + text := func(value string) Term { + return term(FieldText, value) + } + not := func(term Term) Term { + term.Negated = true + return term + } + none := []string{} + cases := []struct { + input string + terms []Term + unknown []string + }{ + {"", nil, none}, + {" ", nil, none}, + {"cloud hosting", []Term{text("cloud"), text("hosting")}, none}, + {`"road repairs" leeds`, []Term{text("road repairs"), text("leeds")}, none}, + {`title:"road repairs"`, []Term{term(FieldTitle, "road repairs")}, none}, + {"TITLE:road Buyer:leeds", []Term{term(FieldTitle, "road"), term(FieldBuyer, "leeds")}, none}, + {"buyer_id:GB-1 cpv:72", []Term{term(FieldBuyerID, "GB-1"), term(FieldCPV, "72")}, none}, + {"!cloud !cpv:food", []Term{not(text("cloud")), not(term(FieldCPV, "food"))}, none}, + {`!title:"a b"`, []Term{not(term(FieldTitle, "a b"))}, none}, + {"12:00", []Term{text("12:00")}, none}, + {`"unclosed phrase`, []Term{text("unclosed phrase")}, none}, + {`title: !!! "" ! cloud`, []Term{text("cloud")}, none}, + {"foo:bar cloud Foo:baz foo:x baz:", []Term{text("cloud")}, []string{"foo", "Foo", "baz"}}, + {"buyer_id:", nil, none}, + } + for _, tc := range cases { + got := Parse(tc.input) + if !reflect.DeepEqual(got.Terms, tc.terms) { + t.Errorf("%q: terms %+v, want %+v", tc.input, got.Terms, tc.terms) + } + if !slices.Equal(got.UnknownPrompts, tc.unknown) { + t.Errorf( + "%q: unknown %q, want %q", + tc.input, got.UnknownPrompts, tc.unknown, + ) + } + if got.UnknownPrompts == nil { + t.Errorf("%q: UnknownPrompts is nil", tc.input) + } + if got.MatchesAll() != (len(tc.terms) == 0) { + t.Errorf("%q: MatchesAll %v", tc.input, got.MatchesAll()) + } + } +} + +func TestTermWordsAndCPVCode(t *testing.T) { + cases := []struct { + value string + words []string + code string + isCPV bool + }{ + {"road-repairs", []string{"road", "repairs"}, "", false}, + {"Café's", []string{"Café", "s"}, "", false}, + {"72", []string{"72"}, "72", true}, + {"72210000-5", []string{"72210000", "5"}, "72210000", true}, + {"72-55", []string{"72", "55"}, "", false}, + {"72a", []string{"72a"}, "", false}, + } + for _, tc := range cases { + term := Term{ + Field: FieldCPV, + Value: tc.value, + } + if got := term.Words(); !slices.Equal(got, tc.words) { + t.Errorf("%q: words %q, want %q", tc.value, got, tc.words) + } + code, ok := term.CPVCode() + if code != tc.code || ok != tc.isCPV { + t.Errorf( + "%q: CPVCode %q %v, want %q %v", + tc.value, code, ok, tc.code, tc.isCPV, + ) + } + } +} + +// Every known prompt has help, and every prompt with help is known. +func TestPromptsHaveHelp(t *testing.T) { + if len(Prompts) != len(prompts) { + t.Errorf("%d prompts with help, %d known", len(Prompts), len(prompts)) + } + for _, prompt := range Prompts { + if _, ok := prompts[prompt.Name]; !ok || prompt.Help == "" { + t.Errorf("prompt %q: known %v, help %q", prompt.Name, ok, prompt.Help) + } + } +} diff --git a/internal/store/README.md b/internal/store/README.md index 0476500..7872aec 100644 --- a/internal/store/README.md +++ b/internal/store/README.md @@ -4,7 +4,7 @@ A SQLite store for opportunities and their OCDS releases, built on sqlx + goose ## Build and test -The title search uses SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`. +The title and general searches use SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`. ```bash go test -tags sqlite_fts5 ./... @@ -24,7 +24,9 @@ internal/store/migrate_test.go version checks, stepping up and down, failed- internal/store/queries.go all SQL, as named constants grouped by table internal/store/queries_test.go prepares every query against the migrated schema internal/store/store.go Store: logic, using the queries via sqlx +internal/store/search.go SearchOpportunities: the general search's query internal/store/store_test.go behavioural tests +internal/store/search_test.go behavioural tests of the general search internal/store/common.go common for migrate, store and direct access ``` @@ -38,7 +40,8 @@ internal/store/common.go common for migrate, store and direct access | `opportunity_release_tags` | (`uuid`, `release_id`, `tag`) | Write-once. (`uuid`, `release_id`) references `opportunity_releases`. | | `classification_codes` | `id` | Shared between opportunities. A CPV code has one row (unique on `code` where `is_cpv`), keeping the longest description it has been given: EU Supply cuts descriptions to 64 characters. A code of any other scheme gets a row per distinct content. The table grows over time to include codes not already seen in fetched opportunities. | | `opportunity_classifications` | (`opportunity_uuid`, `classification_id`) | Mapping table; replaced as a set when the opportunity is updated. | -| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, kept in sync by triggers. | +| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, `description` and `buyer_name`, kept in sync by triggers. | +| `classification_codes_fts` | rowid = `classification_codes.id` | FTS5 index on `classification_codes.classification` (the description), kept in sync by triggers. | | `opportunity_change_batches` | `id`; `hash` (unique) | Write-once. One row per save that adds opportunities or gives them new releases, and per change of marks. `hash` names the batch to clients (random, so unguessable); `created_at` orders the batches. | | `opportunity_change_batch_opportunities` | (`batch_id`, `opportunity_uuid`) | The opportunities in each batch. References both tables, cascading on delete. | @@ -71,6 +74,7 @@ All SQL is in `queries.go` as constants, grouped by table in migration order; `s - **TestQueriesPrepare** prepares every query constant against a freshly migrated database, so a syntax error, an unknown table or unknown column fails the test and names the failing statement. - **TestQueriesAllListed** parses `queries.go` and fails if a constant isn't listed in that test's `checks` map. Adding a query therefore means adding one line there, and it can't be skipped by accident. +- **TestSearchOpportunitiesQueryAllTerms** does the same for `SearchOpportunities`, with a term of every kind, plain and negated. - **TestListOpportunitiesQueryAllFilters** builds the `ListOpportunities` query with every filter set and runs it, checking that the fragments combine and that the arguments line up with the placeholders. - **TestMarksCheckConstraint** checks the database itself rejects `won` and `lost` set together, for writes that bypass the Go validation. @@ -111,6 +115,7 @@ ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{ rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags batch, err := s.LatestChangeBatch(ctx) // hash of the newest change batch, or "" ts, batch, err = s.ListChangedOpportunities(ctx, batch, time.Time{}) // changed since; ErrUnknownBatch +uuids, err := s.SearchOpportunities(ctx, search.Parse("cloud buyer:leeds"), releasedFrom) // never nil ok, err := s.ReleaseExists(ctx, uuid, releaseID) ``` @@ -132,3 +137,9 @@ ok, err := s.ReleaseExists(ctx, uuid, releaseID) **Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted. +**General search.** `SearchOpportunities` runs a `search.Query` (parsed by `internal/search`, which documents the syntax) and returns only the matching UUIDs, for the frontend to filter its own list by. It is assembled by `searchOpportunitiesQuery` from the `searchOpportunity*` and `searchCPV*` fragments: + +- Words without a prompt search `{title description}` in `opportunities_fts`, and must all match. `title:` and `buyer:` search the `title` and `buyer_name` columns; repeats of one prompt are joined by OR into one clause. Every negated full-text term goes into one `NOT IN` clause, joined by OR. Each term is one FTS5 phrase built from `Term.Words`, its last word a prefix, so typed input is never FTS5 syntax. +- `buyer_id:` matches the whole ID, ignoring case (`IN`, expanded by `sqlx.In`). +- CPV terms are checked against each code, inside one `EXISTS` on the opportunity's CPV codes: a code must match a `cpv:` term (a code prefix by `LIKE`, or words in `classification_codes_fts`) and no `!cpv:` term. With only `!cpv:` terms the clause is `NOT EXISTS` a code matching any of them, so an opportunity with no CPV codes passes it. +- `OpportunityFilter.Title` searches only the `title` column of the same index. diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 141f8bf..5ff349b 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "path/filepath" "strings" "testing" @@ -402,3 +403,67 @@ func TestMigration10MergesCPVDuplicates(t *testing.T) { t.Error("a second row for a CPV code was accepted") } } + +// Migration 11 indexes the description, buyer name and classification +// descriptions of rows already stored, and its Down leaves the title +// index working. +func TestMigration11IndexesStoredRows(t *testing.T) { + ctx := context.Background() + m, db := newMigrator(t) + if _, err := m.UpTo(ctx, 10); err != nil { + t.Fatal(err) + } + + setup := []string{ + `INSERT INTO opportunities (uuid, uuid_type, title, status, + description, buyer_name, latest_release_date) + VALUES ('a', 'ocid', 'Cloud', 'active', 'Servers', 'Leeds', + '2026-10-01')`, + `INSERT INTO classification_codes (id, is_cpv, code, classification) + VALUES (1, 1, '15800000', 'Miscellaneous food products')`, + } + for _, statement := range setup { + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatal(err) + } + } + + if _, err := m.UpTo(ctx, 11); err != nil { + t.Fatal(err) + } + + count := func(query string) int { + t.Helper() + var n int + if err := db.QueryRowContext(ctx, query).Scan(&n); err != nil { + t.Fatalf("%s: %v", query, err) + } + return n + } + matches := []struct { + table, match string + want int + }{ + {"opportunities_fts", "description : servers", 1}, + {"opportunities_fts", "buyer_name : leeds", 1}, + {"opportunities_fts", "title : servers", 0}, + {"classification_codes_fts", "food", 1}, + } + for _, tc := range matches { + got := count(fmt.Sprintf( + "SELECT count(*) FROM %s WHERE %[1]s MATCH '%s'", tc.table, tc.match, + )) + if got != tc.want { + t.Errorf("%s MATCH %q: %d; want %d", tc.table, tc.match, got, tc.want) + } + } + + if _, err := m.DownTo(ctx, 10); err != nil { + t.Fatal(err) + } + got := count(`SELECT count(*) FROM opportunities_fts + WHERE opportunities_fts MATCH 'cloud'`) + if got != 1 { + t.Errorf("title search after Down: %d; want 1", got) + } +} diff --git a/internal/store/migrations/00011_create_general_search.sql b/internal/store/migrations/00011_create_general_search.sql new file mode 100644 index 0000000..e509f02 --- /dev/null +++ b/internal/store/migrations/00011_create_general_search.sql @@ -0,0 +1,119 @@ +-- +goose Up +-- +goose StatementBegin +-- The general search: opportunities_fts (migration 00006) now also indexes +-- the description and the buyer's name, and classification_codes_fts +-- indexes classification descriptions. Both are kept in sync by triggers. +DROP TRIGGER IF EXISTS "opportunities_fts_update"; +DROP TRIGGER IF EXISTS "opportunities_fts_delete"; +DROP TRIGGER IF EXISTS "opportunities_fts_insert"; +DROP TABLE IF EXISTS "opportunities_fts"; + +CREATE VIRTUAL TABLE "opportunities_fts" USING fts5( + "title", + "description", + "buyer_name", + content = 'opportunities', + content_rowid = 'id', + tokenize = 'porter unicode61 remove_diacritics 2' +); + +CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name") + VALUES (new."id", new."title", new."description", new."buyer_name"); +END; + +CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" + ("opportunities_fts", "rowid", "title", "description", "buyer_name") + VALUES ('delete', old."id", old."title", old."description", old."buyer_name"); +END; + +CREATE TRIGGER "opportunities_fts_update" + AFTER UPDATE OF "title", "description", "buyer_name" ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" + ("opportunities_fts", "rowid", "title", "description", "buyer_name") + VALUES ('delete', old."id", old."title", old."description", old."buyer_name"); + INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name") + VALUES (new."id", new."title", new."description", new."buyer_name"); +END; + +INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild'); + +CREATE VIRTUAL TABLE "classification_codes_fts" USING fts5( + "classification", + content = 'classification_codes', + content_rowid = 'id', + tokenize = 'porter unicode61 remove_diacritics 2' +); + +CREATE TRIGGER "classification_codes_fts_insert" + AFTER INSERT ON "classification_codes" +BEGIN + INSERT INTO "classification_codes_fts" ("rowid", "classification") + VALUES (new."id", new."classification"); +END; + +CREATE TRIGGER "classification_codes_fts_delete" + AFTER DELETE ON "classification_codes" +BEGIN + INSERT INTO "classification_codes_fts" + ("classification_codes_fts", "rowid", "classification") + VALUES ('delete', old."id", old."classification"); +END; + +CREATE TRIGGER "classification_codes_fts_update" + AFTER UPDATE OF "classification" ON "classification_codes" +BEGIN + INSERT INTO "classification_codes_fts" + ("classification_codes_fts", "rowid", "classification") + VALUES ('delete', old."id", old."classification"); + INSERT INTO "classification_codes_fts" ("rowid", "classification") + VALUES (new."id", new."classification"); +END; + +INSERT INTO "classification_codes_fts" ("classification_codes_fts") + VALUES ('rebuild'); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TRIGGER IF EXISTS "classification_codes_fts_update"; +DROP TRIGGER IF EXISTS "classification_codes_fts_delete"; +DROP TRIGGER IF EXISTS "classification_codes_fts_insert"; +DROP TABLE IF EXISTS "classification_codes_fts"; + +DROP TRIGGER IF EXISTS "opportunities_fts_update"; +DROP TRIGGER IF EXISTS "opportunities_fts_delete"; +DROP TRIGGER IF EXISTS "opportunities_fts_insert"; +DROP TABLE IF EXISTS "opportunities_fts"; + +CREATE VIRTUAL TABLE "opportunities_fts" USING fts5( + "title", + content = 'opportunities', + content_rowid = 'id', + tokenize = 'porter unicode61 remove_diacritics 2' +); + +CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title"); +END; + +CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title") + VALUES ('delete', old."id", old."title"); +END; + +CREATE TRIGGER "opportunities_fts_update" AFTER UPDATE OF "title" ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title") + VALUES ('delete', old."id", old."title"); + INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title"); +END; + +INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild'); +-- +goose StatementEnd diff --git a/internal/store/queries.go b/internal/store/queries.go index f9cb021..369e0a4 100644 --- a/internal/store/queries.go +++ b/internal/store/queries.go @@ -107,7 +107,8 @@ LIMIT ? OFFSET ?` )` // filterOpportunityTitleMatch matches a full-text search of the title. The - // argument must come from ftsQuery, never raw user input. + // argument must be ftsQuery's, limited to the title column (see + // listOpportunitiesQuery), never raw user input. filterOpportunityTitleMatch = `t.id IN ( SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ? )` @@ -150,6 +151,52 @@ LIMIT ? OFFSET ?` )` ) +// SearchOpportunities is assembled by searchOpportunitiesQuery as: +// +// searchOpportunitiesSelect +// [WHERE clause AND clause ...] +// +// Each clause is a searchOpportunity* fragment, perhaps with NOT in front, +// or filterOpportunityReleasedFrom. searchOpportunityCPV's %s is filled +// with searchCPV* fragments joined by OR, AND and NOT. A full-text argument +// must be built from search.Term.Words, never raw user input. +const ( + // searchOpportunitiesSelect selects every opportunity's UUID, as t. + searchOpportunitiesSelect = ` +SELECT t.uuid FROM opportunities t` + + // searchOpportunityText matches a full-text search of the title, + // description and buyer_name columns. + searchOpportunityText = `t.id IN ( + SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ? +)` + + // searchOpportunityBuyerID matches any of several buyer IDs, ignoring + // case. "IN (?)" is expanded by sqlx.In, which must be followed by + // Rebind. + searchOpportunityBuyerID = `t.buyer_id COLLATE NOCASE IN (?)` + + // searchOpportunityCPV matches an opportunity with a CPV code meeting + // the condition put in place of %s, which reads the code as c. + searchOpportunityCPV = `EXISTS ( + SELECT 1 + FROM opportunity_classifications tc + JOIN classification_codes c ON c.id = tc.classification_id + WHERE tc.opportunity_uuid = t.uuid AND c.is_cpv AND %s +)` + + // searchCPVCodePrefix matches a code by a LIKE pattern: its first + // digits followed by %. + searchCPVCodePrefix = `c.code LIKE ?` + + // searchCPVDescription matches a full-text search of the code's + // description. + searchCPVDescription = `c.id IN ( + SELECT rowid FROM classification_codes_fts + WHERE classification_codes_fts MATCH ? +)` +) + // selectOtherSourcesTitles lists the uuid, title and latest_release_date of // the opportunities not from one source whose latest_release_date is in // [from, before); it walks the latest_release_date index. diff --git a/internal/store/queries_test.go b/internal/store/queries_test.go index db9557a..fffd389 100644 --- a/internal/store/queries_test.go +++ b/internal/store/queries_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "go/ast" "go/parser" "go/token" @@ -11,6 +12,8 @@ import ( "time" "github.com/jmoiron/sqlx" + + "git.avsdev.uk/Go/job-search/internal/search" ) // Run with: go test -tags sqlite_fts5 ./... @@ -81,6 +84,24 @@ var checks = map[string]queryCheck{ sql: withFilter(filterOpportunityMarkLost), }, + "searchOpportunitiesSelect": {sql: searchOpportunitiesSelect}, + "searchOpportunityText": { + sql: withSearch(searchOpportunityText), + }, + "searchOpportunityBuyerID": { + sql: withSearch(searchOpportunityBuyerID), + in: true, + }, + "searchOpportunityCPV": { + sql: withSearch(fmt.Sprintf(searchOpportunityCPV, "1")), + }, + "searchCPVCodePrefix": { + sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVCodePrefix)), + }, + "searchCPVDescription": { + sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVDescription)), + }, + // opportunity_ocds_releases "insertRelease": {sql: insertRelease, named: true}, "selectReleasesByOpportunity": {sql: selectReleasesByOpportunity}, @@ -140,6 +161,10 @@ func withFilter(fragment string) string { listOpportunitiesOrderAndPage } +func withSearch(fragment string) string { + return searchOpportunitiesSelect + "\nWHERE " + fragment +} + func openTestDB(t *testing.T) *sqlx.DB { t.Helper() s, err := Open( @@ -223,6 +248,28 @@ func TestListOpportunitiesQueryAllFilters(t *testing.T) { } } +// TestSearchOpportunitiesQueryAllTerms runs the assembled search query with +// a term of every kind, both plain and negated, checking the fragments +// combine and the arguments line up with the placeholders. +func TestSearchOpportunitiesQueryAllTerms(t *testing.T) { + db := openTestDB(t) + query := search.Parse(`cloud "road repairs" title:a title:b buyer:c ` + + `buyer_id:GB-1 buyer_id:GB-2 cpv:72 cpv:food !cloud !title:d ` + + `!buyer:e !buyer_id:GB-3 !cpv:7221 !cpv:fuel`) + statement, args := searchOpportunitiesQuery( + query, time.Now().Add(-time.Hour), + ) + statement, args, err := sqlx.In(statement, args...) + if err != nil { + t.Fatal(err) + } + + var uuids []string + if err := db.Select(&uuids, db.Rebind(statement), args...); err != nil { + t.Errorf("run with %d args:\n%s\n%v", len(args), statement, err) + } +} + // TestQueriesAllListed fails if queries.go declares a constant that checks // doesn't list, so every new statement is prepared by TestQueriesPrepare. func TestQueriesAllListed(t *testing.T) { diff --git a/internal/store/search.go b/internal/store/search.go new file mode 100644 index 0000000..ebed95c --- /dev/null +++ b/internal/store/search.go @@ -0,0 +1,202 @@ +package store + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/jmoiron/sqlx" + + "git.avsdev.uk/Go/job-search/internal/search" +) + +// ftsColumns are the opportunities_fts columns each field's terms search, +// as an FTS5 column filter. +var ftsColumns = map[search.Field]string{ + search.FieldText: "{title description}", + search.FieldTitle: "title", + search.FieldBuyer: "buyer_name", +} + +// anyOfFields are the fields whose terms match if any of them does, in the +// order their clauses are added. +var anyOfFields = []search.Field{search.FieldTitle, search.FieldBuyer} + +// searchClauses holds a search's terms, sorted by how they combine. +type searchClauses struct { + // required are full-text matches (words without a prompt) that must + // all match. + required []string + // anyOf are full-text matches by field, of which one must match. + anyOf map[search.Field][]string + // excluded are full-text matches of which none may match. + excluded []string + + buyerIDs []string + excludedBuyerIDs []string + + // cpv and excludedCPV are conditions on one CPV code (searchCPV* + // fragments), with their arguments. + cpv []string + cpvArgs []any + excludedCPV []string + excludedCPVArgs []any +} + +// SearchOpportunities returns the UUIDs of the opportunities matching +// query, in no particular order and never nil. Unless releasedFrom is zero, +// only opportunities whose latest release is on or after it are returned +// (as OpportunityFilter.ReleasedFrom). A query with no terms matches every +// opportunity. +func (s *Store) SearchOpportunities( + ctx context.Context, query search.Query, releasedFrom time.Time, +) ([]string, error) { + statement, args := searchOpportunitiesQuery(query, releasedFrom) + statement, args, err := sqlx.In(statement, args...) + if err != nil { + return nil, fmt.Errorf("expanding the search query: %w", err) + } + + uuids := []string{} + err = s.db.SelectContext(ctx, &uuids, s.db.Rebind(statement), args...) + if err != nil { + return nil, fmt.Errorf("searching opportunities: %w", err) + } + + return uuids, nil +} + +// searchOpportunitiesQuery assembles the SearchOpportunities query from the +// fragments in queries.go, with its arguments. It must be expanded by +// sqlx.In. +func searchOpportunitiesQuery( + query search.Query, releasedFrom time.Time, +) (string, []any) { + clauses := sortTerms(query.Terms) + var where []string + var args []any + add := func(clause string, arguments ...any) { + where = append(where, clause) + args = append(args, arguments...) + } + + if len(clauses.required) > 0 { + add(searchOpportunityText, strings.Join(clauses.required, " AND ")) + } + for _, field := range anyOfFields { + if matches := clauses.anyOf[field]; len(matches) > 0 { + add(searchOpportunityText, strings.Join(matches, " OR ")) + } + } + if len(clauses.excluded) > 0 { + add( + "NOT "+searchOpportunityText, + strings.Join(clauses.excluded, " OR "), + ) + } + + if len(clauses.buyerIDs) > 0 { + add(searchOpportunityBuyerID, clauses.buyerIDs) + } + if len(clauses.excludedBuyerIDs) > 0 { + add("NOT "+searchOpportunityBuyerID, clauses.excludedBuyerIDs) + } + + if clause, arguments := clauses.cpvClause(); clause != "" { + add(clause, arguments...) + } + + if !releasedFrom.IsZero() { + add(filterOpportunityReleasedFrom, releasedFrom.UTC()) + } + + statement := searchOpportunitiesSelect + if len(where) > 0 { + statement += "\nWHERE " + strings.Join(where, "\n\tAND ") + } + + return statement, args +} + +// sortTerms sorts terms by how they combine into the query. +func sortTerms(terms []search.Term) searchClauses { + clauses := searchClauses{ + anyOf: map[search.Field][]string{}, + } + for _, term := range terms { + switch { + case term.Field == search.FieldBuyerID && term.Negated: + clauses.excludedBuyerIDs = append( + clauses.excludedBuyerIDs, term.Value, + ) + case term.Field == search.FieldBuyerID: + clauses.buyerIDs = append(clauses.buyerIDs, term.Value) + case term.Field == search.FieldCPV && term.Negated: + condition, argument := cpvCondition(term) + clauses.excludedCPV = append(clauses.excludedCPV, condition) + clauses.excludedCPVArgs = append( + clauses.excludedCPVArgs, argument, + ) + case term.Field == search.FieldCPV: + condition, argument := cpvCondition(term) + clauses.cpv = append(clauses.cpv, condition) + clauses.cpvArgs = append(clauses.cpvArgs, argument) + case term.Negated: + clauses.excluded = append(clauses.excluded, textMatch(term)) + case term.Field == search.FieldText: + clauses.required = append(clauses.required, textMatch(term)) + default: + clauses.anyOf[term.Field] = append( + clauses.anyOf[term.Field], textMatch(term), + ) + } + } + + return clauses +} + +// cpvClause returns the clause for the CPV terms, with its arguments, or "" +// if there are none. With cpv: terms, an opportunity needs a code that one +// of them matches and no !cpv: term does; with only !cpv: terms, it needs +// no code that any of them matches. +func (c searchClauses) cpvClause() (string, []any) { + excluded := "(" + strings.Join(c.excludedCPV, " OR ") + ")" + switch { + case len(c.cpv) > 0 && len(c.excludedCPV) > 0: + condition := "(" + strings.Join(c.cpv, " OR ") + ")" + + " AND NOT " + excluded + return fmt.Sprintf(searchOpportunityCPV, condition), + append(append([]any{}, c.cpvArgs...), c.excludedCPVArgs...) + case len(c.cpv) > 0: + condition := "(" + strings.Join(c.cpv, " OR ") + ")" + return fmt.Sprintf(searchOpportunityCPV, condition), c.cpvArgs + case len(c.excludedCPV) > 0: + return "NOT " + fmt.Sprintf(searchOpportunityCPV, excluded), + c.excludedCPVArgs + } + + return "", nil +} + +// cpvCondition returns the condition on one CPV code for a cpv: term, with +// its argument: a code prefix, or a full-text search of its description. +func cpvCondition(term search.Term) (string, any) { + if code, ok := term.CPVCode(); ok { + return searchCPVCodePrefix, code + "%" + } + return searchCPVDescription, ftsPhrase(term.Words()) +} + +// textMatch returns the FTS5 query of opportunities_fts for a full-text +// term: its words as a phrase, in the columns its field searches. +func textMatch(term search.Term) string { + return ftsColumns[term.Field] + " : " + ftsPhrase(term.Words()) +} + +// ftsPhrase returns an FTS5 phrase of words, the last also matching as a +// prefix. The words must be split as search.Term.Words splits them, so +// they hold no quotes or FTS5 syntax. +func ftsPhrase(words []string) string { + return `"` + strings.Join(words, " ") + `"*` +} diff --git a/internal/store/search_test.go b/internal/store/search_test.go new file mode 100644 index 0000000..47b4e77 --- /dev/null +++ b/internal/store/search_test.go @@ -0,0 +1,180 @@ +package store_test + +import ( + "context" + "fmt" + "slices" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/search" + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" +) + +// searchStore returns a store holding five opportunities, s1 to s5, released +// on days 1 to 5, for the search tests. +func searchStore(t *testing.T) *store.Store { + t.Helper() + s := openStore(t) + const software = "Programming services of packaged software products" + cpv := func(code, description string) types.ClassificationCode { + return types.ClassificationCode{ + IsCPV: true, + Code: code, + Classification: description, + } + } + rows := []struct { + title, description, buyer, buyerID string + classifications []types.ClassificationCode + }{ + { + title: "Cloud migration services", + description: "Moving servers to the cloud", + buyer: "Leeds City Council", + buyerID: "GB-LEEDS", + classifications: []types.ClassificationCode{ + cpv("72100000", "Hardware consultancy services"), + cpv("72210000", software), + }, + }, + { + title: "Road repairs", + description: "Pothole repairs in Bradford", + buyer: "Bradford Council", + buyerID: "GB-BRAD", + classifications: []types.ClassificationCode{ + cpv("45233141", "Road-maintenance works"), + }, + }, + { + title: "School meals", + description: "Catering and cloud-based ordering", + buyer: "Leeds Schools Trust", + buyerID: "GB-LST", + classifications: []types.ClassificationCode{ + cpv("55524000", "School catering services"), + cpv("15800000", "Miscellaneous food products"), + }, + }, + { + title: "Software licences", + description: "Licences", + buyer: "York Council", + buyerID: "GB-YORK", + classifications: []types.ClassificationCode{ + cpv("72210000", software), + }, + }, + { + title: "Office cleaning", + description: "Cleaning", + buyer: "Hull Council", + buyerID: "GB-HULL", + }, + } + for i, row := range rows { + o := opportunity(fmt.Sprintf("s%d", i+1), row.title, i+1, "r1") + o.Description = row.description + o.BuyerName = row.buyer + o.BuyerID = row.buyerID + o.Classifications = row.classifications + mustSave(t, s, o) + } + return s +} + +func TestSearchOpportunities(t *testing.T) { + s := searchStore(t) + all := []string{"s1", "s2", "s3", "s4", "s5"} + cases := []struct { + query string + want []string + }{ + {"", all}, + {"title: !!! \"\"", all}, + {"cloud", []string{"s1", "s3"}}, + {"CLOUD", []string{"s1", "s3"}}, + {"migr", []string{"s1"}}, + {"migrating", []string{"s1"}}, + {"cloud servers", []string{"s1"}}, + {"leeds", []string{}}, + {"bradford", []string{"s2"}}, + {`"road repairs"`, []string{"s2"}}, + {`"repairs road"`, []string{}}, + {"title:cloud", []string{"s1"}}, + {"title:pothole", []string{}}, + {"title:cloud title:road", []string{"s1", "s2"}}, + {"buyer:leeds", []string{"s1", "s3"}}, + {"buyer:leeds buyer:bradford", []string{"s1", "s2", "s3"}}, + {`buyer:"leeds city"`, []string{"s1"}}, + {"cloud buyer:leeds", []string{"s1", "s3"}}, + {"title:cloud buyer:leeds", []string{"s1"}}, + {`cloud !buyer:"leeds city"`, []string{"s3"}}, + {"!cloud", []string{"s2", "s4", "s5"}}, + {"!cloud !road", []string{"s4", "s5"}}, + {"buyer_id:gb-leeds", []string{"s1"}}, + {"buyer_id:GB-LEEDS buyer_id:gb-york", []string{"s1", "s4"}}, + {"buyer_id:GB", []string{}}, + {"!buyer_id:GB-LEEDS", []string{"s2", "s3", "s4", "s5"}}, + {"cpv:72", []string{"s1", "s4"}}, + {"cpv:72210000-5", []string{"s1", "s4"}}, + {"cpv:72 !cpv:7221", []string{"s1"}}, + {"!cpv:7221", []string{"s2", "s3", "s5"}}, + {"cpv:food", []string{"s3"}}, + {"!cpv:food", []string{"s1", "s2", "s4", "s5"}}, + {"cpv:catering cpv:road", []string{"s2", "s3"}}, + {"cpv:software !cpv:hardware", []string{"s1", "s4"}}, + {"cpv:72 buyer:york", []string{"s4"}}, + {"foo:cloud", all}, + {`cloud" OR title:road*`, []string{}}, + } + for _, tc := range cases { + got, err := s.SearchOpportunities( + context.Background(), search.Parse(tc.query), time.Time{}, + ) + if err != nil { + t.Fatalf("%q: %v", tc.query, err) + } + slices.Sort(got) + if !slices.Equal(got, tc.want) { + t.Errorf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} + +// released_from bounds the search as it bounds ListOpportunities, and +// changing a searched field updates the index. +func TestSearchOpportunitiesReleasedFromAndUpdates(t *testing.T) { + ctx := context.Background() + s := searchStore(t) + + got, err := s.SearchOpportunities(ctx, search.Parse("buyer:leeds"), day(3)) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, []string{"s3"}) { + t.Errorf("released from day 3: got %v, want [s3]", got) + } + + o := opportunity("s5", "Office cleaning", 6, "r1", "r2") + o.Description = "Window washing" + o.BuyerName = "Wakefield Council" + mustSave(t, s, o) + for query, want := range map[string][]string{ + "washing": {"s5"}, + "buyer:wakefield": {"s5"}, + "cleaning": {"s5"}, + "buyer:hull": {}, + "cpv:agricultural": {"s5"}, + } { + got, err := s.SearchOpportunities(ctx, search.Parse(query), time.Time{}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, want) { + t.Errorf("%q after update: got %v, want %v", query, got, want) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index c6fc514..c1b95a2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -861,7 +861,7 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { add(filterOpportunityClassificationCode, filter.ClassificationCode) } if match := ftsQuery(filter.Title); match != "" { - add(filterOpportunityTitleMatch, match) + add(filterOpportunityTitleMatch, "title : ("+match+")") } if !filter.ReleasedFrom.IsZero() { add(filterOpportunityReleasedFrom, filter.ReleasedFrom.UTC()) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 066c794..625c48a 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -453,6 +453,9 @@ func TestListOpportunitiesFilters(t *testing.T) { {"title case", store.OpportunityFilter{ Title: "ROAD", }, []string{"ocds-2"}}, + {"title only, not description", store.OpportunityFilter{ + Title: "description", + }, []string{}}, { "title syntax is not FTS syntax", store.OpportunityFilter{ diff --git a/web/README.md b/web/README.md index f0cff90..53a70b5 100644 --- a/web/README.md +++ b/web/README.md @@ -1,8 +1,9 @@ # JobSearch - Frontend A Vite + SolidJS + Tailwind CSS (v4) + TypeScript table of opportunities with -title search, status, mark and classification filters, and per-opportunity -mark and hide controls. +a general search (run by the backend, with title:, buyer:, buyer_id: and cpv: +prompts), status, mark and classification filters, and per-opportunity mark +and hide controls. ## Run diff --git a/web/src/App.tsx b/web/src/App.tsx index c1619ff..695472c 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -22,6 +22,7 @@ import { readStored, STORAGE_KEYS, writeStored } from '@/lib/storage'; import { createLiveUpdates } from '@/primitives/createLiveUpdates'; import { createClassifications, createOpportunities } from '@/primitives/createOpportunities'; import { createOpportunityAnnotations } from '@/primitives/createOpportunityAnnotations'; +import { createOpportunitySearch } from '@/primitives/createOpportunitySearch'; import { createTheme } from '@/primitives/createTheme'; import { createToasts } from '@/primitives/createToasts'; @@ -44,7 +45,8 @@ export function App(): JSX.Element { const opportunities = createOpportunities(releasedSince); const classifications = createClassifications(); const annotations = createOpportunityAnnotations(opportunities.items, toasts.show); - createLiveUpdates({ opportunities, classifications, annotations }); + const search = createOpportunitySearch(() => filters().search, releasedSince, toasts.show); + createLiveUpdates({ opportunities, classifications, annotations, search }); const theme = createTheme(); const statuses = createMemo(() => statusOptions(opportunities.items())); @@ -62,7 +64,7 @@ export function App(): JSX.Element { // re-runs this. const rows = createMemo(() => sortOpportunities( - filterOpportunities(opportunities.items(), filters(), annotations.get), + filterOpportunities(opportunities.items(), filters(), annotations.get, search.matches()), sort(), annotations.get, ), @@ -107,6 +109,7 @@ export function App(): JSX.Element { filters={filters()} onChange={setFilters} onClear={clear} + search={search} statuses={statuses()} classifications={pickable()} isLoadingClassifications={classifications.isLoading()} diff --git a/web/src/api/client.test.ts b/web/src/api/client.test.ts index ead12df..af622de 100644 --- a/web/src/api/client.test.ts +++ b/web/src/api/client.test.ts @@ -131,6 +131,41 @@ describe('listChangedOpportunities', () => { }); }); +describe('searchOpportunities', () => { + test('sends the box’s text as typed, encoded, with released_from', async () => { + const { api } = await loadClient(); + const found = { + uuids: ['ocds-1'], + matches_all: false, + unknown_prompts: [], + prompts: [{ name: 'title', help: 'The title only.' }], + syntax: [], + }; + respond = () => json(found); + + const result = await api.searchOpportunities( + 'title:"road repairs" !cpv:72&x', + '2026-10-11T23:00:00.000Z', + ); + + expect(sent[0]?.url).toBe( + '/api/opportunities/search?q=title%3A%22road%20repairs%22%20!cpv%3A72%26x' + + '&released_from=2026-10-11T23%3A00%3A00.000Z', + ); + expect(result).toEqual(found); + }); + + test('leaves out released_from when there is no bound', async () => { + const { api } = await loadClient(); + respond = () => + json({ uuids: [], matches_all: true, unknown_prompts: [], prompts: [], syntax: [] }); + + await api.searchOpportunities('', ''); + + expect(sent[0]?.url).toBe('/api/opportunities/search?q='); + }); +}); + describe('getOpportunity', () => { test('fetches the opportunity by its encoded UUID', async () => { const { api } = await loadClient(); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 4c8956f..45bc682 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -4,6 +4,7 @@ import type { Opportunity, OpportunityMarks, OpportunityPage, + OpportunitySearch, Problem, } from '@/types/Opportunities'; @@ -14,6 +15,8 @@ import type { * GET {base}/opportunities?offset=&limit= -> OpportunityPage * GET {base}/opportunities?changed_after={batch} -> OpportunityPage (changed since) * both with &released_from={RFC 3339 time}: only those released since + * GET {base}/opportunities/search?q={text} -> OpportunitySearch + * also with &released_from * GET {base}/opportunities/{uuid} -> Opportunity (with its releases) * GET {base}/classifications -> ClassificationCode[] * PUT {base}/opportunities/{uuid}/marks/{mark} -> OpportunityMarks (sets it) @@ -149,6 +152,22 @@ export const api = { return { opportunities: page.opportunities, batch: page.batch }; }, + /** + * Searches the opportunities released at or after releasedFrom ("" for + * any) with query, the search box's text as typed: the backend parses it. + * A blank query matches all of them, and still returns the search's help. + */ + searchOpportunities( + query: string, + releasedFrom: string, + signal?: AbortSignal, + ): Promise { + return request( + `/opportunities/search?q=${encodeURIComponent(query)}${releasedFromParam(releasedFrom)}`, + { signal }, + ); + }, + /** One opportunity with all of its releases. An unknown UUID is an ApiError with status 404. */ getOpportunity(uuid: string, signal?: AbortSignal): Promise { return request(opportunityPath(uuid), { signal }); diff --git a/web/src/components/FilterPanel.tsx b/web/src/components/FilterPanel.tsx index 6ef7284..16692e2 100644 --- a/web/src/components/FilterPanel.tsx +++ b/web/src/components/FilterPanel.tsx @@ -9,16 +9,20 @@ import { statusLabel, } from '@/lib/filterOpportunities'; import { readStored, STORAGE_KEYS, writeStored } from '@/lib/storage'; +import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch'; import type { OpportunityMark } from '@/types/Annotations'; import { MARK_LABELS, OPPORTUNITY_MARKS } from '@/types/Annotations'; import type { ClassificationCode } from '@/types/Opportunities'; import { ClassificationPicker } from './ClassificationPicker'; +import { SearchField } from './SearchField'; import { buttonQuiet, fieldLabel, input, MARK_BG } from './styles'; interface Props { filters: OpportunityFilters; onChange: (next: OpportunityFilters) => void; onClear: () => void; + /** The search that filters.search runs. */ + search: OpportunitySearchResults; statuses: string[]; classifications: ClassificationCode[]; isLoadingClassifications: boolean; @@ -43,11 +47,10 @@ function toggled(set: ReadonlySet, value: T): Set { } /** - * The collapsible panel of filters (title, status, marks, classification, + * The collapsible panel of filters (search, status, marks, classification, * hidden), with the count of opportunities shown. */ export function FilterPanel(props: Props): JSX.Element { - const titleId = createUniqueId(); const daysId = createUniqueId(); const bodyId = createUniqueId(); @@ -151,19 +154,11 @@ export function FilterPanel(props: Props): JSX.Element { -
- - update({ title: e.currentTarget.value })} - /> -
+ update({ search })} + search={props.search} + />
Status diff --git a/web/src/components/SearchField.tsx b/web/src/components/SearchField.tsx new file mode 100644 index 0000000..ca3e358 --- /dev/null +++ b/web/src/components/SearchField.tsx @@ -0,0 +1,127 @@ +import CircleQuestionMark from 'lucide-solid/icons/circle-question-mark'; +import type { JSX } from 'solid-js'; +import { createSignal, createUniqueId, For, Match, Show, Switch } from 'solid-js'; + +import { promptText, unknownPromptsText } from '@/lib/search'; +import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch'; +import { input, linkButton } from './styles'; + +interface Props { + /** The box's text. */ + value: string; + onInput: (value: string) => void; + search: OpportunitySearchResults; +} + +const helpButton = + 'flex rounded-md border border-line bg-surface p-1 text-ink hover:border-muted aria-expanded:border-ink'; +const helpTerm = 'font-mono text-[13px] whitespace-nowrap'; + +/** + * The general search box, with its help (from the backend, behind the "?" + * button), a note on prompts the search ignored, and the search's errors. + */ +export function SearchField(props: Props): JSX.Element { + const inputId = createUniqueId(); + const helpId = createUniqueId(); + const noteId = createUniqueId(); + const [isHelpOpen, setIsHelpOpen] = createSignal(false); + + const unknownPrompts = (): string => { + const unknown = props.search.unknownPrompts(); + return unknown.length === 0 + ? '' + : unknownPromptsText(unknown, props.search.help()?.prompts ?? []); + }; + + return ( +
+
+ + +
+ props.onInput(e.currentTarget.value)} + aria-describedby={noteId} + aria-busy={props.search.isSearching() || undefined} + /> + + + {(err) => ( + + )} + +

+ {unknownPrompts()} +

+ + +
+ ); +} diff --git a/web/src/lib/filterOpportunities.test.ts b/web/src/lib/filterOpportunities.test.ts index ac34cd8..9430391 100644 --- a/web/src/lib/filterOpportunities.test.ts +++ b/web/src/lib/filterOpportunities.test.ts @@ -43,10 +43,16 @@ const annotations: Record = { const annotationFor = (uuid: string): Annotation => annotations[uuid] ?? { uuid, marks: [], hidden: false }; -const filtered = (filters: Partial): string[] => - filterOpportunities(opportunities, { ...emptyFilters(), ...filters }, annotationFor).map( - (opportunity) => opportunity.uuid, - ); +const filtered = ( + filters: Partial, + searchMatches?: ReadonlySet, +): string[] => + filterOpportunities( + opportunities, + { ...emptyFilters(), ...filters }, + annotationFor, + searchMatches, + ).map((opportunity) => opportunity.uuid); describe('filterOpportunities', () => { test('hides hidden opportunities unless asked, keeping the order given', () => { @@ -55,7 +61,6 @@ describe('filterOpportunities', () => { }); test.each<{ name: string; filters: Partial; want: string[] }>([ - { name: 'title, ignoring case and spaces', filters: { title: ' CLOUD ' }, want: ['a', 'c'] }, { name: 'statuses', filters: { statuses: new Set(['active', 'planned']) }, want: ['a', 'c'] }, { name: 'a mark', filters: { marks: new Set(['interested']) }, want: ['a'] }, { name: 'no marks', filters: { marks: new Set([NOT_MARKED]) }, want: ['c'] }, @@ -65,18 +70,24 @@ describe('filterOpportunities', () => { expect(filtered(filters)).toEqual(want); }); + test('keeps only the search’s matches, when it has some', () => { + expect(filtered({}, new Set(['c', 'x']))).toEqual(['c']); + expect(filtered({}, new Set())).toEqual([]); + }); + test('combines filters', () => { - expect(filtered({ title: 'cloud', smeSuitable: true })).toEqual(['a']); - expect(filtered({ title: 'road', showHidden: true, smeSuitable: true })).toEqual([]); + expect(filtered({ smeSuitable: true }, new Set(['a', 'c']))).toEqual(['a']); + expect(filtered({ showHidden: true, smeSuitable: true }, new Set(['b']))).toEqual([]); }); }); test('activeFilterCount counts the filters in use', () => { expect(activeFilterCount(emptyFilters())).toBe(0); expect( - activeFilterCount({ ...emptyFilters(), title: ' ', smeSuitable: true, showHidden: true }), + activeFilterCount({ ...emptyFilters(), search: ' ', smeSuitable: true, showHidden: true }), ).toBe(2); expect(activeFilterCount({ ...emptyFilters(), releasedWithinDays: 30 })).toBe(1); + expect(activeFilterCount({ ...emptyFilters(), search: 'title:cloud' })).toBe(1); }); test('releasedFrom is the start of the day that many days ago, in UK time', () => { @@ -104,7 +115,7 @@ describe('stored filters', () => { test('read back as they were stored', () => { const filters: OpportunityFilters = { releasedWithinDays: 30, - title: 'cloud', + search: 'cloud !cpv:72', statuses: new Set(['active', 'withdrawn']), marks: new Set(['interested', NOT_MARKED]), classificationIds: new Set([1, 42]), @@ -122,11 +133,16 @@ describe('stored filters', () => { expect(parseFilters(text)).toEqual(emptyFilters()); }); + test('an older version’s title filter is kept as search words', () => { + expect(parseFilters(JSON.stringify({ title: 'road repairs' })).search).toBe('road repairs'); + expect(parseFilters(JSON.stringify({ title: 'road', search: 'cloud' })).search).toBe('cloud'); + }); + test('fields that aren’t valid keep their starting values', () => { const filters = parseFilters( JSON.stringify({ releasedWithinDays: 0, - title: 5, + search: 5, statuses: 'active', marks: ['won', 'not-a-mark'], classificationIds: [3, '4', 1.5], diff --git a/web/src/lib/filterOpportunities.ts b/web/src/lib/filterOpportunities.ts index 98a4c29..eaf17d3 100644 --- a/web/src/lib/filterOpportunities.ts +++ b/web/src/lib/filterOpportunities.ts @@ -18,7 +18,11 @@ export interface OpportunityFilters { * here. */ releasedWithinDays: number; - title: string; + /** + * The general search box's text, as typed. The backend runs the search + * (see createOpportunitySearch), and its matches are applied here. + */ + search: string; /** Empty means every status. */ statuses: ReadonlySet; /** Empty means any marks. Otherwise an opportunity needs at least one of these. */ @@ -37,7 +41,7 @@ export interface OpportunityFilters { */ export const emptyFilters = (): OpportunityFilters => ({ releasedWithinDays: DEFAULT_RELEASED_WITHIN_DAYS, - title: '', + search: '', statuses: new Set(), marks: new Set(), classificationIds: new Set(), @@ -91,7 +95,12 @@ export function parseFilters(text: string | null): OpportunityFilters { releasedWithinDays: isDays(stored.releasedWithinDays) ? stored.releasedWithinDays : filters.releasedWithinDays, - title: isString(stored.title) ? stored.title : filters.title, + // Older versions stored a title filter; its text is kept as search words. + search: isString(stored.search) + ? stored.search + : isString(stored.title) + ? stored.title + : filters.search, statuses: statuses ? new Set(statuses) : filters.statuses, marks: marks ? new Set(marks) : filters.marks, classificationIds: classificationIds ? new Set(classificationIds) : filters.classificationIds, @@ -107,7 +116,7 @@ export function parseFilters(text: string | null): OpportunityFilters { export const activeFilterCount = (filters: OpportunityFilters): number => [ filters.releasedWithinDays !== DEFAULT_RELEASED_WITHIN_DAYS, - filters.title.trim() !== '', + filters.search.trim() !== '', filters.statuses.size > 0, filters.marks.size > 0, filters.classificationIds.size > 0, @@ -128,18 +137,21 @@ export function releasedFrom(days: number, now: Date = new Date()): string { return new Date(now.getFullYear(), now.getMonth(), now.getDate() - days).toISOString(); } -/** The opportunities that pass filters, in the order given. */ +/** + * The opportunities that pass filters, in the order given. searchMatches is + * the UUIDs the search found, or undefined when it doesn't filter (no + * search, or one with nothing to search for). + */ export function filterOpportunities( opportunities: readonly Opportunity[], filters: OpportunityFilters, annotationFor: (uuid: string) => Annotation, + searchMatches: ReadonlySet | undefined, ): Opportunity[] { - const query = filters.title.trim().toLocaleLowerCase(); - return opportunities.filter((opportunity) => { const annotation = annotationFor(opportunity.uuid); if (!filters.showHidden && annotation.hidden) return false; - if (query && !opportunity.title.toLocaleLowerCase().includes(query)) return false; + if (searchMatches && !searchMatches.has(opportunity.uuid)) return false; if (filters.statuses.size > 0 && !filters.statuses.has(opportunity.status)) return false; if (filters.smeSuitable && !opportunity.sme_suitable) return false; if (filters.marks.size > 0) { diff --git a/web/src/lib/search.test.ts b/web/src/lib/search.test.ts new file mode 100644 index 0000000..3d35a18 --- /dev/null +++ b/web/src/lib/search.test.ts @@ -0,0 +1,32 @@ +import { expect, test } from 'vitest'; + +import { promptText, unknownPromptsText } from './search'; + +const prompts = [ + { name: 'title', help: '' }, + { name: 'buyer', help: '' }, + { name: 'cpv', help: '' }, +]; + +test('promptText is the prompt as typed', () => { + expect(promptText('buyer_id')).toBe('buyer_id:'); +}); + +test.each([ + { + name: 'one unknown prompt', + unknown: ['foo'], + want: '“foo:” isn’t a prompt, so it was ignored. The prompts are title:, buyer: and cpv:.', + }, + { + name: 'several, as typed', + unknown: ['foo', 'Bar'], + want: '“foo:” and “Bar:” aren’t prompts, so they were ignored. The prompts are title:, buyer: and cpv:.', + }, +])('unknownPromptsText names $name, then the prompts', ({ unknown, want }) => { + expect(unknownPromptsText(unknown, prompts)).toBe(want); +}); + +test('unknownPromptsText leaves out the prompts before the help has loaded', () => { + expect(unknownPromptsText(['foo'], [])).toBe('“foo:” isn’t a prompt, so it was ignored.'); +}); diff --git a/web/src/lib/search.ts b/web/src/lib/search.ts new file mode 100644 index 0000000..b93310e --- /dev/null +++ b/web/src/lib/search.ts @@ -0,0 +1,24 @@ +import type { SearchPrompt } from '@/types/Opportunities'; + +const LIST_FORMAT = new Intl.ListFormat('en-GB', { type: 'conjunction' }); + +/** A prompt's name as it's typed in the search box: "title:". */ +export const promptText = (name: string): string => `${name}:`; + +/** + * What the search box says about prompts it doesn't know (their names as + * typed, without the colon), which the search ignored: they're named, then + * the prompts there are, if they're known yet. + */ +export function unknownPromptsText( + unknown: readonly string[], + prompts: readonly SearchPrompt[], +): string { + const names = LIST_FORMAT.format(unknown.map((name) => `“${promptText(name)}”`)); + const ignored = + unknown.length === 1 + ? `${names} isn’t a prompt, so it was ignored.` + : `${names} aren’t prompts, so they were ignored.`; + if (prompts.length === 0) return ignored; + return `${ignored} The prompts are ${LIST_FORMAT.format(prompts.map((p) => promptText(p.name)))}.`; +} diff --git a/web/src/primitives/createLiveUpdates.ts b/web/src/primitives/createLiveUpdates.ts index 6d4e6b1..788a87a 100644 --- a/web/src/primitives/createLiveUpdates.ts +++ b/web/src/primitives/createLiveUpdates.ts @@ -3,6 +3,7 @@ import { onCleanup } from 'solid-js'; import { connectEvents } from '@/api/events'; import { applyClassificationChanges } from '@/lib/classifications'; import type { OpportunityAnnotations } from '@/primitives/createOpportunityAnnotations'; +import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch'; import type { ServerEvent } from '@/types/Events'; import type { ClassificationCode } from '@/types/Opportunities'; import type { ApiList, OpportunityList } from './createOpportunities'; @@ -12,6 +13,7 @@ export interface LiveData { opportunities: OpportunityList; classifications: ApiList; annotations: OpportunityAnnotations; + search: OpportunitySearchResults; } /** @@ -21,10 +23,12 @@ export interface LiveData { * - ClassificationsUpdated changes the classifications in place. * - OpportunityMarksUpdated sets the opportunity's marks. * - OpportunitiesUpdated catches the opportunities up: the changed ones are - * fetched and merged in, as the event only names them. + * fetched and merged in, as the event only names them. The search runs + * again, as they may match it now, or no longer. * * After the connection drops and is made again, events may have been missed: - * the opportunities catch up, and the classifications are reloaded. + * the opportunities catch up, the search runs again, and the classifications + * are reloaded. */ export function createLiveUpdates(data: LiveData): void { const handle = (event: ServerEvent): void => { @@ -39,6 +43,7 @@ export function createLiveUpdates(data: LiveData): void { break; case 'OpportunitiesUpdated': data.opportunities.catchUp(); + data.search.refresh(); break; } }; @@ -47,6 +52,7 @@ export function createLiveUpdates(data: LiveData): void { onEvent: handle, onReconnect: () => { data.opportunities.catchUp(); + data.search.refresh(); data.classifications.reload(); }, }); diff --git a/web/src/primitives/createOpportunitySearch.test.ts b/web/src/primitives/createOpportunitySearch.test.ts new file mode 100644 index 0000000..7d93c3f --- /dev/null +++ b/web/src/primitives/createOpportunitySearch.test.ts @@ -0,0 +1,225 @@ +import { createRoot, createSignal } from 'solid-js'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; + +import type * as clientModule from '@/api/client'; +import type { OpportunitySearch } from '@/types/Opportunities'; +import type { OpportunitySearchResults } from './createOpportunitySearch'; +import { createOpportunitySearch } from './createOpportunitySearch'; + +const requests = vi.hoisted(() => ({ + searchOpportunities: + vi.fn< + (query: string, releasedFrom: string, signal?: AbortSignal) => Promise + >(), +})); + +vi.mock('@/api/client', async (importOriginal) => ({ + ...(await importOriginal()), + api: requests, +})); + +/** A search the test answers when it chooses. */ +interface SentSearch { + query: string; + releasedFrom: string; + signal: AbortSignal | undefined; + resolve: (found: OpportunitySearch) => void; + reject: (err: unknown) => void; +} + +const HELP = { + prompts: [{ name: 'title', help: 'The title only.' }], + syntax: [{ example: '!cloud', help: 'Not cloud.' }], +}; + +const found = (...uuids: string[]): OpportunitySearch => ({ + uuids, + matches_all: false, + unknown_prompts: [], + ...HELP, +}); + +let sent: SentSearch[]; +let errors: string[]; +let search: OpportunitySearchResults; +let dispose: () => void; +let setFilters: (filters: { search: string; days: number }) => void; +let setReleasedFrom: (from: string) => void; + +/** The searches sent with a query: the help's has none. */ +const searches = (): SentSearch[] => sent.filter((s) => s.query !== ''); + +/** Lets pending promises, Solid's effects and timers up to ms run. */ +const wait = async (ms = 0): Promise => { + await vi.advanceTimersByTimeAsync(ms); +}; + +/** Starts the search with a stored query, and answers the help's request. */ +async function start(query: string): Promise { + createRoot((disposeRoot) => { + dispose = disposeRoot; + const [filters, setStored] = createSignal({ search: query, days: 7 }); + const [releasedFrom, setFrom] = createSignal('2026-10-12T00:00:00Z'); + setFilters = setStored; + setReleasedFrom = setFrom; + search = createOpportunitySearch( + () => filters().search, + releasedFrom, + (message) => errors.push(message), + ); + }); + sent.find((s) => s.query === '')?.resolve({ ...found(), matches_all: true }); + await wait(); +} + +const type = (text: string): void => setFilters({ search: text, days: 7 }); + +beforeEach(() => { + vi.useFakeTimers(); + sent = []; + errors = []; + requests.searchOpportunities.mockImplementation( + (query, releasedFrom, signal) => + new Promise((resolve, reject) => { + sent.push({ query, releasedFrom, signal, resolve, reject }); + }), + ); +}); + +afterEach(() => { + dispose(); + vi.clearAllMocks(); + vi.useRealTimers(); +}); + +test('loads the help at the start, with a blank query and no bound', async () => { + await start(''); + + expect(sent.map((s) => [s.query, s.releasedFrom])).toEqual([['', '']]); + expect(search.help()).toEqual(HELP); + expect(search.matches()).toBeUndefined(); +}); + +test('runs a stored search at once, and keeps its matches', async () => { + await start('cloud'); + + expect(searches().map((s) => [s.query, s.releasedFrom])).toEqual([ + ['cloud', '2026-10-12T00:00:00Z'], + ]); + expect(search.isSearching()).toBe(true); + searches()[0]?.resolve(found('a', 'b')); + await wait(); + + expect([...(search.matches() ?? [])]).toEqual(['a', 'b']); + expect(search.isSearching()).toBe(false); +}); + +test('waits for typing to pause, then searches for the latest text', async () => { + await start(''); + + type('c'); + await wait(200); + type('cloud'); + await wait(299); + expect(searches()).toHaveLength(0); + expect(search.isSearching()).toBe(true); + + await wait(1); + expect(searches().map((s) => s.query)).toEqual(['cloud']); +}); + +test('aborts a search under way for the next, and ignores its late answer', async () => { + await start('cloud'); + type('road'); + await wait(300); + + expect(searches()[0]?.signal?.aborted).toBe(true); + searches()[0]?.resolve(found('old')); + searches()[1]?.resolve(found('new')); + await wait(); + + expect([...(search.matches() ?? [])]).toEqual(['new']); +}); + +test('doesn’t filter when nothing in the box can be searched for', async () => { + await start('title:'); + searches()[0]?.resolve({ ...found(), matches_all: true, unknown_prompts: ['foo'] }); + await wait(); + + expect(search.matches()).toBeUndefined(); + expect(search.unknownPrompts()).toEqual(['foo']); +}); + +test('a blank box clears the matches at once, without a request', async () => { + await start('cloud'); + searches()[0]?.resolve(found('a')); + await wait(); + + type(' '); + await wait(300); + + expect(searches()).toHaveLength(1); + expect(search.matches()).toBeUndefined(); + expect(search.isSearching()).toBe(false); +}); + +test('keeps the last matches after a failure, and refresh tries again', async () => { + await start('cloud'); + searches()[0]?.resolve(found('a')); + await wait(); + type('road'); + await wait(300); + searches()[1]?.reject(new Error('offline')); + await wait(); + + expect([...(search.matches() ?? [])]).toEqual(['a']); + expect(search.error()?.message).toBe('offline'); + expect(errors).toEqual(["The search didn't run: offline"]); + + search.refresh(); + searches()[2]?.resolve(found('b')); + await wait(); + expect(searches()[2]?.query).toBe('road'); + expect([...(search.matches() ?? [])]).toEqual(['b']); + expect(search.error()).toBeUndefined(); +}); + +test('runs again at once for a new releasedFrom, but not for other filters', async () => { + await start('cloud'); + + setFilters({ search: 'cloud', days: 30 }); + await wait(300); + expect(searches()).toHaveLength(1); + + setReleasedFrom('2026-10-01T00:00:00Z'); + await wait(); + expect(searches().map((s) => s.releasedFrom)).toEqual([ + '2026-10-12T00:00:00Z', + '2026-10-01T00:00:00Z', + ]); +}); + +test('a search mends help that didn’t load', async () => { + createRoot((disposeRoot) => { + dispose = disposeRoot; + const [query, setQuery] = createSignal(''); + setFilters = (filters) => setQuery(filters.search); + search = createOpportunitySearch( + query, + () => '', + (message) => errors.push(message), + ); + }); + sent[0]?.reject(new Error('offline')); + await wait(); + expect(search.helpError()?.message).toBe('offline'); + expect(search.help()).toBeUndefined(); + + type('cloud'); + await wait(300); + searches()[0]?.resolve(found()); + await wait(); + + expect(search.help()).toEqual(HELP); + expect(search.helpError()).toBeUndefined(); +}); diff --git a/web/src/primitives/createOpportunitySearch.ts b/web/src/primitives/createOpportunitySearch.ts new file mode 100644 index 0000000..a7fd19b --- /dev/null +++ b/web/src/primitives/createOpportunitySearch.ts @@ -0,0 +1,155 @@ +import { createEffect, createMemo, createSignal, on, onCleanup } from 'solid-js'; + +import { api } from '@/api/client'; +import { errorText } from '@/lib/errors'; +import type { OpportunitySearch, SearchExample, SearchPrompt } from '@/types/Opportunities'; + +/** How long typing has to pause for before the search runs, in milliseconds. */ +const SEARCH_DELAY = 300; + +/** The search box's help, from the backend, which parses the search. */ +export interface SearchHelp { + prompts: SearchPrompt[]; + /** Examples of the syntax, in the order to show them. */ + syntax: SearchExample[]; +} + +/** The general search, run by the backend, and its help. */ +export interface OpportunitySearchResults { + /** + * The UUIDs the last search found, or undefined when the search doesn't + * filter: the box is blank, nothing in it can be searched for, or no + * search has finished yet. After a failure, the last search's. + */ + matches: () => ReadonlySet | undefined; + /** The last search's prompts the backend doesn't know, as typed, without the colon. */ + unknownPrompts: () => string[]; + /** Whether a search is waiting for typing to pause, or running. */ + isSearching: () => boolean; + /** Why the last search failed, until one succeeds or the box is cleared. */ + error: () => Error | undefined; + /** Runs the search again now, unless the box is blank: to retry, or as opportunities changed. */ + refresh: () => void; + /** The help, or undefined until it has loaded. */ + help: () => SearchHelp | undefined; + /** Why the help didn't load, until it does. */ + helpError: () => Error | undefined; + reloadHelp: () => void; +} + +const toError = (err: unknown): Error => (err instanceof Error ? err : new Error(errorText(err))); + +/** + * Searches the opportunities released at or after releasedFrom with query, + * the search box's text, as either changes: at once for a new releasedFrom + * or the first query, and after a pause in typing for the others. A search + * under way is aborted by the next. A failure is shown with onError, and + * the last search's matches are kept. The help is loaded once, at the + * start. + */ +export function createOpportunitySearch( + query: () => string, + releasedFrom: () => string, + onError: (message: string) => void, +): OpportunitySearchResults { + const [result, setResult] = createSignal(); + const [isSearching, setIsSearching] = createSignal(false); + const [error, setError] = createSignal(); + const [help, setHelp] = createSignal(); + const [helpError, setHelpError] = createSignal(); + let controller: AbortController | undefined; + let helpController: AbortController | undefined; + let timer: ReturnType | undefined; + onCleanup(() => { + clearTimeout(timer); + controller?.abort(); + helpController?.abort(); + }); + + const keepHelp = (found: OpportunitySearch): void => { + setHelp({ prompts: found.prompts, syntax: found.syntax }); + setHelpError(undefined); + }; + + async function search(): Promise { + clearTimeout(timer); + controller?.abort(); + controller = new AbortController(); + const { signal } = controller; + setIsSearching(true); + try { + const found = await api.searchOpportunities(query(), releasedFrom(), signal); + if (signal.aborted) return; + setResult(found); + setError(undefined); + // Every result carries the help, so a search mends a help that didn't load. + keepHelp(found); + } catch (err) { + if (signal.aborted) return; + const failure = toError(err); + setError(failure); + onError(`The search didn't run: ${failure.message}`); + } finally { + if (!signal.aborted) setIsSearching(false); + } + } + + const clear = (): void => { + clearTimeout(timer); + controller?.abort(); + setResult(undefined); + setError(undefined); + setIsSearching(false); + }; + + createEffect( + on([query, releasedFrom], ([text, from], previous) => { + // query may be read from a larger value, such as all the filters, which + // changes when neither does. + if (previous && previous[0] === text && previous[1] === from) return; + if (text.trim() === '') { + clear(); + } else if (previous && previous[0] !== text) { + clearTimeout(timer); + setIsSearching(true); + timer = setTimeout(() => void search(), SEARCH_DELAY); + } else { + void search(); + } + }), + ); + + async function loadHelp(): Promise { + helpController?.abort(); + helpController = new AbortController(); + const { signal } = helpController; + try { + const found = await api.searchOpportunities('', '', signal); + if (!signal.aborted) keepHelp(found); + } catch (err) { + if (signal.aborted) return; + const failure = toError(err); + setHelpError(failure); + onError(`The search help didn't load: ${failure.message}`); + } + } + void loadHelp(); + + const matches = createMemo(() => { + const found = result(); + return found && !found.matches_all ? new Set(found.uuids) : undefined; + }); + + return { + matches, + unknownPrompts: () => result()?.unknown_prompts ?? [], + isSearching, + error, + refresh: () => { + if (query().trim() !== '') void search(); + }, + help, + helpError, + reloadHelp: () => void loadHelp(), + }; +} diff --git a/web/src/types/Opportunities.ts b/web/src/types/Opportunities.ts index 2d30575..1e018b3 100644 --- a/web/src/types/Opportunities.ts +++ b/web/src/types/Opportunities.ts @@ -41,5 +41,12 @@ export type OpportunityReleaseTag = Schemas['types.OpportunityReleaseTag']; /** A page of GET /api/opportunities. */ export type OpportunityPage = Schemas['api.OpportunityPage']; +/** GET /api/opportunities/search's result: the matches, and the search's help. */ +export type OpportunitySearch = Schemas['api.OpportunitySearch']; +/** A search prompt (title:, cpv:, ...) and what it searches. */ +export type SearchPrompt = Schemas['search.Prompt']; +/** An example of the search's syntax, and what it means. */ +export type SearchExample = Schemas['search.Example']; + /** An RFC 9457 problem details body, as the API returns for errors. */ export type Problem = Schemas['httpx.Problem']; diff --git a/web/src/types/api.ts b/web/src/types/api.ts index 409f9f1..19c8048 100644 --- a/web/src/types/api.ts +++ b/web/src/types/api.ts @@ -140,6 +140,77 @@ export interface paths { patch?: never; trace?: never; }; + "/opportunities/search": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Search opportunities + * @description The UUIDs of the opportunities matching a general search, + * in no particular order. q is words and "quoted phrases", + * each optionally with a prompt (title:, buyer:, buyer_id:, + * cpv:) and a leading ! to leave out what it matches; prompts + * and syntax explain it. A search with nothing to search for + * (blank, or only unknown prompts) has matches_all set and no + * uuids. unknown_prompts lists the prompts that aren't known, + * as typed; their terms are ignored. released_from (RFC 3339, + * inclusive) leaves out opportunities whose latest release is + * before it. + */ + get: { + parameters: { + query?: { + /** @description The search */ + q?: string; + /** @description Only opportunities whose latest release is at or after this time */ + released_from?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["api.OpportunitySearch"]; + }; + }; + /** @description Invalid released_from */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["httpx.Problem"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["httpx.Problem"]; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/opportunities/{uuid}": { parameters: { query?: never; @@ -351,6 +422,13 @@ export interface components { offset: number; opportunities: components["schemas"]["types.Opportunity"][]; }; + "api.OpportunitySearch": { + matches_all: boolean; + prompts: components["schemas"]["search.Prompt"][]; + syntax: components["schemas"]["search.Example"][]; + unknown_prompts: string[]; + uuids: string[]; + }; "httpx.Problem": { detail?: string; instance?: string; @@ -359,6 +437,14 @@ export interface components { title: string; type?: string; }; + "search.Example": { + example: string; + help: string; + }; + "search.Prompt": { + help: string; + name: string; + }; "types.ClassificationCode": { classification: string; code: string;