Compare commits

4 Commits
Author SHA1 Message Date
bwilliams b0230e6847 Replaced the Title filter with a general search, run by the backend
GET /api/opportunities/search?q= returns the UUIDs of the matching
opportunities, and the frontend filters its list by them. Words without
a prompt search the title and description; title:, buyer:, buyer_id:
and cpv: search only their own field, and a leading ! leaves out what a
term matches. Repeats of one prompt are OR, everything else AND. CPV
terms are checked per code: cpv:72 !cpv:7221 needs a code starting 72
that doesn't start 7221.

Unknown prompts are reported, and every response carries the help that
the search box's "?" panel shows. Migration 00011 adds the description
and buyer name to the full-text index, and indexes CPV descriptions.
2026-10-22 13:10:25 +01:00
bwilliams 0a777792d8 A CPV code is now stored once, keeping its longest description 2026-10-22 09:56:01 +01:00
bwilliams b0f93fd437 config init now leaves out the test sources unless given --all
A section tagged init:"hidden" isn't written to the file config init
generates; fts_mock and sleeper are. config init --all writes them too.
They still load, and the config commands read and set them as before.
2026-10-21 16:22:45 +01:00
bwilliams f467c6c7da The release tarball's files are now owned by root, not the user who built it 2026-10-21 16:07:21 +01:00
38 changed files with 2255 additions and 72 deletions

No files matched your search

+2 -1
View File
@@ -105,7 +105,7 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and
| Command | Purpose |
|---|---|
| `job-search config init [path]` | Create a config file |
| `job-search config init [path]` | Create a config file holding the defaults (`--all` also writes the test sources, `fts_mock` and `sleeper`) |
| `job-search config show` | Print the configuration (`--json` or `--yaml` for those formats) |
| `job-search config get <key>` | Print one value, e.g. `logging.level` |
| `job-search config set <key> <value>` | Validate and write one value to the config file |
@@ -234,6 +234,7 @@ second one fails with "address already in use".
| `internal/cli` | Cobra commands and shared CLI setup |
| `internal/config` | Configuration loading, validation and editing |
| `internal/fetcher` | Runs a worker per enabled source |
| `internal/search` | Parsing of the general opportunity search (`title:`, `buyer:`, `buyer_id:`, `cpv:`, `!`) and its help |
| `internal/skiprules` | The rules that mark new opportunities Skipped (see [internal/skiprules](internal/skiprules/README.md)) |
| `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) |
| `internal/findatender` | FTS API client and conversion from OCDS to JobSearch types |
+72
View File
@@ -13,6 +13,7 @@ import (
"git.avsdev.uk/Go/job-search/internal/events"
"git.avsdev.uk/Go/job-search/internal/httpx"
"git.avsdev.uk/Go/job-search/internal/search"
"git.avsdev.uk/Go/job-search/internal/store"
"git.avsdev.uk/Go/job-search/internal/types"
)
@@ -38,6 +39,21 @@ type OpportunityPage struct {
Batch string `json:"batch"`
}
// OpportunitySearch is the result of a general search, as GET
// /opportunities/search returns it: the UUIDs of the opportunities that
// match, in no particular order. MatchesAll reports a search with nothing to
// search for (blank, or only unknown prompts), which matches every
// opportunity; UUIDs is then empty. UnknownPrompts are the prompts in the
// search that aren't known, as typed, without their colon. Prompts and
// Syntax are the help for the search box.
type OpportunitySearch struct {
UUIDs []string `json:"uuids"`
MatchesAll bool `json:"matches_all"`
UnknownPrompts []string `json:"unknown_prompts"`
Prompts []search.Prompt `json:"prompts"`
Syntax []search.Example `json:"syntax"`
}
// opportunityRoutes wires HTTP handlers to the opportunity store.
// Opportunities are read-only; only their marks can be changed.
type opportunityRoutes struct {
@@ -64,6 +80,7 @@ func newOpportunityRoutes(
// Routes registers the opportunity routes on router (mounted under /api).
func (h *opportunityRoutes) Routes(router chi.Router) {
router.Get("/opportunities", h.list)
router.Get("/opportunities/search", h.search)
router.Get("/opportunities/{uuid}", h.get)
router.Put("/opportunities/{uuid}/marks/{mark}", h.setMark)
router.Delete("/opportunities/{uuid}/marks/{mark}", h.clearMark)
@@ -203,6 +220,61 @@ func (h *opportunityRoutes) listChanged(
})
}
// search serves GET /opportunities/search: the UUIDs of the opportunities
// matching the general search in the q query parameter (see package
// search), with the help for the search box. released_from bounds it as it
// bounds list.
//
// @Summary Search opportunities
// @Description The UUIDs of the opportunities matching a general search,
// @Description in no particular order. q is words and "quoted phrases",
// @Description each optionally with a prompt (title:, buyer:, buyer_id:,
// @Description cpv:) and a leading ! to leave out what it matches; prompts
// @Description and syntax explain it. A search with nothing to search for
// @Description (blank, or only unknown prompts) has matches_all set and no
// @Description uuids. unknown_prompts lists the prompts that aren't known,
// @Description as typed; their terms are ignored. released_from (RFC 3339,
// @Description inclusive) leaves out opportunities whose latest release is
// @Description before it.
// @Tags opportunities
// @Produce json
// @Param q query string false "The search"
// @Param released_from query string false "Only opportunities whose latest release is at or after this time" format(date-time)
// @Success 200 {object} OpportunitySearch
// @Failure 400 {object} httpx.Problem "Invalid released_from"
// @Failure 500 {object} httpx.Problem
// @Router /opportunities/search [get]
func (h *opportunityRoutes) search(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
releasedFrom, err := timeParam(
"released_from", query.Get("released_from"),
)
if err != nil {
httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error())
return
}
parsed := search.Parse(query.Get("q"))
result := OpportunitySearch{
UUIDs: []string{},
MatchesAll: parsed.MatchesAll(),
UnknownPrompts: parsed.UnknownPrompts,
Prompts: search.Prompts,
Syntax: search.Syntax,
}
if !result.MatchesAll {
result.UUIDs, err = h.store.SearchOpportunities(
r.Context(), parsed, releasedFrom,
)
if err != nil {
httpx.WriteProblem(w, r, http.StatusInternalServerError, "")
return
}
}
httpx.WriteJSON(w, http.StatusOK, result)
}
// get serves GET /opportunities/{uuid}: one opportunity with its
// classifications, marks and releases.
//
+52
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
@@ -483,3 +484,54 @@ func TestChangeMarkPublishes(t *testing.T) {
}
}
}
// GET /opportunities/search returns the matching UUIDs, bounded by
// released_from, with the help; a search with nothing to search for matches
// all, and unknown prompts are listed.
func TestSearchOpportunities(t *testing.T) {
url := opportunitiesServer(t, 5) + "/opportunities/search"
from3 := "&released_from=2026-10-03T00:00:00Z"
cases := []struct {
query string
uuids []string
matchesAll bool
unknown []string
}{
{"", []string{}, true, []string{}},
{"?q=", []string{}, true, []string{}},
{"?q=cpv:72", []string{"ocds-1", "ocds-3", "ocds-5"}, false, []string{}},
{"?q=cpv:72" + from3, []string{"ocds-3", "ocds-5"}, false, []string{}},
{"?q=opportunity%20!cpv:72", []string{"ocds-2", "ocds-4"}, false, []string{}},
{"?q=nothing", []string{}, false, []string{}},
{"?q=foo:x%20Bar:y", []string{}, true, []string{"foo", "Bar"}},
{"?q=foo:x%20title:3", []string{"ocds-3"}, false, []string{"foo"}},
}
for _, tc := range cases {
var got OpportunitySearch
status := request(t, http.MethodGet, url+tc.query, &got)
slices.Sort(got.UUIDs)
if status != http.StatusOK || !slices.Equal(got.UUIDs, tc.uuids) ||
got.MatchesAll != tc.matchesAll ||
!slices.Equal(got.UnknownPrompts, tc.unknown) {
t.Errorf("%s: status %d, %+v; want %v, matches_all %v, unknown %v",
tc.query, status, got, tc.uuids, tc.matchesAll, tc.unknown)
}
if len(got.Prompts) == 0 || len(got.Syntax) == 0 {
t.Errorf("%s: no help: %+v", tc.query, got)
}
}
// Empty lists are [], never null.
var raw map[string]json.RawMessage
request(t, http.MethodGet, url+"?q=nothing", &raw)
for _, field := range []string{"uuids", "unknown_prompts"} {
if string(raw[field]) != "[]" {
t.Errorf("%s = %s; want []", field, raw[field])
}
}
status := request(t, http.MethodGet, url+"?q=x&released_from=x", nil)
if status != http.StatusBadRequest {
t.Errorf("bad released_from: status %d; want 400", status)
}
}
+26
View File
@@ -214,6 +214,32 @@ func TestConfigInit(t *testing.T) {
}
}
// The test sources are left out of the file unless --all is given.
func TestConfigInitTestSources(t *testing.T) {
cmdtest.Setup(t, "")
mustRun(t, "config", "init", "./default.yaml")
written := cmdtest.ReadFile(t, "default.yaml")
cmdtest.Contains(t, written, "find_a_tender:", "eu_supply:")
for _, hidden := range []string{"fts_mock", "sleeper"} {
if strings.Contains(written, hidden) {
t.Errorf("init wrote %s without --all:\n%s", hidden, written)
}
}
mustRun(t, "config", "init", "--all", "./all.yaml")
cmdtest.Contains(
t, cmdtest.ReadFile(t, "all.yaml"), "fts_mock:", "sleeper:",
)
// The test sources still load and are set as usual.
mustRun(
t, "--config", "default.yaml",
"config", "set", "sources.sleeper.enabled", "true",
)
cmdtest.Contains(t, cmdtest.ReadFile(t, "default.yaml"), "sleeper:")
}
// With no path (or a blank one), init searches the default locations. The
// working directory is one, so its config.yaml is found. (Not finding one
// would write to /etc/job-search, which a test can't do.)
+22 -15
View File
@@ -17,7 +17,11 @@ func commandInit() *cobra.Command {
cmd := &cobra.Command{
Use: "init [config_path]",
Args: cobra.MatchAll(cobra.MinimumNArgs(0), cobra.MaximumNArgs(1)),
Short: "Create an empty configuration file",
Short: "Create a configuration file holding the defaults",
Long: `Create a configuration file holding the defaults, at config_path or,
without one, the first config.yaml found in /etc/job-search then the working
directory (/etc/job-search/config.yaml if there is none). The test sources
(fts_mock and sleeper) are left out unless --all is given.`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
slog.Debug("command persistent pre-run", "command", "config.init")
},
@@ -64,26 +68,29 @@ func commandInit() *cobra.Command {
return err
}
// Load default configuration
config.LoadDefaultConfig()
if viper.ConfigFileUsed() == "" {
viper.SetConfigFile(config.EtcConfigPath + "/config.yaml")
path := viper.ConfigFileUsed()
if path == "" {
path = config.EtcConfigPath + "/config.yaml"
}
fmt.Print(
"writing default configuration to file: " +
viper.ConfigFileUsed() +
"\n",
)
if err := viper.WriteConfig(); err != nil {
return err
// A Viper of its own, holding only the keys to write: the global
// one would write every default, the hidden sections' included.
all, _ := cmd.Flags().GetBool("all")
generated := viper.New()
for key, value := range config.InitDefaults(all) {
generated.Set(key, value)
}
generated.SetConfigFile(path)
return nil
fmt.Print("writing default configuration to file: " + path + "\n")
return generated.WriteConfig()
},
}
cmd.Flags().Bool(
"all", false, "also write the test sources (fts_mock and sleeper)",
)
// Config flag should be hidden as the config path should be provided
cmd.SetHelpFunc(func(command *cobra.Command, args []string) {
command.Flags().MarkHidden("config") //nolint:errcheck,gosec // a root flag, so it always exists
+6 -1
View File
@@ -267,6 +267,10 @@ per key:
of the configuration.
- `default:"value"` is the field's default, parsed as if it were a string
passed to `SetValue` (e.g. `"10m"`, `"true"`).
- `init:"hidden"` on a nested section's field leaves the section out of the
file `config init` writes, unless `--all` is given (`InitDefaults`). The
test sources, `fts_mock` and `sleeper`, have it. The section is otherwise
as any other: loaded, and read and set by the `config` commands.
- A tagged field whose type is a struct is a nested section, addressed as
`section.key`. An embedded struct has its fields promoted, as Go does;
every source embeds `JSCSourcesSource` for `enabled` and `interval`.
@@ -324,7 +328,8 @@ To add a new section:
`JSCRateLimitRule`, has no constant and passes `""` to `flattenOf`.
2. Add the struct as a tagged field of its parent: `JobSearchConfig` in
`config.go`, or `JSCSources` in `conf_sources.go` for a source,
etc. The field's `cfg` tag must match the `JSCK*` constant.
etc. The field's `cfg` tag must match the `JSCK*` constant. Add
`init:"hidden"` if it's only for testing.
Nothing in `section.go` needs changing.
+20
View File
@@ -288,6 +288,26 @@ func TestConcreteSourceSections(t *testing.T) {
}
}
// InitDefaults leaves out the sections tagged init:"hidden" unless asked.
func TestInitDefaults(t *testing.T) {
all := InitDefaults(true)
shown := InitDefaults(false)
if all["sources.sleeper.enabled"] != false ||
all["sources.fts_mock.interval"] != "10m0s" {
t.Errorf("all = %v; want the test sources' defaults", all)
}
for key := range all {
hidden := strings.HasPrefix(key, "sources.fts_mock.") ||
strings.HasPrefix(key, "sources.sleeper.")
if _, found := shown[key]; found == hidden {
t.Errorf("%s: in the generated defaults = %v", key, found)
}
}
if shown["sources.eu_supply.base_url"] != "https://uk.eu-supply.com" {
t.Errorf("eu_supply missing: %v", shown)
}
}
func TestEUSupplySection(t *testing.T) {
conf := defaultConfig(t)
got := conf.Sources.EUSupply
+4 -2
View File
@@ -13,15 +13,17 @@ const JSCKSources string = "sources"
// JSCSources configures each source of opportunities, one field
// per source. Each field's cfg tag must match the section's JSCK* constant.
// The test sources (FTSMock and Sleeper) are tagged init:"hidden", so
// `config init` only writes them with --all.
// SkipRulesFile is the JSON file of rules that mark new opportunities
// Skipped (see internal/skiprules), read again when it changes; empty means
// none.
type JSCSources struct {
SkipRulesFile string `cfg:"skip_rules_file"`
FindATender JSCSourceFindATender `cfg:"find_a_tender"`
FTSMock JSCSourceFTSMock `cfg:"fts_mock"`
FTSMock JSCSourceFTSMock `cfg:"fts_mock" init:"hidden"`
EUSupply JSCSourceEUSupply `cfg:"eu_supply"`
Sleeper JSCSourceSleeper `cfg:"sleeper"`
Sleeper JSCSourceSleeper `cfg:"sleeper" init:"hidden"`
}
// ---------- Section methods (see Section) ----------
+24
View File
@@ -40,6 +40,30 @@ func LoadDefaultConfig() {
LoadToViper(conf)
}
// InitDefaults returns the default configuration as dotted keys, as
// Flatten gives it, for `config init` to write. Sections tagged
// init:"hidden" (the test sources) are left out unless includeHidden is set.
func InitDefaults(includeHidden bool) map[string]any {
var conf JobSearchConfig
if err := applyDefaults(&conf); err != nil {
panic(fmt.Sprintf("config: %v", err))
}
flat := conf.Flatten()
if includeHidden {
return flat
}
hidden := hiddenFromInitKeys("", reflect.TypeFor[JobSearchConfig]())
for _, section := range hidden {
for key := range flat {
if strings.HasPrefix(key, section+".") {
delete(flat, key)
}
}
}
return flat
}
// ---------- Section methods (see Section) ----------
// Map implements Section.
+28 -2
View File
@@ -50,6 +50,10 @@ const (
keyTag = "cfg"
// defaultTag holds a field's default value.
defaultTag = "default"
// initTag set to initHidden on a section field leaves the section out of
// the file `config init` writes (see InitDefaults), as for test sources.
initTag = "init"
initHidden = "hidden"
)
// The unknown key error and the field types section.go handles specially.
@@ -109,6 +113,8 @@ type field struct {
section bool
defaultValue string
hasDefault bool
// hiddenFromInit is set by initTag (see InitDefaults).
hiddenFromInit bool
}
// isSectionList reports whether listType is a list of sections, []S for a
@@ -146,8 +152,9 @@ func fieldsOf(structType reflect.Type) []field {
fieldType: structField.Type,
section: structField.Type.Kind() == reflect.Struct &&
structField.Type != timeType,
defaultValue: defaultValue,
hasDefault: hasDefault,
defaultValue: defaultValue,
hasDefault: hasDefault,
hiddenFromInit: structField.Tag.Get(initTag) == initHidden,
})
}
@@ -244,6 +251,25 @@ func flattenInto(flat map[string]any, prefix string, section reflect.Value) {
}
}
// hiddenFromInitKeys returns the dotted keys of the sections under section
// (prefixed with prefix, if non-empty) whose field has initTag set to
// initHidden, descending into nested sections.
func hiddenFromInitKeys(prefix string, sectionType reflect.Type) []string {
var hidden []string
for _, configField := range fieldsOf(sectionType) {
if !configField.section {
continue
}
key := joinKey(prefix, configField.key)
if configField.hiddenFromInit {
hidden = append(hidden, key)
continue
}
hidden = append(hidden, hiddenFromInitKeys(key, configField.fieldType)...)
}
return hidden
}
// exportValue returns a field's value as Map, Flatten and ValueOf give it
// out: durations and byte sizes become strings such as "1m30s" and "32KiB",
// so values agree with kindOf and can be passed back to SetValue; []string is
+222
View File
@@ -0,0 +1,222 @@
// Package search parses the general opportunity search: words and quoted
// phrases, each optionally with a prompt naming the field it searches
// (title:, buyer:, buyer_id:, cpv:) and a leading "!" to negate it. The
// store runs a parsed Query (store.SearchOpportunities).
package search
import (
"regexp"
"strings"
"unicode"
)
// Field is what a term searches.
type Field int
// Values of Field.
const (
// FieldText is a term with no prompt: the title or the description.
FieldText Field = iota
// FieldTitle is title:, the title only.
FieldTitle
// FieldBuyer is buyer:, the buyer's name.
FieldBuyer
// FieldBuyerID is buyer_id:, the buyer's ID, matched whole.
FieldBuyerID
// FieldCPV is cpv:, a CPV code prefix or words of a CPV description.
FieldCPV
)
// Prompt is a known prompt, with a line of help for the user.
type Prompt struct {
Name string `json:"name"`
Help string `json:"help"`
}
// Example is an example of the search syntax, with a line of help for the
// user.
type Example struct {
Example string `json:"example"`
Help string `json:"help"`
}
// Prompts are the known prompts, in the order to show them.
var Prompts = []Prompt{
{Name: "title", Help: "Words in the title only."},
{Name: "buyer", Help: "Words in the buyer's name."},
{
Name: "buyer_id",
Help: "The buyer's ID, whole and in any case (e.g. GB-FTS-148803).",
},
{
Name: "cpv",
Help: "The start of a CPV code (cpv:72), or words in a CPV code's description (cpv:food).",
},
}
// Syntax explains the search syntax, in the order to show it.
var Syntax = []Example{
{
Example: "cloud hosting",
Help: "Words without a prompt are searched for in the title and the description. Every word must be found, though not necessarily in the same one.",
},
{
Example: "migr",
Help: "A word also matches the start of a longer word, and other forms of it: migr finds migration, and migrating finds migration.",
},
{
Example: `"road repairs"`,
Help: "Quotes search for a phrase: its words together, in that order.",
},
{
Example: `title:"road repairs"`,
Help: "A prompt searches only its own field, for one word or one quoted phrase.",
},
{
Example: "buyer:leeds buyer:bradford",
Help: "Repeats of the same prompt match if any of them does: Leeds or Bradford.",
},
{
Example: "cloud buyer:leeds cpv:72",
Help: "Different prompts, and words without one, must all match.",
},
{
Example: "!cloud !buyer:leeds",
Help: "! in front of a word, a phrase or a prompt leaves out what it matches.",
},
{
Example: "cpv:72 !cpv:7221",
Help: "CPV terms are checked against each of an opportunity's codes: this needs a code starting 72 that doesn't start 7221. !cpv: without cpv: leaves out every opportunity with a matching code.",
},
}
// prompts maps each known prompt's name to its field.
var prompts = map[string]Field{
"title": FieldTitle,
"buyer": FieldBuyer,
"buyer_id": FieldBuyerID,
"cpv": FieldCPV,
}
// promptPattern matches a prompt at the start of a term: a name of letters
// and underscores, then a colon. A colon after anything else (12:00) is
// part of the term.
var promptPattern = regexp.MustCompile(`^([A-Za-z_]+):`)
// cpvCodePattern matches a CPV code or the start of one, with an optional
// check digit.
var cpvCodePattern = regexp.MustCompile(`^([0-9]+)(?:-[0-9])?$`)
// Term is one search term: a word or a quoted phrase, as typed.
type Term struct {
Field Field
Negated bool
Value string
}
// Words returns the term's words, split as the store's full-text index
// splits them: on anything but letters and numbers.
func (t Term) Words() []string {
return strings.FieldsFunc(t.Value, func(char rune) bool {
return !unicode.IsLetter(char) && !unicode.IsNumber(char)
})
}
// CPVCode returns the digits of a cpv: term that is a code or the start of
// one, without its check digit, and reports whether it is one. Any other
// cpv: term searches CPV descriptions.
func (t Term) CPVCode() (string, bool) {
match := cpvCodePattern.FindStringSubmatch(t.Value)
if match == nil {
return "", false
}
return match[1], true
}
// usable reports whether the term can be searched for: a buyer ID that
// isn't blank, or any other term with at least one word.
func (t Term) usable() bool {
if t.Field == FieldBuyerID {
return strings.TrimSpace(t.Value) != ""
}
return len(t.Words()) > 0
}
// Query is a parsed search.
type Query struct {
// Terms are the terms to search for, all usable.
Terms []Term
// UnknownPrompts are the prompts that aren't known, as typed, without
// their colon, once each. Their terms are left out of Terms.
UnknownPrompts []string
}
// MatchesAll reports whether the query has no terms, so it matches every
// opportunity.
func (q Query) MatchesAll() bool {
return len(q.Terms) == 0
}
// Parse parses text as a search. It never fails: text that isn't valid
// syntax, such as an unclosed quote, is read as plain words, and a term
// with no words (title: or !!!) is left out.
func Parse(text string) Query {
query := Query{
UnknownPrompts: []string{},
}
unknown := map[string]bool{}
for rest := strings.TrimSpace(text); rest != ""; {
var term Term
var prompt string
var known bool
term, prompt, known, rest = nextTerm(rest)
rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
switch {
case !known:
if !unknown[prompt] {
unknown[prompt] = true
query.UnknownPrompts = append(query.UnknownPrompts, prompt)
}
case term.usable():
query.Terms = append(query.Terms, term)
}
}
return query
}
// nextTerm reads the term at the start of text, which mustn't start with
// space. It returns the term, its prompt as typed ("" if none), whether the
// prompt is known, and the text after the term.
func nextTerm(
text string,
) (term Term, prompt string, known bool, rest string) {
if strings.HasPrefix(text, "!") {
term.Negated = true
text = text[1:]
}
if match := promptPattern.FindStringSubmatch(text); match != nil {
prompt = match[1]
text = text[len(match[0]):]
}
term.Field, known = prompts[strings.ToLower(prompt)]
if prompt == "" {
term.Field, known = FieldText, true
}
if strings.HasPrefix(text, `"`) {
term.Value, rest, _ = strings.Cut(text[1:], `"`)
return term, prompt, known, rest
}
end := strings.IndexFunc(text, unicode.IsSpace)
if end < 0 {
end = len(text)
}
term.Value = text[:end]
return term, prompt, known, text[end:]
}
+106
View File
@@ -0,0 +1,106 @@
package search
import (
"reflect"
"slices"
"testing"
)
func TestParse(t *testing.T) {
term := func(field Field, value string) Term {
return Term{
Field: field,
Value: value,
}
}
text := func(value string) Term {
return term(FieldText, value)
}
not := func(term Term) Term {
term.Negated = true
return term
}
none := []string{}
cases := []struct {
input string
terms []Term
unknown []string
}{
{"", nil, none},
{" ", nil, none},
{"cloud hosting", []Term{text("cloud"), text("hosting")}, none},
{`"road repairs" leeds`, []Term{text("road repairs"), text("leeds")}, none},
{`title:"road repairs"`, []Term{term(FieldTitle, "road repairs")}, none},
{"TITLE:road Buyer:leeds", []Term{term(FieldTitle, "road"), term(FieldBuyer, "leeds")}, none},
{"buyer_id:GB-1 cpv:72", []Term{term(FieldBuyerID, "GB-1"), term(FieldCPV, "72")}, none},
{"!cloud !cpv:food", []Term{not(text("cloud")), not(term(FieldCPV, "food"))}, none},
{`!title:"a b"`, []Term{not(term(FieldTitle, "a b"))}, none},
{"12:00", []Term{text("12:00")}, none},
{`"unclosed phrase`, []Term{text("unclosed phrase")}, none},
{`title: !!! "" ! cloud`, []Term{text("cloud")}, none},
{"foo:bar cloud Foo:baz foo:x baz:", []Term{text("cloud")}, []string{"foo", "Foo", "baz"}},
{"buyer_id:", nil, none},
}
for _, tc := range cases {
got := Parse(tc.input)
if !reflect.DeepEqual(got.Terms, tc.terms) {
t.Errorf("%q: terms %+v, want %+v", tc.input, got.Terms, tc.terms)
}
if !slices.Equal(got.UnknownPrompts, tc.unknown) {
t.Errorf(
"%q: unknown %q, want %q",
tc.input, got.UnknownPrompts, tc.unknown,
)
}
if got.UnknownPrompts == nil {
t.Errorf("%q: UnknownPrompts is nil", tc.input)
}
if got.MatchesAll() != (len(tc.terms) == 0) {
t.Errorf("%q: MatchesAll %v", tc.input, got.MatchesAll())
}
}
}
func TestTermWordsAndCPVCode(t *testing.T) {
cases := []struct {
value string
words []string
code string
isCPV bool
}{
{"road-repairs", []string{"road", "repairs"}, "", false},
{"Café's", []string{"Café", "s"}, "", false},
{"72", []string{"72"}, "72", true},
{"72210000-5", []string{"72210000", "5"}, "72210000", true},
{"72-55", []string{"72", "55"}, "", false},
{"72a", []string{"72a"}, "", false},
}
for _, tc := range cases {
term := Term{
Field: FieldCPV,
Value: tc.value,
}
if got := term.Words(); !slices.Equal(got, tc.words) {
t.Errorf("%q: words %q, want %q", tc.value, got, tc.words)
}
code, ok := term.CPVCode()
if code != tc.code || ok != tc.isCPV {
t.Errorf(
"%q: CPVCode %q %v, want %q %v",
tc.value, code, ok, tc.code, tc.isCPV,
)
}
}
}
// Every known prompt has help, and every prompt with help is known.
func TestPromptsHaveHelp(t *testing.T) {
if len(Prompts) != len(prompts) {
t.Errorf("%d prompts with help, %d known", len(Prompts), len(prompts))
}
for _, prompt := range Prompts {
if _, ok := prompts[prompt.Name]; !ok || prompt.Help == "" {
t.Errorf("prompt %q: known %v, help %q", prompt.Name, ok, prompt.Help)
}
}
}
+15 -4
View File
@@ -4,7 +4,7 @@ A SQLite store for opportunities and their OCDS releases, built on sqlx + goose
## Build and test
The title search uses SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
The title and general searches use SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
```bash
go test -tags sqlite_fts5 ./...
@@ -24,7 +24,9 @@ internal/store/migrate_test.go version checks, stepping up and down, failed-
internal/store/queries.go all SQL, as named constants grouped by table
internal/store/queries_test.go prepares every query against the migrated schema
internal/store/store.go Store: logic, using the queries via sqlx
internal/store/search.go SearchOpportunities: the general search's query
internal/store/store_test.go behavioural tests
internal/store/search_test.go behavioural tests of the general search
internal/store/common.go common for migrate, store and direct access
```
@@ -36,15 +38,16 @@ internal/store/common.go common for migrate, store and direct access
| `opportunity_marks` | `uuid` | `uuid` references `opportunities`. The user's own marks (`hidden`, `skipped`, `interested`, `bid_submitted`, `won`, `lost`), each `BOOLEAN NOT NULL DEFAULT 0`. A `CHECK` constraint stops `won` and `lost` both being set. |
| `opportunity_releases` | (`uuid`, `release_id`) | Write-once. `uuid` references `opportunities`. `data` is TEXT, so `json_extract` works on it. |
| `opportunity_release_tags` | (`uuid`, `release_id`, `tag`) | Write-once. (`uuid`, `release_id`) references `opportunity_releases`. |
| `classification_codes` | `id` | Shared between opportunities. A row is reused only when every content column matches. The table grows over time to include codes not already seen in fetched opportunities. |
| `classification_codes` | `id` | Shared between opportunities. A CPV code has one row (unique on `code` where `is_cpv`), keeping the longest description it has been given: EU Supply cuts descriptions to 64 characters. A code of any other scheme gets a row per distinct content. The table grows over time to include codes not already seen in fetched opportunities. |
| `opportunity_classifications` | (`opportunity_uuid`, `classification_id`) | Mapping table; replaced as a set when the opportunity is updated. |
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, kept in sync by triggers. |
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, `description` and `buyer_name`, kept in sync by triggers. |
| `classification_codes_fts` | rowid = `classification_codes.id` | FTS5 index on `classification_codes.classification` (the description), kept in sync by triggers. |
| `opportunity_change_batches` | `id`; `hash` (unique) | Write-once. One row per save that adds opportunities or gives them new releases, and per change of marks. `hash` names the batch to clients (random, so unguessable); `created_at` orders the batches. |
| `opportunity_change_batch_opportunities` | (`batch_id`, `opportunity_uuid`) | The opportunities in each batch. References both tables, cascading on delete. |
Foreign keys are enforced (`_foreign_keys=on` in the connection string) and cascade on delete from `opportunities`. The connection also enables WAL mode, a 5s busy timeout, and `_txlock=immediate`, so a save takes the write lock up front rather than failing halfway through.
**When adding a classification scheme flag** (e.g. `is_unspsc`), the migration adding the column must drop and recreate the `classification_codes_content` unique index with the new column included. Otherwise rows differing only by that flag would be merged.
**When adding a classification scheme flag** (e.g. `is_unspsc`), the migration adding the column must drop and recreate the `classification_codes_content` unique index with the new column included. Otherwise rows differing only by that flag would be merged. A scheme whose codes have one official description, like CPV, can instead have a partial unique index on `code` (as `classification_codes_cpv` does), with its own `ON CONFLICT` clause in `upsertClassificationCodeReturningID`.
**internal/types**: The structs contain `db` tags to allow sqlx to map table columns to struct members. Any struct members not mapped to a column (including child slices) are referenced by `db:"-"`.
@@ -71,6 +74,7 @@ All SQL is in `queries.go` as constants, grouped by table in migration order; `s
- **TestQueriesPrepare** prepares every query constant against a freshly migrated database, so a syntax error, an unknown table or unknown column fails the test and names the failing statement.
- **TestQueriesAllListed** parses `queries.go` and fails if a constant isn't listed in that test's `checks` map. Adding a query therefore means adding one line there, and it can't be skipped by accident.
- **TestSearchOpportunitiesQueryAllTerms** does the same for `SearchOpportunities`, with a term of every kind, plain and negated.
- **TestListOpportunitiesQueryAllFilters** builds the `ListOpportunities` query with every filter set and runs it, checking that the fragments combine and that the arguments line up with the placeholders.
- **TestMarksCheckConstraint** checks the database itself rejects `won` and `lost` set together, for writes that bypass the Go validation.
@@ -111,6 +115,7 @@ ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags
batch, err := s.LatestChangeBatch(ctx) // hash of the newest change batch, or ""
ts, batch, err = s.ListChangedOpportunities(ctx, batch, time.Time{}) // changed since; ErrUnknownBatch
uuids, err := s.SearchOpportunities(ctx, search.Parse("cloud buyer:leeds"), releasedFrom) // never nil
ok, err := s.ReleaseExists(ctx, uuid, releaseID)
```
@@ -132,3 +137,9 @@ ok, err := s.ReleaseExists(ctx, uuid, releaseID)
**Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted.
**General search.** `SearchOpportunities` runs a `search.Query` (parsed by `internal/search`, which documents the syntax) and returns only the matching UUIDs, for the frontend to filter its own list by. It is assembled by `searchOpportunitiesQuery` from the `searchOpportunity*` and `searchCPV*` fragments:
- Words without a prompt search `{title description}` in `opportunities_fts`, and must all match. `title:` and `buyer:` search the `title` and `buyer_name` columns; repeats of one prompt are joined by OR into one clause. Every negated full-text term goes into one `NOT IN` clause, joined by OR. Each term is one FTS5 phrase built from `Term.Words`, its last word a prefix, so typed input is never FTS5 syntax.
- `buyer_id:` matches the whole ID, ignoring case (`IN`, expanded by `sqlx.In`).
- CPV terms are checked against each code, inside one `EXISTS` on the opportunity's CPV codes: a code must match a `cpv:` term (a code prefix by `LIKE`, or words in `classification_codes_fts`) and no `!cpv:` term. With only `!cpv:` terms the clause is `NOT EXISTS` a code matching any of them, so an opportunity with no CPV codes passes it.
- `OpportunityFilter.Title` searches only the `title` column of the same index.
+140
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
@@ -327,3 +328,142 @@ func TestFailedMigrationIsRolledBack(t *testing.T) {
t.Errorf("failed migration left %d objects behind", indexes)
}
}
// Migration 10 merges each CPV code's rows into the one with the longest
// description, moving the links to it (one link if an opportunity had both),
// and leaves other schemes' rows alone.
func TestMigration10MergesCPVDuplicates(t *testing.T) {
ctx := context.Background()
m, db := newMigrator(t)
if _, err := m.UpTo(ctx, 9); err != nil {
t.Fatal(err)
}
const full = "Petroleum products, fuel, electricity and other sources of energy"
setup := []string{
`INSERT INTO opportunities (uuid, uuid_type, title, status, latest_release_date)
VALUES ('a', 'ocid', 'A', 'active', '2026-10-01'),
('b', 'ocid', 'B', 'active', '2026-10-01')`,
`INSERT INTO classification_codes (id, is_cpv, code, classification) VALUES
(1, 1, '09000000', '` + full + `'),
(2, 1, '09000000', '` + full[:64] + `'),
(3, 0, '123', 'Scheme X'),
(4, 0, '123', 'Scheme X, amended')`,
// a links to both CPV rows, b only to the cut one.
`INSERT INTO opportunity_classifications VALUES
('a', 1), ('a', 2), ('b', 2), ('a', 3), ('b', 4)`,
}
for _, statement := range setup {
if _, err := db.ExecContext(ctx, statement); err != nil {
t.Fatal(err)
}
}
if _, err := m.UpTo(ctx, 10); err != nil {
t.Fatal(err)
}
var codes []string
rows, err := db.QueryContext(ctx,
`SELECT id || ':' || classification FROM classification_codes ORDER BY id`)
if err != nil {
t.Fatal(err)
}
for rows.Next() {
var code string
rows.Scan(&code)
codes = append(codes, code)
}
rows.Close()
want := "1:" + full + "|3:Scheme X|4:Scheme X, amended"
if got := strings.Join(codes, "|"); got != want {
t.Errorf("codes = %s; want %s", got, want)
}
var links []string
rows, err = db.QueryContext(ctx, `SELECT opportunity_uuid || classification_id
FROM opportunity_classifications ORDER BY 1`)
if err != nil {
t.Fatal(err)
}
for rows.Next() {
var link string
rows.Scan(&link)
links = append(links, link)
}
rows.Close()
if got := strings.Join(links, ","); got != "a1,a3,b1,b4" {
t.Errorf("links = %s; want a1,a3,b1,b4", got)
}
// A second row for the code is now refused.
_, err = db.ExecContext(ctx, `INSERT INTO classification_codes
(is_cpv, code, classification) VALUES (1, '09000000', 'Other')`)
if err == nil {
t.Error("a second row for a CPV code was accepted")
}
}
// Migration 11 indexes the description, buyer name and classification
// descriptions of rows already stored, and its Down leaves the title
// index working.
func TestMigration11IndexesStoredRows(t *testing.T) {
ctx := context.Background()
m, db := newMigrator(t)
if _, err := m.UpTo(ctx, 10); err != nil {
t.Fatal(err)
}
setup := []string{
`INSERT INTO opportunities (uuid, uuid_type, title, status,
description, buyer_name, latest_release_date)
VALUES ('a', 'ocid', 'Cloud', 'active', 'Servers', 'Leeds',
'2026-10-01')`,
`INSERT INTO classification_codes (id, is_cpv, code, classification)
VALUES (1, 1, '15800000', 'Miscellaneous food products')`,
}
for _, statement := range setup {
if _, err := db.ExecContext(ctx, statement); err != nil {
t.Fatal(err)
}
}
if _, err := m.UpTo(ctx, 11); err != nil {
t.Fatal(err)
}
count := func(query string) int {
t.Helper()
var n int
if err := db.QueryRowContext(ctx, query).Scan(&n); err != nil {
t.Fatalf("%s: %v", query, err)
}
return n
}
matches := []struct {
table, match string
want int
}{
{"opportunities_fts", "description : servers", 1},
{"opportunities_fts", "buyer_name : leeds", 1},
{"opportunities_fts", "title : servers", 0},
{"classification_codes_fts", "food", 1},
}
for _, tc := range matches {
got := count(fmt.Sprintf(
"SELECT count(*) FROM %s WHERE %[1]s MATCH '%s'", tc.table, tc.match,
))
if got != tc.want {
t.Errorf("%s MATCH %q: %d; want %d", tc.table, tc.match, got, tc.want)
}
}
if _, err := m.DownTo(ctx, 10); err != nil {
t.Fatal(err)
}
got := count(`SELECT count(*) FROM opportunities_fts
WHERE opportunities_fts MATCH 'cloud'`)
if got != 1 {
t.Errorf("title search after Down: %d; want 1", got)
}
}
@@ -0,0 +1,60 @@
-- +goose Up
-- +goose StatementBegin
-- A CPV code has one official description, so it is now one row whatever
-- description it arrives with: EU Supply cuts descriptions to 64 characters,
-- which made a second row for each code it sent. The duplicates are merged
-- into the row with the longest description, and the upsert keeps the
-- longest from now on. Other schemes still get a row per distinct content.
-- The row each CPV row merges into.
CREATE TEMP TABLE "cpv_merge" AS
SELECT
"c"."id" AS "id",
(
SELECT "k"."id" FROM "classification_codes" "k"
WHERE "k"."is_cpv" AND "k"."code" = "c"."code"
ORDER BY length("k"."classification") DESC, "k"."id"
LIMIT 1
) AS "keep_id"
FROM "classification_codes" "c"
WHERE "c"."is_cpv";
-- An opportunity linked to both rows keeps only its link to the kept one.
DELETE FROM "opportunity_classifications"
WHERE "classification_id" IN (
SELECT "id" FROM "cpv_merge" WHERE "id" <> "keep_id"
)
AND EXISTS (
SELECT 1
FROM "cpv_merge" "m"
JOIN "opportunity_classifications" "kept"
ON "kept"."classification_id" = "m"."keep_id"
WHERE "m"."id" = "opportunity_classifications"."classification_id"
AND "kept"."opportunity_uuid"
= "opportunity_classifications"."opportunity_uuid"
);
UPDATE "opportunity_classifications"
SET "classification_id" = (
SELECT "keep_id" FROM "cpv_merge"
WHERE "id" = "opportunity_classifications"."classification_id"
)
WHERE "classification_id" IN (
SELECT "id" FROM "cpv_merge" WHERE "id" <> "keep_id"
);
DELETE FROM "classification_codes"
WHERE "id" IN (SELECT "id" FROM "cpv_merge" WHERE "id" <> "keep_id");
DROP TABLE "cpv_merge";
CREATE UNIQUE INDEX IF NOT EXISTS "classification_codes_cpv"
ON "classification_codes" ("code") WHERE "is_cpv";
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- The merged duplicates aren't restored: they only differed by a cut-off
-- description.
DROP INDEX IF EXISTS "classification_codes_cpv";
-- +goose StatementEnd
@@ -0,0 +1,119 @@
-- +goose Up
-- +goose StatementBegin
-- The general search: opportunities_fts (migration 00006) now also indexes
-- the description and the buyer's name, and classification_codes_fts
-- indexes classification descriptions. Both are kept in sync by triggers.
DROP TRIGGER IF EXISTS "opportunities_fts_update";
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
DROP TABLE IF EXISTS "opportunities_fts";
CREATE VIRTUAL TABLE "opportunities_fts" USING fts5(
"title",
"description",
"buyer_name",
content = 'opportunities',
content_rowid = 'id',
tokenize = 'porter unicode61 remove_diacritics 2'
);
CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name")
VALUES (new."id", new."title", new."description", new."buyer_name");
END;
CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts"
("opportunities_fts", "rowid", "title", "description", "buyer_name")
VALUES ('delete', old."id", old."title", old."description", old."buyer_name");
END;
CREATE TRIGGER "opportunities_fts_update"
AFTER UPDATE OF "title", "description", "buyer_name" ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts"
("opportunities_fts", "rowid", "title", "description", "buyer_name")
VALUES ('delete', old."id", old."title", old."description", old."buyer_name");
INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name")
VALUES (new."id", new."title", new."description", new."buyer_name");
END;
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
CREATE VIRTUAL TABLE "classification_codes_fts" USING fts5(
"classification",
content = 'classification_codes',
content_rowid = 'id',
tokenize = 'porter unicode61 remove_diacritics 2'
);
CREATE TRIGGER "classification_codes_fts_insert"
AFTER INSERT ON "classification_codes"
BEGIN
INSERT INTO "classification_codes_fts" ("rowid", "classification")
VALUES (new."id", new."classification");
END;
CREATE TRIGGER "classification_codes_fts_delete"
AFTER DELETE ON "classification_codes"
BEGIN
INSERT INTO "classification_codes_fts"
("classification_codes_fts", "rowid", "classification")
VALUES ('delete', old."id", old."classification");
END;
CREATE TRIGGER "classification_codes_fts_update"
AFTER UPDATE OF "classification" ON "classification_codes"
BEGIN
INSERT INTO "classification_codes_fts"
("classification_codes_fts", "rowid", "classification")
VALUES ('delete', old."id", old."classification");
INSERT INTO "classification_codes_fts" ("rowid", "classification")
VALUES (new."id", new."classification");
END;
INSERT INTO "classification_codes_fts" ("classification_codes_fts")
VALUES ('rebuild');
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TRIGGER IF EXISTS "classification_codes_fts_update";
DROP TRIGGER IF EXISTS "classification_codes_fts_delete";
DROP TRIGGER IF EXISTS "classification_codes_fts_insert";
DROP TABLE IF EXISTS "classification_codes_fts";
DROP TRIGGER IF EXISTS "opportunities_fts_update";
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
DROP TABLE IF EXISTS "opportunities_fts";
CREATE VIRTUAL TABLE "opportunities_fts" USING fts5(
"title",
content = 'opportunities',
content_rowid = 'id',
tokenize = 'porter unicode61 remove_diacritics 2'
);
CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
END;
CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
VALUES ('delete', old."id", old."title");
END;
CREATE TRIGGER "opportunities_fts_update" AFTER UPDATE OF "title" ON "opportunities"
BEGIN
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
VALUES ('delete', old."id", old."title");
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
END;
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
-- +goose StatementEnd
+61 -5
View File
@@ -107,7 +107,8 @@ LIMIT ? OFFSET ?`
)`
// filterOpportunityTitleMatch matches a full-text search of the title. The
// argument must come from ftsQuery, never raw user input.
// argument must be ftsQuery's, limited to the title column (see
// listOpportunitiesQuery), never raw user input.
filterOpportunityTitleMatch = `t.id IN (
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
)`
@@ -150,6 +151,52 @@ LIMIT ? OFFSET ?`
)`
)
// SearchOpportunities is assembled by searchOpportunitiesQuery as:
//
// searchOpportunitiesSelect
// [WHERE clause AND clause ...]
//
// Each clause is a searchOpportunity* fragment, perhaps with NOT in front,
// or filterOpportunityReleasedFrom. searchOpportunityCPV's %s is filled
// with searchCPV* fragments joined by OR, AND and NOT. A full-text argument
// must be built from search.Term.Words, never raw user input.
const (
// searchOpportunitiesSelect selects every opportunity's UUID, as t.
searchOpportunitiesSelect = `
SELECT t.uuid FROM opportunities t`
// searchOpportunityText matches a full-text search of the title,
// description and buyer_name columns.
searchOpportunityText = `t.id IN (
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
)`
// searchOpportunityBuyerID matches any of several buyer IDs, ignoring
// case. "IN (?)" is expanded by sqlx.In, which must be followed by
// Rebind.
searchOpportunityBuyerID = `t.buyer_id COLLATE NOCASE IN (?)`
// searchOpportunityCPV matches an opportunity with a CPV code meeting
// the condition put in place of %s, which reads the code as c.
searchOpportunityCPV = `EXISTS (
SELECT 1
FROM opportunity_classifications tc
JOIN classification_codes c ON c.id = tc.classification_id
WHERE tc.opportunity_uuid = t.uuid AND c.is_cpv AND %s
)`
// searchCPVCodePrefix matches a code by a LIKE pattern: its first
// digits followed by %.
searchCPVCodePrefix = `c.code LIKE ?`
// searchCPVDescription matches a full-text search of the code's
// description.
searchCPVDescription = `c.id IN (
SELECT rowid FROM classification_codes_fts
WHERE classification_codes_fts MATCH ?
)`
)
// selectOtherSourcesTitles lists the uuid, title and latest_release_date of
// the opportunities not from one source whose latest_release_date is in
// [from, before); it walks the latest_release_date index.
@@ -197,15 +244,24 @@ ORDER BY release_id, tag`
// ---------- classification_codes ----------
// upsertClassificationCodeReturningID returns the id of the row matching
// every content column, inserting it first if needed. Named parameters
// upsertClassificationCodeReturningID returns the id of the row for a
// classification, inserting it first if needed. Named parameters
// (types.ClassificationCode).
//
// The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING yield the
// existing row's id on conflict.
// A CPV code has one row whatever its description (migration 00010): the
// row keeps the longest description it has been given, as EU Supply cuts
// them to 64 characters. Any other scheme's row matches every content
// column. The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING
// yield the existing row's id on conflict.
const upsertClassificationCodeReturningID = `
INSERT INTO classification_codes (is_cpv, code, classification)
VALUES (:is_cpv, :code, :classification)
ON CONFLICT (code) WHERE is_cpv
DO UPDATE SET classification = CASE
WHEN length(excluded.classification) > length(classification)
THEN excluded.classification
ELSE classification
END
ON CONFLICT (is_cpv, code, classification)
DO UPDATE SET is_cpv = excluded.is_cpv
RETURNING id`
+47
View File
@@ -2,6 +2,7 @@ package store
import (
"context"
"fmt"
"go/ast"
"go/parser"
"go/token"
@@ -11,6 +12,8 @@ import (
"time"
"github.com/jmoiron/sqlx"
"git.avsdev.uk/Go/job-search/internal/search"
)
// Run with: go test -tags sqlite_fts5 ./...
@@ -81,6 +84,24 @@ var checks = map[string]queryCheck{
sql: withFilter(filterOpportunityMarkLost),
},
"searchOpportunitiesSelect": {sql: searchOpportunitiesSelect},
"searchOpportunityText": {
sql: withSearch(searchOpportunityText),
},
"searchOpportunityBuyerID": {
sql: withSearch(searchOpportunityBuyerID),
in: true,
},
"searchOpportunityCPV": {
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, "1")),
},
"searchCPVCodePrefix": {
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVCodePrefix)),
},
"searchCPVDescription": {
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVDescription)),
},
// opportunity_ocds_releases
"insertRelease": {sql: insertRelease, named: true},
"selectReleasesByOpportunity": {sql: selectReleasesByOpportunity},
@@ -140,6 +161,10 @@ func withFilter(fragment string) string {
listOpportunitiesOrderAndPage
}
func withSearch(fragment string) string {
return searchOpportunitiesSelect + "\nWHERE " + fragment
}
func openTestDB(t *testing.T) *sqlx.DB {
t.Helper()
s, err := Open(
@@ -223,6 +248,28 @@ func TestListOpportunitiesQueryAllFilters(t *testing.T) {
}
}
// TestSearchOpportunitiesQueryAllTerms runs the assembled search query with
// a term of every kind, both plain and negated, checking the fragments
// combine and the arguments line up with the placeholders.
func TestSearchOpportunitiesQueryAllTerms(t *testing.T) {
db := openTestDB(t)
query := search.Parse(`cloud "road repairs" title:a title:b buyer:c ` +
`buyer_id:GB-1 buyer_id:GB-2 cpv:72 cpv:food !cloud !title:d ` +
`!buyer:e !buyer_id:GB-3 !cpv:7221 !cpv:fuel`)
statement, args := searchOpportunitiesQuery(
query, time.Now().Add(-time.Hour),
)
statement, args, err := sqlx.In(statement, args...)
if err != nil {
t.Fatal(err)
}
var uuids []string
if err := db.Select(&uuids, db.Rebind(statement), args...); err != nil {
t.Errorf("run with %d args:\n%s\n%v", len(args), statement, err)
}
}
// TestQueriesAllListed fails if queries.go declares a constant that checks
// doesn't list, so every new statement is prepared by TestQueriesPrepare.
func TestQueriesAllListed(t *testing.T) {
+202
View File
@@ -0,0 +1,202 @@
package store
import (
"context"
"fmt"
"strings"
"time"
"github.com/jmoiron/sqlx"
"git.avsdev.uk/Go/job-search/internal/search"
)
// ftsColumns are the opportunities_fts columns each field's terms search,
// as an FTS5 column filter.
var ftsColumns = map[search.Field]string{
search.FieldText: "{title description}",
search.FieldTitle: "title",
search.FieldBuyer: "buyer_name",
}
// anyOfFields are the fields whose terms match if any of them does, in the
// order their clauses are added.
var anyOfFields = []search.Field{search.FieldTitle, search.FieldBuyer}
// searchClauses holds a search's terms, sorted by how they combine.
type searchClauses struct {
// required are full-text matches (words without a prompt) that must
// all match.
required []string
// anyOf are full-text matches by field, of which one must match.
anyOf map[search.Field][]string
// excluded are full-text matches of which none may match.
excluded []string
buyerIDs []string
excludedBuyerIDs []string
// cpv and excludedCPV are conditions on one CPV code (searchCPV*
// fragments), with their arguments.
cpv []string
cpvArgs []any
excludedCPV []string
excludedCPVArgs []any
}
// SearchOpportunities returns the UUIDs of the opportunities matching
// query, in no particular order and never nil. Unless releasedFrom is zero,
// only opportunities whose latest release is on or after it are returned
// (as OpportunityFilter.ReleasedFrom). A query with no terms matches every
// opportunity.
func (s *Store) SearchOpportunities(
ctx context.Context, query search.Query, releasedFrom time.Time,
) ([]string, error) {
statement, args := searchOpportunitiesQuery(query, releasedFrom)
statement, args, err := sqlx.In(statement, args...)
if err != nil {
return nil, fmt.Errorf("expanding the search query: %w", err)
}
uuids := []string{}
err = s.db.SelectContext(ctx, &uuids, s.db.Rebind(statement), args...)
if err != nil {
return nil, fmt.Errorf("searching opportunities: %w", err)
}
return uuids, nil
}
// searchOpportunitiesQuery assembles the SearchOpportunities query from the
// fragments in queries.go, with its arguments. It must be expanded by
// sqlx.In.
func searchOpportunitiesQuery(
query search.Query, releasedFrom time.Time,
) (string, []any) {
clauses := sortTerms(query.Terms)
var where []string
var args []any
add := func(clause string, arguments ...any) {
where = append(where, clause)
args = append(args, arguments...)
}
if len(clauses.required) > 0 {
add(searchOpportunityText, strings.Join(clauses.required, " AND "))
}
for _, field := range anyOfFields {
if matches := clauses.anyOf[field]; len(matches) > 0 {
add(searchOpportunityText, strings.Join(matches, " OR "))
}
}
if len(clauses.excluded) > 0 {
add(
"NOT "+searchOpportunityText,
strings.Join(clauses.excluded, " OR "),
)
}
if len(clauses.buyerIDs) > 0 {
add(searchOpportunityBuyerID, clauses.buyerIDs)
}
if len(clauses.excludedBuyerIDs) > 0 {
add("NOT "+searchOpportunityBuyerID, clauses.excludedBuyerIDs)
}
if clause, arguments := clauses.cpvClause(); clause != "" {
add(clause, arguments...)
}
if !releasedFrom.IsZero() {
add(filterOpportunityReleasedFrom, releasedFrom.UTC())
}
statement := searchOpportunitiesSelect
if len(where) > 0 {
statement += "\nWHERE " + strings.Join(where, "\n\tAND ")
}
return statement, args
}
// sortTerms sorts terms by how they combine into the query.
func sortTerms(terms []search.Term) searchClauses {
clauses := searchClauses{
anyOf: map[search.Field][]string{},
}
for _, term := range terms {
switch {
case term.Field == search.FieldBuyerID && term.Negated:
clauses.excludedBuyerIDs = append(
clauses.excludedBuyerIDs, term.Value,
)
case term.Field == search.FieldBuyerID:
clauses.buyerIDs = append(clauses.buyerIDs, term.Value)
case term.Field == search.FieldCPV && term.Negated:
condition, argument := cpvCondition(term)
clauses.excludedCPV = append(clauses.excludedCPV, condition)
clauses.excludedCPVArgs = append(
clauses.excludedCPVArgs, argument,
)
case term.Field == search.FieldCPV:
condition, argument := cpvCondition(term)
clauses.cpv = append(clauses.cpv, condition)
clauses.cpvArgs = append(clauses.cpvArgs, argument)
case term.Negated:
clauses.excluded = append(clauses.excluded, textMatch(term))
case term.Field == search.FieldText:
clauses.required = append(clauses.required, textMatch(term))
default:
clauses.anyOf[term.Field] = append(
clauses.anyOf[term.Field], textMatch(term),
)
}
}
return clauses
}
// cpvClause returns the clause for the CPV terms, with its arguments, or ""
// if there are none. With cpv: terms, an opportunity needs a code that one
// of them matches and no !cpv: term does; with only !cpv: terms, it needs
// no code that any of them matches.
func (c searchClauses) cpvClause() (string, []any) {
excluded := "(" + strings.Join(c.excludedCPV, " OR ") + ")"
switch {
case len(c.cpv) > 0 && len(c.excludedCPV) > 0:
condition := "(" + strings.Join(c.cpv, " OR ") + ")" +
" AND NOT " + excluded
return fmt.Sprintf(searchOpportunityCPV, condition),
append(append([]any{}, c.cpvArgs...), c.excludedCPVArgs...)
case len(c.cpv) > 0:
condition := "(" + strings.Join(c.cpv, " OR ") + ")"
return fmt.Sprintf(searchOpportunityCPV, condition), c.cpvArgs
case len(c.excludedCPV) > 0:
return "NOT " + fmt.Sprintf(searchOpportunityCPV, excluded),
c.excludedCPVArgs
}
return "", nil
}
// cpvCondition returns the condition on one CPV code for a cpv: term, with
// its argument: a code prefix, or a full-text search of its description.
func cpvCondition(term search.Term) (string, any) {
if code, ok := term.CPVCode(); ok {
return searchCPVCodePrefix, code + "%"
}
return searchCPVDescription, ftsPhrase(term.Words())
}
// textMatch returns the FTS5 query of opportunities_fts for a full-text
// term: its words as a phrase, in the columns its field searches.
func textMatch(term search.Term) string {
return ftsColumns[term.Field] + " : " + ftsPhrase(term.Words())
}
// ftsPhrase returns an FTS5 phrase of words, the last also matching as a
// prefix. The words must be split as search.Term.Words splits them, so
// they hold no quotes or FTS5 syntax.
func ftsPhrase(words []string) string {
return `"` + strings.Join(words, " ") + `"*`
}
+180
View File
@@ -0,0 +1,180 @@
package store_test
import (
"context"
"fmt"
"slices"
"testing"
"time"
"git.avsdev.uk/Go/job-search/internal/search"
"git.avsdev.uk/Go/job-search/internal/store"
"git.avsdev.uk/Go/job-search/internal/types"
)
// searchStore returns a store holding five opportunities, s1 to s5, released
// on days 1 to 5, for the search tests.
func searchStore(t *testing.T) *store.Store {
t.Helper()
s := openStore(t)
const software = "Programming services of packaged software products"
cpv := func(code, description string) types.ClassificationCode {
return types.ClassificationCode{
IsCPV: true,
Code: code,
Classification: description,
}
}
rows := []struct {
title, description, buyer, buyerID string
classifications []types.ClassificationCode
}{
{
title: "Cloud migration services",
description: "Moving servers to the cloud",
buyer: "Leeds City Council",
buyerID: "GB-LEEDS",
classifications: []types.ClassificationCode{
cpv("72100000", "Hardware consultancy services"),
cpv("72210000", software),
},
},
{
title: "Road repairs",
description: "Pothole repairs in Bradford",
buyer: "Bradford Council",
buyerID: "GB-BRAD",
classifications: []types.ClassificationCode{
cpv("45233141", "Road-maintenance works"),
},
},
{
title: "School meals",
description: "Catering and cloud-based ordering",
buyer: "Leeds Schools Trust",
buyerID: "GB-LST",
classifications: []types.ClassificationCode{
cpv("55524000", "School catering services"),
cpv("15800000", "Miscellaneous food products"),
},
},
{
title: "Software licences",
description: "Licences",
buyer: "York Council",
buyerID: "GB-YORK",
classifications: []types.ClassificationCode{
cpv("72210000", software),
},
},
{
title: "Office cleaning",
description: "Cleaning",
buyer: "Hull Council",
buyerID: "GB-HULL",
},
}
for i, row := range rows {
o := opportunity(fmt.Sprintf("s%d", i+1), row.title, i+1, "r1")
o.Description = row.description
o.BuyerName = row.buyer
o.BuyerID = row.buyerID
o.Classifications = row.classifications
mustSave(t, s, o)
}
return s
}
func TestSearchOpportunities(t *testing.T) {
s := searchStore(t)
all := []string{"s1", "s2", "s3", "s4", "s5"}
cases := []struct {
query string
want []string
}{
{"", all},
{"title: !!! \"\"", all},
{"cloud", []string{"s1", "s3"}},
{"CLOUD", []string{"s1", "s3"}},
{"migr", []string{"s1"}},
{"migrating", []string{"s1"}},
{"cloud servers", []string{"s1"}},
{"leeds", []string{}},
{"bradford", []string{"s2"}},
{`"road repairs"`, []string{"s2"}},
{`"repairs road"`, []string{}},
{"title:cloud", []string{"s1"}},
{"title:pothole", []string{}},
{"title:cloud title:road", []string{"s1", "s2"}},
{"buyer:leeds", []string{"s1", "s3"}},
{"buyer:leeds buyer:bradford", []string{"s1", "s2", "s3"}},
{`buyer:"leeds city"`, []string{"s1"}},
{"cloud buyer:leeds", []string{"s1", "s3"}},
{"title:cloud buyer:leeds", []string{"s1"}},
{`cloud !buyer:"leeds city"`, []string{"s3"}},
{"!cloud", []string{"s2", "s4", "s5"}},
{"!cloud !road", []string{"s4", "s5"}},
{"buyer_id:gb-leeds", []string{"s1"}},
{"buyer_id:GB-LEEDS buyer_id:gb-york", []string{"s1", "s4"}},
{"buyer_id:GB", []string{}},
{"!buyer_id:GB-LEEDS", []string{"s2", "s3", "s4", "s5"}},
{"cpv:72", []string{"s1", "s4"}},
{"cpv:72210000-5", []string{"s1", "s4"}},
{"cpv:72 !cpv:7221", []string{"s1"}},
{"!cpv:7221", []string{"s2", "s3", "s5"}},
{"cpv:food", []string{"s3"}},
{"!cpv:food", []string{"s1", "s2", "s4", "s5"}},
{"cpv:catering cpv:road", []string{"s2", "s3"}},
{"cpv:software !cpv:hardware", []string{"s1", "s4"}},
{"cpv:72 buyer:york", []string{"s4"}},
{"foo:cloud", all},
{`cloud" OR title:road*`, []string{}},
}
for _, tc := range cases {
got, err := s.SearchOpportunities(
context.Background(), search.Parse(tc.query), time.Time{},
)
if err != nil {
t.Fatalf("%q: %v", tc.query, err)
}
slices.Sort(got)
if !slices.Equal(got, tc.want) {
t.Errorf("%q: got %v, want %v", tc.query, got, tc.want)
}
}
}
// released_from bounds the search as it bounds ListOpportunities, and
// changing a searched field updates the index.
func TestSearchOpportunitiesReleasedFromAndUpdates(t *testing.T) {
ctx := context.Background()
s := searchStore(t)
got, err := s.SearchOpportunities(ctx, search.Parse("buyer:leeds"), day(3))
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, []string{"s3"}) {
t.Errorf("released from day 3: got %v, want [s3]", got)
}
o := opportunity("s5", "Office cleaning", 6, "r1", "r2")
o.Description = "Window washing"
o.BuyerName = "Wakefield Council"
mustSave(t, s, o)
for query, want := range map[string][]string{
"washing": {"s5"},
"buyer:wakefield": {"s5"},
"cleaning": {"s5"},
"buyer:hull": {},
"cpv:agricultural": {"s5"},
} {
got, err := s.SearchOpportunities(ctx, search.Parse(query), time.Time{})
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, want) {
t.Errorf("%q after update: got %v, want %v", query, got, want)
}
}
}
+1 -1
View File
@@ -861,7 +861,7 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) {
add(filterOpportunityClassificationCode, filter.ClassificationCode)
}
if match := ftsQuery(filter.Title); match != "" {
add(filterOpportunityTitleMatch, match)
add(filterOpportunityTitleMatch, "title : ("+match+")")
}
if !filter.ReleasedFrom.IsZero() {
add(filterOpportunityReleasedFrom, filter.ReleasedFrom.UTC())
+62 -2
View File
@@ -3,6 +3,7 @@ package store_test
import (
"context"
"errors"
"fmt"
"path/filepath"
"slices"
"strings"
@@ -273,8 +274,9 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
got.Classifications[0].Classification != "IT services (amended)" {
t.Fatalf("classifications not replaced: %+v", got.Classifications)
}
if got.Classifications[0].ID == oldIT {
t.Errorf("changed description should create a new classification row")
// A CPV code is one row: the longer description replaces the shorter.
if got.Classifications[0].ID != oldIT {
t.Errorf("a CPV code's new description should keep its row")
}
for title, want := range map[string]int{"cloud": 0, "redesign": 1} {
@@ -295,6 +297,61 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
}
}
// A CPV code keeps one row and its longest description, whichever order
// the descriptions arrive in (EU Supply cuts them to 64 characters); a code
// of another scheme still gets a row per description.
func TestCPVCodeKeepsLongestDescription(t *testing.T) {
const full = "Petroleum products, fuel, electricity and other sources of energy"
cut := full[:64]
for name, order := range map[string][]string{
"full first": {full, cut},
"cut first": {cut, full},
} {
t.Run(name, func(t *testing.T) {
s := openStore(t)
var ids []int64
for i, description := range order {
o := opportunity(fmt.Sprintf("ocds-%d", i), "Fuel", 1, "r1")
o.Classifications = []types.ClassificationCode{
{
IsCPV: true,
Code: "09000000",
Classification: description,
},
}
mustSave(t, s, o)
ids = append(ids, o.Classifications[0].ID)
}
if ids[0] != ids[1] {
t.Errorf("ids = %v; want one row for the code", ids)
}
for _, uuid := range []string{"ocds-0", "ocds-1"} {
got := mustGet(t, s, uuid).Classifications[0].Classification
if got != full {
t.Errorf("%s: description = %q; want the full one", uuid, got)
}
}
})
}
s := openStore(t)
var ids []int64
for i, description := range []string{"Scheme X", "Scheme X, amended"} {
o := opportunity(fmt.Sprintf("other-%d", i), "Other", 1, "r1")
o.Classifications = []types.ClassificationCode{
{
Code: "123",
Classification: description,
},
}
mustSave(t, s, o)
ids = append(ids, o.Classifications[0].ID)
}
if ids[0] == ids[1] {
t.Errorf("a non-CPV code's descriptions share a row: %v", ids)
}
}
func TestClassificationRowsAreShared(t *testing.T) {
s := openStore(t)
a := opportunity("ocds-a", "A", 1, "r1")
@@ -396,6 +453,9 @@ func TestListOpportunitiesFilters(t *testing.T) {
{"title case", store.OpportunityFilter{
Title: "ROAD",
}, []string{"ocds-2"}},
{"title only, not description", store.OpportunityFilter{
Title: "description",
}, []string{}},
{
"title syntax is not FTS syntax",
store.OpportunityFilter{
+2
View File
@@ -70,7 +70,9 @@ if [[ $DO_RELEASE -eq 1 ]]; then
install -DT -m 644 deploy/job-search.service $RELEASE_DIR/job-search.service
install -DT -m 644 deploy/job-search.logrotate $RELEASE_DIR/job-search.logrotate
ARCH=`dpkg --print-architecture`
# Owned by root (0:0), not the user who built it, with no names
tar -czf $RELEASE_DIR/job-search-$VERSION-$ARCH.tar.gz \
--owner=0 --group=0 --numeric-owner \
--transform "s,^$RELEASE_DIR/,job-search-$VERSION-$ARCH/," \
$RELEASE_DIR/job-search $RELEASE_DIR/job-search.service \
$RELEASE_DIR/job-search.logrotate
+3 -2
View File
@@ -1,8 +1,9 @@
# JobSearch - Frontend
A Vite + SolidJS + Tailwind CSS (v4) + TypeScript table of opportunities with
title search, status, mark and classification filters, and per-opportunity
mark and hide controls.
a general search (run by the backend, with title:, buyer:, buyer_id: and cpv:
prompts), status, mark and classification filters, and per-opportunity mark
and hide controls.
## Run
+5 -2
View File
@@ -22,6 +22,7 @@ import { readStored, STORAGE_KEYS, writeStored } from '@/lib/storage';
import { createLiveUpdates } from '@/primitives/createLiveUpdates';
import { createClassifications, createOpportunities } from '@/primitives/createOpportunities';
import { createOpportunityAnnotations } from '@/primitives/createOpportunityAnnotations';
import { createOpportunitySearch } from '@/primitives/createOpportunitySearch';
import { createTheme } from '@/primitives/createTheme';
import { createToasts } from '@/primitives/createToasts';
@@ -44,7 +45,8 @@ export function App(): JSX.Element {
const opportunities = createOpportunities(releasedSince);
const classifications = createClassifications();
const annotations = createOpportunityAnnotations(opportunities.items, toasts.show);
createLiveUpdates({ opportunities, classifications, annotations });
const search = createOpportunitySearch(() => filters().search, releasedSince, toasts.show);
createLiveUpdates({ opportunities, classifications, annotations, search });
const theme = createTheme();
const statuses = createMemo(() => statusOptions(opportunities.items()));
@@ -62,7 +64,7 @@ export function App(): JSX.Element {
// re-runs this.
const rows = createMemo(() =>
sortOpportunities(
filterOpportunities(opportunities.items(), filters(), annotations.get),
filterOpportunities(opportunities.items(), filters(), annotations.get, search.matches()),
sort(),
annotations.get,
),
@@ -107,6 +109,7 @@ export function App(): JSX.Element {
filters={filters()}
onChange={setFilters}
onClear={clear}
search={search}
statuses={statuses()}
classifications={pickable()}
isLoadingClassifications={classifications.isLoading()}
+35
View File
@@ -131,6 +131,41 @@ describe('listChangedOpportunities', () => {
});
});
describe('searchOpportunities', () => {
test('sends the box’s text as typed, encoded, with released_from', async () => {
const { api } = await loadClient();
const found = {
uuids: ['ocds-1'],
matches_all: false,
unknown_prompts: [],
prompts: [{ name: 'title', help: 'The title only.' }],
syntax: [],
};
respond = () => json(found);
const result = await api.searchOpportunities(
'title:"road repairs" !cpv:72&x',
'2026-10-11T23:00:00.000Z',
);
expect(sent[0]?.url).toBe(
'/api/opportunities/search?q=title%3A%22road%20repairs%22%20!cpv%3A72%26x' +
'&released_from=2026-10-11T23%3A00%3A00.000Z',
);
expect(result).toEqual(found);
});
test('leaves out released_from when there is no bound', async () => {
const { api } = await loadClient();
respond = () =>
json({ uuids: [], matches_all: true, unknown_prompts: [], prompts: [], syntax: [] });
await api.searchOpportunities('', '');
expect(sent[0]?.url).toBe('/api/opportunities/search?q=');
});
});
describe('getOpportunity', () => {
test('fetches the opportunity by its encoded UUID', async () => {
const { api } = await loadClient();
+19
View File
@@ -4,6 +4,7 @@ import type {
Opportunity,
OpportunityMarks,
OpportunityPage,
OpportunitySearch,
Problem,
} from '@/types/Opportunities';
@@ -14,6 +15,8 @@ import type {
* GET {base}/opportunities?offset=&limit= -> OpportunityPage
* GET {base}/opportunities?changed_after={batch} -> OpportunityPage (changed since)
* both with &released_from={RFC 3339 time}: only those released since
* GET {base}/opportunities/search?q={text} -> OpportunitySearch
* also with &released_from
* GET {base}/opportunities/{uuid} -> Opportunity (with its releases)
* GET {base}/classifications -> ClassificationCode[]
* PUT {base}/opportunities/{uuid}/marks/{mark} -> OpportunityMarks (sets it)
@@ -149,6 +152,22 @@ export const api = {
return { opportunities: page.opportunities, batch: page.batch };
},
/**
* Searches the opportunities released at or after releasedFrom ("" for
* any) with query, the search box's text as typed: the backend parses it.
* A blank query matches all of them, and still returns the search's help.
*/
searchOpportunities(
query: string,
releasedFrom: string,
signal?: AbortSignal,
): Promise<OpportunitySearch> {
return request<OpportunitySearch>(
`/opportunities/search?q=${encodeURIComponent(query)}${releasedFromParam(releasedFrom)}`,
{ signal },
);
},
/** One opportunity with all of its releases. An unknown UUID is an ApiError with status 404. */
getOpportunity(uuid: string, signal?: AbortSignal): Promise<Opportunity> {
return request<Opportunity>(opportunityPath(uuid), { signal });
+10 -15
View File
@@ -9,16 +9,20 @@ import {
statusLabel,
} from '@/lib/filterOpportunities';
import { readStored, STORAGE_KEYS, writeStored } from '@/lib/storage';
import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch';
import type { OpportunityMark } from '@/types/Annotations';
import { MARK_LABELS, OPPORTUNITY_MARKS } from '@/types/Annotations';
import type { ClassificationCode } from '@/types/Opportunities';
import { ClassificationPicker } from './ClassificationPicker';
import { SearchField } from './SearchField';
import { buttonQuiet, fieldLabel, input, MARK_BG } from './styles';
interface Props {
filters: OpportunityFilters;
onChange: (next: OpportunityFilters) => void;
onClear: () => void;
/** The search that filters.search runs. */
search: OpportunitySearchResults;
statuses: string[];
classifications: ClassificationCode[];
isLoadingClassifications: boolean;
@@ -43,11 +47,10 @@ function toggled<T>(set: ReadonlySet<T>, value: T): Set<T> {
}
/**
* The collapsible panel of filters (title, status, marks, classification,
* The collapsible panel of filters (search, status, marks, classification,
* hidden), with the count of opportunities shown.
*/
export function FilterPanel(props: Props): JSX.Element {
const titleId = createUniqueId();
const daysId = createUniqueId();
const bodyId = createUniqueId();
@@ -151,19 +154,11 @@ export function FilterPanel(props: Props): JSX.Element {
</span>
</div>
<div>
<label class={fieldLabel} for={titleId}>
Title
</label>
<input
id={titleId}
type="search"
class={`${input} px-3 py-2.5 text-base`}
placeholder="Search opportunity titles"
value={props.filters.title}
onInput={(e) => update({ title: e.currentTarget.value })}
/>
</div>
<SearchField
value={props.filters.search}
onInput={(search) => update({ search })}
search={props.search}
/>
<fieldset class="m-0 min-w-0 border-0 p-0">
<legend class={fieldLabel}>Status</legend>
+127
View File
@@ -0,0 +1,127 @@
import CircleQuestionMark from 'lucide-solid/icons/circle-question-mark';
import type { JSX } from 'solid-js';
import { createSignal, createUniqueId, For, Match, Show, Switch } from 'solid-js';
import { promptText, unknownPromptsText } from '@/lib/search';
import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch';
import { input, linkButton } from './styles';
interface Props {
/** The box's text. */
value: string;
onInput: (value: string) => void;
search: OpportunitySearchResults;
}
const helpButton =
'flex rounded-md border border-line bg-surface p-1 text-ink hover:border-muted aria-expanded:border-ink';
const helpTerm = 'font-mono text-[13px] whitespace-nowrap';
/**
* The general search box, with its help (from the backend, behind the "?"
* button), a note on prompts the search ignored, and the search's errors.
*/
export function SearchField(props: Props): JSX.Element {
const inputId = createUniqueId();
const helpId = createUniqueId();
const noteId = createUniqueId();
const [isHelpOpen, setIsHelpOpen] = createSignal(false);
const unknownPrompts = (): string => {
const unknown = props.search.unknownPrompts();
return unknown.length === 0
? ''
: unknownPromptsText(unknown, props.search.help()?.prompts ?? []);
};
return (
<div>
<div class="mb-1.5 flex items-center gap-1.5">
<label class="text-sm font-semibold" for={inputId}>
Search
</label>
<button
type="button"
class={helpButton}
aria-label="Search help"
title="Search help"
aria-expanded={isHelpOpen()}
aria-controls={helpId}
onClick={() => setIsHelpOpen(!isHelpOpen())}
>
<CircleQuestionMark size={14} strokeWidth={2} />
</button>
</div>
<input
id={inputId}
type="search"
class={`${input} px-3 py-2.5 text-base`}
placeholder="Words, “phrases” and prompts such as title:"
value={props.value}
onInput={(e) => props.onInput(e.currentTarget.value)}
aria-describedby={noteId}
aria-busy={props.search.isSearching() || undefined}
/>
<Show when={props.search.error()}>
{(err) => (
<p class="mt-1.5 text-sm text-danger" role="alert">
The search didn't run: {err().message}{' '}
<button type="button" class={linkButton} onClick={() => props.search.refresh()}>
Try again
</button>
</p>
)}
</Show>
<p id={noteId} class="mt-1.5 text-sm text-muted empty:hidden">
{unknownPrompts()}
</p>
<div
id={helpId}
class="mt-2 rounded-md border border-line-soft bg-paper p-3 text-sm"
hidden={!isHelpOpen()}
>
<Switch fallback={<p class="text-muted">Loading the search help…</p>}>
<Match when={props.search.help()}>
{(help) => (
<>
<dl class="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1.5">
<For each={help().syntax}>
{(syntax) => (
<>
<dt class={helpTerm}>{syntax.example}</dt>
<dd>{syntax.help}</dd>
</>
)}
</For>
</dl>
<h3 class="mt-3 mb-1.5 font-semibold">Prompts</h3>
<dl class="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1.5">
<For each={help().prompts}>
{(prompt) => (
<>
<dt class={helpTerm}>{promptText(prompt.name)}</dt>
<dd>{prompt.help}</dd>
</>
)}
</For>
</dl>
</>
)}
</Match>
<Match when={props.search.helpError()}>
{(err) => (
<p class="text-danger" role="alert">
The search help didn't load: {err().message}{' '}
<button type="button" class={linkButton} onClick={() => props.search.reloadHelp()}>
Try again
</button>
</p>
)}
</Match>
</Switch>
</div>
</div>
);
}
+26 -10
View File
@@ -43,10 +43,16 @@ const annotations: Record<string, Annotation> = {
const annotationFor = (uuid: string): Annotation =>
annotations[uuid] ?? { uuid, marks: [], hidden: false };
const filtered = (filters: Partial<OpportunityFilters>): string[] =>
filterOpportunities(opportunities, { ...emptyFilters(), ...filters }, annotationFor).map(
(opportunity) => opportunity.uuid,
);
const filtered = (
filters: Partial<OpportunityFilters>,
searchMatches?: ReadonlySet<string>,
): string[] =>
filterOpportunities(
opportunities,
{ ...emptyFilters(), ...filters },
annotationFor,
searchMatches,
).map((opportunity) => opportunity.uuid);
describe('filterOpportunities', () => {
test('hides hidden opportunities unless asked, keeping the order given', () => {
@@ -55,7 +61,6 @@ describe('filterOpportunities', () => {
});
test.each<{ name: string; filters: Partial<OpportunityFilters>; want: string[] }>([
{ name: 'title, ignoring case and spaces', filters: { title: ' CLOUD ' }, want: ['a', 'c'] },
{ name: 'statuses', filters: { statuses: new Set(['active', 'planned']) }, want: ['a', 'c'] },
{ name: 'a mark', filters: { marks: new Set(['interested']) }, want: ['a'] },
{ name: 'no marks', filters: { marks: new Set([NOT_MARKED]) }, want: ['c'] },
@@ -65,18 +70,24 @@ describe('filterOpportunities', () => {
expect(filtered(filters)).toEqual(want);
});
test('keeps only the search’s matches, when it has some', () => {
expect(filtered({}, new Set(['c', 'x']))).toEqual(['c']);
expect(filtered({}, new Set())).toEqual([]);
});
test('combines filters', () => {
expect(filtered({ title: 'cloud', smeSuitable: true })).toEqual(['a']);
expect(filtered({ title: 'road', showHidden: true, smeSuitable: true })).toEqual([]);
expect(filtered({ smeSuitable: true }, new Set(['a', 'c']))).toEqual(['a']);
expect(filtered({ showHidden: true, smeSuitable: true }, new Set(['b']))).toEqual([]);
});
});
test('activeFilterCount counts the filters in use', () => {
expect(activeFilterCount(emptyFilters())).toBe(0);
expect(
activeFilterCount({ ...emptyFilters(), title: ' ', smeSuitable: true, showHidden: true }),
activeFilterCount({ ...emptyFilters(), search: ' ', smeSuitable: true, showHidden: true }),
).toBe(2);
expect(activeFilterCount({ ...emptyFilters(), releasedWithinDays: 30 })).toBe(1);
expect(activeFilterCount({ ...emptyFilters(), search: 'title:cloud' })).toBe(1);
});
test('releasedFrom is the start of the day that many days ago, in UK time', () => {
@@ -104,7 +115,7 @@ describe('stored filters', () => {
test('read back as they were stored', () => {
const filters: OpportunityFilters = {
releasedWithinDays: 30,
title: 'cloud',
search: 'cloud !cpv:72',
statuses: new Set(['active', 'withdrawn']),
marks: new Set(['interested', NOT_MARKED]),
classificationIds: new Set([1, 42]),
@@ -122,11 +133,16 @@ describe('stored filters', () => {
expect(parseFilters(text)).toEqual(emptyFilters());
});
test('an older version’s title filter is kept as search words', () => {
expect(parseFilters(JSON.stringify({ title: 'road repairs' })).search).toBe('road repairs');
expect(parseFilters(JSON.stringify({ title: 'road', search: 'cloud' })).search).toBe('cloud');
});
test('fields that aren’t valid keep their starting values', () => {
const filters = parseFilters(
JSON.stringify({
releasedWithinDays: 0,
title: 5,
search: 5,
statuses: 'active',
marks: ['won', 'not-a-mark'],
classificationIds: [3, '4', 1.5],
+20 -8
View File
@@ -18,7 +18,11 @@ export interface OpportunityFilters {
* here.
*/
releasedWithinDays: number;
title: string;
/**
* The general search box's text, as typed. The backend runs the search
* (see createOpportunitySearch), and its matches are applied here.
*/
search: string;
/** Empty means every status. */
statuses: ReadonlySet<string>;
/** Empty means any marks. Otherwise an opportunity needs at least one of these. */
@@ -37,7 +41,7 @@ export interface OpportunityFilters {
*/
export const emptyFilters = (): OpportunityFilters => ({
releasedWithinDays: DEFAULT_RELEASED_WITHIN_DAYS,
title: '',
search: '',
statuses: new Set(),
marks: new Set(),
classificationIds: new Set(),
@@ -91,7 +95,12 @@ export function parseFilters(text: string | null): OpportunityFilters {
releasedWithinDays: isDays(stored.releasedWithinDays)
? stored.releasedWithinDays
: filters.releasedWithinDays,
title: isString(stored.title) ? stored.title : filters.title,
// Older versions stored a title filter; its text is kept as search words.
search: isString(stored.search)
? stored.search
: isString(stored.title)
? stored.title
: filters.search,
statuses: statuses ? new Set(statuses) : filters.statuses,
marks: marks ? new Set(marks) : filters.marks,
classificationIds: classificationIds ? new Set(classificationIds) : filters.classificationIds,
@@ -107,7 +116,7 @@ export function parseFilters(text: string | null): OpportunityFilters {
export const activeFilterCount = (filters: OpportunityFilters): number =>
[
filters.releasedWithinDays !== DEFAULT_RELEASED_WITHIN_DAYS,
filters.title.trim() !== '',
filters.search.trim() !== '',
filters.statuses.size > 0,
filters.marks.size > 0,
filters.classificationIds.size > 0,
@@ -128,18 +137,21 @@ export function releasedFrom(days: number, now: Date = new Date()): string {
return new Date(now.getFullYear(), now.getMonth(), now.getDate() - days).toISOString();
}
/** The opportunities that pass filters, in the order given. */
/**
* The opportunities that pass filters, in the order given. searchMatches is
* the UUIDs the search found, or undefined when it doesn't filter (no
* search, or one with nothing to search for).
*/
export function filterOpportunities(
opportunities: readonly Opportunity[],
filters: OpportunityFilters,
annotationFor: (uuid: string) => Annotation,
searchMatches: ReadonlySet<string> | undefined,
): Opportunity[] {
const query = filters.title.trim().toLocaleLowerCase();
return opportunities.filter((opportunity) => {
const annotation = annotationFor(opportunity.uuid);
if (!filters.showHidden && annotation.hidden) return false;
if (query && !opportunity.title.toLocaleLowerCase().includes(query)) return false;
if (searchMatches && !searchMatches.has(opportunity.uuid)) return false;
if (filters.statuses.size > 0 && !filters.statuses.has(opportunity.status)) return false;
if (filters.smeSuitable && !opportunity.sme_suitable) return false;
if (filters.marks.size > 0) {
+32
View File
@@ -0,0 +1,32 @@
import { expect, test } from 'vitest';
import { promptText, unknownPromptsText } from './search';
const prompts = [
{ name: 'title', help: '' },
{ name: 'buyer', help: '' },
{ name: 'cpv', help: '' },
];
test('promptText is the prompt as typed', () => {
expect(promptText('buyer_id')).toBe('buyer_id:');
});
test.each([
{
name: 'one unknown prompt',
unknown: ['foo'],
want: '“foo:” isn’t a prompt, so it was ignored. The prompts are title:, buyer: and cpv:.',
},
{
name: 'several, as typed',
unknown: ['foo', 'Bar'],
want: '“foo:” and “Bar:” aren’t prompts, so they were ignored. The prompts are title:, buyer: and cpv:.',
},
])('unknownPromptsText names $name, then the prompts', ({ unknown, want }) => {
expect(unknownPromptsText(unknown, prompts)).toBe(want);
});
test('unknownPromptsText leaves out the prompts before the help has loaded', () => {
expect(unknownPromptsText(['foo'], [])).toBe('“foo:” isn’t a prompt, so it was ignored.');
});
+24
View File
@@ -0,0 +1,24 @@
import type { SearchPrompt } from '@/types/Opportunities';
const LIST_FORMAT = new Intl.ListFormat('en-GB', { type: 'conjunction' });
/** A prompt's name as it's typed in the search box: "title:". */
export const promptText = (name: string): string => `${name}:`;
/**
* What the search box says about prompts it doesn't know (their names as
* typed, without the colon), which the search ignored: they're named, then
* the prompts there are, if they're known yet.
*/
export function unknownPromptsText(
unknown: readonly string[],
prompts: readonly SearchPrompt[],
): string {
const names = LIST_FORMAT.format(unknown.map((name) => `“${promptText(name)}”`));
const ignored =
unknown.length === 1
? `${names} isn’t a prompt, so it was ignored.`
: `${names} aren’t prompts, so they were ignored.`;
if (prompts.length === 0) return ignored;
return `${ignored} The prompts are ${LIST_FORMAT.format(prompts.map((p) => promptText(p.name)))}.`;
}
+8 -2
View File
@@ -3,6 +3,7 @@ import { onCleanup } from 'solid-js';
import { connectEvents } from '@/api/events';
import { applyClassificationChanges } from '@/lib/classifications';
import type { OpportunityAnnotations } from '@/primitives/createOpportunityAnnotations';
import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch';
import type { ServerEvent } from '@/types/Events';
import type { ClassificationCode } from '@/types/Opportunities';
import type { ApiList, OpportunityList } from './createOpportunities';
@@ -12,6 +13,7 @@ export interface LiveData {
opportunities: OpportunityList;
classifications: ApiList<ClassificationCode>;
annotations: OpportunityAnnotations;
search: OpportunitySearchResults;
}
/**
@@ -21,10 +23,12 @@ export interface LiveData {
* - ClassificationsUpdated changes the classifications in place.
* - OpportunityMarksUpdated sets the opportunity's marks.
* - OpportunitiesUpdated catches the opportunities up: the changed ones are
* fetched and merged in, as the event only names them.
* fetched and merged in, as the event only names them. The search runs
* again, as they may match it now, or no longer.
*
* After the connection drops and is made again, events may have been missed:
* the opportunities catch up, and the classifications are reloaded.
* the opportunities catch up, the search runs again, and the classifications
* are reloaded.
*/
export function createLiveUpdates(data: LiveData): void {
const handle = (event: ServerEvent): void => {
@@ -39,6 +43,7 @@ export function createLiveUpdates(data: LiveData): void {
break;
case 'OpportunitiesUpdated':
data.opportunities.catchUp();
data.search.refresh();
break;
}
};
@@ -47,6 +52,7 @@ export function createLiveUpdates(data: LiveData): void {
onEvent: handle,
onReconnect: () => {
data.opportunities.catchUp();
data.search.refresh();
data.classifications.reload();
},
});
@@ -0,0 +1,225 @@
import { createRoot, createSignal } from 'solid-js';
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
import type * as clientModule from '@/api/client';
import type { OpportunitySearch } from '@/types/Opportunities';
import type { OpportunitySearchResults } from './createOpportunitySearch';
import { createOpportunitySearch } from './createOpportunitySearch';
const requests = vi.hoisted(() => ({
searchOpportunities:
vi.fn<
(query: string, releasedFrom: string, signal?: AbortSignal) => Promise<OpportunitySearch>
>(),
}));
vi.mock('@/api/client', async (importOriginal) => ({
...(await importOriginal<typeof clientModule>()),
api: requests,
}));
/** A search the test answers when it chooses. */
interface SentSearch {
query: string;
releasedFrom: string;
signal: AbortSignal | undefined;
resolve: (found: OpportunitySearch) => void;
reject: (err: unknown) => void;
}
const HELP = {
prompts: [{ name: 'title', help: 'The title only.' }],
syntax: [{ example: '!cloud', help: 'Not cloud.' }],
};
const found = (...uuids: string[]): OpportunitySearch => ({
uuids,
matches_all: false,
unknown_prompts: [],
...HELP,
});
let sent: SentSearch[];
let errors: string[];
let search: OpportunitySearchResults;
let dispose: () => void;
let setFilters: (filters: { search: string; days: number }) => void;
let setReleasedFrom: (from: string) => void;
/** The searches sent with a query: the help's has none. */
const searches = (): SentSearch[] => sent.filter((s) => s.query !== '');
/** Lets pending promises, Solid's effects and timers up to ms run. */
const wait = async (ms = 0): Promise<void> => {
await vi.advanceTimersByTimeAsync(ms);
};
/** Starts the search with a stored query, and answers the help's request. */
async function start(query: string): Promise<void> {
createRoot((disposeRoot) => {
dispose = disposeRoot;
const [filters, setStored] = createSignal({ search: query, days: 7 });
const [releasedFrom, setFrom] = createSignal('2026-10-12T00:00:00Z');
setFilters = setStored;
setReleasedFrom = setFrom;
search = createOpportunitySearch(
() => filters().search,
releasedFrom,
(message) => errors.push(message),
);
});
sent.find((s) => s.query === '')?.resolve({ ...found(), matches_all: true });
await wait();
}
const type = (text: string): void => setFilters({ search: text, days: 7 });
beforeEach(() => {
vi.useFakeTimers();
sent = [];
errors = [];
requests.searchOpportunities.mockImplementation(
(query, releasedFrom, signal) =>
new Promise((resolve, reject) => {
sent.push({ query, releasedFrom, signal, resolve, reject });
}),
);
});
afterEach(() => {
dispose();
vi.clearAllMocks();
vi.useRealTimers();
});
test('loads the help at the start, with a blank query and no bound', async () => {
await start('');
expect(sent.map((s) => [s.query, s.releasedFrom])).toEqual([['', '']]);
expect(search.help()).toEqual(HELP);
expect(search.matches()).toBeUndefined();
});
test('runs a stored search at once, and keeps its matches', async () => {
await start('cloud');
expect(searches().map((s) => [s.query, s.releasedFrom])).toEqual([
['cloud', '2026-10-12T00:00:00Z'],
]);
expect(search.isSearching()).toBe(true);
searches()[0]?.resolve(found('a', 'b'));
await wait();
expect([...(search.matches() ?? [])]).toEqual(['a', 'b']);
expect(search.isSearching()).toBe(false);
});
test('waits for typing to pause, then searches for the latest text', async () => {
await start('');
type('c');
await wait(200);
type('cloud');
await wait(299);
expect(searches()).toHaveLength(0);
expect(search.isSearching()).toBe(true);
await wait(1);
expect(searches().map((s) => s.query)).toEqual(['cloud']);
});
test('aborts a search under way for the next, and ignores its late answer', async () => {
await start('cloud');
type('road');
await wait(300);
expect(searches()[0]?.signal?.aborted).toBe(true);
searches()[0]?.resolve(found('old'));
searches()[1]?.resolve(found('new'));
await wait();
expect([...(search.matches() ?? [])]).toEqual(['new']);
});
test('doesn’t filter when nothing in the box can be searched for', async () => {
await start('title:');
searches()[0]?.resolve({ ...found(), matches_all: true, unknown_prompts: ['foo'] });
await wait();
expect(search.matches()).toBeUndefined();
expect(search.unknownPrompts()).toEqual(['foo']);
});
test('a blank box clears the matches at once, without a request', async () => {
await start('cloud');
searches()[0]?.resolve(found('a'));
await wait();
type(' ');
await wait(300);
expect(searches()).toHaveLength(1);
expect(search.matches()).toBeUndefined();
expect(search.isSearching()).toBe(false);
});
test('keeps the last matches after a failure, and refresh tries again', async () => {
await start('cloud');
searches()[0]?.resolve(found('a'));
await wait();
type('road');
await wait(300);
searches()[1]?.reject(new Error('offline'));
await wait();
expect([...(search.matches() ?? [])]).toEqual(['a']);
expect(search.error()?.message).toBe('offline');
expect(errors).toEqual(["The search didn't run: offline"]);
search.refresh();
searches()[2]?.resolve(found('b'));
await wait();
expect(searches()[2]?.query).toBe('road');
expect([...(search.matches() ?? [])]).toEqual(['b']);
expect(search.error()).toBeUndefined();
});
test('runs again at once for a new releasedFrom, but not for other filters', async () => {
await start('cloud');
setFilters({ search: 'cloud', days: 30 });
await wait(300);
expect(searches()).toHaveLength(1);
setReleasedFrom('2026-10-01T00:00:00Z');
await wait();
expect(searches().map((s) => s.releasedFrom)).toEqual([
'2026-10-12T00:00:00Z',
'2026-10-01T00:00:00Z',
]);
});
test('a search mends help that didn’t load', async () => {
createRoot((disposeRoot) => {
dispose = disposeRoot;
const [query, setQuery] = createSignal('');
setFilters = (filters) => setQuery(filters.search);
search = createOpportunitySearch(
query,
() => '',
(message) => errors.push(message),
);
});
sent[0]?.reject(new Error('offline'));
await wait();
expect(search.helpError()?.message).toBe('offline');
expect(search.help()).toBeUndefined();
type('cloud');
await wait(300);
searches()[0]?.resolve(found());
await wait();
expect(search.help()).toEqual(HELP);
expect(search.helpError()).toBeUndefined();
});
@@ -0,0 +1,155 @@
import { createEffect, createMemo, createSignal, on, onCleanup } from 'solid-js';
import { api } from '@/api/client';
import { errorText } from '@/lib/errors';
import type { OpportunitySearch, SearchExample, SearchPrompt } from '@/types/Opportunities';
/** How long typing has to pause for before the search runs, in milliseconds. */
const SEARCH_DELAY = 300;
/** The search box's help, from the backend, which parses the search. */
export interface SearchHelp {
prompts: SearchPrompt[];
/** Examples of the syntax, in the order to show them. */
syntax: SearchExample[];
}
/** The general search, run by the backend, and its help. */
export interface OpportunitySearchResults {
/**
* The UUIDs the last search found, or undefined when the search doesn't
* filter: the box is blank, nothing in it can be searched for, or no
* search has finished yet. After a failure, the last search's.
*/
matches: () => ReadonlySet<string> | undefined;
/** The last search's prompts the backend doesn't know, as typed, without the colon. */
unknownPrompts: () => string[];
/** Whether a search is waiting for typing to pause, or running. */
isSearching: () => boolean;
/** Why the last search failed, until one succeeds or the box is cleared. */
error: () => Error | undefined;
/** Runs the search again now, unless the box is blank: to retry, or as opportunities changed. */
refresh: () => void;
/** The help, or undefined until it has loaded. */
help: () => SearchHelp | undefined;
/** Why the help didn't load, until it does. */
helpError: () => Error | undefined;
reloadHelp: () => void;
}
const toError = (err: unknown): Error => (err instanceof Error ? err : new Error(errorText(err)));
/**
* Searches the opportunities released at or after releasedFrom with query,
* the search box's text, as either changes: at once for a new releasedFrom
* or the first query, and after a pause in typing for the others. A search
* under way is aborted by the next. A failure is shown with onError, and
* the last search's matches are kept. The help is loaded once, at the
* start.
*/
export function createOpportunitySearch(
query: () => string,
releasedFrom: () => string,
onError: (message: string) => void,
): OpportunitySearchResults {
const [result, setResult] = createSignal<OpportunitySearch>();
const [isSearching, setIsSearching] = createSignal(false);
const [error, setError] = createSignal<Error>();
const [help, setHelp] = createSignal<SearchHelp>();
const [helpError, setHelpError] = createSignal<Error>();
let controller: AbortController | undefined;
let helpController: AbortController | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
onCleanup(() => {
clearTimeout(timer);
controller?.abort();
helpController?.abort();
});
const keepHelp = (found: OpportunitySearch): void => {
setHelp({ prompts: found.prompts, syntax: found.syntax });
setHelpError(undefined);
};
async function search(): Promise<void> {
clearTimeout(timer);
controller?.abort();
controller = new AbortController();
const { signal } = controller;
setIsSearching(true);
try {
const found = await api.searchOpportunities(query(), releasedFrom(), signal);
if (signal.aborted) return;
setResult(found);
setError(undefined);
// Every result carries the help, so a search mends a help that didn't load.
keepHelp(found);
} catch (err) {
if (signal.aborted) return;
const failure = toError(err);
setError(failure);
onError(`The search didn't run: ${failure.message}`);
} finally {
if (!signal.aborted) setIsSearching(false);
}
}
const clear = (): void => {
clearTimeout(timer);
controller?.abort();
setResult(undefined);
setError(undefined);
setIsSearching(false);
};
createEffect(
on([query, releasedFrom], ([text, from], previous) => {
// query may be read from a larger value, such as all the filters, which
// changes when neither does.
if (previous && previous[0] === text && previous[1] === from) return;
if (text.trim() === '') {
clear();
} else if (previous && previous[0] !== text) {
clearTimeout(timer);
setIsSearching(true);
timer = setTimeout(() => void search(), SEARCH_DELAY);
} else {
void search();
}
}),
);
async function loadHelp(): Promise<void> {
helpController?.abort();
helpController = new AbortController();
const { signal } = helpController;
try {
const found = await api.searchOpportunities('', '', signal);
if (!signal.aborted) keepHelp(found);
} catch (err) {
if (signal.aborted) return;
const failure = toError(err);
setHelpError(failure);
onError(`The search help didn't load: ${failure.message}`);
}
}
void loadHelp();
const matches = createMemo(() => {
const found = result();
return found && !found.matches_all ? new Set(found.uuids) : undefined;
});
return {
matches,
unknownPrompts: () => result()?.unknown_prompts ?? [],
isSearching,
error,
refresh: () => {
if (query().trim() !== '') void search();
},
help,
helpError,
reloadHelp: () => void loadHelp(),
};
}
+7
View File
@@ -41,5 +41,12 @@ export type OpportunityReleaseTag = Schemas['types.OpportunityReleaseTag'];
/** A page of GET /api/opportunities. */
export type OpportunityPage = Schemas['api.OpportunityPage'];
/** GET /api/opportunities/search's result: the matches, and the search's help. */
export type OpportunitySearch = Schemas['api.OpportunitySearch'];
/** A search prompt (title:, cpv:, ...) and what it searches. */
export type SearchPrompt = Schemas['search.Prompt'];
/** An example of the search's syntax, and what it means. */
export type SearchExample = Schemas['search.Example'];
/** An RFC 9457 problem details body, as the API returns for errors. */
export type Problem = Schemas['httpx.Problem'];
+86
View File
@@ -140,6 +140,77 @@ export interface paths {
patch?: never;
trace?: never;
};
"/opportunities/search": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Search opportunities
* @description The UUIDs of the opportunities matching a general search,
* in no particular order. q is words and "quoted phrases",
* each optionally with a prompt (title:, buyer:, buyer_id:,
* cpv:) and a leading ! to leave out what it matches; prompts
* and syntax explain it. A search with nothing to search for
* (blank, or only unknown prompts) has matches_all set and no
* uuids. unknown_prompts lists the prompts that aren't known,
* as typed; their terms are ignored. released_from (RFC 3339,
* inclusive) leaves out opportunities whose latest release is
* before it.
*/
get: {
parameters: {
query?: {
/** @description The search */
q?: string;
/** @description Only opportunities whose latest release is at or after this time */
released_from?: string;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description OK */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["api.OpportunitySearch"];
};
};
/** @description Invalid released_from */
400: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["httpx.Problem"];
};
};
/** @description Internal Server Error */
500: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["httpx.Problem"];
};
};
};
};
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/opportunities/{uuid}": {
parameters: {
query?: never;
@@ -351,6 +422,13 @@ export interface components {
offset: number;
opportunities: components["schemas"]["types.Opportunity"][];
};
"api.OpportunitySearch": {
matches_all: boolean;
prompts: components["schemas"]["search.Prompt"][];
syntax: components["schemas"]["search.Example"][];
unknown_prompts: string[];
uuids: string[];
};
"httpx.Problem": {
detail?: string;
instance?: string;
@@ -359,6 +437,14 @@ export interface components {
title: string;
type?: string;
};
"search.Example": {
example: string;
help: string;
};
"search.Prompt": {
help: string;
name: string;
};
"types.ClassificationCode": {
classification: string;
code: string;