Compare commits
3
Commits
f467c6c7da
...
b0230e6847
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b0230e6847 | ||
|
|
0a777792d8 | ||
|
|
b0f93fd437 |
No files matched your search
@@ -105,7 +105,7 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and
|
||||
|
||||
| Command | Purpose |
|
||||
|---|---|
|
||||
| `job-search config init [path]` | Create a config file |
|
||||
| `job-search config init [path]` | Create a config file holding the defaults (`--all` also writes the test sources, `fts_mock` and `sleeper`) |
|
||||
| `job-search config show` | Print the configuration (`--json` or `--yaml` for those formats) |
|
||||
| `job-search config get <key>` | Print one value, e.g. `logging.level` |
|
||||
| `job-search config set <key> <value>` | Validate and write one value to the config file |
|
||||
@@ -234,6 +234,7 @@ second one fails with "address already in use".
|
||||
| `internal/cli` | Cobra commands and shared CLI setup |
|
||||
| `internal/config` | Configuration loading, validation and editing |
|
||||
| `internal/fetcher` | Runs a worker per enabled source |
|
||||
| `internal/search` | Parsing of the general opportunity search (`title:`, `buyer:`, `buyer_id:`, `cpv:`, `!`) and its help |
|
||||
| `internal/skiprules` | The rules that mark new opportunities Skipped (see [internal/skiprules](internal/skiprules/README.md)) |
|
||||
| `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) |
|
||||
| `internal/findatender` | FTS API client and conversion from OCDS to JobSearch types |
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/events"
|
||||
"git.avsdev.uk/Go/job-search/internal/httpx"
|
||||
"git.avsdev.uk/Go/job-search/internal/search"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
@@ -38,6 +39,21 @@ type OpportunityPage struct {
|
||||
Batch string `json:"batch"`
|
||||
}
|
||||
|
||||
// OpportunitySearch is the result of a general search, as GET
|
||||
// /opportunities/search returns it: the UUIDs of the opportunities that
|
||||
// match, in no particular order. MatchesAll reports a search with nothing to
|
||||
// search for (blank, or only unknown prompts), which matches every
|
||||
// opportunity; UUIDs is then empty. UnknownPrompts are the prompts in the
|
||||
// search that aren't known, as typed, without their colon. Prompts and
|
||||
// Syntax are the help for the search box.
|
||||
type OpportunitySearch struct {
|
||||
UUIDs []string `json:"uuids"`
|
||||
MatchesAll bool `json:"matches_all"`
|
||||
UnknownPrompts []string `json:"unknown_prompts"`
|
||||
Prompts []search.Prompt `json:"prompts"`
|
||||
Syntax []search.Example `json:"syntax"`
|
||||
}
|
||||
|
||||
// opportunityRoutes wires HTTP handlers to the opportunity store.
|
||||
// Opportunities are read-only; only their marks can be changed.
|
||||
type opportunityRoutes struct {
|
||||
@@ -64,6 +80,7 @@ func newOpportunityRoutes(
|
||||
// Routes registers the opportunity routes on router (mounted under /api).
|
||||
func (h *opportunityRoutes) Routes(router chi.Router) {
|
||||
router.Get("/opportunities", h.list)
|
||||
router.Get("/opportunities/search", h.search)
|
||||
router.Get("/opportunities/{uuid}", h.get)
|
||||
router.Put("/opportunities/{uuid}/marks/{mark}", h.setMark)
|
||||
router.Delete("/opportunities/{uuid}/marks/{mark}", h.clearMark)
|
||||
@@ -203,6 +220,61 @@ func (h *opportunityRoutes) listChanged(
|
||||
})
|
||||
}
|
||||
|
||||
// search serves GET /opportunities/search: the UUIDs of the opportunities
|
||||
// matching the general search in the q query parameter (see package
|
||||
// search), with the help for the search box. released_from bounds it as it
|
||||
// bounds list.
|
||||
//
|
||||
// @Summary Search opportunities
|
||||
// @Description The UUIDs of the opportunities matching a general search,
|
||||
// @Description in no particular order. q is words and "quoted phrases",
|
||||
// @Description each optionally with a prompt (title:, buyer:, buyer_id:,
|
||||
// @Description cpv:) and a leading ! to leave out what it matches; prompts
|
||||
// @Description and syntax explain it. A search with nothing to search for
|
||||
// @Description (blank, or only unknown prompts) has matches_all set and no
|
||||
// @Description uuids. unknown_prompts lists the prompts that aren't known,
|
||||
// @Description as typed; their terms are ignored. released_from (RFC 3339,
|
||||
// @Description inclusive) leaves out opportunities whose latest release is
|
||||
// @Description before it.
|
||||
// @Tags opportunities
|
||||
// @Produce json
|
||||
// @Param q query string false "The search"
|
||||
// @Param released_from query string false "Only opportunities whose latest release is at or after this time" format(date-time)
|
||||
// @Success 200 {object} OpportunitySearch
|
||||
// @Failure 400 {object} httpx.Problem "Invalid released_from"
|
||||
// @Failure 500 {object} httpx.Problem
|
||||
// @Router /opportunities/search [get]
|
||||
func (h *opportunityRoutes) search(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
releasedFrom, err := timeParam(
|
||||
"released_from", query.Get("released_from"),
|
||||
)
|
||||
if err != nil {
|
||||
httpx.WriteProblem(w, r, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
parsed := search.Parse(query.Get("q"))
|
||||
result := OpportunitySearch{
|
||||
UUIDs: []string{},
|
||||
MatchesAll: parsed.MatchesAll(),
|
||||
UnknownPrompts: parsed.UnknownPrompts,
|
||||
Prompts: search.Prompts,
|
||||
Syntax: search.Syntax,
|
||||
}
|
||||
if !result.MatchesAll {
|
||||
result.UUIDs, err = h.store.SearchOpportunities(
|
||||
r.Context(), parsed, releasedFrom,
|
||||
)
|
||||
if err != nil {
|
||||
httpx.WriteProblem(w, r, http.StatusInternalServerError, "")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
httpx.WriteJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
// get serves GET /opportunities/{uuid}: one opportunity with its
|
||||
// classifications, marks and releases.
|
||||
//
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -483,3 +484,54 @@ func TestChangeMarkPublishes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GET /opportunities/search returns the matching UUIDs, bounded by
|
||||
// released_from, with the help; a search with nothing to search for matches
|
||||
// all, and unknown prompts are listed.
|
||||
func TestSearchOpportunities(t *testing.T) {
|
||||
url := opportunitiesServer(t, 5) + "/opportunities/search"
|
||||
from3 := "&released_from=2026-10-03T00:00:00Z"
|
||||
cases := []struct {
|
||||
query string
|
||||
uuids []string
|
||||
matchesAll bool
|
||||
unknown []string
|
||||
}{
|
||||
{"", []string{}, true, []string{}},
|
||||
{"?q=", []string{}, true, []string{}},
|
||||
{"?q=cpv:72", []string{"ocds-1", "ocds-3", "ocds-5"}, false, []string{}},
|
||||
{"?q=cpv:72" + from3, []string{"ocds-3", "ocds-5"}, false, []string{}},
|
||||
{"?q=opportunity%20!cpv:72", []string{"ocds-2", "ocds-4"}, false, []string{}},
|
||||
{"?q=nothing", []string{}, false, []string{}},
|
||||
{"?q=foo:x%20Bar:y", []string{}, true, []string{"foo", "Bar"}},
|
||||
{"?q=foo:x%20title:3", []string{"ocds-3"}, false, []string{"foo"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
var got OpportunitySearch
|
||||
status := request(t, http.MethodGet, url+tc.query, &got)
|
||||
slices.Sort(got.UUIDs)
|
||||
if status != http.StatusOK || !slices.Equal(got.UUIDs, tc.uuids) ||
|
||||
got.MatchesAll != tc.matchesAll ||
|
||||
!slices.Equal(got.UnknownPrompts, tc.unknown) {
|
||||
t.Errorf("%s: status %d, %+v; want %v, matches_all %v, unknown %v",
|
||||
tc.query, status, got, tc.uuids, tc.matchesAll, tc.unknown)
|
||||
}
|
||||
if len(got.Prompts) == 0 || len(got.Syntax) == 0 {
|
||||
t.Errorf("%s: no help: %+v", tc.query, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Empty lists are [], never null.
|
||||
var raw map[string]json.RawMessage
|
||||
request(t, http.MethodGet, url+"?q=nothing", &raw)
|
||||
for _, field := range []string{"uuids", "unknown_prompts"} {
|
||||
if string(raw[field]) != "[]" {
|
||||
t.Errorf("%s = %s; want []", field, raw[field])
|
||||
}
|
||||
}
|
||||
|
||||
status := request(t, http.MethodGet, url+"?q=x&released_from=x", nil)
|
||||
if status != http.StatusBadRequest {
|
||||
t.Errorf("bad released_from: status %d; want 400", status)
|
||||
}
|
||||
}
|
||||
@@ -214,6 +214,32 @@ func TestConfigInit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The test sources are left out of the file unless --all is given.
|
||||
func TestConfigInitTestSources(t *testing.T) {
|
||||
cmdtest.Setup(t, "")
|
||||
|
||||
mustRun(t, "config", "init", "./default.yaml")
|
||||
written := cmdtest.ReadFile(t, "default.yaml")
|
||||
cmdtest.Contains(t, written, "find_a_tender:", "eu_supply:")
|
||||
for _, hidden := range []string{"fts_mock", "sleeper"} {
|
||||
if strings.Contains(written, hidden) {
|
||||
t.Errorf("init wrote %s without --all:\n%s", hidden, written)
|
||||
}
|
||||
}
|
||||
|
||||
mustRun(t, "config", "init", "--all", "./all.yaml")
|
||||
cmdtest.Contains(
|
||||
t, cmdtest.ReadFile(t, "all.yaml"), "fts_mock:", "sleeper:",
|
||||
)
|
||||
|
||||
// The test sources still load and are set as usual.
|
||||
mustRun(
|
||||
t, "--config", "default.yaml",
|
||||
"config", "set", "sources.sleeper.enabled", "true",
|
||||
)
|
||||
cmdtest.Contains(t, cmdtest.ReadFile(t, "default.yaml"), "sleeper:")
|
||||
}
|
||||
|
||||
// With no path (or a blank one), init searches the default locations. The
|
||||
// working directory is one, so its config.yaml is found. (Not finding one
|
||||
// would write to /etc/job-search, which a test can't do.)
|
||||
|
||||
+22
-15
@@ -17,7 +17,11 @@ func commandInit() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "init [config_path]",
|
||||
Args: cobra.MatchAll(cobra.MinimumNArgs(0), cobra.MaximumNArgs(1)),
|
||||
Short: "Create an empty configuration file",
|
||||
Short: "Create a configuration file holding the defaults",
|
||||
Long: `Create a configuration file holding the defaults, at config_path or,
|
||||
without one, the first config.yaml found in /etc/job-search then the working
|
||||
directory (/etc/job-search/config.yaml if there is none). The test sources
|
||||
(fts_mock and sleeper) are left out unless --all is given.`,
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("command persistent pre-run", "command", "config.init")
|
||||
},
|
||||
@@ -64,26 +68,29 @@ func commandInit() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
|
||||
// Load default configuration
|
||||
config.LoadDefaultConfig()
|
||||
|
||||
if viper.ConfigFileUsed() == "" {
|
||||
viper.SetConfigFile(config.EtcConfigPath + "/config.yaml")
|
||||
path := viper.ConfigFileUsed()
|
||||
if path == "" {
|
||||
path = config.EtcConfigPath + "/config.yaml"
|
||||
}
|
||||
|
||||
fmt.Print(
|
||||
"writing default configuration to file: " +
|
||||
viper.ConfigFileUsed() +
|
||||
"\n",
|
||||
)
|
||||
|
||||
if err := viper.WriteConfig(); err != nil {
|
||||
return err
|
||||
// A Viper of its own, holding only the keys to write: the global
|
||||
// one would write every default, the hidden sections' included.
|
||||
all, _ := cmd.Flags().GetBool("all")
|
||||
generated := viper.New()
|
||||
for key, value := range config.InitDefaults(all) {
|
||||
generated.Set(key, value)
|
||||
}
|
||||
generated.SetConfigFile(path)
|
||||
|
||||
return nil
|
||||
fmt.Print("writing default configuration to file: " + path + "\n")
|
||||
|
||||
return generated.WriteConfig()
|
||||
},
|
||||
}
|
||||
cmd.Flags().Bool(
|
||||
"all", false, "also write the test sources (fts_mock and sleeper)",
|
||||
)
|
||||
|
||||
// Config flag should be hidden as the config path should be provided
|
||||
cmd.SetHelpFunc(func(command *cobra.Command, args []string) {
|
||||
command.Flags().MarkHidden("config") //nolint:errcheck,gosec // a root flag, so it always exists
|
||||
|
||||
@@ -267,6 +267,10 @@ per key:
|
||||
of the configuration.
|
||||
- `default:"value"` is the field's default, parsed as if it were a string
|
||||
passed to `SetValue` (e.g. `"10m"`, `"true"`).
|
||||
- `init:"hidden"` on a nested section's field leaves the section out of the
|
||||
file `config init` writes, unless `--all` is given (`InitDefaults`). The
|
||||
test sources, `fts_mock` and `sleeper`, have it. The section is otherwise
|
||||
as any other: loaded, and read and set by the `config` commands.
|
||||
- A tagged field whose type is a struct is a nested section, addressed as
|
||||
`section.key`. An embedded struct has its fields promoted, as Go does;
|
||||
every source embeds `JSCSourcesSource` for `enabled` and `interval`.
|
||||
@@ -324,7 +328,8 @@ To add a new section:
|
||||
`JSCRateLimitRule`, has no constant and passes `""` to `flattenOf`.
|
||||
2. Add the struct as a tagged field of its parent: `JobSearchConfig` in
|
||||
`config.go`, or `JSCSources` in `conf_sources.go` for a source,
|
||||
etc. The field's `cfg` tag must match the `JSCK*` constant.
|
||||
etc. The field's `cfg` tag must match the `JSCK*` constant. Add
|
||||
`init:"hidden"` if it's only for testing.
|
||||
|
||||
Nothing in `section.go` needs changing.
|
||||
|
||||
|
||||
@@ -288,6 +288,26 @@ func TestConcreteSourceSections(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// InitDefaults leaves out the sections tagged init:"hidden" unless asked.
|
||||
func TestInitDefaults(t *testing.T) {
|
||||
all := InitDefaults(true)
|
||||
shown := InitDefaults(false)
|
||||
if all["sources.sleeper.enabled"] != false ||
|
||||
all["sources.fts_mock.interval"] != "10m0s" {
|
||||
t.Errorf("all = %v; want the test sources' defaults", all)
|
||||
}
|
||||
for key := range all {
|
||||
hidden := strings.HasPrefix(key, "sources.fts_mock.") ||
|
||||
strings.HasPrefix(key, "sources.sleeper.")
|
||||
if _, found := shown[key]; found == hidden {
|
||||
t.Errorf("%s: in the generated defaults = %v", key, found)
|
||||
}
|
||||
}
|
||||
if shown["sources.eu_supply.base_url"] != "https://uk.eu-supply.com" {
|
||||
t.Errorf("eu_supply missing: %v", shown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEUSupplySection(t *testing.T) {
|
||||
conf := defaultConfig(t)
|
||||
got := conf.Sources.EUSupply
|
||||
|
||||
@@ -13,15 +13,17 @@ const JSCKSources string = "sources"
|
||||
|
||||
// JSCSources configures each source of opportunities, one field
|
||||
// per source. Each field's cfg tag must match the section's JSCK* constant.
|
||||
// The test sources (FTSMock and Sleeper) are tagged init:"hidden", so
|
||||
// `config init` only writes them with --all.
|
||||
// SkipRulesFile is the JSON file of rules that mark new opportunities
|
||||
// Skipped (see internal/skiprules), read again when it changes; empty means
|
||||
// none.
|
||||
type JSCSources struct {
|
||||
SkipRulesFile string `cfg:"skip_rules_file"`
|
||||
FindATender JSCSourceFindATender `cfg:"find_a_tender"`
|
||||
FTSMock JSCSourceFTSMock `cfg:"fts_mock"`
|
||||
FTSMock JSCSourceFTSMock `cfg:"fts_mock" init:"hidden"`
|
||||
EUSupply JSCSourceEUSupply `cfg:"eu_supply"`
|
||||
Sleeper JSCSourceSleeper `cfg:"sleeper"`
|
||||
Sleeper JSCSourceSleeper `cfg:"sleeper" init:"hidden"`
|
||||
}
|
||||
|
||||
// ---------- Section methods (see Section) ----------
|
||||
|
||||
@@ -40,6 +40,30 @@ func LoadDefaultConfig() {
|
||||
LoadToViper(conf)
|
||||
}
|
||||
|
||||
// InitDefaults returns the default configuration as dotted keys, as
|
||||
// Flatten gives it, for `config init` to write. Sections tagged
|
||||
// init:"hidden" (the test sources) are left out unless includeHidden is set.
|
||||
func InitDefaults(includeHidden bool) map[string]any {
|
||||
var conf JobSearchConfig
|
||||
if err := applyDefaults(&conf); err != nil {
|
||||
panic(fmt.Sprintf("config: %v", err))
|
||||
}
|
||||
flat := conf.Flatten()
|
||||
if includeHidden {
|
||||
return flat
|
||||
}
|
||||
|
||||
hidden := hiddenFromInitKeys("", reflect.TypeFor[JobSearchConfig]())
|
||||
for _, section := range hidden {
|
||||
for key := range flat {
|
||||
if strings.HasPrefix(key, section+".") {
|
||||
delete(flat, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return flat
|
||||
}
|
||||
|
||||
// ---------- Section methods (see Section) ----------
|
||||
|
||||
// Map implements Section.
|
||||
|
||||
@@ -50,6 +50,10 @@ const (
|
||||
keyTag = "cfg"
|
||||
// defaultTag holds a field's default value.
|
||||
defaultTag = "default"
|
||||
// initTag set to initHidden on a section field leaves the section out of
|
||||
// the file `config init` writes (see InitDefaults), as for test sources.
|
||||
initTag = "init"
|
||||
initHidden = "hidden"
|
||||
)
|
||||
|
||||
// The unknown key error and the field types section.go handles specially.
|
||||
@@ -109,6 +113,8 @@ type field struct {
|
||||
section bool
|
||||
defaultValue string
|
||||
hasDefault bool
|
||||
// hiddenFromInit is set by initTag (see InitDefaults).
|
||||
hiddenFromInit bool
|
||||
}
|
||||
|
||||
// isSectionList reports whether listType is a list of sections, []S for a
|
||||
@@ -146,8 +152,9 @@ func fieldsOf(structType reflect.Type) []field {
|
||||
fieldType: structField.Type,
|
||||
section: structField.Type.Kind() == reflect.Struct &&
|
||||
structField.Type != timeType,
|
||||
defaultValue: defaultValue,
|
||||
hasDefault: hasDefault,
|
||||
defaultValue: defaultValue,
|
||||
hasDefault: hasDefault,
|
||||
hiddenFromInit: structField.Tag.Get(initTag) == initHidden,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -244,6 +251,25 @@ func flattenInto(flat map[string]any, prefix string, section reflect.Value) {
|
||||
}
|
||||
}
|
||||
|
||||
// hiddenFromInitKeys returns the dotted keys of the sections under section
|
||||
// (prefixed with prefix, if non-empty) whose field has initTag set to
|
||||
// initHidden, descending into nested sections.
|
||||
func hiddenFromInitKeys(prefix string, sectionType reflect.Type) []string {
|
||||
var hidden []string
|
||||
for _, configField := range fieldsOf(sectionType) {
|
||||
if !configField.section {
|
||||
continue
|
||||
}
|
||||
key := joinKey(prefix, configField.key)
|
||||
if configField.hiddenFromInit {
|
||||
hidden = append(hidden, key)
|
||||
continue
|
||||
}
|
||||
hidden = append(hidden, hiddenFromInitKeys(key, configField.fieldType)...)
|
||||
}
|
||||
return hidden
|
||||
}
|
||||
|
||||
// exportValue returns a field's value as Map, Flatten and ValueOf give it
|
||||
// out: durations and byte sizes become strings such as "1m30s" and "32KiB",
|
||||
// so values agree with kindOf and can be passed back to SetValue; []string is
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
// Package search parses the general opportunity search: words and quoted
|
||||
// phrases, each optionally with a prompt naming the field it searches
|
||||
// (title:, buyer:, buyer_id:, cpv:) and a leading "!" to negate it. The
|
||||
// store runs a parsed Query (store.SearchOpportunities).
|
||||
package search
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Field is what a term searches.
|
||||
type Field int
|
||||
|
||||
// Values of Field.
|
||||
const (
|
||||
// FieldText is a term with no prompt: the title or the description.
|
||||
FieldText Field = iota
|
||||
// FieldTitle is title:, the title only.
|
||||
FieldTitle
|
||||
// FieldBuyer is buyer:, the buyer's name.
|
||||
FieldBuyer
|
||||
// FieldBuyerID is buyer_id:, the buyer's ID, matched whole.
|
||||
FieldBuyerID
|
||||
// FieldCPV is cpv:, a CPV code prefix or words of a CPV description.
|
||||
FieldCPV
|
||||
)
|
||||
|
||||
// Prompt is a known prompt, with a line of help for the user.
|
||||
type Prompt struct {
|
||||
Name string `json:"name"`
|
||||
Help string `json:"help"`
|
||||
}
|
||||
|
||||
// Example is an example of the search syntax, with a line of help for the
|
||||
// user.
|
||||
type Example struct {
|
||||
Example string `json:"example"`
|
||||
Help string `json:"help"`
|
||||
}
|
||||
|
||||
// Prompts are the known prompts, in the order to show them.
|
||||
var Prompts = []Prompt{
|
||||
{Name: "title", Help: "Words in the title only."},
|
||||
{Name: "buyer", Help: "Words in the buyer's name."},
|
||||
{
|
||||
Name: "buyer_id",
|
||||
Help: "The buyer's ID, whole and in any case (e.g. GB-FTS-148803).",
|
||||
},
|
||||
{
|
||||
Name: "cpv",
|
||||
Help: "The start of a CPV code (cpv:72), or words in a CPV code's description (cpv:food).",
|
||||
},
|
||||
}
|
||||
|
||||
// Syntax explains the search syntax, in the order to show it.
|
||||
var Syntax = []Example{
|
||||
{
|
||||
Example: "cloud hosting",
|
||||
Help: "Words without a prompt are searched for in the title and the description. Every word must be found, though not necessarily in the same one.",
|
||||
},
|
||||
{
|
||||
Example: "migr",
|
||||
Help: "A word also matches the start of a longer word, and other forms of it: migr finds migration, and migrating finds migration.",
|
||||
},
|
||||
{
|
||||
Example: `"road repairs"`,
|
||||
Help: "Quotes search for a phrase: its words together, in that order.",
|
||||
},
|
||||
{
|
||||
Example: `title:"road repairs"`,
|
||||
Help: "A prompt searches only its own field, for one word or one quoted phrase.",
|
||||
},
|
||||
{
|
||||
Example: "buyer:leeds buyer:bradford",
|
||||
Help: "Repeats of the same prompt match if any of them does: Leeds or Bradford.",
|
||||
},
|
||||
{
|
||||
Example: "cloud buyer:leeds cpv:72",
|
||||
Help: "Different prompts, and words without one, must all match.",
|
||||
},
|
||||
{
|
||||
Example: "!cloud !buyer:leeds",
|
||||
Help: "! in front of a word, a phrase or a prompt leaves out what it matches.",
|
||||
},
|
||||
{
|
||||
Example: "cpv:72 !cpv:7221",
|
||||
Help: "CPV terms are checked against each of an opportunity's codes: this needs a code starting 72 that doesn't start 7221. !cpv: without cpv: leaves out every opportunity with a matching code.",
|
||||
},
|
||||
}
|
||||
|
||||
// prompts maps each known prompt's name to its field.
|
||||
var prompts = map[string]Field{
|
||||
"title": FieldTitle,
|
||||
"buyer": FieldBuyer,
|
||||
"buyer_id": FieldBuyerID,
|
||||
"cpv": FieldCPV,
|
||||
}
|
||||
|
||||
// promptPattern matches a prompt at the start of a term: a name of letters
|
||||
// and underscores, then a colon. A colon after anything else (12:00) is
|
||||
// part of the term.
|
||||
var promptPattern = regexp.MustCompile(`^([A-Za-z_]+):`)
|
||||
|
||||
// cpvCodePattern matches a CPV code or the start of one, with an optional
|
||||
// check digit.
|
||||
var cpvCodePattern = regexp.MustCompile(`^([0-9]+)(?:-[0-9])?$`)
|
||||
|
||||
// Term is one search term: a word or a quoted phrase, as typed.
|
||||
type Term struct {
|
||||
Field Field
|
||||
Negated bool
|
||||
Value string
|
||||
}
|
||||
|
||||
// Words returns the term's words, split as the store's full-text index
|
||||
// splits them: on anything but letters and numbers.
|
||||
func (t Term) Words() []string {
|
||||
return strings.FieldsFunc(t.Value, func(char rune) bool {
|
||||
return !unicode.IsLetter(char) && !unicode.IsNumber(char)
|
||||
})
|
||||
}
|
||||
|
||||
// CPVCode returns the digits of a cpv: term that is a code or the start of
|
||||
// one, without its check digit, and reports whether it is one. Any other
|
||||
// cpv: term searches CPV descriptions.
|
||||
func (t Term) CPVCode() (string, bool) {
|
||||
match := cpvCodePattern.FindStringSubmatch(t.Value)
|
||||
if match == nil {
|
||||
return "", false
|
||||
}
|
||||
return match[1], true
|
||||
}
|
||||
|
||||
// usable reports whether the term can be searched for: a buyer ID that
|
||||
// isn't blank, or any other term with at least one word.
|
||||
func (t Term) usable() bool {
|
||||
if t.Field == FieldBuyerID {
|
||||
return strings.TrimSpace(t.Value) != ""
|
||||
}
|
||||
return len(t.Words()) > 0
|
||||
}
|
||||
|
||||
// Query is a parsed search.
|
||||
type Query struct {
|
||||
// Terms are the terms to search for, all usable.
|
||||
Terms []Term
|
||||
// UnknownPrompts are the prompts that aren't known, as typed, without
|
||||
// their colon, once each. Their terms are left out of Terms.
|
||||
UnknownPrompts []string
|
||||
}
|
||||
|
||||
// MatchesAll reports whether the query has no terms, so it matches every
|
||||
// opportunity.
|
||||
func (q Query) MatchesAll() bool {
|
||||
return len(q.Terms) == 0
|
||||
}
|
||||
|
||||
// Parse parses text as a search. It never fails: text that isn't valid
|
||||
// syntax, such as an unclosed quote, is read as plain words, and a term
|
||||
// with no words (title: or !!!) is left out.
|
||||
func Parse(text string) Query {
|
||||
query := Query{
|
||||
UnknownPrompts: []string{},
|
||||
}
|
||||
unknown := map[string]bool{}
|
||||
|
||||
for rest := strings.TrimSpace(text); rest != ""; {
|
||||
var term Term
|
||||
var prompt string
|
||||
var known bool
|
||||
term, prompt, known, rest = nextTerm(rest)
|
||||
rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
|
||||
|
||||
switch {
|
||||
case !known:
|
||||
if !unknown[prompt] {
|
||||
unknown[prompt] = true
|
||||
query.UnknownPrompts = append(query.UnknownPrompts, prompt)
|
||||
}
|
||||
case term.usable():
|
||||
query.Terms = append(query.Terms, term)
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// nextTerm reads the term at the start of text, which mustn't start with
|
||||
// space. It returns the term, its prompt as typed ("" if none), whether the
|
||||
// prompt is known, and the text after the term.
|
||||
func nextTerm(
|
||||
text string,
|
||||
) (term Term, prompt string, known bool, rest string) {
|
||||
if strings.HasPrefix(text, "!") {
|
||||
term.Negated = true
|
||||
text = text[1:]
|
||||
}
|
||||
|
||||
if match := promptPattern.FindStringSubmatch(text); match != nil {
|
||||
prompt = match[1]
|
||||
text = text[len(match[0]):]
|
||||
}
|
||||
term.Field, known = prompts[strings.ToLower(prompt)]
|
||||
if prompt == "" {
|
||||
term.Field, known = FieldText, true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(text, `"`) {
|
||||
term.Value, rest, _ = strings.Cut(text[1:], `"`)
|
||||
return term, prompt, known, rest
|
||||
}
|
||||
|
||||
end := strings.IndexFunc(text, unicode.IsSpace)
|
||||
if end < 0 {
|
||||
end = len(text)
|
||||
}
|
||||
term.Value = text[:end]
|
||||
|
||||
return term, prompt, known, text[end:]
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
term := func(field Field, value string) Term {
|
||||
return Term{
|
||||
Field: field,
|
||||
Value: value,
|
||||
}
|
||||
}
|
||||
text := func(value string) Term {
|
||||
return term(FieldText, value)
|
||||
}
|
||||
not := func(term Term) Term {
|
||||
term.Negated = true
|
||||
return term
|
||||
}
|
||||
none := []string{}
|
||||
cases := []struct {
|
||||
input string
|
||||
terms []Term
|
||||
unknown []string
|
||||
}{
|
||||
{"", nil, none},
|
||||
{" ", nil, none},
|
||||
{"cloud hosting", []Term{text("cloud"), text("hosting")}, none},
|
||||
{`"road repairs" leeds`, []Term{text("road repairs"), text("leeds")}, none},
|
||||
{`title:"road repairs"`, []Term{term(FieldTitle, "road repairs")}, none},
|
||||
{"TITLE:road Buyer:leeds", []Term{term(FieldTitle, "road"), term(FieldBuyer, "leeds")}, none},
|
||||
{"buyer_id:GB-1 cpv:72", []Term{term(FieldBuyerID, "GB-1"), term(FieldCPV, "72")}, none},
|
||||
{"!cloud !cpv:food", []Term{not(text("cloud")), not(term(FieldCPV, "food"))}, none},
|
||||
{`!title:"a b"`, []Term{not(term(FieldTitle, "a b"))}, none},
|
||||
{"12:00", []Term{text("12:00")}, none},
|
||||
{`"unclosed phrase`, []Term{text("unclosed phrase")}, none},
|
||||
{`title: !!! "" ! cloud`, []Term{text("cloud")}, none},
|
||||
{"foo:bar cloud Foo:baz foo:x baz:", []Term{text("cloud")}, []string{"foo", "Foo", "baz"}},
|
||||
{"buyer_id:", nil, none},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := Parse(tc.input)
|
||||
if !reflect.DeepEqual(got.Terms, tc.terms) {
|
||||
t.Errorf("%q: terms %+v, want %+v", tc.input, got.Terms, tc.terms)
|
||||
}
|
||||
if !slices.Equal(got.UnknownPrompts, tc.unknown) {
|
||||
t.Errorf(
|
||||
"%q: unknown %q, want %q",
|
||||
tc.input, got.UnknownPrompts, tc.unknown,
|
||||
)
|
||||
}
|
||||
if got.UnknownPrompts == nil {
|
||||
t.Errorf("%q: UnknownPrompts is nil", tc.input)
|
||||
}
|
||||
if got.MatchesAll() != (len(tc.terms) == 0) {
|
||||
t.Errorf("%q: MatchesAll %v", tc.input, got.MatchesAll())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTermWordsAndCPVCode(t *testing.T) {
|
||||
cases := []struct {
|
||||
value string
|
||||
words []string
|
||||
code string
|
||||
isCPV bool
|
||||
}{
|
||||
{"road-repairs", []string{"road", "repairs"}, "", false},
|
||||
{"Café's", []string{"Café", "s"}, "", false},
|
||||
{"72", []string{"72"}, "72", true},
|
||||
{"72210000-5", []string{"72210000", "5"}, "72210000", true},
|
||||
{"72-55", []string{"72", "55"}, "", false},
|
||||
{"72a", []string{"72a"}, "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
term := Term{
|
||||
Field: FieldCPV,
|
||||
Value: tc.value,
|
||||
}
|
||||
if got := term.Words(); !slices.Equal(got, tc.words) {
|
||||
t.Errorf("%q: words %q, want %q", tc.value, got, tc.words)
|
||||
}
|
||||
code, ok := term.CPVCode()
|
||||
if code != tc.code || ok != tc.isCPV {
|
||||
t.Errorf(
|
||||
"%q: CPVCode %q %v, want %q %v",
|
||||
tc.value, code, ok, tc.code, tc.isCPV,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every known prompt has help, and every prompt with help is known.
|
||||
func TestPromptsHaveHelp(t *testing.T) {
|
||||
if len(Prompts) != len(prompts) {
|
||||
t.Errorf("%d prompts with help, %d known", len(Prompts), len(prompts))
|
||||
}
|
||||
for _, prompt := range Prompts {
|
||||
if _, ok := prompts[prompt.Name]; !ok || prompt.Help == "" {
|
||||
t.Errorf("prompt %q: known %v, help %q", prompt.Name, ok, prompt.Help)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ A SQLite store for opportunities and their OCDS releases, built on sqlx + goose
|
||||
|
||||
## Build and test
|
||||
|
||||
The title search uses SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
|
||||
The title and general searches use SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
|
||||
|
||||
```bash
|
||||
go test -tags sqlite_fts5 ./...
|
||||
@@ -24,7 +24,9 @@ internal/store/migrate_test.go version checks, stepping up and down, failed-
|
||||
internal/store/queries.go all SQL, as named constants grouped by table
|
||||
internal/store/queries_test.go prepares every query against the migrated schema
|
||||
internal/store/store.go Store: logic, using the queries via sqlx
|
||||
internal/store/search.go SearchOpportunities: the general search's query
|
||||
internal/store/store_test.go behavioural tests
|
||||
internal/store/search_test.go behavioural tests of the general search
|
||||
internal/store/common.go common for migrate, store and direct access
|
||||
```
|
||||
|
||||
@@ -36,15 +38,16 @@ internal/store/common.go common for migrate, store and direct access
|
||||
| `opportunity_marks` | `uuid` | `uuid` references `opportunities`. The user's own marks (`hidden`, `skipped`, `interested`, `bid_submitted`, `won`, `lost`), each `BOOLEAN NOT NULL DEFAULT 0`. A `CHECK` constraint stops `won` and `lost` both being set. |
|
||||
| `opportunity_releases` | (`uuid`, `release_id`) | Write-once. `uuid` references `opportunities`. `data` is TEXT, so `json_extract` works on it. |
|
||||
| `opportunity_release_tags` | (`uuid`, `release_id`, `tag`) | Write-once. (`uuid`, `release_id`) references `opportunity_releases`. |
|
||||
| `classification_codes` | `id` | Shared between opportunities. A row is reused only when every content column matches. The table grows over time to include codes not already seen in fetched opportunities. |
|
||||
| `classification_codes` | `id` | Shared between opportunities. A CPV code has one row (unique on `code` where `is_cpv`), keeping the longest description it has been given: EU Supply cuts descriptions to 64 characters. A code of any other scheme gets a row per distinct content. The table grows over time to include codes not already seen in fetched opportunities. |
|
||||
| `opportunity_classifications` | (`opportunity_uuid`, `classification_id`) | Mapping table; replaced as a set when the opportunity is updated. |
|
||||
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, kept in sync by triggers. |
|
||||
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, `description` and `buyer_name`, kept in sync by triggers. |
|
||||
| `classification_codes_fts` | rowid = `classification_codes.id` | FTS5 index on `classification_codes.classification` (the description), kept in sync by triggers. |
|
||||
| `opportunity_change_batches` | `id`; `hash` (unique) | Write-once. One row per save that adds opportunities or gives them new releases, and per change of marks. `hash` names the batch to clients (random, so unguessable); `created_at` orders the batches. |
|
||||
| `opportunity_change_batch_opportunities` | (`batch_id`, `opportunity_uuid`) | The opportunities in each batch. References both tables, cascading on delete. |
|
||||
|
||||
Foreign keys are enforced (`_foreign_keys=on` in the connection string) and cascade on delete from `opportunities`. The connection also enables WAL mode, a 5s busy timeout, and `_txlock=immediate`, so a save takes the write lock up front rather than failing halfway through.
|
||||
|
||||
**When adding a classification scheme flag** (e.g. `is_unspsc`), the migration adding the column must drop and recreate the `classification_codes_content` unique index with the new column included. Otherwise rows differing only by that flag would be merged.
|
||||
**When adding a classification scheme flag** (e.g. `is_unspsc`), the migration adding the column must drop and recreate the `classification_codes_content` unique index with the new column included. Otherwise rows differing only by that flag would be merged. A scheme whose codes have one official description, like CPV, can instead have a partial unique index on `code` (as `classification_codes_cpv` does), with its own `ON CONFLICT` clause in `upsertClassificationCodeReturningID`.
|
||||
|
||||
**internal/types**: The structs contain `db` tags to allow sqlx to map table columns to struct members. Any struct members not mapped to a column (including child slices) are referenced by `db:"-"`.
|
||||
|
||||
@@ -71,6 +74,7 @@ All SQL is in `queries.go` as constants, grouped by table in migration order; `s
|
||||
|
||||
- **TestQueriesPrepare** prepares every query constant against a freshly migrated database, so a syntax error, an unknown table or unknown column fails the test and names the failing statement.
|
||||
- **TestQueriesAllListed** parses `queries.go` and fails if a constant isn't listed in that test's `checks` map. Adding a query therefore means adding one line there, and it can't be skipped by accident.
|
||||
- **TestSearchOpportunitiesQueryAllTerms** does the same for `SearchOpportunities`, with a term of every kind, plain and negated.
|
||||
- **TestListOpportunitiesQueryAllFilters** builds the `ListOpportunities` query with every filter set and runs it, checking that the fragments combine and that the arguments line up with the placeholders.
|
||||
- **TestMarksCheckConstraint** checks the database itself rejects `won` and `lost` set together, for writes that bypass the Go validation.
|
||||
|
||||
@@ -111,6 +115,7 @@ ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
||||
rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags
|
||||
batch, err := s.LatestChangeBatch(ctx) // hash of the newest change batch, or ""
|
||||
ts, batch, err = s.ListChangedOpportunities(ctx, batch, time.Time{}) // changed since; ErrUnknownBatch
|
||||
uuids, err := s.SearchOpportunities(ctx, search.Parse("cloud buyer:leeds"), releasedFrom) // never nil
|
||||
ok, err := s.ReleaseExists(ctx, uuid, releaseID)
|
||||
```
|
||||
|
||||
@@ -132,3 +137,9 @@ ok, err := s.ReleaseExists(ctx, uuid, releaseID)
|
||||
|
||||
**Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted.
|
||||
|
||||
**General search.** `SearchOpportunities` runs a `search.Query` (parsed by `internal/search`, which documents the syntax) and returns only the matching UUIDs, for the frontend to filter its own list by. It is assembled by `searchOpportunitiesQuery` from the `searchOpportunity*` and `searchCPV*` fragments:
|
||||
|
||||
- Words without a prompt search `{title description}` in `opportunities_fts`, and must all match. `title:` and `buyer:` search the `title` and `buyer_name` columns; repeats of one prompt are joined by OR into one clause. Every negated full-text term goes into one `NOT IN` clause, joined by OR. Each term is one FTS5 phrase built from `Term.Words`, its last word a prefix, so typed input is never FTS5 syntax.
|
||||
- `buyer_id:` matches the whole ID, ignoring case (`IN`, expanded by `sqlx.In`).
|
||||
- CPV terms are checked against each code, inside one `EXISTS` on the opportunity's CPV codes: a code must match a `cpv:` term (a code prefix by `LIKE`, or words in `classification_codes_fts`) and no `!cpv:` term. With only `!cpv:` terms the clause is `NOT EXISTS` a code matching any of them, so an opportunity with no CPV codes passes it.
|
||||
- `OpportunityFilter.Title` searches only the `title` column of the same index.
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -327,3 +328,142 @@ func TestFailedMigrationIsRolledBack(t *testing.T) {
|
||||
t.Errorf("failed migration left %d objects behind", indexes)
|
||||
}
|
||||
}
|
||||
|
||||
// Migration 10 merges each CPV code's rows into the one with the longest
|
||||
// description, moving the links to it (one link if an opportunity had both),
|
||||
// and leaves other schemes' rows alone.
|
||||
func TestMigration10MergesCPVDuplicates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, db := newMigrator(t)
|
||||
if _, err := m.UpTo(ctx, 9); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const full = "Petroleum products, fuel, electricity and other sources of energy"
|
||||
setup := []string{
|
||||
`INSERT INTO opportunities (uuid, uuid_type, title, status, latest_release_date)
|
||||
VALUES ('a', 'ocid', 'A', 'active', '2026-10-01'),
|
||||
('b', 'ocid', 'B', 'active', '2026-10-01')`,
|
||||
`INSERT INTO classification_codes (id, is_cpv, code, classification) VALUES
|
||||
(1, 1, '09000000', '` + full + `'),
|
||||
(2, 1, '09000000', '` + full[:64] + `'),
|
||||
(3, 0, '123', 'Scheme X'),
|
||||
(4, 0, '123', 'Scheme X, amended')`,
|
||||
// a links to both CPV rows, b only to the cut one.
|
||||
`INSERT INTO opportunity_classifications VALUES
|
||||
('a', 1), ('a', 2), ('b', 2), ('a', 3), ('b', 4)`,
|
||||
}
|
||||
for _, statement := range setup {
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := m.UpTo(ctx, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var codes []string
|
||||
rows, err := db.QueryContext(ctx,
|
||||
`SELECT id || ':' || classification FROM classification_codes ORDER BY id`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var code string
|
||||
rows.Scan(&code)
|
||||
codes = append(codes, code)
|
||||
}
|
||||
rows.Close()
|
||||
want := "1:" + full + "|3:Scheme X|4:Scheme X, amended"
|
||||
if got := strings.Join(codes, "|"); got != want {
|
||||
t.Errorf("codes = %s; want %s", got, want)
|
||||
}
|
||||
|
||||
var links []string
|
||||
rows, err = db.QueryContext(ctx, `SELECT opportunity_uuid || classification_id
|
||||
FROM opportunity_classifications ORDER BY 1`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var link string
|
||||
rows.Scan(&link)
|
||||
links = append(links, link)
|
||||
}
|
||||
rows.Close()
|
||||
if got := strings.Join(links, ","); got != "a1,a3,b1,b4" {
|
||||
t.Errorf("links = %s; want a1,a3,b1,b4", got)
|
||||
}
|
||||
|
||||
// A second row for the code is now refused.
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO classification_codes
|
||||
(is_cpv, code, classification) VALUES (1, '09000000', 'Other')`)
|
||||
if err == nil {
|
||||
t.Error("a second row for a CPV code was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Migration 11 indexes the description, buyer name and classification
|
||||
// descriptions of rows already stored, and its Down leaves the title
|
||||
// index working.
|
||||
func TestMigration11IndexesStoredRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, db := newMigrator(t)
|
||||
if _, err := m.UpTo(ctx, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
setup := []string{
|
||||
`INSERT INTO opportunities (uuid, uuid_type, title, status,
|
||||
description, buyer_name, latest_release_date)
|
||||
VALUES ('a', 'ocid', 'Cloud', 'active', 'Servers', 'Leeds',
|
||||
'2026-10-01')`,
|
||||
`INSERT INTO classification_codes (id, is_cpv, code, classification)
|
||||
VALUES (1, 1, '15800000', 'Miscellaneous food products')`,
|
||||
}
|
||||
for _, statement := range setup {
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := m.UpTo(ctx, 11); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := func(query string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := db.QueryRowContext(ctx, query).Scan(&n); err != nil {
|
||||
t.Fatalf("%s: %v", query, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
matches := []struct {
|
||||
table, match string
|
||||
want int
|
||||
}{
|
||||
{"opportunities_fts", "description : servers", 1},
|
||||
{"opportunities_fts", "buyer_name : leeds", 1},
|
||||
{"opportunities_fts", "title : servers", 0},
|
||||
{"classification_codes_fts", "food", 1},
|
||||
}
|
||||
for _, tc := range matches {
|
||||
got := count(fmt.Sprintf(
|
||||
"SELECT count(*) FROM %s WHERE %[1]s MATCH '%s'", tc.table, tc.match,
|
||||
))
|
||||
if got != tc.want {
|
||||
t.Errorf("%s MATCH %q: %d; want %d", tc.table, tc.match, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := m.DownTo(ctx, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := count(`SELECT count(*) FROM opportunities_fts
|
||||
WHERE opportunities_fts MATCH 'cloud'`)
|
||||
if got != 1 {
|
||||
t.Errorf("title search after Down: %d; want 1", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- A CPV code has one official description, so it is now one row whatever
|
||||
-- description it arrives with: EU Supply cuts descriptions to 64 characters,
|
||||
-- which made a second row for each code it sent. The duplicates are merged
|
||||
-- into the row with the longest description, and the upsert keeps the
|
||||
-- longest from now on. Other schemes still get a row per distinct content.
|
||||
|
||||
-- The row each CPV row merges into.
|
||||
CREATE TEMP TABLE "cpv_merge" AS
|
||||
SELECT
|
||||
"c"."id" AS "id",
|
||||
(
|
||||
SELECT "k"."id" FROM "classification_codes" "k"
|
||||
WHERE "k"."is_cpv" AND "k"."code" = "c"."code"
|
||||
ORDER BY length("k"."classification") DESC, "k"."id"
|
||||
LIMIT 1
|
||||
) AS "keep_id"
|
||||
FROM "classification_codes" "c"
|
||||
WHERE "c"."is_cpv";
|
||||
|
||||
-- An opportunity linked to both rows keeps only its link to the kept one.
|
||||
DELETE FROM "opportunity_classifications"
|
||||
WHERE "classification_id" IN (
|
||||
SELECT "id" FROM "cpv_merge" WHERE "id" <> "keep_id"
|
||||
)
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM "cpv_merge" "m"
|
||||
JOIN "opportunity_classifications" "kept"
|
||||
ON "kept"."classification_id" = "m"."keep_id"
|
||||
WHERE "m"."id" = "opportunity_classifications"."classification_id"
|
||||
AND "kept"."opportunity_uuid"
|
||||
= "opportunity_classifications"."opportunity_uuid"
|
||||
);
|
||||
|
||||
UPDATE "opportunity_classifications"
|
||||
SET "classification_id" = (
|
||||
SELECT "keep_id" FROM "cpv_merge"
|
||||
WHERE "id" = "opportunity_classifications"."classification_id"
|
||||
)
|
||||
WHERE "classification_id" IN (
|
||||
SELECT "id" FROM "cpv_merge" WHERE "id" <> "keep_id"
|
||||
);
|
||||
|
||||
DELETE FROM "classification_codes"
|
||||
WHERE "id" IN (SELECT "id" FROM "cpv_merge" WHERE "id" <> "keep_id");
|
||||
|
||||
DROP TABLE "cpv_merge";
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "classification_codes_cpv"
|
||||
ON "classification_codes" ("code") WHERE "is_cpv";
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- The merged duplicates aren't restored: they only differed by a cut-off
|
||||
-- description.
|
||||
DROP INDEX IF EXISTS "classification_codes_cpv";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,119 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- The general search: opportunities_fts (migration 00006) now also indexes
|
||||
-- the description and the buyer's name, and classification_codes_fts
|
||||
-- indexes classification descriptions. Both are kept in sync by triggers.
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_update";
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
|
||||
DROP TABLE IF EXISTS "opportunities_fts";
|
||||
|
||||
CREATE VIRTUAL TABLE "opportunities_fts" USING fts5(
|
||||
"title",
|
||||
"description",
|
||||
"buyer_name",
|
||||
content = 'opportunities',
|
||||
content_rowid = 'id',
|
||||
tokenize = 'porter unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name")
|
||||
VALUES (new."id", new."title", new."description", new."buyer_name");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts"
|
||||
("opportunities_fts", "rowid", "title", "description", "buyer_name")
|
||||
VALUES ('delete', old."id", old."title", old."description", old."buyer_name");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER "opportunities_fts_update"
|
||||
AFTER UPDATE OF "title", "description", "buyer_name" ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts"
|
||||
("opportunities_fts", "rowid", "title", "description", "buyer_name")
|
||||
VALUES ('delete', old."id", old."title", old."description", old."buyer_name");
|
||||
INSERT INTO "opportunities_fts" ("rowid", "title", "description", "buyer_name")
|
||||
VALUES (new."id", new."title", new."description", new."buyer_name");
|
||||
END;
|
||||
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
|
||||
|
||||
CREATE VIRTUAL TABLE "classification_codes_fts" USING fts5(
|
||||
"classification",
|
||||
content = 'classification_codes',
|
||||
content_rowid = 'id',
|
||||
tokenize = 'porter unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TRIGGER "classification_codes_fts_insert"
|
||||
AFTER INSERT ON "classification_codes"
|
||||
BEGIN
|
||||
INSERT INTO "classification_codes_fts" ("rowid", "classification")
|
||||
VALUES (new."id", new."classification");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER "classification_codes_fts_delete"
|
||||
AFTER DELETE ON "classification_codes"
|
||||
BEGIN
|
||||
INSERT INTO "classification_codes_fts"
|
||||
("classification_codes_fts", "rowid", "classification")
|
||||
VALUES ('delete', old."id", old."classification");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER "classification_codes_fts_update"
|
||||
AFTER UPDATE OF "classification" ON "classification_codes"
|
||||
BEGIN
|
||||
INSERT INTO "classification_codes_fts"
|
||||
("classification_codes_fts", "rowid", "classification")
|
||||
VALUES ('delete', old."id", old."classification");
|
||||
INSERT INTO "classification_codes_fts" ("rowid", "classification")
|
||||
VALUES (new."id", new."classification");
|
||||
END;
|
||||
|
||||
INSERT INTO "classification_codes_fts" ("classification_codes_fts")
|
||||
VALUES ('rebuild');
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TRIGGER IF EXISTS "classification_codes_fts_update";
|
||||
DROP TRIGGER IF EXISTS "classification_codes_fts_delete";
|
||||
DROP TRIGGER IF EXISTS "classification_codes_fts_insert";
|
||||
DROP TABLE IF EXISTS "classification_codes_fts";
|
||||
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_update";
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
|
||||
DROP TABLE IF EXISTS "opportunities_fts";
|
||||
|
||||
CREATE VIRTUAL TABLE "opportunities_fts" USING fts5(
|
||||
"title",
|
||||
content = 'opportunities',
|
||||
content_rowid = 'id',
|
||||
tokenize = 'porter unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TRIGGER "opportunities_fts_insert" AFTER INSERT ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER "opportunities_fts_delete" AFTER DELETE ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
|
||||
VALUES ('delete', old."id", old."title");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER "opportunities_fts_update" AFTER UPDATE OF "title" ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
|
||||
VALUES ('delete', old."id", old."title");
|
||||
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
|
||||
END;
|
||||
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
|
||||
-- +goose StatementEnd
|
||||
@@ -107,7 +107,8 @@ LIMIT ? OFFSET ?`
|
||||
)`
|
||||
|
||||
// filterOpportunityTitleMatch matches a full-text search of the title. The
|
||||
// argument must come from ftsQuery, never raw user input.
|
||||
// argument must be ftsQuery's, limited to the title column (see
|
||||
// listOpportunitiesQuery), never raw user input.
|
||||
filterOpportunityTitleMatch = `t.id IN (
|
||||
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
|
||||
)`
|
||||
@@ -150,6 +151,52 @@ LIMIT ? OFFSET ?`
|
||||
)`
|
||||
)
|
||||
|
||||
// SearchOpportunities is assembled by searchOpportunitiesQuery as:
|
||||
//
|
||||
// searchOpportunitiesSelect
|
||||
// [WHERE clause AND clause ...]
|
||||
//
|
||||
// Each clause is a searchOpportunity* fragment, perhaps with NOT in front,
|
||||
// or filterOpportunityReleasedFrom. searchOpportunityCPV's %s is filled
|
||||
// with searchCPV* fragments joined by OR, AND and NOT. A full-text argument
|
||||
// must be built from search.Term.Words, never raw user input.
|
||||
const (
|
||||
// searchOpportunitiesSelect selects every opportunity's UUID, as t.
|
||||
searchOpportunitiesSelect = `
|
||||
SELECT t.uuid FROM opportunities t`
|
||||
|
||||
// searchOpportunityText matches a full-text search of the title,
|
||||
// description and buyer_name columns.
|
||||
searchOpportunityText = `t.id IN (
|
||||
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
|
||||
)`
|
||||
|
||||
// searchOpportunityBuyerID matches any of several buyer IDs, ignoring
|
||||
// case. "IN (?)" is expanded by sqlx.In, which must be followed by
|
||||
// Rebind.
|
||||
searchOpportunityBuyerID = `t.buyer_id COLLATE NOCASE IN (?)`
|
||||
|
||||
// searchOpportunityCPV matches an opportunity with a CPV code meeting
|
||||
// the condition put in place of %s, which reads the code as c.
|
||||
searchOpportunityCPV = `EXISTS (
|
||||
SELECT 1
|
||||
FROM opportunity_classifications tc
|
||||
JOIN classification_codes c ON c.id = tc.classification_id
|
||||
WHERE tc.opportunity_uuid = t.uuid AND c.is_cpv AND %s
|
||||
)`
|
||||
|
||||
// searchCPVCodePrefix matches a code by a LIKE pattern: its first
|
||||
// digits followed by %.
|
||||
searchCPVCodePrefix = `c.code LIKE ?`
|
||||
|
||||
// searchCPVDescription matches a full-text search of the code's
|
||||
// description.
|
||||
searchCPVDescription = `c.id IN (
|
||||
SELECT rowid FROM classification_codes_fts
|
||||
WHERE classification_codes_fts MATCH ?
|
||||
)`
|
||||
)
|
||||
|
||||
// selectOtherSourcesTitles lists the uuid, title and latest_release_date of
|
||||
// the opportunities not from one source whose latest_release_date is in
|
||||
// [from, before); it walks the latest_release_date index.
|
||||
@@ -197,15 +244,24 @@ ORDER BY release_id, tag`
|
||||
|
||||
// ---------- classification_codes ----------
|
||||
|
||||
// upsertClassificationCodeReturningID returns the id of the row matching
|
||||
// every content column, inserting it first if needed. Named parameters
|
||||
// upsertClassificationCodeReturningID returns the id of the row for a
|
||||
// classification, inserting it first if needed. Named parameters
|
||||
// (types.ClassificationCode).
|
||||
//
|
||||
// The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING yield the
|
||||
// existing row's id on conflict.
|
||||
// A CPV code has one row whatever its description (migration 00010): the
|
||||
// row keeps the longest description it has been given, as EU Supply cuts
|
||||
// them to 64 characters. Any other scheme's row matches every content
|
||||
// column. The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING
|
||||
// yield the existing row's id on conflict.
|
||||
const upsertClassificationCodeReturningID = `
|
||||
INSERT INTO classification_codes (is_cpv, code, classification)
|
||||
VALUES (:is_cpv, :code, :classification)
|
||||
ON CONFLICT (code) WHERE is_cpv
|
||||
DO UPDATE SET classification = CASE
|
||||
WHEN length(excluded.classification) > length(classification)
|
||||
THEN excluded.classification
|
||||
ELSE classification
|
||||
END
|
||||
ON CONFLICT (is_cpv, code, classification)
|
||||
DO UPDATE SET is_cpv = excluded.is_cpv
|
||||
RETURNING id`
|
||||
|
||||
@@ -2,6 +2,7 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
@@ -11,6 +12,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/search"
|
||||
)
|
||||
|
||||
// Run with: go test -tags sqlite_fts5 ./...
|
||||
@@ -81,6 +84,24 @@ var checks = map[string]queryCheck{
|
||||
sql: withFilter(filterOpportunityMarkLost),
|
||||
},
|
||||
|
||||
"searchOpportunitiesSelect": {sql: searchOpportunitiesSelect},
|
||||
"searchOpportunityText": {
|
||||
sql: withSearch(searchOpportunityText),
|
||||
},
|
||||
"searchOpportunityBuyerID": {
|
||||
sql: withSearch(searchOpportunityBuyerID),
|
||||
in: true,
|
||||
},
|
||||
"searchOpportunityCPV": {
|
||||
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, "1")),
|
||||
},
|
||||
"searchCPVCodePrefix": {
|
||||
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVCodePrefix)),
|
||||
},
|
||||
"searchCPVDescription": {
|
||||
sql: withSearch(fmt.Sprintf(searchOpportunityCPV, searchCPVDescription)),
|
||||
},
|
||||
|
||||
// opportunity_ocds_releases
|
||||
"insertRelease": {sql: insertRelease, named: true},
|
||||
"selectReleasesByOpportunity": {sql: selectReleasesByOpportunity},
|
||||
@@ -140,6 +161,10 @@ func withFilter(fragment string) string {
|
||||
listOpportunitiesOrderAndPage
|
||||
}
|
||||
|
||||
func withSearch(fragment string) string {
|
||||
return searchOpportunitiesSelect + "\nWHERE " + fragment
|
||||
}
|
||||
|
||||
func openTestDB(t *testing.T) *sqlx.DB {
|
||||
t.Helper()
|
||||
s, err := Open(
|
||||
@@ -223,6 +248,28 @@ func TestListOpportunitiesQueryAllFilters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchOpportunitiesQueryAllTerms runs the assembled search query with
|
||||
// a term of every kind, both plain and negated, checking the fragments
|
||||
// combine and the arguments line up with the placeholders.
|
||||
func TestSearchOpportunitiesQueryAllTerms(t *testing.T) {
|
||||
db := openTestDB(t)
|
||||
query := search.Parse(`cloud "road repairs" title:a title:b buyer:c ` +
|
||||
`buyer_id:GB-1 buyer_id:GB-2 cpv:72 cpv:food !cloud !title:d ` +
|
||||
`!buyer:e !buyer_id:GB-3 !cpv:7221 !cpv:fuel`)
|
||||
statement, args := searchOpportunitiesQuery(
|
||||
query, time.Now().Add(-time.Hour),
|
||||
)
|
||||
statement, args, err := sqlx.In(statement, args...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var uuids []string
|
||||
if err := db.Select(&uuids, db.Rebind(statement), args...); err != nil {
|
||||
t.Errorf("run with %d args:\n%s\n%v", len(args), statement, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueriesAllListed fails if queries.go declares a constant that checks
|
||||
// doesn't list, so every new statement is prepared by TestQueriesPrepare.
|
||||
func TestQueriesAllListed(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/search"
|
||||
)
|
||||
|
||||
// ftsColumns are the opportunities_fts columns each field's terms search,
|
||||
// as an FTS5 column filter.
|
||||
var ftsColumns = map[search.Field]string{
|
||||
search.FieldText: "{title description}",
|
||||
search.FieldTitle: "title",
|
||||
search.FieldBuyer: "buyer_name",
|
||||
}
|
||||
|
||||
// anyOfFields are the fields whose terms match if any of them does, in the
|
||||
// order their clauses are added.
|
||||
var anyOfFields = []search.Field{search.FieldTitle, search.FieldBuyer}
|
||||
|
||||
// searchClauses holds a search's terms, sorted by how they combine.
|
||||
type searchClauses struct {
|
||||
// required are full-text matches (words without a prompt) that must
|
||||
// all match.
|
||||
required []string
|
||||
// anyOf are full-text matches by field, of which one must match.
|
||||
anyOf map[search.Field][]string
|
||||
// excluded are full-text matches of which none may match.
|
||||
excluded []string
|
||||
|
||||
buyerIDs []string
|
||||
excludedBuyerIDs []string
|
||||
|
||||
// cpv and excludedCPV are conditions on one CPV code (searchCPV*
|
||||
// fragments), with their arguments.
|
||||
cpv []string
|
||||
cpvArgs []any
|
||||
excludedCPV []string
|
||||
excludedCPVArgs []any
|
||||
}
|
||||
|
||||
// SearchOpportunities returns the UUIDs of the opportunities matching
|
||||
// query, in no particular order and never nil. Unless releasedFrom is zero,
|
||||
// only opportunities whose latest release is on or after it are returned
|
||||
// (as OpportunityFilter.ReleasedFrom). A query with no terms matches every
|
||||
// opportunity.
|
||||
func (s *Store) SearchOpportunities(
|
||||
ctx context.Context, query search.Query, releasedFrom time.Time,
|
||||
) ([]string, error) {
|
||||
statement, args := searchOpportunitiesQuery(query, releasedFrom)
|
||||
statement, args, err := sqlx.In(statement, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("expanding the search query: %w", err)
|
||||
}
|
||||
|
||||
uuids := []string{}
|
||||
err = s.db.SelectContext(ctx, &uuids, s.db.Rebind(statement), args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("searching opportunities: %w", err)
|
||||
}
|
||||
|
||||
return uuids, nil
|
||||
}
|
||||
|
||||
// searchOpportunitiesQuery assembles the SearchOpportunities query from the
|
||||
// fragments in queries.go, with its arguments. It must be expanded by
|
||||
// sqlx.In.
|
||||
func searchOpportunitiesQuery(
|
||||
query search.Query, releasedFrom time.Time,
|
||||
) (string, []any) {
|
||||
clauses := sortTerms(query.Terms)
|
||||
var where []string
|
||||
var args []any
|
||||
add := func(clause string, arguments ...any) {
|
||||
where = append(where, clause)
|
||||
args = append(args, arguments...)
|
||||
}
|
||||
|
||||
if len(clauses.required) > 0 {
|
||||
add(searchOpportunityText, strings.Join(clauses.required, " AND "))
|
||||
}
|
||||
for _, field := range anyOfFields {
|
||||
if matches := clauses.anyOf[field]; len(matches) > 0 {
|
||||
add(searchOpportunityText, strings.Join(matches, " OR "))
|
||||
}
|
||||
}
|
||||
if len(clauses.excluded) > 0 {
|
||||
add(
|
||||
"NOT "+searchOpportunityText,
|
||||
strings.Join(clauses.excluded, " OR "),
|
||||
)
|
||||
}
|
||||
|
||||
if len(clauses.buyerIDs) > 0 {
|
||||
add(searchOpportunityBuyerID, clauses.buyerIDs)
|
||||
}
|
||||
if len(clauses.excludedBuyerIDs) > 0 {
|
||||
add("NOT "+searchOpportunityBuyerID, clauses.excludedBuyerIDs)
|
||||
}
|
||||
|
||||
if clause, arguments := clauses.cpvClause(); clause != "" {
|
||||
add(clause, arguments...)
|
||||
}
|
||||
|
||||
if !releasedFrom.IsZero() {
|
||||
add(filterOpportunityReleasedFrom, releasedFrom.UTC())
|
||||
}
|
||||
|
||||
statement := searchOpportunitiesSelect
|
||||
if len(where) > 0 {
|
||||
statement += "\nWHERE " + strings.Join(where, "\n\tAND ")
|
||||
}
|
||||
|
||||
return statement, args
|
||||
}
|
||||
|
||||
// sortTerms sorts terms by how they combine into the query.
|
||||
func sortTerms(terms []search.Term) searchClauses {
|
||||
clauses := searchClauses{
|
||||
anyOf: map[search.Field][]string{},
|
||||
}
|
||||
for _, term := range terms {
|
||||
switch {
|
||||
case term.Field == search.FieldBuyerID && term.Negated:
|
||||
clauses.excludedBuyerIDs = append(
|
||||
clauses.excludedBuyerIDs, term.Value,
|
||||
)
|
||||
case term.Field == search.FieldBuyerID:
|
||||
clauses.buyerIDs = append(clauses.buyerIDs, term.Value)
|
||||
case term.Field == search.FieldCPV && term.Negated:
|
||||
condition, argument := cpvCondition(term)
|
||||
clauses.excludedCPV = append(clauses.excludedCPV, condition)
|
||||
clauses.excludedCPVArgs = append(
|
||||
clauses.excludedCPVArgs, argument,
|
||||
)
|
||||
case term.Field == search.FieldCPV:
|
||||
condition, argument := cpvCondition(term)
|
||||
clauses.cpv = append(clauses.cpv, condition)
|
||||
clauses.cpvArgs = append(clauses.cpvArgs, argument)
|
||||
case term.Negated:
|
||||
clauses.excluded = append(clauses.excluded, textMatch(term))
|
||||
case term.Field == search.FieldText:
|
||||
clauses.required = append(clauses.required, textMatch(term))
|
||||
default:
|
||||
clauses.anyOf[term.Field] = append(
|
||||
clauses.anyOf[term.Field], textMatch(term),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return clauses
|
||||
}
|
||||
|
||||
// cpvClause returns the clause for the CPV terms, with its arguments, or ""
|
||||
// if there are none. With cpv: terms, an opportunity needs a code that one
|
||||
// of them matches and no !cpv: term does; with only !cpv: terms, it needs
|
||||
// no code that any of them matches.
|
||||
func (c searchClauses) cpvClause() (string, []any) {
|
||||
excluded := "(" + strings.Join(c.excludedCPV, " OR ") + ")"
|
||||
switch {
|
||||
case len(c.cpv) > 0 && len(c.excludedCPV) > 0:
|
||||
condition := "(" + strings.Join(c.cpv, " OR ") + ")" +
|
||||
" AND NOT " + excluded
|
||||
return fmt.Sprintf(searchOpportunityCPV, condition),
|
||||
append(append([]any{}, c.cpvArgs...), c.excludedCPVArgs...)
|
||||
case len(c.cpv) > 0:
|
||||
condition := "(" + strings.Join(c.cpv, " OR ") + ")"
|
||||
return fmt.Sprintf(searchOpportunityCPV, condition), c.cpvArgs
|
||||
case len(c.excludedCPV) > 0:
|
||||
return "NOT " + fmt.Sprintf(searchOpportunityCPV, excluded),
|
||||
c.excludedCPVArgs
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// cpvCondition returns the condition on one CPV code for a cpv: term, with
|
||||
// its argument: a code prefix, or a full-text search of its description.
|
||||
func cpvCondition(term search.Term) (string, any) {
|
||||
if code, ok := term.CPVCode(); ok {
|
||||
return searchCPVCodePrefix, code + "%"
|
||||
}
|
||||
return searchCPVDescription, ftsPhrase(term.Words())
|
||||
}
|
||||
|
||||
// textMatch returns the FTS5 query of opportunities_fts for a full-text
|
||||
// term: its words as a phrase, in the columns its field searches.
|
||||
func textMatch(term search.Term) string {
|
||||
return ftsColumns[term.Field] + " : " + ftsPhrase(term.Words())
|
||||
}
|
||||
|
||||
// ftsPhrase returns an FTS5 phrase of words, the last also matching as a
|
||||
// prefix. The words must be split as search.Term.Words splits them, so
|
||||
// they hold no quotes or FTS5 syntax.
|
||||
func ftsPhrase(words []string) string {
|
||||
return `"` + strings.Join(words, " ") + `"*`
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/search"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
// searchStore returns a store holding five opportunities, s1 to s5, released
|
||||
// on days 1 to 5, for the search tests.
|
||||
func searchStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
s := openStore(t)
|
||||
const software = "Programming services of packaged software products"
|
||||
cpv := func(code, description string) types.ClassificationCode {
|
||||
return types.ClassificationCode{
|
||||
IsCPV: true,
|
||||
Code: code,
|
||||
Classification: description,
|
||||
}
|
||||
}
|
||||
rows := []struct {
|
||||
title, description, buyer, buyerID string
|
||||
classifications []types.ClassificationCode
|
||||
}{
|
||||
{
|
||||
title: "Cloud migration services",
|
||||
description: "Moving servers to the cloud",
|
||||
buyer: "Leeds City Council",
|
||||
buyerID: "GB-LEEDS",
|
||||
classifications: []types.ClassificationCode{
|
||||
cpv("72100000", "Hardware consultancy services"),
|
||||
cpv("72210000", software),
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Road repairs",
|
||||
description: "Pothole repairs in Bradford",
|
||||
buyer: "Bradford Council",
|
||||
buyerID: "GB-BRAD",
|
||||
classifications: []types.ClassificationCode{
|
||||
cpv("45233141", "Road-maintenance works"),
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "School meals",
|
||||
description: "Catering and cloud-based ordering",
|
||||
buyer: "Leeds Schools Trust",
|
||||
buyerID: "GB-LST",
|
||||
classifications: []types.ClassificationCode{
|
||||
cpv("55524000", "School catering services"),
|
||||
cpv("15800000", "Miscellaneous food products"),
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Software licences",
|
||||
description: "Licences",
|
||||
buyer: "York Council",
|
||||
buyerID: "GB-YORK",
|
||||
classifications: []types.ClassificationCode{
|
||||
cpv("72210000", software),
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Office cleaning",
|
||||
description: "Cleaning",
|
||||
buyer: "Hull Council",
|
||||
buyerID: "GB-HULL",
|
||||
},
|
||||
}
|
||||
for i, row := range rows {
|
||||
o := opportunity(fmt.Sprintf("s%d", i+1), row.title, i+1, "r1")
|
||||
o.Description = row.description
|
||||
o.BuyerName = row.buyer
|
||||
o.BuyerID = row.buyerID
|
||||
o.Classifications = row.classifications
|
||||
mustSave(t, s, o)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestSearchOpportunities(t *testing.T) {
|
||||
s := searchStore(t)
|
||||
all := []string{"s1", "s2", "s3", "s4", "s5"}
|
||||
cases := []struct {
|
||||
query string
|
||||
want []string
|
||||
}{
|
||||
{"", all},
|
||||
{"title: !!! \"\"", all},
|
||||
{"cloud", []string{"s1", "s3"}},
|
||||
{"CLOUD", []string{"s1", "s3"}},
|
||||
{"migr", []string{"s1"}},
|
||||
{"migrating", []string{"s1"}},
|
||||
{"cloud servers", []string{"s1"}},
|
||||
{"leeds", []string{}},
|
||||
{"bradford", []string{"s2"}},
|
||||
{`"road repairs"`, []string{"s2"}},
|
||||
{`"repairs road"`, []string{}},
|
||||
{"title:cloud", []string{"s1"}},
|
||||
{"title:pothole", []string{}},
|
||||
{"title:cloud title:road", []string{"s1", "s2"}},
|
||||
{"buyer:leeds", []string{"s1", "s3"}},
|
||||
{"buyer:leeds buyer:bradford", []string{"s1", "s2", "s3"}},
|
||||
{`buyer:"leeds city"`, []string{"s1"}},
|
||||
{"cloud buyer:leeds", []string{"s1", "s3"}},
|
||||
{"title:cloud buyer:leeds", []string{"s1"}},
|
||||
{`cloud !buyer:"leeds city"`, []string{"s3"}},
|
||||
{"!cloud", []string{"s2", "s4", "s5"}},
|
||||
{"!cloud !road", []string{"s4", "s5"}},
|
||||
{"buyer_id:gb-leeds", []string{"s1"}},
|
||||
{"buyer_id:GB-LEEDS buyer_id:gb-york", []string{"s1", "s4"}},
|
||||
{"buyer_id:GB", []string{}},
|
||||
{"!buyer_id:GB-LEEDS", []string{"s2", "s3", "s4", "s5"}},
|
||||
{"cpv:72", []string{"s1", "s4"}},
|
||||
{"cpv:72210000-5", []string{"s1", "s4"}},
|
||||
{"cpv:72 !cpv:7221", []string{"s1"}},
|
||||
{"!cpv:7221", []string{"s2", "s3", "s5"}},
|
||||
{"cpv:food", []string{"s3"}},
|
||||
{"!cpv:food", []string{"s1", "s2", "s4", "s5"}},
|
||||
{"cpv:catering cpv:road", []string{"s2", "s3"}},
|
||||
{"cpv:software !cpv:hardware", []string{"s1", "s4"}},
|
||||
{"cpv:72 buyer:york", []string{"s4"}},
|
||||
{"foo:cloud", all},
|
||||
{`cloud" OR title:road*`, []string{}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, err := s.SearchOpportunities(
|
||||
context.Background(), search.Parse(tc.query), time.Time{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("%q: %v", tc.query, err)
|
||||
}
|
||||
slices.Sort(got)
|
||||
if !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%q: got %v, want %v", tc.query, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// released_from bounds the search as it bounds ListOpportunities, and
|
||||
// changing a searched field updates the index.
|
||||
func TestSearchOpportunitiesReleasedFromAndUpdates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := searchStore(t)
|
||||
|
||||
got, err := s.SearchOpportunities(ctx, search.Parse("buyer:leeds"), day(3))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(got, []string{"s3"}) {
|
||||
t.Errorf("released from day 3: got %v, want [s3]", got)
|
||||
}
|
||||
|
||||
o := opportunity("s5", "Office cleaning", 6, "r1", "r2")
|
||||
o.Description = "Window washing"
|
||||
o.BuyerName = "Wakefield Council"
|
||||
mustSave(t, s, o)
|
||||
for query, want := range map[string][]string{
|
||||
"washing": {"s5"},
|
||||
"buyer:wakefield": {"s5"},
|
||||
"cleaning": {"s5"},
|
||||
"buyer:hull": {},
|
||||
"cpv:agricultural": {"s5"},
|
||||
} {
|
||||
got, err := s.SearchOpportunities(ctx, search.Parse(query), time.Time{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("%q after update: got %v, want %v", query, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -861,7 +861,7 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) {
|
||||
add(filterOpportunityClassificationCode, filter.ClassificationCode)
|
||||
}
|
||||
if match := ftsQuery(filter.Title); match != "" {
|
||||
add(filterOpportunityTitleMatch, match)
|
||||
add(filterOpportunityTitleMatch, "title : ("+match+")")
|
||||
}
|
||||
if !filter.ReleasedFrom.IsZero() {
|
||||
add(filterOpportunityReleasedFrom, filter.ReleasedFrom.UTC())
|
||||
|
||||
@@ -3,6 +3,7 @@ package store_test
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -273,8 +274,9 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
|
||||
got.Classifications[0].Classification != "IT services (amended)" {
|
||||
t.Fatalf("classifications not replaced: %+v", got.Classifications)
|
||||
}
|
||||
if got.Classifications[0].ID == oldIT {
|
||||
t.Errorf("changed description should create a new classification row")
|
||||
// A CPV code is one row: the longer description replaces the shorter.
|
||||
if got.Classifications[0].ID != oldIT {
|
||||
t.Errorf("a CPV code's new description should keep its row")
|
||||
}
|
||||
|
||||
for title, want := range map[string]int{"cloud": 0, "redesign": 1} {
|
||||
@@ -295,6 +297,61 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A CPV code keeps one row and its longest description, whichever order
|
||||
// the descriptions arrive in (EU Supply cuts them to 64 characters); a code
|
||||
// of another scheme still gets a row per description.
|
||||
func TestCPVCodeKeepsLongestDescription(t *testing.T) {
|
||||
const full = "Petroleum products, fuel, electricity and other sources of energy"
|
||||
cut := full[:64]
|
||||
for name, order := range map[string][]string{
|
||||
"full first": {full, cut},
|
||||
"cut first": {cut, full},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s := openStore(t)
|
||||
var ids []int64
|
||||
for i, description := range order {
|
||||
o := opportunity(fmt.Sprintf("ocds-%d", i), "Fuel", 1, "r1")
|
||||
o.Classifications = []types.ClassificationCode{
|
||||
{
|
||||
IsCPV: true,
|
||||
Code: "09000000",
|
||||
Classification: description,
|
||||
},
|
||||
}
|
||||
mustSave(t, s, o)
|
||||
ids = append(ids, o.Classifications[0].ID)
|
||||
}
|
||||
if ids[0] != ids[1] {
|
||||
t.Errorf("ids = %v; want one row for the code", ids)
|
||||
}
|
||||
for _, uuid := range []string{"ocds-0", "ocds-1"} {
|
||||
got := mustGet(t, s, uuid).Classifications[0].Classification
|
||||
if got != full {
|
||||
t.Errorf("%s: description = %q; want the full one", uuid, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
s := openStore(t)
|
||||
var ids []int64
|
||||
for i, description := range []string{"Scheme X", "Scheme X, amended"} {
|
||||
o := opportunity(fmt.Sprintf("other-%d", i), "Other", 1, "r1")
|
||||
o.Classifications = []types.ClassificationCode{
|
||||
{
|
||||
Code: "123",
|
||||
Classification: description,
|
||||
},
|
||||
}
|
||||
mustSave(t, s, o)
|
||||
ids = append(ids, o.Classifications[0].ID)
|
||||
}
|
||||
if ids[0] == ids[1] {
|
||||
t.Errorf("a non-CPV code's descriptions share a row: %v", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassificationRowsAreShared(t *testing.T) {
|
||||
s := openStore(t)
|
||||
a := opportunity("ocds-a", "A", 1, "r1")
|
||||
@@ -396,6 +453,9 @@ func TestListOpportunitiesFilters(t *testing.T) {
|
||||
{"title case", store.OpportunityFilter{
|
||||
Title: "ROAD",
|
||||
}, []string{"ocds-2"}},
|
||||
{"title only, not description", store.OpportunityFilter{
|
||||
Title: "description",
|
||||
}, []string{}},
|
||||
{
|
||||
"title syntax is not FTS syntax",
|
||||
store.OpportunityFilter{
|
||||
|
||||
+3
-2
@@ -1,8 +1,9 @@
|
||||
# JobSearch - Frontend
|
||||
|
||||
A Vite + SolidJS + Tailwind CSS (v4) + TypeScript table of opportunities with
|
||||
title search, status, mark and classification filters, and per-opportunity
|
||||
mark and hide controls.
|
||||
a general search (run by the backend, with title:, buyer:, buyer_id: and cpv:
|
||||
prompts), status, mark and classification filters, and per-opportunity mark
|
||||
and hide controls.
|
||||
|
||||
## Run
|
||||
|
||||
|
||||
+5
-2
@@ -22,6 +22,7 @@ import { readStored, STORAGE_KEYS, writeStored } from '@/lib/storage';
|
||||
import { createLiveUpdates } from '@/primitives/createLiveUpdates';
|
||||
import { createClassifications, createOpportunities } from '@/primitives/createOpportunities';
|
||||
import { createOpportunityAnnotations } from '@/primitives/createOpportunityAnnotations';
|
||||
import { createOpportunitySearch } from '@/primitives/createOpportunitySearch';
|
||||
import { createTheme } from '@/primitives/createTheme';
|
||||
import { createToasts } from '@/primitives/createToasts';
|
||||
|
||||
@@ -44,7 +45,8 @@ export function App(): JSX.Element {
|
||||
const opportunities = createOpportunities(releasedSince);
|
||||
const classifications = createClassifications();
|
||||
const annotations = createOpportunityAnnotations(opportunities.items, toasts.show);
|
||||
createLiveUpdates({ opportunities, classifications, annotations });
|
||||
const search = createOpportunitySearch(() => filters().search, releasedSince, toasts.show);
|
||||
createLiveUpdates({ opportunities, classifications, annotations, search });
|
||||
const theme = createTheme();
|
||||
|
||||
const statuses = createMemo(() => statusOptions(opportunities.items()));
|
||||
@@ -62,7 +64,7 @@ export function App(): JSX.Element {
|
||||
// re-runs this.
|
||||
const rows = createMemo(() =>
|
||||
sortOpportunities(
|
||||
filterOpportunities(opportunities.items(), filters(), annotations.get),
|
||||
filterOpportunities(opportunities.items(), filters(), annotations.get, search.matches()),
|
||||
sort(),
|
||||
annotations.get,
|
||||
),
|
||||
@@ -107,6 +109,7 @@ export function App(): JSX.Element {
|
||||
filters={filters()}
|
||||
onChange={setFilters}
|
||||
onClear={clear}
|
||||
search={search}
|
||||
statuses={statuses()}
|
||||
classifications={pickable()}
|
||||
isLoadingClassifications={classifications.isLoading()}
|
||||
|
||||
@@ -131,6 +131,41 @@ describe('listChangedOpportunities', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchOpportunities', () => {
|
||||
test('sends the box’s text as typed, encoded, with released_from', async () => {
|
||||
const { api } = await loadClient();
|
||||
const found = {
|
||||
uuids: ['ocds-1'],
|
||||
matches_all: false,
|
||||
unknown_prompts: [],
|
||||
prompts: [{ name: 'title', help: 'The title only.' }],
|
||||
syntax: [],
|
||||
};
|
||||
respond = () => json(found);
|
||||
|
||||
const result = await api.searchOpportunities(
|
||||
'title:"road repairs" !cpv:72&x',
|
||||
'2026-10-11T23:00:00.000Z',
|
||||
);
|
||||
|
||||
expect(sent[0]?.url).toBe(
|
||||
'/api/opportunities/search?q=title%3A%22road%20repairs%22%20!cpv%3A72%26x' +
|
||||
'&released_from=2026-10-11T23%3A00%3A00.000Z',
|
||||
);
|
||||
expect(result).toEqual(found);
|
||||
});
|
||||
|
||||
test('leaves out released_from when there is no bound', async () => {
|
||||
const { api } = await loadClient();
|
||||
respond = () =>
|
||||
json({ uuids: [], matches_all: true, unknown_prompts: [], prompts: [], syntax: [] });
|
||||
|
||||
await api.searchOpportunities('', '');
|
||||
|
||||
expect(sent[0]?.url).toBe('/api/opportunities/search?q=');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getOpportunity', () => {
|
||||
test('fetches the opportunity by its encoded UUID', async () => {
|
||||
const { api } = await loadClient();
|
||||
|
||||
@@ -4,6 +4,7 @@ import type {
|
||||
Opportunity,
|
||||
OpportunityMarks,
|
||||
OpportunityPage,
|
||||
OpportunitySearch,
|
||||
Problem,
|
||||
} from '@/types/Opportunities';
|
||||
|
||||
@@ -14,6 +15,8 @@ import type {
|
||||
* GET {base}/opportunities?offset=&limit= -> OpportunityPage
|
||||
* GET {base}/opportunities?changed_after={batch} -> OpportunityPage (changed since)
|
||||
* both with &released_from={RFC 3339 time}: only those released since
|
||||
* GET {base}/opportunities/search?q={text} -> OpportunitySearch
|
||||
* also with &released_from
|
||||
* GET {base}/opportunities/{uuid} -> Opportunity (with its releases)
|
||||
* GET {base}/classifications -> ClassificationCode[]
|
||||
* PUT {base}/opportunities/{uuid}/marks/{mark} -> OpportunityMarks (sets it)
|
||||
@@ -149,6 +152,22 @@ export const api = {
|
||||
return { opportunities: page.opportunities, batch: page.batch };
|
||||
},
|
||||
|
||||
/**
|
||||
* Searches the opportunities released at or after releasedFrom ("" for
|
||||
* any) with query, the search box's text as typed: the backend parses it.
|
||||
* A blank query matches all of them, and still returns the search's help.
|
||||
*/
|
||||
searchOpportunities(
|
||||
query: string,
|
||||
releasedFrom: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<OpportunitySearch> {
|
||||
return request<OpportunitySearch>(
|
||||
`/opportunities/search?q=${encodeURIComponent(query)}${releasedFromParam(releasedFrom)}`,
|
||||
{ signal },
|
||||
);
|
||||
},
|
||||
|
||||
/** One opportunity with all of its releases. An unknown UUID is an ApiError with status 404. */
|
||||
getOpportunity(uuid: string, signal?: AbortSignal): Promise<Opportunity> {
|
||||
return request<Opportunity>(opportunityPath(uuid), { signal });
|
||||
|
||||
@@ -9,16 +9,20 @@ import {
|
||||
statusLabel,
|
||||
} from '@/lib/filterOpportunities';
|
||||
import { readStored, STORAGE_KEYS, writeStored } from '@/lib/storage';
|
||||
import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch';
|
||||
import type { OpportunityMark } from '@/types/Annotations';
|
||||
import { MARK_LABELS, OPPORTUNITY_MARKS } from '@/types/Annotations';
|
||||
import type { ClassificationCode } from '@/types/Opportunities';
|
||||
import { ClassificationPicker } from './ClassificationPicker';
|
||||
import { SearchField } from './SearchField';
|
||||
import { buttonQuiet, fieldLabel, input, MARK_BG } from './styles';
|
||||
|
||||
interface Props {
|
||||
filters: OpportunityFilters;
|
||||
onChange: (next: OpportunityFilters) => void;
|
||||
onClear: () => void;
|
||||
/** The search that filters.search runs. */
|
||||
search: OpportunitySearchResults;
|
||||
statuses: string[];
|
||||
classifications: ClassificationCode[];
|
||||
isLoadingClassifications: boolean;
|
||||
@@ -43,11 +47,10 @@ function toggled<T>(set: ReadonlySet<T>, value: T): Set<T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* The collapsible panel of filters (title, status, marks, classification,
|
||||
* The collapsible panel of filters (search, status, marks, classification,
|
||||
* hidden), with the count of opportunities shown.
|
||||
*/
|
||||
export function FilterPanel(props: Props): JSX.Element {
|
||||
const titleId = createUniqueId();
|
||||
const daysId = createUniqueId();
|
||||
const bodyId = createUniqueId();
|
||||
|
||||
@@ -151,19 +154,11 @@ export function FilterPanel(props: Props): JSX.Element {
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class={fieldLabel} for={titleId}>
|
||||
Title
|
||||
</label>
|
||||
<input
|
||||
id={titleId}
|
||||
type="search"
|
||||
class={`${input} px-3 py-2.5 text-base`}
|
||||
placeholder="Search opportunity titles"
|
||||
value={props.filters.title}
|
||||
onInput={(e) => update({ title: e.currentTarget.value })}
|
||||
/>
|
||||
</div>
|
||||
<SearchField
|
||||
value={props.filters.search}
|
||||
onInput={(search) => update({ search })}
|
||||
search={props.search}
|
||||
/>
|
||||
|
||||
<fieldset class="m-0 min-w-0 border-0 p-0">
|
||||
<legend class={fieldLabel}>Status</legend>
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import CircleQuestionMark from 'lucide-solid/icons/circle-question-mark';
|
||||
import type { JSX } from 'solid-js';
|
||||
import { createSignal, createUniqueId, For, Match, Show, Switch } from 'solid-js';
|
||||
|
||||
import { promptText, unknownPromptsText } from '@/lib/search';
|
||||
import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch';
|
||||
import { input, linkButton } from './styles';
|
||||
|
||||
interface Props {
|
||||
/** The box's text. */
|
||||
value: string;
|
||||
onInput: (value: string) => void;
|
||||
search: OpportunitySearchResults;
|
||||
}
|
||||
|
||||
const helpButton =
|
||||
'flex rounded-md border border-line bg-surface p-1 text-ink hover:border-muted aria-expanded:border-ink';
|
||||
const helpTerm = 'font-mono text-[13px] whitespace-nowrap';
|
||||
|
||||
/**
|
||||
* The general search box, with its help (from the backend, behind the "?"
|
||||
* button), a note on prompts the search ignored, and the search's errors.
|
||||
*/
|
||||
export function SearchField(props: Props): JSX.Element {
|
||||
const inputId = createUniqueId();
|
||||
const helpId = createUniqueId();
|
||||
const noteId = createUniqueId();
|
||||
const [isHelpOpen, setIsHelpOpen] = createSignal(false);
|
||||
|
||||
const unknownPrompts = (): string => {
|
||||
const unknown = props.search.unknownPrompts();
|
||||
return unknown.length === 0
|
||||
? ''
|
||||
: unknownPromptsText(unknown, props.search.help()?.prompts ?? []);
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div class="mb-1.5 flex items-center gap-1.5">
|
||||
<label class="text-sm font-semibold" for={inputId}>
|
||||
Search
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
class={helpButton}
|
||||
aria-label="Search help"
|
||||
title="Search help"
|
||||
aria-expanded={isHelpOpen()}
|
||||
aria-controls={helpId}
|
||||
onClick={() => setIsHelpOpen(!isHelpOpen())}
|
||||
>
|
||||
<CircleQuestionMark size={14} strokeWidth={2} />
|
||||
</button>
|
||||
</div>
|
||||
<input
|
||||
id={inputId}
|
||||
type="search"
|
||||
class={`${input} px-3 py-2.5 text-base`}
|
||||
placeholder="Words, “phrases” and prompts such as title:"
|
||||
value={props.value}
|
||||
onInput={(e) => props.onInput(e.currentTarget.value)}
|
||||
aria-describedby={noteId}
|
||||
aria-busy={props.search.isSearching() || undefined}
|
||||
/>
|
||||
|
||||
<Show when={props.search.error()}>
|
||||
{(err) => (
|
||||
<p class="mt-1.5 text-sm text-danger" role="alert">
|
||||
The search didn't run: {err().message}{' '}
|
||||
<button type="button" class={linkButton} onClick={() => props.search.refresh()}>
|
||||
Try again
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
</Show>
|
||||
<p id={noteId} class="mt-1.5 text-sm text-muted empty:hidden">
|
||||
{unknownPrompts()}
|
||||
</p>
|
||||
|
||||
<div
|
||||
id={helpId}
|
||||
class="mt-2 rounded-md border border-line-soft bg-paper p-3 text-sm"
|
||||
hidden={!isHelpOpen()}
|
||||
>
|
||||
<Switch fallback={<p class="text-muted">Loading the search help…</p>}>
|
||||
<Match when={props.search.help()}>
|
||||
{(help) => (
|
||||
<>
|
||||
<dl class="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1.5">
|
||||
<For each={help().syntax}>
|
||||
{(syntax) => (
|
||||
<>
|
||||
<dt class={helpTerm}>{syntax.example}</dt>
|
||||
<dd>{syntax.help}</dd>
|
||||
</>
|
||||
)}
|
||||
</For>
|
||||
</dl>
|
||||
<h3 class="mt-3 mb-1.5 font-semibold">Prompts</h3>
|
||||
<dl class="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1.5">
|
||||
<For each={help().prompts}>
|
||||
{(prompt) => (
|
||||
<>
|
||||
<dt class={helpTerm}>{promptText(prompt.name)}</dt>
|
||||
<dd>{prompt.help}</dd>
|
||||
</>
|
||||
)}
|
||||
</For>
|
||||
</dl>
|
||||
</>
|
||||
)}
|
||||
</Match>
|
||||
<Match when={props.search.helpError()}>
|
||||
{(err) => (
|
||||
<p class="text-danger" role="alert">
|
||||
The search help didn't load: {err().message}{' '}
|
||||
<button type="button" class={linkButton} onClick={() => props.search.reloadHelp()}>
|
||||
Try again
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
</Match>
|
||||
</Switch>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -43,10 +43,16 @@ const annotations: Record<string, Annotation> = {
|
||||
const annotationFor = (uuid: string): Annotation =>
|
||||
annotations[uuid] ?? { uuid, marks: [], hidden: false };
|
||||
|
||||
const filtered = (filters: Partial<OpportunityFilters>): string[] =>
|
||||
filterOpportunities(opportunities, { ...emptyFilters(), ...filters }, annotationFor).map(
|
||||
(opportunity) => opportunity.uuid,
|
||||
);
|
||||
const filtered = (
|
||||
filters: Partial<OpportunityFilters>,
|
||||
searchMatches?: ReadonlySet<string>,
|
||||
): string[] =>
|
||||
filterOpportunities(
|
||||
opportunities,
|
||||
{ ...emptyFilters(), ...filters },
|
||||
annotationFor,
|
||||
searchMatches,
|
||||
).map((opportunity) => opportunity.uuid);
|
||||
|
||||
describe('filterOpportunities', () => {
|
||||
test('hides hidden opportunities unless asked, keeping the order given', () => {
|
||||
@@ -55,7 +61,6 @@ describe('filterOpportunities', () => {
|
||||
});
|
||||
|
||||
test.each<{ name: string; filters: Partial<OpportunityFilters>; want: string[] }>([
|
||||
{ name: 'title, ignoring case and spaces', filters: { title: ' CLOUD ' }, want: ['a', 'c'] },
|
||||
{ name: 'statuses', filters: { statuses: new Set(['active', 'planned']) }, want: ['a', 'c'] },
|
||||
{ name: 'a mark', filters: { marks: new Set(['interested']) }, want: ['a'] },
|
||||
{ name: 'no marks', filters: { marks: new Set([NOT_MARKED]) }, want: ['c'] },
|
||||
@@ -65,18 +70,24 @@ describe('filterOpportunities', () => {
|
||||
expect(filtered(filters)).toEqual(want);
|
||||
});
|
||||
|
||||
test('keeps only the search’s matches, when it has some', () => {
|
||||
expect(filtered({}, new Set(['c', 'x']))).toEqual(['c']);
|
||||
expect(filtered({}, new Set())).toEqual([]);
|
||||
});
|
||||
|
||||
test('combines filters', () => {
|
||||
expect(filtered({ title: 'cloud', smeSuitable: true })).toEqual(['a']);
|
||||
expect(filtered({ title: 'road', showHidden: true, smeSuitable: true })).toEqual([]);
|
||||
expect(filtered({ smeSuitable: true }, new Set(['a', 'c']))).toEqual(['a']);
|
||||
expect(filtered({ showHidden: true, smeSuitable: true }, new Set(['b']))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
test('activeFilterCount counts the filters in use', () => {
|
||||
expect(activeFilterCount(emptyFilters())).toBe(0);
|
||||
expect(
|
||||
activeFilterCount({ ...emptyFilters(), title: ' ', smeSuitable: true, showHidden: true }),
|
||||
activeFilterCount({ ...emptyFilters(), search: ' ', smeSuitable: true, showHidden: true }),
|
||||
).toBe(2);
|
||||
expect(activeFilterCount({ ...emptyFilters(), releasedWithinDays: 30 })).toBe(1);
|
||||
expect(activeFilterCount({ ...emptyFilters(), search: 'title:cloud' })).toBe(1);
|
||||
});
|
||||
|
||||
test('releasedFrom is the start of the day that many days ago, in UK time', () => {
|
||||
@@ -104,7 +115,7 @@ describe('stored filters', () => {
|
||||
test('read back as they were stored', () => {
|
||||
const filters: OpportunityFilters = {
|
||||
releasedWithinDays: 30,
|
||||
title: 'cloud',
|
||||
search: 'cloud !cpv:72',
|
||||
statuses: new Set(['active', 'withdrawn']),
|
||||
marks: new Set(['interested', NOT_MARKED]),
|
||||
classificationIds: new Set([1, 42]),
|
||||
@@ -122,11 +133,16 @@ describe('stored filters', () => {
|
||||
expect(parseFilters(text)).toEqual(emptyFilters());
|
||||
});
|
||||
|
||||
test('an older version’s title filter is kept as search words', () => {
|
||||
expect(parseFilters(JSON.stringify({ title: 'road repairs' })).search).toBe('road repairs');
|
||||
expect(parseFilters(JSON.stringify({ title: 'road', search: 'cloud' })).search).toBe('cloud');
|
||||
});
|
||||
|
||||
test('fields that aren’t valid keep their starting values', () => {
|
||||
const filters = parseFilters(
|
||||
JSON.stringify({
|
||||
releasedWithinDays: 0,
|
||||
title: 5,
|
||||
search: 5,
|
||||
statuses: 'active',
|
||||
marks: ['won', 'not-a-mark'],
|
||||
classificationIds: [3, '4', 1.5],
|
||||
|
||||
@@ -18,7 +18,11 @@ export interface OpportunityFilters {
|
||||
* here.
|
||||
*/
|
||||
releasedWithinDays: number;
|
||||
title: string;
|
||||
/**
|
||||
* The general search box's text, as typed. The backend runs the search
|
||||
* (see createOpportunitySearch), and its matches are applied here.
|
||||
*/
|
||||
search: string;
|
||||
/** Empty means every status. */
|
||||
statuses: ReadonlySet<string>;
|
||||
/** Empty means any marks. Otherwise an opportunity needs at least one of these. */
|
||||
@@ -37,7 +41,7 @@ export interface OpportunityFilters {
|
||||
*/
|
||||
export const emptyFilters = (): OpportunityFilters => ({
|
||||
releasedWithinDays: DEFAULT_RELEASED_WITHIN_DAYS,
|
||||
title: '',
|
||||
search: '',
|
||||
statuses: new Set(),
|
||||
marks: new Set(),
|
||||
classificationIds: new Set(),
|
||||
@@ -91,7 +95,12 @@ export function parseFilters(text: string | null): OpportunityFilters {
|
||||
releasedWithinDays: isDays(stored.releasedWithinDays)
|
||||
? stored.releasedWithinDays
|
||||
: filters.releasedWithinDays,
|
||||
title: isString(stored.title) ? stored.title : filters.title,
|
||||
// Older versions stored a title filter; its text is kept as search words.
|
||||
search: isString(stored.search)
|
||||
? stored.search
|
||||
: isString(stored.title)
|
||||
? stored.title
|
||||
: filters.search,
|
||||
statuses: statuses ? new Set(statuses) : filters.statuses,
|
||||
marks: marks ? new Set(marks) : filters.marks,
|
||||
classificationIds: classificationIds ? new Set(classificationIds) : filters.classificationIds,
|
||||
@@ -107,7 +116,7 @@ export function parseFilters(text: string | null): OpportunityFilters {
|
||||
export const activeFilterCount = (filters: OpportunityFilters): number =>
|
||||
[
|
||||
filters.releasedWithinDays !== DEFAULT_RELEASED_WITHIN_DAYS,
|
||||
filters.title.trim() !== '',
|
||||
filters.search.trim() !== '',
|
||||
filters.statuses.size > 0,
|
||||
filters.marks.size > 0,
|
||||
filters.classificationIds.size > 0,
|
||||
@@ -128,18 +137,21 @@ export function releasedFrom(days: number, now: Date = new Date()): string {
|
||||
return new Date(now.getFullYear(), now.getMonth(), now.getDate() - days).toISOString();
|
||||
}
|
||||
|
||||
/** The opportunities that pass filters, in the order given. */
|
||||
/**
|
||||
* The opportunities that pass filters, in the order given. searchMatches is
|
||||
* the UUIDs the search found, or undefined when it doesn't filter (no
|
||||
* search, or one with nothing to search for).
|
||||
*/
|
||||
export function filterOpportunities(
|
||||
opportunities: readonly Opportunity[],
|
||||
filters: OpportunityFilters,
|
||||
annotationFor: (uuid: string) => Annotation,
|
||||
searchMatches: ReadonlySet<string> | undefined,
|
||||
): Opportunity[] {
|
||||
const query = filters.title.trim().toLocaleLowerCase();
|
||||
|
||||
return opportunities.filter((opportunity) => {
|
||||
const annotation = annotationFor(opportunity.uuid);
|
||||
if (!filters.showHidden && annotation.hidden) return false;
|
||||
if (query && !opportunity.title.toLocaleLowerCase().includes(query)) return false;
|
||||
if (searchMatches && !searchMatches.has(opportunity.uuid)) return false;
|
||||
if (filters.statuses.size > 0 && !filters.statuses.has(opportunity.status)) return false;
|
||||
if (filters.smeSuitable && !opportunity.sme_suitable) return false;
|
||||
if (filters.marks.size > 0) {
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { expect, test } from 'vitest';
|
||||
|
||||
import { promptText, unknownPromptsText } from './search';
|
||||
|
||||
const prompts = [
|
||||
{ name: 'title', help: '' },
|
||||
{ name: 'buyer', help: '' },
|
||||
{ name: 'cpv', help: '' },
|
||||
];
|
||||
|
||||
test('promptText is the prompt as typed', () => {
|
||||
expect(promptText('buyer_id')).toBe('buyer_id:');
|
||||
});
|
||||
|
||||
test.each([
|
||||
{
|
||||
name: 'one unknown prompt',
|
||||
unknown: ['foo'],
|
||||
want: '“foo:” isn’t a prompt, so it was ignored. The prompts are title:, buyer: and cpv:.',
|
||||
},
|
||||
{
|
||||
name: 'several, as typed',
|
||||
unknown: ['foo', 'Bar'],
|
||||
want: '“foo:” and “Bar:” aren’t prompts, so they were ignored. The prompts are title:, buyer: and cpv:.',
|
||||
},
|
||||
])('unknownPromptsText names $name, then the prompts', ({ unknown, want }) => {
|
||||
expect(unknownPromptsText(unknown, prompts)).toBe(want);
|
||||
});
|
||||
|
||||
test('unknownPromptsText leaves out the prompts before the help has loaded', () => {
|
||||
expect(unknownPromptsText(['foo'], [])).toBe('“foo:” isn’t a prompt, so it was ignored.');
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { SearchPrompt } from '@/types/Opportunities';
|
||||
|
||||
const LIST_FORMAT = new Intl.ListFormat('en-GB', { type: 'conjunction' });
|
||||
|
||||
/** A prompt's name as it's typed in the search box: "title:". */
|
||||
export const promptText = (name: string): string => `${name}:`;
|
||||
|
||||
/**
|
||||
* What the search box says about prompts it doesn't know (their names as
|
||||
* typed, without the colon), which the search ignored: they're named, then
|
||||
* the prompts there are, if they're known yet.
|
||||
*/
|
||||
export function unknownPromptsText(
|
||||
unknown: readonly string[],
|
||||
prompts: readonly SearchPrompt[],
|
||||
): string {
|
||||
const names = LIST_FORMAT.format(unknown.map((name) => `“${promptText(name)}”`));
|
||||
const ignored =
|
||||
unknown.length === 1
|
||||
? `${names} isn’t a prompt, so it was ignored.`
|
||||
: `${names} aren’t prompts, so they were ignored.`;
|
||||
if (prompts.length === 0) return ignored;
|
||||
return `${ignored} The prompts are ${LIST_FORMAT.format(prompts.map((p) => promptText(p.name)))}.`;
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { onCleanup } from 'solid-js';
|
||||
import { connectEvents } from '@/api/events';
|
||||
import { applyClassificationChanges } from '@/lib/classifications';
|
||||
import type { OpportunityAnnotations } from '@/primitives/createOpportunityAnnotations';
|
||||
import type { OpportunitySearchResults } from '@/primitives/createOpportunitySearch';
|
||||
import type { ServerEvent } from '@/types/Events';
|
||||
import type { ClassificationCode } from '@/types/Opportunities';
|
||||
import type { ApiList, OpportunityList } from './createOpportunities';
|
||||
@@ -12,6 +13,7 @@ export interface LiveData {
|
||||
opportunities: OpportunityList;
|
||||
classifications: ApiList<ClassificationCode>;
|
||||
annotations: OpportunityAnnotations;
|
||||
search: OpportunitySearchResults;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -21,10 +23,12 @@ export interface LiveData {
|
||||
* - ClassificationsUpdated changes the classifications in place.
|
||||
* - OpportunityMarksUpdated sets the opportunity's marks.
|
||||
* - OpportunitiesUpdated catches the opportunities up: the changed ones are
|
||||
* fetched and merged in, as the event only names them.
|
||||
* fetched and merged in, as the event only names them. The search runs
|
||||
* again, as they may match it now, or no longer.
|
||||
*
|
||||
* After the connection drops and is made again, events may have been missed:
|
||||
* the opportunities catch up, and the classifications are reloaded.
|
||||
* the opportunities catch up, the search runs again, and the classifications
|
||||
* are reloaded.
|
||||
*/
|
||||
export function createLiveUpdates(data: LiveData): void {
|
||||
const handle = (event: ServerEvent): void => {
|
||||
@@ -39,6 +43,7 @@ export function createLiveUpdates(data: LiveData): void {
|
||||
break;
|
||||
case 'OpportunitiesUpdated':
|
||||
data.opportunities.catchUp();
|
||||
data.search.refresh();
|
||||
break;
|
||||
}
|
||||
};
|
||||
@@ -47,6 +52,7 @@ export function createLiveUpdates(data: LiveData): void {
|
||||
onEvent: handle,
|
||||
onReconnect: () => {
|
||||
data.opportunities.catchUp();
|
||||
data.search.refresh();
|
||||
data.classifications.reload();
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
import { createRoot, createSignal } from 'solid-js';
|
||||
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
|
||||
|
||||
import type * as clientModule from '@/api/client';
|
||||
import type { OpportunitySearch } from '@/types/Opportunities';
|
||||
import type { OpportunitySearchResults } from './createOpportunitySearch';
|
||||
import { createOpportunitySearch } from './createOpportunitySearch';
|
||||
|
||||
const requests = vi.hoisted(() => ({
|
||||
searchOpportunities:
|
||||
vi.fn<
|
||||
(query: string, releasedFrom: string, signal?: AbortSignal) => Promise<OpportunitySearch>
|
||||
>(),
|
||||
}));
|
||||
|
||||
vi.mock('@/api/client', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof clientModule>()),
|
||||
api: requests,
|
||||
}));
|
||||
|
||||
/** A search the test answers when it chooses. */
|
||||
interface SentSearch {
|
||||
query: string;
|
||||
releasedFrom: string;
|
||||
signal: AbortSignal | undefined;
|
||||
resolve: (found: OpportunitySearch) => void;
|
||||
reject: (err: unknown) => void;
|
||||
}
|
||||
|
||||
const HELP = {
|
||||
prompts: [{ name: 'title', help: 'The title only.' }],
|
||||
syntax: [{ example: '!cloud', help: 'Not cloud.' }],
|
||||
};
|
||||
|
||||
const found = (...uuids: string[]): OpportunitySearch => ({
|
||||
uuids,
|
||||
matches_all: false,
|
||||
unknown_prompts: [],
|
||||
...HELP,
|
||||
});
|
||||
|
||||
let sent: SentSearch[];
|
||||
let errors: string[];
|
||||
let search: OpportunitySearchResults;
|
||||
let dispose: () => void;
|
||||
let setFilters: (filters: { search: string; days: number }) => void;
|
||||
let setReleasedFrom: (from: string) => void;
|
||||
|
||||
/** The searches sent with a query: the help's has none. */
|
||||
const searches = (): SentSearch[] => sent.filter((s) => s.query !== '');
|
||||
|
||||
/** Lets pending promises, Solid's effects and timers up to ms run. */
|
||||
const wait = async (ms = 0): Promise<void> => {
|
||||
await vi.advanceTimersByTimeAsync(ms);
|
||||
};
|
||||
|
||||
/** Starts the search with a stored query, and answers the help's request. */
|
||||
async function start(query: string): Promise<void> {
|
||||
createRoot((disposeRoot) => {
|
||||
dispose = disposeRoot;
|
||||
const [filters, setStored] = createSignal({ search: query, days: 7 });
|
||||
const [releasedFrom, setFrom] = createSignal('2026-10-12T00:00:00Z');
|
||||
setFilters = setStored;
|
||||
setReleasedFrom = setFrom;
|
||||
search = createOpportunitySearch(
|
||||
() => filters().search,
|
||||
releasedFrom,
|
||||
(message) => errors.push(message),
|
||||
);
|
||||
});
|
||||
sent.find((s) => s.query === '')?.resolve({ ...found(), matches_all: true });
|
||||
await wait();
|
||||
}
|
||||
|
||||
const type = (text: string): void => setFilters({ search: text, days: 7 });
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
sent = [];
|
||||
errors = [];
|
||||
requests.searchOpportunities.mockImplementation(
|
||||
(query, releasedFrom, signal) =>
|
||||
new Promise((resolve, reject) => {
|
||||
sent.push({ query, releasedFrom, signal, resolve, reject });
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
dispose();
|
||||
vi.clearAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
test('loads the help at the start, with a blank query and no bound', async () => {
|
||||
await start('');
|
||||
|
||||
expect(sent.map((s) => [s.query, s.releasedFrom])).toEqual([['', '']]);
|
||||
expect(search.help()).toEqual(HELP);
|
||||
expect(search.matches()).toBeUndefined();
|
||||
});
|
||||
|
||||
test('runs a stored search at once, and keeps its matches', async () => {
|
||||
await start('cloud');
|
||||
|
||||
expect(searches().map((s) => [s.query, s.releasedFrom])).toEqual([
|
||||
['cloud', '2026-10-12T00:00:00Z'],
|
||||
]);
|
||||
expect(search.isSearching()).toBe(true);
|
||||
searches()[0]?.resolve(found('a', 'b'));
|
||||
await wait();
|
||||
|
||||
expect([...(search.matches() ?? [])]).toEqual(['a', 'b']);
|
||||
expect(search.isSearching()).toBe(false);
|
||||
});
|
||||
|
||||
test('waits for typing to pause, then searches for the latest text', async () => {
|
||||
await start('');
|
||||
|
||||
type('c');
|
||||
await wait(200);
|
||||
type('cloud');
|
||||
await wait(299);
|
||||
expect(searches()).toHaveLength(0);
|
||||
expect(search.isSearching()).toBe(true);
|
||||
|
||||
await wait(1);
|
||||
expect(searches().map((s) => s.query)).toEqual(['cloud']);
|
||||
});
|
||||
|
||||
test('aborts a search under way for the next, and ignores its late answer', async () => {
|
||||
await start('cloud');
|
||||
type('road');
|
||||
await wait(300);
|
||||
|
||||
expect(searches()[0]?.signal?.aborted).toBe(true);
|
||||
searches()[0]?.resolve(found('old'));
|
||||
searches()[1]?.resolve(found('new'));
|
||||
await wait();
|
||||
|
||||
expect([...(search.matches() ?? [])]).toEqual(['new']);
|
||||
});
|
||||
|
||||
test('doesn’t filter when nothing in the box can be searched for', async () => {
|
||||
await start('title:');
|
||||
searches()[0]?.resolve({ ...found(), matches_all: true, unknown_prompts: ['foo'] });
|
||||
await wait();
|
||||
|
||||
expect(search.matches()).toBeUndefined();
|
||||
expect(search.unknownPrompts()).toEqual(['foo']);
|
||||
});
|
||||
|
||||
test('a blank box clears the matches at once, without a request', async () => {
|
||||
await start('cloud');
|
||||
searches()[0]?.resolve(found('a'));
|
||||
await wait();
|
||||
|
||||
type(' ');
|
||||
await wait(300);
|
||||
|
||||
expect(searches()).toHaveLength(1);
|
||||
expect(search.matches()).toBeUndefined();
|
||||
expect(search.isSearching()).toBe(false);
|
||||
});
|
||||
|
||||
test('keeps the last matches after a failure, and refresh tries again', async () => {
|
||||
await start('cloud');
|
||||
searches()[0]?.resolve(found('a'));
|
||||
await wait();
|
||||
type('road');
|
||||
await wait(300);
|
||||
searches()[1]?.reject(new Error('offline'));
|
||||
await wait();
|
||||
|
||||
expect([...(search.matches() ?? [])]).toEqual(['a']);
|
||||
expect(search.error()?.message).toBe('offline');
|
||||
expect(errors).toEqual(["The search didn't run: offline"]);
|
||||
|
||||
search.refresh();
|
||||
searches()[2]?.resolve(found('b'));
|
||||
await wait();
|
||||
expect(searches()[2]?.query).toBe('road');
|
||||
expect([...(search.matches() ?? [])]).toEqual(['b']);
|
||||
expect(search.error()).toBeUndefined();
|
||||
});
|
||||
|
||||
test('runs again at once for a new releasedFrom, but not for other filters', async () => {
|
||||
await start('cloud');
|
||||
|
||||
setFilters({ search: 'cloud', days: 30 });
|
||||
await wait(300);
|
||||
expect(searches()).toHaveLength(1);
|
||||
|
||||
setReleasedFrom('2026-10-01T00:00:00Z');
|
||||
await wait();
|
||||
expect(searches().map((s) => s.releasedFrom)).toEqual([
|
||||
'2026-10-12T00:00:00Z',
|
||||
'2026-10-01T00:00:00Z',
|
||||
]);
|
||||
});
|
||||
|
||||
test('a search mends help that didn’t load', async () => {
|
||||
createRoot((disposeRoot) => {
|
||||
dispose = disposeRoot;
|
||||
const [query, setQuery] = createSignal('');
|
||||
setFilters = (filters) => setQuery(filters.search);
|
||||
search = createOpportunitySearch(
|
||||
query,
|
||||
() => '',
|
||||
(message) => errors.push(message),
|
||||
);
|
||||
});
|
||||
sent[0]?.reject(new Error('offline'));
|
||||
await wait();
|
||||
expect(search.helpError()?.message).toBe('offline');
|
||||
expect(search.help()).toBeUndefined();
|
||||
|
||||
type('cloud');
|
||||
await wait(300);
|
||||
searches()[0]?.resolve(found());
|
||||
await wait();
|
||||
|
||||
expect(search.help()).toEqual(HELP);
|
||||
expect(search.helpError()).toBeUndefined();
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { createEffect, createMemo, createSignal, on, onCleanup } from 'solid-js';
|
||||
|
||||
import { api } from '@/api/client';
|
||||
import { errorText } from '@/lib/errors';
|
||||
import type { OpportunitySearch, SearchExample, SearchPrompt } from '@/types/Opportunities';
|
||||
|
||||
/** How long typing has to pause for before the search runs, in milliseconds. */
|
||||
const SEARCH_DELAY = 300;
|
||||
|
||||
/** The search box's help, from the backend, which parses the search. */
|
||||
export interface SearchHelp {
|
||||
prompts: SearchPrompt[];
|
||||
/** Examples of the syntax, in the order to show them. */
|
||||
syntax: SearchExample[];
|
||||
}
|
||||
|
||||
/** The general search, run by the backend, and its help. */
|
||||
export interface OpportunitySearchResults {
|
||||
/**
|
||||
* The UUIDs the last search found, or undefined when the search doesn't
|
||||
* filter: the box is blank, nothing in it can be searched for, or no
|
||||
* search has finished yet. After a failure, the last search's.
|
||||
*/
|
||||
matches: () => ReadonlySet<string> | undefined;
|
||||
/** The last search's prompts the backend doesn't know, as typed, without the colon. */
|
||||
unknownPrompts: () => string[];
|
||||
/** Whether a search is waiting for typing to pause, or running. */
|
||||
isSearching: () => boolean;
|
||||
/** Why the last search failed, until one succeeds or the box is cleared. */
|
||||
error: () => Error | undefined;
|
||||
/** Runs the search again now, unless the box is blank: to retry, or as opportunities changed. */
|
||||
refresh: () => void;
|
||||
/** The help, or undefined until it has loaded. */
|
||||
help: () => SearchHelp | undefined;
|
||||
/** Why the help didn't load, until it does. */
|
||||
helpError: () => Error | undefined;
|
||||
reloadHelp: () => void;
|
||||
}
|
||||
|
||||
const toError = (err: unknown): Error => (err instanceof Error ? err : new Error(errorText(err)));
|
||||
|
||||
/**
|
||||
* Searches the opportunities released at or after releasedFrom with query,
|
||||
* the search box's text, as either changes: at once for a new releasedFrom
|
||||
* or the first query, and after a pause in typing for the others. A search
|
||||
* under way is aborted by the next. A failure is shown with onError, and
|
||||
* the last search's matches are kept. The help is loaded once, at the
|
||||
* start.
|
||||
*/
|
||||
export function createOpportunitySearch(
|
||||
query: () => string,
|
||||
releasedFrom: () => string,
|
||||
onError: (message: string) => void,
|
||||
): OpportunitySearchResults {
|
||||
const [result, setResult] = createSignal<OpportunitySearch>();
|
||||
const [isSearching, setIsSearching] = createSignal(false);
|
||||
const [error, setError] = createSignal<Error>();
|
||||
const [help, setHelp] = createSignal<SearchHelp>();
|
||||
const [helpError, setHelpError] = createSignal<Error>();
|
||||
let controller: AbortController | undefined;
|
||||
let helpController: AbortController | undefined;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
onCleanup(() => {
|
||||
clearTimeout(timer);
|
||||
controller?.abort();
|
||||
helpController?.abort();
|
||||
});
|
||||
|
||||
const keepHelp = (found: OpportunitySearch): void => {
|
||||
setHelp({ prompts: found.prompts, syntax: found.syntax });
|
||||
setHelpError(undefined);
|
||||
};
|
||||
|
||||
async function search(): Promise<void> {
|
||||
clearTimeout(timer);
|
||||
controller?.abort();
|
||||
controller = new AbortController();
|
||||
const { signal } = controller;
|
||||
setIsSearching(true);
|
||||
try {
|
||||
const found = await api.searchOpportunities(query(), releasedFrom(), signal);
|
||||
if (signal.aborted) return;
|
||||
setResult(found);
|
||||
setError(undefined);
|
||||
// Every result carries the help, so a search mends a help that didn't load.
|
||||
keepHelp(found);
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
const failure = toError(err);
|
||||
setError(failure);
|
||||
onError(`The search didn't run: ${failure.message}`);
|
||||
} finally {
|
||||
if (!signal.aborted) setIsSearching(false);
|
||||
}
|
||||
}
|
||||
|
||||
const clear = (): void => {
|
||||
clearTimeout(timer);
|
||||
controller?.abort();
|
||||
setResult(undefined);
|
||||
setError(undefined);
|
||||
setIsSearching(false);
|
||||
};
|
||||
|
||||
createEffect(
|
||||
on([query, releasedFrom], ([text, from], previous) => {
|
||||
// query may be read from a larger value, such as all the filters, which
|
||||
// changes when neither does.
|
||||
if (previous && previous[0] === text && previous[1] === from) return;
|
||||
if (text.trim() === '') {
|
||||
clear();
|
||||
} else if (previous && previous[0] !== text) {
|
||||
clearTimeout(timer);
|
||||
setIsSearching(true);
|
||||
timer = setTimeout(() => void search(), SEARCH_DELAY);
|
||||
} else {
|
||||
void search();
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
async function loadHelp(): Promise<void> {
|
||||
helpController?.abort();
|
||||
helpController = new AbortController();
|
||||
const { signal } = helpController;
|
||||
try {
|
||||
const found = await api.searchOpportunities('', '', signal);
|
||||
if (!signal.aborted) keepHelp(found);
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
const failure = toError(err);
|
||||
setHelpError(failure);
|
||||
onError(`The search help didn't load: ${failure.message}`);
|
||||
}
|
||||
}
|
||||
void loadHelp();
|
||||
|
||||
const matches = createMemo(() => {
|
||||
const found = result();
|
||||
return found && !found.matches_all ? new Set(found.uuids) : undefined;
|
||||
});
|
||||
|
||||
return {
|
||||
matches,
|
||||
unknownPrompts: () => result()?.unknown_prompts ?? [],
|
||||
isSearching,
|
||||
error,
|
||||
refresh: () => {
|
||||
if (query().trim() !== '') void search();
|
||||
},
|
||||
help,
|
||||
helpError,
|
||||
reloadHelp: () => void loadHelp(),
|
||||
};
|
||||
}
|
||||
@@ -41,5 +41,12 @@ export type OpportunityReleaseTag = Schemas['types.OpportunityReleaseTag'];
|
||||
/** A page of GET /api/opportunities. */
|
||||
export type OpportunityPage = Schemas['api.OpportunityPage'];
|
||||
|
||||
/** GET /api/opportunities/search's result: the matches, and the search's help. */
|
||||
export type OpportunitySearch = Schemas['api.OpportunitySearch'];
|
||||
/** A search prompt (title:, cpv:, ...) and what it searches. */
|
||||
export type SearchPrompt = Schemas['search.Prompt'];
|
||||
/** An example of the search's syntax, and what it means. */
|
||||
export type SearchExample = Schemas['search.Example'];
|
||||
|
||||
/** An RFC 9457 problem details body, as the API returns for errors. */
|
||||
export type Problem = Schemas['httpx.Problem'];
|
||||
@@ -140,6 +140,77 @@ export interface paths {
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/opportunities/search": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Search opportunities
|
||||
* @description The UUIDs of the opportunities matching a general search,
|
||||
* in no particular order. q is words and "quoted phrases",
|
||||
* each optionally with a prompt (title:, buyer:, buyer_id:,
|
||||
* cpv:) and a leading ! to leave out what it matches; prompts
|
||||
* and syntax explain it. A search with nothing to search for
|
||||
* (blank, or only unknown prompts) has matches_all set and no
|
||||
* uuids. unknown_prompts lists the prompts that aren't known,
|
||||
* as typed; their terms are ignored. released_from (RFC 3339,
|
||||
* inclusive) leaves out opportunities whose latest release is
|
||||
* before it.
|
||||
*/
|
||||
get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
/** @description The search */
|
||||
q?: string;
|
||||
/** @description Only opportunities whose latest release is at or after this time */
|
||||
released_from?: string;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description OK */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["api.OpportunitySearch"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid released_from */
|
||||
400: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["httpx.Problem"];
|
||||
};
|
||||
};
|
||||
/** @description Internal Server Error */
|
||||
500: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["httpx.Problem"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/opportunities/{uuid}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
@@ -351,6 +422,13 @@ export interface components {
|
||||
offset: number;
|
||||
opportunities: components["schemas"]["types.Opportunity"][];
|
||||
};
|
||||
"api.OpportunitySearch": {
|
||||
matches_all: boolean;
|
||||
prompts: components["schemas"]["search.Prompt"][];
|
||||
syntax: components["schemas"]["search.Example"][];
|
||||
unknown_prompts: string[];
|
||||
uuids: string[];
|
||||
};
|
||||
"httpx.Problem": {
|
||||
detail?: string;
|
||||
instance?: string;
|
||||
@@ -359,6 +437,14 @@ export interface components {
|
||||
title: string;
|
||||
type?: string;
|
||||
};
|
||||
"search.Example": {
|
||||
example: string;
|
||||
help: string;
|
||||
};
|
||||
"search.Prompt": {
|
||||
help: string;
|
||||
name: string;
|
||||
};
|
||||
"types.ClassificationCode": {
|
||||
classification: string;
|
||||
code: string;
|
||||
|
||||
Reference in new issue
Block a user