From 045492a14897ed4907ff1b7d03cdd703f24656fe Mon Sep 17 00:00:00 2001 From: Craig Williams Date: Tue, 13 Oct 2026 12:52:17 +0100 Subject: [PATCH] First pass implementation of an SQLite backing store for fetched opportunities --- README.md | 30 +- cmd/jobsearch/main.go | 2 + go.mod | 9 +- go.sum | 43 +- internal/cli/confirmAction.go | 34 + internal/cli/migrate/common.go | 139 ++++ internal/cli/migrate/down.go | 77 +++ internal/cli/migrate/migrate.go | 42 ++ internal/cli/migrate/status.go | 94 +++ internal/cli/migrate/up.go | 73 ++ internal/config/README.md | 4 + internal/config/conf_database.go | 35 + internal/config/config.go | 5 +- internal/store/README.md | 114 +++ internal/store/common.go | 43 ++ internal/store/migrate.go | 200 ++++++ internal/store/migrate_test.go | 321 +++++++++ .../00001_create_opportunities_table.sql | 40 ++ ...0002_create_opportunity_releases_table.sql | 19 + ..._create_opportunity_release_tags_table.sql | 18 + ...0004_create_classification_codes_table.sql | 26 + ...eate_opportunity_classifications_table.sql | 20 + ...0006_create_opportunities_title_search.sql | 40 ++ .../00007_create_opportunity_marks.sql | 22 + internal/store/queries.go | 234 +++++++ internal/store/queries_test.go | 211 ++++++ internal/store/store.go | 596 ++++++++++++++++ internal/store/store_test.go | 647 ++++++++++++++++++ scripts/build.sh | 3 + 29 files changed, 3132 insertions(+), 9 deletions(-) create mode 100644 internal/cli/confirmAction.go create mode 100644 internal/cli/migrate/common.go create mode 100644 internal/cli/migrate/down.go create mode 100644 internal/cli/migrate/migrate.go create mode 100644 internal/cli/migrate/status.go create mode 100644 internal/cli/migrate/up.go create mode 100644 internal/config/conf_database.go create mode 100644 internal/store/README.md create mode 100644 internal/store/common.go create mode 100644 internal/store/migrate.go create mode 100644 internal/store/migrate_test.go create mode 100644 internal/store/migrations/00001_create_opportunities_table.sql create mode 100644 internal/store/migrations/00002_create_opportunity_releases_table.sql create mode 100644 internal/store/migrations/00003_create_opportunity_release_tags_table.sql create mode 100644 internal/store/migrations/00004_create_classification_codes_table.sql create mode 100644 internal/store/migrations/00005_create_opportunity_classifications_table.sql create mode 100644 internal/store/migrations/00006_create_opportunities_title_search.sql create mode 100644 internal/store/migrations/00007_create_opportunity_marks.sql create mode 100644 internal/store/queries.go create mode 100644 internal/store/queries_test.go create mode 100644 internal/store/store.go create mode 100644 internal/store/store_test.go diff --git a/README.md b/README.md index 9d56276..ad585a1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,9 @@ The build script runs `go generate ./...` and stamps the version from the latest git tag. A build from untagged commits is marked as a development build and isn't packaged unless `--force` is also given. -During development, `go run ./cmd/jobsearch` works as well. +During development, `go run -tags sqlite_fts5 ./cmd/jobsearch` works as well. +The `sqlite_fts5` tag is needed for the database's title search (see +[Testing](#testing)). ## Configuration @@ -37,6 +39,8 @@ Example `config.yaml`: logging: level: info file: /var/log/job-search.log +database: + file: /var/lib/job-search/jobsearch.db sources: find_a_tender: enabled: true @@ -60,6 +64,10 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and | `job-search config get ` | Print one value, e.g. `logging.level` | | `job-search config set ` | Validate and write one value to the config file | | `job-search config set --unset ` | Remove a value from the config file | +| `job-search migrate` | Apply every pending database migration (same as `migrate up`) | +| `job-search migrate up` | Apply pending migrations (`--steps N` for only N; `--create` to create a missing database file without asking) | +| `job-search migrate down --steps N` | Roll back the last N migrations (`--steps 0` rolls back all of them, deleting all data, and needs `--yes`) | +| `job-search migrate status` | Show the database version and which migrations are applied or pending | | `job-search version` | Print version information | `fetch service` logs to `logging.file` if it's set, or to the terminal with @@ -68,6 +76,10 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and `--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead of waiting. +The `migrate` commands work on the database in `database.file`. The store +refuses to open a database that isn't at this build's schema version, so run +`job-search migrate` after installing a new version. + ## Mock Find a Tender server To develop without using the real API, run the mock and enable the `fts_mock` @@ -95,6 +107,7 @@ See [internal/mock](internal/mock/README.md) for its options and behaviour. | `internal/cli` | Cobra commands and shared CLI setup | | `internal/config` | Configuration loading, validation and editing | | `internal/fetcher` | Runs a worker per enabled source | +| `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) | | `internal/findATender` | FTS API client and conversion from OCDS to JobSearch types | | `internal/types` | JobSearch's own opportunity types | | `internal/util` | The periodic `Worker` used by the fetcher | @@ -104,9 +117,14 @@ See [internal/mock](internal/mock/README.md) for its options and behaviour. ## Testing +The store's title search needs SQLite's FTS5 module, which is only compiled in +with the `sqlite_fts5` build tag, so tests (and lint) need it. Without it the +store tests fail with `no such module: fts5`. + ```bash -go test -race ./internal/... -go test -coverprofile=cover.out ./internal/... && go tool cover -func=cover.out +go test -race -tags sqlite_fts5 ./internal/... +go test -tags sqlite_fts5 -coverprofile=cover.out ./internal/... && go tool cover -func=cover.out +golangci-lint run --build-tags sqlite_fts5 ./... ``` Tests don't need network access; HTTP calls go to local `httptest` servers. @@ -168,7 +186,13 @@ To install the tools: ```bash go install github.com/spf13/cobra-cli@latest go install git.avsdev.uk/Go/go-ocds/cmd/ocdsgen@latest +go install github.com/pressly/goose/v3/cmd/goose@latest +go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest ``` +`goose` creates new migration files (see `internal/store`); the migrations +themselves are applied by `job-search migrate`. `golangci-lint` uses the +project's `.golangci.json`. + Ensure that the GOBIN path ($HOME/go/bin) is added to the environment before running any of the tools. diff --git a/cmd/jobsearch/main.go b/cmd/jobsearch/main.go index ad55cd7..59619eb 100644 --- a/cmd/jobsearch/main.go +++ b/cmd/jobsearch/main.go @@ -26,6 +26,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" "git.avsdev.uk/Go/job-search/internal/cli/config" "git.avsdev.uk/Go/job-search/internal/cli/fetch" + "git.avsdev.uk/Go/job-search/internal/cli/migrate" "git.avsdev.uk/Go/job-search/internal/cli/version" "github.com/spf13/cobra" ) @@ -73,6 +74,7 @@ opportunities. version.NewCommand(), config.NewCommand(opts), fetch.NewCommand(opts), + migrate.NewCommand(opts), ) return root diff --git a/go.mod b/go.mod index efa569d..4b0a20c 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module git.avsdev.uk/Go/job-search go 1.26.8 require ( - git.avsdev.uk/Go/go-ocds v1.0.0 + github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 github.com/lmittmann/tint v1.2.0 + github.com/mattn/go-sqlite3 v1.14.52 + github.com/pressly/goose/v3 v3.28.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 go.yaml.in/yaml/v3 v3.0.5 @@ -16,15 +18,18 @@ require ( github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sethvargo/go-retry v0.4.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stretchr/testify v1.12.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index 6d7ed93..cf7653a 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,58 @@ -git.avsdev.uk/Go/go-ocds v1.0.0 h1:X/CoCyDcVy8ObGZ+UK4t6viPGj2bqATTEYc4Q6/2DKA= -git.avsdev.uk/Go/go-ocds v1.0.0/go.mod h1:YPnqdf7ft6vPHiHLzeL/R7RtWYNKZJF1NXu/YEBpEbA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lmittmann/tint v1.2.0 h1:AogHRHy8HUJUnNJBHJlYa+fR4YY8mko2cnCp67xn9JY= github.com/lmittmann/tint v1.2.0/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pressly/goose/v3 v3.28.0 h1:D2M+iL31GmpZxSHOhX8mqyqAT3CXnokUmm0eKoSP+Vc= +github.com/pressly/goose/v3 v3.28.0/go.mod h1:v26MOuB8bL3kzzrt3Vqhb3R0PRVsl8hFQKdrht/L6Rk= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= +github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -43,15 +70,27 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= resty.dev/v3 v3.0.0-rc.4 h1:n3360yEcFiyrX5Vkc9XG3Y8kMM24alRBxJ9Ev1mLmqs= resty.dev/v3 v3.0.0-rc.4/go.mod h1:NTOerrC/4T7/FE6tXIZGIysXXBdgNqwMZuKtxpea9NM= diff --git a/internal/cli/confirmAction.go b/internal/cli/confirmAction.go new file mode 100644 index 0000000..4b0c12e --- /dev/null +++ b/internal/cli/confirmAction.go @@ -0,0 +1,34 @@ +package cli + +import ( + "bufio" + "fmt" + "log" + "os" + "strings" +) + +// Confirm asks the user the yes/no question s on stdin and reports whether +// they answered yes. "y" and "yes" confirm, "n" and "no" refuse, in any case; +// anything else asks again. It exits the program if stdin can't be read. +func Confirm(s string) bool { + reader := bufio.NewReader(os.Stdin) + + for { + fmt.Printf("%s [y/n]: ", s) + + response, err := reader.ReadString('\n') + if err != nil { + log.Fatal(err) + } + + response = strings.ToLower(strings.TrimSpace(response)) + + switch response { + case "y", "yes": + return true + case "n", "no": + return false + } + } +} diff --git a/internal/cli/migrate/common.go b/internal/cli/migrate/common.go new file mode 100644 index 0000000..7e7ce65 --- /dev/null +++ b/internal/cli/migrate/common.go @@ -0,0 +1,139 @@ +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/store" + "github.com/pressly/goose/v3" + "github.com/spf13/cobra" +) + +// openMigrator loads the configuration and returns a migrator over the +// store's embedded migrations for the configured database, opened with the +// store's connection settings. The caller must close the returned *sql.DB. +// +// If the database file doesn't exist, create decides between asking whether +// to create it (up) and failing (down, status: nothing to work on). +func openMigrator( + cmd *cobra.Command, opts *cli.Options, create bool, createConfirmed bool, +) (*goose.Provider, *sql.DB, error) { + conf, err := opts.RunSetup(cmd, true, true) + if err != nil { + return nil, nil, fmt.Errorf("no configuration loaded (%w)", err) + } + + file := conf.Database.File + if len(file) == 0 { + return nil, nil, errors.New( + "no database file specified. Update your configuration", + ) + } + + if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) { + if !create { + return nil, nil, fmt.Errorf("database file '%s' does not exist", file) + } + if !createConfirmed { + question := fmt.Sprintf( + "Database file '%s' does not exist. Would you like to create it?", file, + ) + if !cli.Confirm(question) { + return nil, nil, errors.New("migrate: no database file") + } + } + } else if err != nil { + return nil, nil, err + } + + db, err := store.OpenDB(file) + if err != nil { + return nil, nil, err + } + + m, err := store.NewMigrator(db) + if err != nil { + return nil, nil, errors.Join(err, db.Close()) + } + + return m, db, nil +} + +// closeDB closes db when deferred, adding any error to *err. +func closeDB(db *sql.DB, err *error) { + *err = errors.Join(*err, db.Close()) +} + +// errWriter writes until a write fails, then keeps the error, so a run of +// prints needs checking only once. +type errWriter struct { + w io.Writer + err error +} + +// printf writes like fmt.Fprintf, unless an earlier write failed. +func (e *errWriter) printf(format string, a ...any) { + if e.err == nil { + _, e.err = fmt.Fprintf(e.w, format, a...) + } +} + +// printResults lists migrations that ran, e.g. +// +// applied 00003_create_release_tags_table.sql (2ms) +func printResults(w io.Writer, results []*goose.MigrationResult) error { + ew := &errWriter{w: w} + for _, r := range results { + verb := "applied" + if r.Direction == "down" { + verb = "reverted" + } + ew.printf(" %-9s %s (%s)\n", + verb, filepath.Base(r.Source.Path), r.Duration.Round(time.Millisecond)) + } + return ew.err +} + +// failure prints what ran before a failed migration and describes the +// failure. Each migration runs in a transaction, so the failed one has been +// rolled back and the database is at the last version that succeeded. +func failure(w io.Writer, err error) error { + var pe *goose.PartialError + if !errors.As(err, &pe) { + return err + } + + printErr := printResults(w, pe.Applied) + + name := "unknown migration" + if pe.Failed != nil && pe.Failed.Source != nil { + name = filepath.Base(pe.Failed.Source.Path) + } + + return errors.Join(fmt.Errorf( + "migration %s failed and was rolled back; the database is at the last "+ + "version that succeeded.\nMigration error: %w", + name, pe.Err), printErr) +} + +// printVersion reports where the database ended up. +func printVersion(ctx context.Context, w io.Writer, m *goose.Provider) error { + v, err := m.GetDBVersion(ctx) + if err != nil { + return err + } + + if v == 0 { + _, err = fmt.Fprintln(w, "Database cleared") + } else { + _, err = fmt.Fprintf(w, "Database migrated to version %d\n", v) + } + return err +} diff --git a/internal/cli/migrate/down.go b/internal/cli/migrate/down.go new file mode 100644 index 0000000..9fedb6a --- /dev/null +++ b/internal/cli/migrate/down.go @@ -0,0 +1,77 @@ +package migrate + +import ( + "errors" + "fmt" + "log/slog" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/store" + "github.com/spf13/cobra" +) + +func commandDown(opts *cli.Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "down", + Short: "Migrate SQLite database to a previous version", + Long: `Roll back the most recent migrations. + +--steps 0 rolls back every migration which deletes all data and must be +confirmed by providing -yes.`, + Args: cobra.NoArgs, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.down.PersistentPreRun") + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.down.PersistentPostRun") + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + slog.Debug("migrate.down.Run") + + yes, _ := cmd.Flags().GetBool("yes") + steps, _ := cmd.Flags().GetUint("steps") + + m, db, err := openMigrator(cmd, opts, false, false) + if err != nil { + return err + } + defer closeDB(db, &err) + + results, err := store.StepDown(cmd.Context(), m, int(steps), yes) + + if len(results) == 0 && err == nil { + fmt.Println("No changes made to database (empty)") + return nil + } + if err == store.ErrWouldEmptyDatabase { + return fmt.Errorf( + "%w\n\nTo confirm you wish to proceed, pass the --yes flag", err, + ) + } + + if printErr := printResults(cmd.OutOrStdout(), results); printErr != nil { + return errors.Join(err, printErr) + } + + if err != nil { + return failure(cmd.OutOrStdout(), err) + } + + return printVersion(cmd.Context(), cmd.OutOrStdout(), m) + }, + } + cmd.Flags().Uint( + "steps", + 0, + "number of migrations to roll back. 0 = all (deletes all data)", + ) + if err := cmd.MarkFlagRequired("steps"); err != nil { + panic(err) // only fails if the flag isn't defined + } + cmd.Flags().BoolP( + "yes", "y", + false, + "confirm you are intending to roll back every migration", + ) + return cmd +} diff --git a/internal/cli/migrate/migrate.go b/internal/cli/migrate/migrate.go new file mode 100644 index 0000000..4bf1122 --- /dev/null +++ b/internal/cli/migrate/migrate.go @@ -0,0 +1,42 @@ +// Package migrate provides the "migrate" command and its subcommands, which +// apply, roll back and report the store's migrations (goose, embedded in the +// store package) on the configured SQLite database. +// +// goose runs each migration in its own transaction, so a failed migration is +// rolled back and leaves the database at the previous version. +package migrate + +import ( + "log/slog" + + "git.avsdev.uk/Go/job-search/internal/cli" + "github.com/spf13/cobra" +) + +// NewCommand returns the "migrate" command, which migrates the database schema (see +// internal/store). +func NewCommand(opts *cli.Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Migrate SQLite database to latest version", + Long: `Migrate the SQLite database to the latest version. + +Running without a subcommand is the same as running "migrate up": every pending +migration is applied.`, + Args: cobra.NoArgs, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.PersistentPreRun") + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.PersistentPostRun") + }, + RunE: func(cmd *cobra.Command, args []string) error { + slog.Debug("migrate.Run") + return runStepUp(cmd, opts, 0, false) + }, + } + cmd.AddCommand(commandUp(opts)) + cmd.AddCommand(commandDown(opts)) + cmd.AddCommand(commandStatus(opts)) + return cmd +} diff --git a/internal/cli/migrate/status.go b/internal/cli/migrate/status.go new file mode 100644 index 0000000..e3270f4 --- /dev/null +++ b/internal/cli/migrate/status.go @@ -0,0 +1,94 @@ +package migrate + +import ( + "context" + "errors" + "io" + "log/slog" + "path/filepath" + "text/tabwriter" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/store" + "github.com/pressly/goose/v3" + "github.com/spf13/cobra" +) + +// printStatus lists every migration as applied (with when) or pending, +// after a summary line, e.g. +// +// Database version 4 of 6 (2 pending) +// 1 applied 2026-10-07 10:12:01 00001_create_opportunities_table.sql +// ... +// 5 pending 00005_create_opportunity_classifications_table.sql +func printStatus(ctx context.Context, w io.Writer, m *goose.Provider) error { + statuses, err := m.Status(ctx) + if err != nil { + return err + } + + current, err := m.GetDBVersion(ctx) + if err != nil { + return err + } + + latest := store.LatestVersion(m) + + pending := 0 + for _, s := range statuses { + if s.State == goose.StatePending { + pending++ + } + } + + ew := &errWriter{w: w} + switch { + case current > latest: + ew.printf("Database version %d is newer than this build (latest %d)\n", + current, latest) + case pending == 0: + ew.printf("Database version %d of %d (up-to-date)\n", current, latest) + default: + ew.printf("Database version %d of %d (%d pending)\n", current, latest, pending) + } + if ew.err != nil { + return ew.err + } + + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + ew = &errWriter{w: tw} + for _, s := range statuses { + applied := "" + if s.State == goose.StateApplied { + applied = s.AppliedAt.Local().Format("2006-01-02 15:04:05") + } + ew.printf(" %d\t%s\t%s\t%s\n", + s.Source.Version, s.State, applied, filepath.Base(s.Source.Path)) + } + return errors.Join(ew.err, tw.Flush()) +} + +func commandStatus(opts *cli.Options) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Show which migrations have been applied to the SQLite database", + Args: cobra.NoArgs, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.status.PersistentPreRun") + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.status.PersistentPostRun") + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + slog.Debug("migrate.status.Run") + + m, db, err := openMigrator(cmd, opts, false, false) + if err != nil { + return err + } + defer closeDB(db, &err) + + return printStatus(cmd.Context(), cmd.OutOrStdout(), m) + }, + } +} diff --git a/internal/cli/migrate/up.go b/internal/cli/migrate/up.go new file mode 100644 index 0000000..d688bfc --- /dev/null +++ b/internal/cli/migrate/up.go @@ -0,0 +1,73 @@ +package migrate + +import ( + "fmt" + "log/slog" + + "git.avsdev.uk/Go/job-search/internal/cli" + "git.avsdev.uk/Go/job-search/internal/store" + "github.com/spf13/cobra" +) + +// runStepUp applies steps pending migrations (0 means all), printing each one +// and the resulting version. create says the user already agreed to create a +// missing database file; otherwise they are asked. +func runStepUp( + cmd *cobra.Command, opts *cli.Options, steps int, create bool, +) (err error) { + m, db, err := openMigrator(cmd, opts, true, create) + if err != nil { + return err + } + defer closeDB(db, &err) + + results, err := store.StepUp(cmd.Context(), m, int(steps)) + + if len(results) == 0 && err == nil { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "No changes made to database (up-to-date)") + return err + } + + if err := printResults(cmd.OutOrStdout(), results); err != nil { + return err + } + + if err != nil { + return failure(cmd.OutOrStdout(), err) + } + + return printVersion(cmd.Context(), cmd.OutOrStdout(), m) +} + +func commandUp(opts *cli.Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "up", + Short: "Migrate SQLite database to the latest version, or by a number of steps", + Args: cobra.NoArgs, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.up.PersistentPreRun") + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + slog.Debug("migrate.up.PersistentPostRun") + }, + RunE: func(cmd *cobra.Command, args []string) error { + slog.Debug("migrate.up.Run") + + steps, _ := cmd.Flags().GetUint("steps") + create, _ := cmd.Flags().GetBool("create") + + return runStepUp(cmd, opts, int(steps), create) + }, + } + cmd.Flags().Uint( + "steps", + 0, + "number of migrations to apply. 0 = all pending", + ) + cmd.Flags().Bool( + "create", + false, + "create the database file if it does not already exist", + ) + return cmd +} diff --git a/internal/config/README.md b/internal/config/README.md index 92b1925..afc268e 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -34,6 +34,7 @@ is kept. |---|---|---|---| | `logging.level` | string | `warn` | One of `debug`, `info`, `warn`, `error` | | `logging.file` | string | | Log file for `fetch service` (unless `--foreground`); other commands log to the terminal | +| `database.file` | string | | Path to the SQLite database file, used by the `migrate` commands | | `sources..enabled` | bool | `false` | For every source | | `sources..interval` | duration | `10m` | For every source; e.g. `30s`, `1h` | | `sources.find_a_tender.api_root` | string | `https://www.find-tender.service.gov.uk/api/1.0` | URL of the Find a Tender API | @@ -49,6 +50,8 @@ Example `config.yaml`: logging: level: info file: "" +database: + file: ./jobsearch.db sources: find_a_tender: enabled: true @@ -126,6 +129,7 @@ Nothing in `section.go` needs changing. | File | Contents | |---|---| | `config.go` | `JobSearchConfig`, defaults, Viper loading, text output | +| `conf_database.go` | `database` section | | `load.go` | `Init`, `.env` and config file loading | | `modify.go` | `StoreValue` and `PurgeValue` | | `section.go` | The `Section` interface and the struct tag reflection shared by every section | diff --git a/internal/config/conf_database.go b/internal/config/conf_database.go new file mode 100644 index 0000000..04465a4 --- /dev/null +++ b/internal/config/conf_database.go @@ -0,0 +1,35 @@ +package config + +import "reflect" + +// JSCKDatabase is the database section's configuration key. +const JSCKDatabase string = "database" + +// JobSearchConfigDatabase configures the SQLite database. File is the path to +// the database file. +type JobSearchConfigDatabase struct { + File string `cfg:"file"` +} + +// ---------- Section methods (see Section) ---------- + +// Keys returns the keys directly in the section. +func (c *JobSearchConfigDatabase) Keys() []string { return keysOf(c) } + +func (c JobSearchConfigDatabase) Map() map[string]any { return mapOf(c) } + +func (c JobSearchConfigDatabase) Flatten() map[string]any { + return flattenOf(JSCKDatabase, c) +} + +func (c JobSearchConfigDatabase) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +func (c JobSearchConfigDatabase) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +func (c *JobSearchConfigDatabase) SetValue(key string, value any) error { + return setValue(c, key, value) +} diff --git a/internal/config/config.go b/internal/config/config.go index 83dffc4..56e18b0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,8 +22,9 @@ const EtcConfigPath string = "/etc/job-search" // JobSearchConfig is the application's whole configuration, one field per // section. Each field's cfg tag must match the section's JSCK* constant. type JobSearchConfig struct { - Logging JobSearchConfigLogging `cfg:"logging"` - Sources JobSearchConfigSources `cfg:"sources"` + Logging JobSearchConfigLogging `cfg:"logging"` + Database JobSearchConfigDatabase `cfg:"database"` + Sources JobSearchConfigSources `cfg:"sources"` } // LoadDefaultConfig sets Viper's defaults from the configuration's default diff --git a/internal/store/README.md b/internal/store/README.md new file mode 100644 index 0000000..003c8db --- /dev/null +++ b/internal/store/README.md @@ -0,0 +1,114 @@ +# Store + +A SQLite store for opportunities and their OCDS releases, built on sqlx + goose and mattn/go-sqlite3. + +## Build and test + +The title search uses SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`. + +```bash +go test -tags sqlite_fts5 ./... +go run -tags sqlite_fts5 ./cmd/jobsearch migrate status +``` + +The tag can be made the default for every build with +`go env -w GOFLAGS=-tags=sqlite_fts5`. + +## Layout + +``` +internal/types/ Go struct definitions for tables (opportunities.go, classifications.go) +internal/store/migrations/ NNNNN_name.sql (Up and Down in one file) +internal/store/migrate.go OpenDB, NewMigrator (goose), schema version checks +internal/store/migrate_test.go version checks, stepping up and down, failed-migration rollback +internal/store/queries.go all SQL, as named constants grouped by table +internal/store/queries_test.go prepares every query against the migrated schema +internal/store/store.go Store: logic, using the queries via sqlx +internal/store/store_test.go behavioural tests +internal/store/common.go common for migrate, store and direct access +``` + +## Schema + +| Table | Key | Notes | +|---|---|---| +| `opportunities` | `uuid` (unique) | Updatable. `id INTEGER PRIMARY KEY` exists only to give the search index a stable rowid (fts5). | +| `opportunity_marks` | `uuid` | `uuid` references `opportunities`. The user's own marks (`hidden`, `skipped`, `interested`, `bid_submitted`, `won`, `lost`), each `BOOLEAN NOT NULL DEFAULT 0`. A `CHECK` constraint stops `won` and `lost` both being set. | +| `opportunity_releases` | (`uuid`, `release_id`) | Write-once. `uuid` references `opportunities`. `data` is TEXT, so `json_extract` works on it. | +| `opportunity_release_tags` | (`uuid`, `release_id`, `tag`) | Write-once. (`uuid`, `release_id`) references `opportunity_releases`. | +| `classification_codes` | `id` | Shared between opportunities. A row is reused only when every content column matches. The table grows over time to include codes not already seen in fetched opportunities. | +| `opportunity_classifications` | (`opportunity_uuid`, `classification_id`) | Mapping table; replaced as a set when the opportunity is updated. | +| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, kept in sync by triggers. | + +Foreign keys are enforced (`_foreign_keys=on` in the connection string) and cascade on delete from `opportunities`. The connection also enables WAL mode, a 5s busy timeout, and `_txlock=immediate`, so a save takes the write lock up front rather than failing halfway through. + +**When adding a classification scheme flag** (e.g. `is_unspsc`), the migration adding the column must drop and recreate the `classification_codes_content` unique index with the new column included. Otherwise rows differing only by that flag would be merged. + +**internal/types**: The structs contain `db` tags to allow sqlx to map table columns to struct members. Any struct members not mapped to a column (including child slices) are referenced by `db:"-"`. + + +## Migrations and Goose + +Migrations are embedded in the store package and applied explicitly, by the project's `migrate` command. `store.Open` doesn't migrate: it checks the schema version and fails with + +- `ErrNeedsMigration` if migrations are pending (resolved by running `migrate`); +- `ErrSchemaTooNew` if a newer build has migrated the database. + +`store.WithAutoMigrate()` makes `Open` apply pending migrations instead, for tests and throwaway databases. For tooling, `store.OpenDB(path)` opens the database with the store's connection settings, and `store.NewMigrator(db)` returns a goose `Provider` over the embedded migrations. + +Each migration runs in its own transaction, so a failed migration is rolled back and leaves the database at the previous version. goose records applied migrations in `goose_db_version`. Each file has `-- +goose Up` and `-- +goose Down` sections. Each section is one `StatementBegin`/`StatementEnd` block, which sends it to SQLite in a single call; that's what lets trigger bodies (which contain `;`) work. A new migration is created with: +```bash +goose -dir internal/store/migrations create add_something sql +``` + +## Queries and sqlx + +All SQL is in `queries.go` as constants, grouped by table in migration order; `store.go` holds only the logic. Inserts use sqlx named parameters (`:column`, matched to the `db` struct tags in `internal/types`). sqlx treats any colon followed by a name inside named SQL as a parameter, including inside SQL comments, so comments go on the Go constant, never inside the string. + +`queries_test.go` checks the SQL without needing a test for every code path: + +- **TestQueriesPrepare** prepares every query constant against a freshly migrated database, so a syntax error, an unknown table or unknown column fails the test and names the failing statement. +- **TestQueriesAllListed** parses `queries.go` and fails if a constant isn't listed in that test's `checks` map. Adding a query therefore means adding one line there, and it can't be skipped by accident. +- **TestListOpportunitiesQueryAllFilters** builds the `ListOpportunities` query with every filter set and runs it, checking that the fragments combine and that the arguments line up with the placeholders. +- **TestMarksCheckConstraint** checks the database itself rejects `won` and `lost` set together, for writes that bypass the Go validation. + +Adding a table involves adding its migration, putting its queries in a new section of `queries.go`, listing them in `checks`, and adding the store methods. + +Reads select whole rows (`SELECT *`, `t.*`, `c.*`) which map to a Go struct. sqlx refuses to read a result column that has no matching struct field, so when a migration adds a column, reads fail with `missing destination name ` until it is mapped: the field is added to the type in `internal/types` if callers should see it, or to the store's row wrapper if it is internal (like `opportunities.id` in `opportunityRow`). This means that queries should only return data for a single struct type at a time (nested structs are not supported - subsequent queries must be run to populate the referenced structs). + +`ListOpportunities` is assembled by `listOpportunitiesQuery` from fragments in `queries.go`: a base `SELECT`, one `filterOpportunity*` clause per filter that is set, and the ordering and paging tail. Each filter fragment takes exactly one argument, except the `filterOpportunityMark*` fragments, which take none (`MarkUnset` puts `NOT` in front of them). Adding a filter means one fragment and one line in `listOpportunitiesQuery`. The classifications and the marks for a page of opportunities are then loaded with one `IN (...)` query each. + +## Store API + +```go +s, err := store.Open(ctx, "jobsearch.db") // fails with ErrNeedsMigration if not migrated +s, err := store.Open(ctx, path, store.WithAutoMigrate()) // tests: migrate on open +res, err := s.SaveOpportunity(ctx, &opportunity) // SaveResult{OpportunityUpdated, NewReleases} +res, err := s.SaveOpportunities(ctx, opportunities) // all in one transaction; SaveMultipleResult +err = s.SetMarks(ctx, types.OpportunityMarks{UUID: uuid, Interested: true}) // ErrNotFound +t, err := s.GetOpportunity(ctx, uuid) // classifications + marks + releases + tags; ErrNotFound +ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{ + Title: "cloud migr", + Hidden: store.MarkUnset, // MarkAny (default), MarkSet or MarkUnset per mark + Interested: store.MarkSet, + Limit: 20, +}) +rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags +ok, err := s.ReleaseExists(ctx, uuid, releaseID) +``` + +**Saving.** `SaveOpportunity` writes everything in one transaction. `SaveOpportunities` does the same for several opportunities in a single transaction, reporting totals (`NewOpportunities`, `UpdatedOpportunities`, `NewReleases`) and each opportunity's `SaveResult`: + +- **Opportunity normalisation.** The opportunity is normalised in place. Releases, Marks and tags get the opportunity's `UUID` and their `ReleaseID`, times become UTC, `LatestReleaseDate` becomes the newest release date, and classifications get their database `ID`s. +- **Minimal writing.** Releases and tags are write-once. A release already stored (same `uuid` and `release_id`) is skipped, and `NewReleases` counts only the new ones. `ReleaseExists` lets a fetcher skip converting a release it already has. +- **Opportunity row updates.** The opportunity row and its classifications are replaced only if the save's newest release date is at least the stored `latest_release_date`. An older release arriving late is still stored, but never overwrites newer opportunity data; `OpportunityUpdated` reports which happened. +- **Marks are kept.** An opportunity's marks are stored when it is first saved; saving it again never changes them, so fetched data (which carries no marks) can't reset what the user has set. + +**Marks.** `SetMarks` replaces an opportunity's marks, returning `ErrNotFound` if the opportunity isn't stored. `won` and `lost` exclude each other: `SetMarks`, `SaveOpportunity` and `SaveOpportunities` reject marks with both set with `types.ErrWonAndLost` (from `OpportunityMarks.Validate`), and the database's `CHECK` constraint backs this up. + + +**Listing.** `ListOpportunities` filters by status, classification code, title text, a range on `LatestReleaseDate` and each mark (`MarkSet` or `MarkUnset`; an opportunity with no marks stored has every mark unset), newest first, 50 per page by default. It loads classifications and marks but not releases (which can be large); those are loaded by `ListReleases` or `GetOpportunity`. + + +**Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted. + diff --git a/internal/store/common.go b/internal/store/common.go new file mode 100644 index 0000000..490f90b --- /dev/null +++ b/internal/store/common.go @@ -0,0 +1,43 @@ +package store + +import ( + "database/sql" + "fmt" +) + +// Option configures Open. +type Option func(*options) + +type options struct { + autoMigrate bool +} + +// OpenDB opens the SQLite database at path with the store's connection +// settings (foreign keys, WAL, busy timeout, immediate transactions). The +// file is created on first use. Use it for anything that works on the store's +// database directly, such as running migrations. +func OpenDB(path string) (*sql.DB, error) { + return sql.Open("sqlite3", dsn(path)) +} + +// WithAutoMigrate makes Open apply pending migrations instead of returning +// ErrNeedsMigration. Intended for tests and throwaway databases; applications +// should migrate explicitly with the migrate command. +func WithAutoMigrate() Option { + return func(o *options) { o.autoMigrate = true } +} + +// dsn builds the mattn/go-sqlite3 connection string. +// +// - _foreign_keys: SQLite ignores REFERENCES clauses unless this is on. +// - _journal_mode=WAL: readers don't block the writer or each other. +// - _busy_timeout: wait for a lock rather than failing immediately. +// - _txlock=immediate: transactions take the write lock at BEGIN, so a +// read-then-write transaction (as in SaveOpportunity) can't fail with +// SQLITE_BUSY halfway through. +func dsn(path string) string { + return fmt.Sprintf( + "file:%s?_foreign_keys=on&_journal_mode=WAL&_busy_timeout=5000&_txlock=immediate", + path, + ) +} diff --git a/internal/store/migrate.go b/internal/store/migrate.go new file mode 100644 index 0000000..66bf7e4 --- /dev/null +++ b/internal/store/migrate.go @@ -0,0 +1,200 @@ +package store + +import ( + "context" + "database/sql" + "embed" + "errors" + "fmt" + "io/fs" + + "github.com/pressly/goose/v3" +) + +// Each file holds both directions, marked with -- +goose Up / -- +goose Down. +// Create new ones with: goose -dir internal/store/migrations create NAME sql +// +//go:embed migrations/*.sql +var migrationsFS embed.FS + +// ErrNeedsMigration is returned by Open when the database has pending +// migrations. Apply them with the migrate command, or pass +// WithAutoMigrate. +var ErrNeedsMigration = errors.New("store: database needs migrating") + +// ErrSchemaTooNew is returned by Open when the database has migrations +// this build doesn't know about, i.e. it was migrated by a newer build. +var ErrSchemaTooNew = errors.New("store: database schema is newer than this build") + +// ErrWouldEmptyDatabase is returned by StepDown when the migration would drop +// all of the tables and thus empty the database. +var ErrWouldEmptyDatabase = errors.New( + "migration cancelled; database would be emptied by undoing all migrations", +) + +// NewMigrator returns a goose Provider over the store's embedded migrations, +// for applying, rolling back or inspecting them. Each migration runs in its +// own transaction, so a failed migration leaves the database at the previous +// version (goose has no "dirty" state). +func NewMigrator(db *sql.DB) (*goose.Provider, error) { + fsys, err := fs.Sub(migrationsFS, "migrations") + if err != nil { + return nil, err + } + + return goose.NewProvider(goose.DialectSQLite3, db, fsys) +} + +// LatestVersion is the highest migration version in this build. +func LatestVersion(p *goose.Provider) int64 { + sources := p.ListSources() // sorted ascending + if len(sources) == 0 { + return 0 + } + return sources[len(sources)-1].Version +} + +// checkVersion returns ErrNeedsMigration or ErrSchemaTooNew unless the +// database is exactly at this build's schema. +func checkVersion(ctx context.Context, p *goose.Provider) error { + current, err := p.GetDBVersion(ctx) + if err != nil { + return err + } + + latest := LatestVersion(p) + if current > latest { + return fmt.Errorf("%w (database is at version %d, this build knows up to %d)", + ErrSchemaTooNew, current, latest) + } + + pending, err := p.HasPending(ctx) + if err != nil { + return err + } + + if pending { + return fmt.Errorf("%w (database is at version %d, latest is %d)", + ErrNeedsMigration, current, latest) + } + + return nil +} + +// checkNotNewer refuses to work on a database migrated by a newer build: +// this build has neither the newer migrations nor their down steps. +func checkNotNewer(ctx context.Context, m *goose.Provider) error { + current, err := m.GetDBVersion(ctx) + if err != nil { + return err + } + if latest := LatestVersion(m); current > latest { + return fmt.Errorf( + "database is at version %d but this build only knows up to %d; use a newer build", + current, latest) + } + return nil +} + +// prepareSchema migrates (with WithAutoMigrate) or verifies the schema +// version. +func prepareSchema(ctx context.Context, db *sql.DB, o options) error { + p, err := NewMigrator(db) + if err != nil { + return err + } + + if o.autoMigrate { + if _, err := p.Up(ctx); err != nil { + return fmt.Errorf("store: migrating database: %w", err) + } + } + + return checkVersion(ctx, p) +} + +// StepUp applies steps pending migrations, or all of them when steps is 0. +func StepUp( + ctx context.Context, m *goose.Provider, steps int, +) ([]*goose.MigrationResult, error) { + var results []*goose.MigrationResult + + if err := checkNotNewer(ctx, m); err != nil { + return results, err + } + + var err error + if steps == 0 { + results, err = m.Up(ctx) + } else { + for range steps { + var r *goose.MigrationResult + r, err = m.UpByOne(ctx) + if errors.Is(err, goose.ErrNoNextVersion) { + err = nil + break + } + if err != nil { + break + } + results = append(results, r) + } + } + + return results, err +} + +// StepDown rolls back steps applied migrations, or all of them when steps +// is 0. Rolling everything back deletes all data, so confirmDrop is checked +// first in that case. +func StepDown( + ctx context.Context, m *goose.Provider, steps int, confirmDrop bool, +) ([]*goose.MigrationResult, error) { + var results []*goose.MigrationResult + + if err := checkNotNewer(ctx, m); err != nil { + return results, err + } + + current, err := m.GetDBVersion(ctx) + if err != nil { + return results, err + } + if current == 0 { + return results, nil + } + + statuses, err := m.Status(ctx) + if err != nil { + return results, err + } + applied := 0 + for _, s := range statuses { + if s.State == goose.StateApplied { + applied++ + } + } + all := steps == 0 || steps >= applied + if all && !confirmDrop { + return results, ErrWouldEmptyDatabase + } + + if all { + results, err = m.DownTo(ctx, 0) + } else { + for range steps { + var r *goose.MigrationResult + r, err = m.Down(ctx) + if errors.Is(err, goose.ErrNoNextVersion) { + err = nil + break + } + if err != nil { + break + } + results = append(results, r) + } + } + + return results, err +} diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go new file mode 100644 index 0000000..4661b8a --- /dev/null +++ b/internal/store/migrate_test.go @@ -0,0 +1,321 @@ +package store_test + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "strings" + "testing" + + "git.avsdev.uk/Go/job-search/internal/store" + "github.com/pressly/goose/v3" +) + +// Run with: go test -tags sqlite_fts5 ./... + +func tempDBPath(t *testing.T) string { + return filepath.Join(t.TempDir(), "migrate.db") +} + +func newMigrator(t *testing.T) (*goose.Provider, *sql.DB) { + t.Helper() + db, err := store.OpenDB(filepath.Join(t.TempDir(), "migrate.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + m, err := store.NewMigrator(db) + if err != nil { + t.Fatal(err) + } + return m, db +} + +func version(t *testing.T, m *goose.Provider) int64 { + t.Helper() + v, err := m.GetDBVersion(context.Background()) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestOpenRequiresMigration(t *testing.T) { + ctx := context.Background() + path := tempDBPath(t) + + if _, err := store.Open(ctx, path); !errors.Is(err, store.ErrNeedsMigration) { + t.Fatalf("fresh database: err = %v, want ErrNeedsMigration", err) + } + + db, err := store.OpenDB(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + m, err := store.NewMigrator(db) + if err != nil { + t.Fatal(err) + } + + if _, err := m.UpByOne(ctx); err != nil { + t.Fatal(err) + } + if _, err := store.Open(ctx, path); !errors.Is(err, store.ErrNeedsMigration) { + t.Fatalf("partly migrated: err = %v, want ErrNeedsMigration", err) + } + + if _, err := m.Up(ctx); err != nil { + t.Fatal(err) + } + s, err := store.Open(ctx, path) + if err != nil { + t.Fatalf("fully migrated: %v", err) + } + s.Close() +} + +func TestOpenRejectsNewerSchema(t *testing.T) { + ctx := context.Background() + path := tempDBPath(t) + s, err := store.Open(ctx, path, store.WithAutoMigrate()) + if err != nil { + t.Fatal(err) + } + s.Close() + + // Simulate a migration applied by a newer build. + db, err := store.OpenDB(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec( + `INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`); err != nil { + t.Fatal(err) + } + + for name, opts := range map[string][]store.Option{ + "check": nil, + "auto-migrate": {store.WithAutoMigrate()}, + } { + if _, err := store.Open(ctx, path, opts...); !errors.Is(err, store.ErrSchemaTooNew) { + t.Errorf("%s: err = %v, want ErrSchemaTooNew", name, err) + } + } +} + +func TestMigrateDownAllAndUpAgain(t *testing.T) { + ctx := context.Background() + path := tempDBPath(t) + db, err := store.OpenDB(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + m, err := store.NewMigrator(db) + if err != nil { + t.Fatal(err) + } + + if _, err := m.Up(ctx); err != nil { + t.Fatal(err) + } + if v, _ := m.GetDBVersion(ctx); v != store.LatestVersion(m) { + t.Fatalf("after up: version %d, want %d", v, store.LatestVersion(m)) + } + + if _, err := m.DownTo(ctx, 0); err != nil { + t.Fatalf("down to 0: %v", err) + } + var tables int + if err := db.QueryRow(` + SELECT count(*) FROM sqlite_master + WHERE type IN ('table', 'index', 'trigger') + AND name NOT LIKE 'sqlite_%' AND name != 'goose_db_version'`).Scan(&tables); err != nil { + t.Fatal(err) + } + if tables != 0 { + t.Errorf("after down to 0: %d schema objects remain", tables) + } + + if _, err := m.Up(ctx); err != nil { + t.Fatalf("up again: %v", err) + } + s, err := store.Open(ctx, path) + if err != nil { + t.Fatalf("open after round trip: %v", err) + } + s.Close() +} + +func TestUpAllThenUpToDate(t *testing.T) { + ctx := context.Background() + m, _ := newMigrator(t) + latest := store.LatestVersion(m) + + results, err := store.StepUp(ctx, m, 0) + if err != nil { + t.Fatal(err) + } + if got := len(results); got != int(latest) { + t.Fatalf("applied %d migrations, want %d.", got, latest) + } + + results, err = store.StepUp(ctx, m, 0) + if err != nil { + t.Fatal(err) + } + + if len(results) != 0 { + t.Fatalf("expected no further migrations, got %d.", len(results)) + } +} + +func TestUpBySteps(t *testing.T) { + ctx := context.Background() + m, _ := newMigrator(t) + + nToApply := 2 + + _, err := store.StepUp(ctx, m, nToApply) + if err != nil { + t.Fatal(err) + } + if v := version(t, m); v != int64(nToApply) { + t.Fatalf("after 2 steps: version %d", v) + } + // More steps than pending stops at the latest version. + _, err = store.StepUp(ctx, m, 50) + if err != nil { + t.Fatal(err) + } + if v := version(t, m); v != store.LatestVersion(m) { + t.Fatalf("after 50 steps: version %d", v) + } +} + +func TestDownBySteps(t *testing.T) { + ctx := context.Background() + m, _ := newMigrator(t) + + _, err := store.StepUp(ctx, m, 0) + if err != nil { + t.Fatal(err) + } + + _, err = store.StepDown(ctx, m, 2, false) + if err != nil { + t.Fatal(err) + } + if v, want := version(t, m), store.LatestVersion(m)-2; v != want { + t.Fatalf("after 2 down steps: version %d, want %d", v, want) + } +} + +func TestDownAllRequiresConfirmation(t *testing.T) { + ctx := context.Background() + m, _ := newMigrator(t) + + _, err := store.StepUp(ctx, m, 0) + if err != nil { + t.Fatal(err) + } + + hasErrored := false + for name, steps := range map[string]int{"steps 0": 0, "more steps than applied": 99} { + _, err := store.StepDown(ctx, m, steps, false) + if !errors.Is(err, store.ErrWouldEmptyDatabase) { + t.Errorf("%s: %s", name, err) + hasErrored = true + } + if v := version(t, m); v != store.LatestVersion(m) { + t.Errorf("%s: version changed to %d", name, v) + hasErrored = true + } + } + if hasErrored { + t.FailNow() + } + + _, err = store.StepDown(ctx, m, 0, true) + if err != nil { + t.Fatal(err) + } + if v := version(t, m); v != 0 { + t.Fatalf("after confirmed down: version %d", v) + } +} + +func TestDownOnEmptyDatabase(t *testing.T) { + m, _ := newMigrator(t) + results, err := store.StepDown(context.Background(), m, 0, false) + if err != nil { + t.Fatal(err) + } + if len(results) != 0 { + t.Fatalf("expected no further migrations, got %d.", len(results)) + } +} + +func TestNewerDatabaseIsRefused(t *testing.T) { + ctx := context.Background() + m, db := newMigrator(t) + + if _, err := store.StepUp(ctx, m, 0); err != nil { + t.Fatal(err) + } + if _, err := db.Exec( + `INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`); err != nil { + t.Fatal(err) + } + + if _, err := store.StepUp( + ctx, + m, + 0, + ); err == nil || + !strings.Contains(err.Error(), "newer build") { + t.Errorf("up: err = %v, want a newer-build error", err) + } + if _, err := store.StepDown( + ctx, + m, + 1, + false, + ); err == nil || + !strings.Contains(err.Error(), "newer build") { + t.Errorf("down: err = %v, want a newer-build error", err) + } +} + +func TestFailedMigrationIsRolledBack(t *testing.T) { + ctx := context.Background() + m, db := newMigrator(t) + + if _, err := store.StepUp(ctx, m, 1); err != nil { + t.Fatal(err) + } + + // Occupy migration 4's table name with a view. Its CREATE TABLE IF NOT + // EXISTS then skips, and the CREATE INDEX that follows fails because + // views can't be indexed, part-way through the migration. + if _, err := db.Exec(`CREATE VIEW classification_codes AS SELECT 1 AS x`); err != nil { + t.Fatal(err) + } + + _, err := store.StepUp(ctx, m, 0) + if err == nil { + t.Fatal("expected migration 4 to fail") + } + if v := version(t, m); v != 3 { + t.Errorf("version after failure: %d, want 3", v) + } + var indexes int + db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name LIKE 'classification_codes_%'`). + Scan(&indexes) + if indexes != 0 { + t.Errorf("failed migration left %d objects behind", indexes) + } +} diff --git a/internal/store/migrations/00001_create_opportunities_table.sql b/internal/store/migrations/00001_create_opportunities_table.sql new file mode 100644 index 0000000..7fa90a8 --- /dev/null +++ b/internal/store/migrations/00001_create_opportunities_table.sql @@ -0,0 +1,40 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS "opportunities" ( + -- Surrogate key. "uuid" is the real identifier; "id" exists only to give + -- the title search index (opportunities_fts) a stable rowid to point at. + "id" INTEGER PRIMARY KEY, + "uuid" TEXT NOT NULL UNIQUE, + "uuid_type" TEXT NOT NULL, + "title" TEXT NOT NULL, + "status" TEXT NOT NULL, + "description" TEXT NOT NULL DEFAULT '', + "buyer_name" TEXT NOT NULL DEFAULT '', + -- classifications: see tender_classifications + "buyer_id" TEXT NOT NULL DEFAULT '', + "main_category" TEXT NOT NULL DEFAULT '', + "procurement_method" TEXT NOT NULL DEFAULT '', + "multi_lot" BOOLEAN NOT NULL DEFAULT FALSE, + "sme_suitable" BOOLEAN NOT NULL DEFAULT FALSE, + "vcse_suitable" BOOLEAN NOT NULL DEFAULT FALSE, + "start_date" DATETIME, + "end_date" DATETIME, + "gross_value" REAL NOT NULL DEFAULT 0, + "has_documents" BOOLEAN NOT NULL DEFAULT FALSE, + "source" TEXT NOT NULL DEFAULT '', + -- Release date of the newest release applied to this row. A release + -- older than this never overwrites the tender. + "latest_release_date" DATETIME NOT NULL + -- releases: see tender_ocds_releases +); + +CREATE INDEX IF NOT EXISTS "opportunities_status" + ON "opportunities" ("status"); +CREATE INDEX IF NOT EXISTS "opportunities_latest_release_date" + ON "opportunities" ("latest_release_date"); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "opportunities"; +-- +goose StatementEnd diff --git a/internal/store/migrations/00002_create_opportunity_releases_table.sql b/internal/store/migrations/00002_create_opportunity_releases_table.sql new file mode 100644 index 0000000..1c4abdf --- /dev/null +++ b/internal/store/migrations/00002_create_opportunity_releases_table.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +-- Write-once: a release is never modified after it is stored. +-- "release_id" is the OCDS release id, unique only within its OCID ("uuid"). +CREATE TABLE IF NOT EXISTS "opportunity_releases" ( + "uuid" TEXT NOT NULL + REFERENCES "opportunities" ("uuid") ON DELETE CASCADE, + "release_id" TEXT NOT NULL, + "release_date" DATETIME NOT NULL, + -- tags: see release_tags + "data" TEXT NOT NULL, + PRIMARY KEY ("uuid", "release_id") +) WITHOUT ROWID; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "opportunity_releases"; +-- +goose StatementEnd diff --git a/internal/store/migrations/00003_create_opportunity_release_tags_table.sql b/internal/store/migrations/00003_create_opportunity_release_tags_table.sql new file mode 100644 index 0000000..98bce40 --- /dev/null +++ b/internal/store/migrations/00003_create_opportunity_release_tags_table.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- +goose StatementBegin +-- Write-once, inserted together with the release it belongs to. +CREATE TABLE IF NOT EXISTS "opportunity_release_tags" ( + "uuid" TEXT NOT NULL, + "release_id" TEXT NOT NULL, + "tag" TEXT NOT NULL, + PRIMARY KEY ("uuid", "release_id", "tag"), + FOREIGN KEY ("uuid", "release_id") + REFERENCES "opportunity_releases" ("uuid", "release_id") + ON DELETE CASCADE +) WITHOUT ROWID; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "opportunity_release_tags"; +-- +goose StatementEnd diff --git a/internal/store/migrations/00004_create_classification_codes_table.sql b/internal/store/migrations/00004_create_classification_codes_table.sql new file mode 100644 index 0000000..5f660e3 --- /dev/null +++ b/internal/store/migrations/00004_create_classification_codes_table.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +-- Shared, write-once lookup of classifications. A row is reused only when +-- every content column matches; any difference (e.g. a changed description) +-- creates a new row. +-- +-- When adding a new scheme flag column (e.g. "is_unspsc"), the same migration +-- must drop and recreate "classification_codes_content" with the new column +-- included, otherwise rows differing only by that flag would be merged. +CREATE TABLE IF NOT EXISTS "classification_codes" ( + "id" INTEGER PRIMARY KEY, + "is_cpv" BOOLEAN NOT NULL, + "code" TEXT NOT NULL, + "classification" TEXT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "classification_codes_content" + ON "classification_codes" ("is_cpv", "code", "classification"); +CREATE INDEX IF NOT EXISTS "classification_codes_code" + ON "classification_codes" ("code"); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "classification_codes"; +-- +goose StatementEnd diff --git a/internal/store/migrations/00005_create_opportunity_classifications_table.sql b/internal/store/migrations/00005_create_opportunity_classifications_table.sql new file mode 100644 index 0000000..87bd3de --- /dev/null +++ b/internal/store/migrations/00005_create_opportunity_classifications_table.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- +goose StatementBegin +-- Maps opportunities to classification codes. Replaced as a set whenever the +-- opportunity row is updated. +CREATE TABLE IF NOT EXISTS "opportunity_classifications" ( + "opportunity_uuid" TEXT NOT NULL + REFERENCES "opportunities" ("uuid") ON DELETE CASCADE, + "classification_id" INTEGER NOT NULL + REFERENCES "classification_codes" ("id"), + PRIMARY KEY ("opportunity_uuid", "classification_id") +) WITHOUT ROWID; + +CREATE INDEX IF NOT EXISTS "opportunity_classifications_classification_id" + ON "opportunity_classifications" ("classification_id"); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "opportunity_classifications"; +-- +goose StatementEnd diff --git a/internal/store/migrations/00006_create_opportunities_title_search.sql b/internal/store/migrations/00006_create_opportunities_title_search.sql new file mode 100644 index 0000000..03bd5f5 --- /dev/null +++ b/internal/store/migrations/00006_create_opportunities_title_search.sql @@ -0,0 +1,40 @@ +-- +goose Up +-- +goose StatementBegin +-- Full-text index over opportunities.title. Requires mattn/go-sqlite3 to be +-- built with -tags sqlite_fts5. Kept in sync by the triggers below. +CREATE VIRTUAL TABLE IF NOT EXISTS "opportunities_fts" USING fts5( + "title", + content = 'opportunities', + content_rowid = 'id', + tokenize = 'porter unicode61 remove_diacritics 2' +); + +CREATE TRIGGER IF NOT EXISTS "opportunities_fts_insert" AFTER INSERT ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title"); +END; + +CREATE TRIGGER IF NOT EXISTS "opportunities_fts_delete" AFTER DELETE ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title") + VALUES ('delete', old."id", old."title"); +END; + +CREATE TRIGGER IF NOT EXISTS "opportunities_fts_update" AFTER UPDATE OF "title" ON "opportunities" +BEGIN + INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title") + VALUES ('delete', old."id", old."title"); + INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title"); +END; + +-- Index any opportunities that existed before this migration. +INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild'); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TRIGGER IF EXISTS "opportunities_fts_update"; +DROP TRIGGER IF EXISTS "opportunities_fts_delete"; +DROP TRIGGER IF EXISTS "opportunities_fts_insert"; +DROP TABLE IF EXISTS "opportunities_fts"; +-- +goose StatementEnd diff --git a/internal/store/migrations/00007_create_opportunity_marks.sql b/internal/store/migrations/00007_create_opportunity_marks.sql new file mode 100644 index 0000000..62f36d5 --- /dev/null +++ b/internal/store/migrations/00007_create_opportunity_marks.sql @@ -0,0 +1,22 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS "opportunity_marks" ( + "uuid" TEXT NOT NULL UNIQUE, + "hidden" BOOLEAN NOT NULL DEFAULT 0, + "skipped" BOOLEAN NOT NULL DEFAULT 0, + "interested" BOOLEAN NOT NULL DEFAULT 0, + "bid_submitted" BOOLEAN NOT NULL DEFAULT 0, + "lost" BOOLEAN NOT NULL DEFAULT 0, + "won" BOOLEAN NOT NULL DEFAULT 0, + CHECK (NOT ("won" AND "lost")), + PRIMARY KEY ("uuid"), + FOREIGN KEY ("uuid") + REFERENCES "opportunities" ("uuid") + ON DELETE CASCADE +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "opportunity_marks"; +-- +goose StatementEnd diff --git a/internal/store/queries.go b/internal/store/queries.go new file mode 100644 index 0000000..8763d51 --- /dev/null +++ b/internal/store/queries.go @@ -0,0 +1,234 @@ +package store + +// All SQL used by the store, grouped by table in migration order. +// +// Conventions: +// - Statements are named , e.g. insertRelease. Fragments +// combined at run time are named after what they build (listOpportunities*, +// filterOpportunity*). +// - Named-parameter statements (":column", bound from a struct's db tags) +// say so in their comment. Everything else uses "?" placeholders. +// - Reads select whole rows (*, t.*, c.*) so a column added by a migration +// but not mapped to a struct field fails loudly; see opportunityRow. +// - Keep comments out of the SQL strings: sqlx reads a colon followed by a +// name as a parameter, even inside an SQL comment. +// - Every constant here is prepared against the migrated schema by +// TestQueriesPrepare (queries_test.go), which also fails if a constant is +// added without being listed there. + +// ---------- opportunities ---------- + +// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares against. +const selectOpportunityLatestReleaseDate = ` +SELECT latest_release_date FROM opportunities WHERE uuid = ?` + +// upsertOpportunity inserts or updates a opportunity. Named parameters (types.Opportunity). +// +// An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first, +// cascading to the opportunity's releases and changing its id. +const upsertOpportunity = ` +INSERT INTO opportunities ( + uuid, uuid_type, title, status, description, buyer_name, buyer_id, + main_category, procurement_method, multi_lot, sme_suitable, vcse_suitable, + start_date, end_date, gross_value, has_documents, latest_release_date +) VALUES ( + :uuid, :uuid_type, :title, :status, :description, :buyer_name, :buyer_id, + :main_category, :procurement_method, :multi_lot, :sme_suitable, :vcse_suitable, + :start_date, :end_date, :gross_value, :has_documents, :latest_release_date +) +ON CONFLICT (uuid) DO UPDATE SET + uuid_type = excluded.uuid_type, + title = excluded.title, + status = excluded.status, + description = excluded.description, + buyer_name = excluded.buyer_name, + buyer_id = excluded.buyer_id, + main_category = excluded.main_category, + procurement_method = excluded.procurement_method, + multi_lot = excluded.multi_lot, + sme_suitable = excluded.sme_suitable, + vcse_suitable = excluded.vcse_suitable, + start_date = excluded.start_date, + end_date = excluded.end_date, + gross_value = excluded.gross_value, + has_documents = excluded.has_documents, + latest_release_date = excluded.latest_release_date` + +// selectOpportunityByUUID reads one opportunity into a opportunityRow. +const selectOpportunityByUUID = ` +SELECT t.* FROM opportunities t WHERE t.uuid = ?` + +// ListOpportunities is assembled by listOpportunitiesQuery as: +// +// listOpportunitiesSelect [WHERE filter AND filter ...] listOpportunitiesOrderAndPage +// +// Each filterOpportunity* fragment takes exactly one "?" argument, except +// filterOpportunityMark* fragments, which take none. A mark fragment is true +// when the mark is set and is negated with NOT to match it unset; an +// opportunity with no marks row has every mark unset. +const ( + listOpportunitiesSelect = ` +SELECT t.* FROM opportunities t` + + listOpportunitiesOrderAndPage = ` +ORDER BY t.latest_release_date DESC, t.uuid +LIMIT ? OFFSET ?` + + filterOpportunityStatus = `t.status = ?` + + filterOpportunityClassificationCode = `EXISTS ( + SELECT 1 + FROM opportunity_classifications tc + JOIN classification_codes c ON c.id = tc.classification_id + WHERE tc.opportunity_uuid = t.uuid AND c.code = ? +)` + + // The argument must come from ftsQuery, never raw user input. + filterOpportunityTitleMatch = `t.id IN ( + SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ? +)` + + filterOpportunityReleasedFrom = `t.latest_release_date >= ?` + + filterOpportunityReleasedBefore = `t.latest_release_date < ?` + + filterOpportunityMarkHidden = `EXISTS ( + SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.hidden +)` + + filterOpportunityMarkSkipped = `EXISTS ( + SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.skipped +)` + + filterOpportunityMarkInterested = `EXISTS ( + SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.interested +)` + + filterOpportunityMarkBidSubmitted = `EXISTS ( + SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.bid_submitted +)` + + filterOpportunityMarkWon = `EXISTS ( + SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.won +)` + + filterOpportunityMarkLost = `EXISTS ( + SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.lost +)` +) + +// ---------- opportunity_releases ---------- + +// insertRelease stores a release; 0 rows affected means it was already +// stored. Named parameters (types.OCDSReleaseJSON). +const insertRelease = ` +INSERT INTO opportunity_releases (uuid, release_id, release_date, data) +VALUES (:uuid, :release_id, :release_date, :data) +ON CONFLICT DO NOTHING` + +// selectReleasesByOpportunity reads a opportunity's releases, oldest first. +const selectReleasesByOpportunity = ` +SELECT * +FROM opportunity_releases +WHERE uuid = ? +ORDER BY release_date, release_id` + +// selectReleaseExists returns a single boolean. +const selectReleaseExists = ` +SELECT EXISTS ( + SELECT 1 FROM opportunity_releases WHERE uuid = ? AND release_id = ? +)` + +// ---------- opportunity_release_tags ---------- + +// insertReleaseTag stores a tag. Named parameters (types.OCDSReleaseTags). +const insertReleaseTag = ` +INSERT INTO opportunity_release_tags (uuid, release_id, tag) +VALUES (:uuid, :release_id, :tag) +ON CONFLICT DO NOTHING` + +// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's releases. +const selectReleaseTagsByOpportunity = ` +SELECT * +FROM opportunity_release_tags +WHERE uuid = ? +ORDER BY release_id, tag` + +// ---------- classification_codes ---------- + +// upsertClassificationCodeReturningID returns the id of the row matching +// every content column, inserting it first if needed. Named parameters +// (types.ClassificationCode). +// +// The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING yield the +// existing row's id on conflict. +const upsertClassificationCodeReturningID = ` +INSERT INTO classification_codes (is_cpv, code, classification) +VALUES (:is_cpv, :code, :classification) +ON CONFLICT (is_cpv, code, classification) DO UPDATE SET is_cpv = excluded.is_cpv +RETURNING id` + +// ---------- opportunity_classifications ---------- + +// deleteOpportunityClassifications clears a opportunity's mappings before they are +// rewritten. +const deleteOpportunityClassifications = ` +DELETE FROM opportunity_classifications WHERE opportunity_uuid = ?` + +// insertOpportunityClassification maps a opportunity to a classification code. +const insertOpportunityClassification = ` +INSERT INTO opportunity_classifications (opportunity_uuid, classification_id) +VALUES (?, ?) +ON CONFLICT DO NOTHING` + +// selectClassificationsForOpportunities reads the classifications of several +// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by Rebind. +// c.* must map entirely onto types.ClassificationCode. +const selectClassificationsForOpportunities = ` +SELECT tc.opportunity_uuid, c.* +FROM opportunity_classifications tc +JOIN classification_codes c ON c.id = tc.classification_id +WHERE tc.opportunity_uuid IN (?) +ORDER BY tc.opportunity_uuid, c.code, c.id` + +// ---------- opportunity_marks ---------- + +// insertOpportunityMarks gives a new opportunity its marks row; an existing +// row is left alone, so saving fetched data never resets marks set by the +// user. Named parameters (types.OpportunityMarks). +const insertOpportunityMarks = ` +INSERT INTO opportunity_marks ( + uuid, hidden, interested, skipped, bid_submitted, won, lost +) +VALUES ( + :uuid, :hidden, :interested, :skipped, :bid_submitted, :won, :lost +) +ON CONFLICT (uuid) DO NOTHING` + +// upsertOpportunityMarks inserts or updates an opportunity's marks. Named +// parameters (types.OpportunityMarks). +// +// An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first +// which is unnecessary. +const upsertOpportunityMarks = ` +INSERT INTO opportunity_marks ( + uuid, hidden, interested, skipped, bid_submitted, won, lost +) +VALUES ( + :uuid, :hidden, :interested, :skipped, :bid_submitted, :won, :lost +) +ON CONFLICT (uuid) DO UPDATE SET + hidden = excluded.hidden, + interested = excluded.interested, + skipped = excluded.skipped, + bid_submitted = excluded.bid_submitted, + won = excluded.won, + lost = excluded.lost` + +// selectMarksForOpportunities reads the marks of several opportunities. +// "IN (?)" is expanded by sqlx.In, which must be followed by Rebind. +const selectMarksForOpportunities = ` +SELECT * +FROM opportunity_marks +WHERE uuid IN (?) +ORDER BY uuid` diff --git a/internal/store/queries_test.go b/internal/store/queries_test.go new file mode 100644 index 0000000..aa3d988 --- /dev/null +++ b/internal/store/queries_test.go @@ -0,0 +1,211 @@ +package store + +import ( + "context" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/jmoiron/sqlx" +) + +// Run with: go test -tags sqlite_fts5 ./... + +// queryCheck is one statement to prepare. Preparing it makes SQLite parse it +// against the migrated schema, catching syntax errors and unknown tables or +// columns without running it. (Whether result columns map onto struct +// fields is covered by the behavioural tests in store_test.go.) +type queryCheck struct { + sql string + named bool // ":column" parameters, prepared via sqlx's named support + in bool // contains "IN (?)" for sqlx.In +} + +// checks lists every constant in queries.go by name. Fragments are checked +// inside a complete statement. TestQueriesAllListed fails if a constant is +// missing here. +var checks = map[string]queryCheck{ + // opportunities + "selectOpportunityLatestReleaseDate": {sql: selectOpportunityLatestReleaseDate}, + "upsertOpportunity": {sql: upsertOpportunity, named: true}, + "selectOpportunityByUUID": {sql: selectOpportunityByUUID}, + "listOpportunitiesSelect": { + sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage, + }, + "listOpportunitiesOrderAndPage": { + sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage, + }, + "filterOpportunityStatus": {sql: withFilter(filterOpportunityStatus)}, + "filterOpportunityClassificationCode": {sql: withFilter(filterOpportunityClassificationCode)}, + "filterOpportunityTitleMatch": {sql: withFilter(filterOpportunityTitleMatch)}, + "filterOpportunityReleasedFrom": {sql: withFilter(filterOpportunityReleasedFrom)}, + "filterOpportunityReleasedBefore": {sql: withFilter(filterOpportunityReleasedBefore)}, + "filterOpportunityMarkHidden": {sql: withFilter(filterOpportunityMarkHidden)}, + "filterOpportunityMarkSkipped": {sql: withFilter(filterOpportunityMarkSkipped)}, + "filterOpportunityMarkInterested": {sql: withFilter(filterOpportunityMarkInterested)}, + "filterOpportunityMarkBidSubmitted": {sql: withFilter(filterOpportunityMarkBidSubmitted)}, + "filterOpportunityMarkWon": {sql: withFilter(filterOpportunityMarkWon)}, + "filterOpportunityMarkLost": {sql: withFilter(filterOpportunityMarkLost)}, + + // opportunity_ocds_releases + "insertRelease": {sql: insertRelease, named: true}, + "selectReleasesByOpportunity": {sql: selectReleasesByOpportunity}, + "selectReleaseExists": {sql: selectReleaseExists}, + + // release_tags + "insertReleaseTag": {sql: insertReleaseTag, named: true}, + "selectReleaseTagsByOpportunity": {sql: selectReleaseTagsByOpportunity}, + + // classification_codes + "upsertClassificationCodeReturningID": {sql: upsertClassificationCodeReturningID, named: true}, + + // opportunity_classifications + "deleteOpportunityClassifications": {sql: deleteOpportunityClassifications}, + "insertOpportunityClassification": {sql: insertOpportunityClassification}, + "selectClassificationsForOpportunities": {sql: selectClassificationsForOpportunities, in: true}, + + // opportunity_marks + "insertOpportunityMarks": {sql: insertOpportunityMarks, named: true}, + "upsertOpportunityMarks": {sql: upsertOpportunityMarks, named: true}, + "selectMarksForOpportunities": {sql: selectMarksForOpportunities, in: true}, +} + +func withFilter(fragment string) string { + return listOpportunitiesSelect + "\nWHERE " + fragment + listOpportunitiesOrderAndPage +} + +func openTestDB(t *testing.T) *sqlx.DB { + t.Helper() + s, err := Open(context.Background(), filepath.Join(t.TempDir(), "queries.db"), WithAutoMigrate()) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s.db +} + +func prepare(db *sqlx.DB, c queryCheck) error { + var err error + switch { + case c.named: + var stmt *sqlx.NamedStmt + if stmt, err = db.PrepareNamed(c.sql); err == nil { + stmt.Close() + } + default: + query := c.sql + if c.in { + var expanded string + if expanded, _, err = sqlx.In(query, []string{"a", "b"}); err != nil { + return err + } + query = db.Rebind(expanded) + } + var stmt *sqlx.Stmt + if stmt, err = db.Preparex(query); err == nil { + stmt.Close() + } + } + return err +} + +func TestQueriesPrepare(t *testing.T) { + db := openTestDB(t) + for name, c := range checks { + if err := prepare(db, c); err != nil { + t.Errorf("%s: %v", name, err) + } + } +} + +// TestListOpportunitiesQueryAllFilters prepares the assembled query with every +// filter set, checking the fragments combine correctly and that there is +// exactly one argument per placeholder. +func TestListOpportunitiesQueryAllFilters(t *testing.T) { + db := openTestDB(t) + query, args := listOpportunitiesQuery(OpportunityFilter{ + Status: "active", + ClassificationCode: "72000000", + Title: "cloud migration", + ReleasedFrom: time.Now().Add(-time.Hour), + ReleasedBefore: time.Now(), + Hidden: MarkUnset, + Skipped: MarkUnset, + Interested: MarkSet, + BidSubmitted: MarkSet, + Won: MarkUnset, + Lost: MarkUnset, + Limit: 10, + Offset: 5, + }) + stmt, err := db.Preparex(query) + if err != nil { + t.Fatalf("prepare:\n%s\n%v", query, err) + } + defer stmt.Close() + var rows []opportunityRow + if err := stmt.Select(&rows, args...); err != nil { + t.Errorf("run with %d args:\n%s\n%v", len(args), query, err) + } +} + +// TestQueriesAllListed fails if queries.go declares a constant that checks +// doesn't list, so every new statement is prepared by TestQueriesPrepare. +func TestQueriesAllListed(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "queries.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var declared []string + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.CONST { + continue + } + for _, spec := range gen.Specs { + for _, name := range spec.(*ast.ValueSpec).Names { + declared = append(declared, name.Name) + } + } + } + sort.Strings(declared) + if len(declared) == 0 { + t.Fatal("found no constants in queries.go") + } + for _, name := range declared { + if _, ok := checks[name]; !ok { + t.Errorf("queries.go constant %s is not listed in checks", name) + } + } + for name := range checks { + if i := sort.SearchStrings(declared, name); i == len(declared) || declared[i] != name { + t.Errorf("checks lists %s, which queries.go does not declare", name) + } + } +} + +// TestMarksCheckConstraint checks the database itself rejects won and lost +// together, for writes that bypass OpportunityMarks.Validate. +func TestMarksCheckConstraint(t *testing.T) { + db := openTestDB(t) + if _, err := db.Exec(`INSERT INTO opportunities + (uuid, uuid_type, title, status, latest_release_date) + VALUES ('ocds-1', 'ocid', 'T', 'active', '2026-01-01T00:00:00Z')`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won, lost) + VALUES ('ocds-1', 1, 1)`); err == nil { + t.Error("inserting won and lost together succeeded") + } + if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won) + VALUES ('ocds-1', 1)`); err != nil { + t.Fatalf("inserting won: %v", err) + } + if _, err := db.Exec(`UPDATE opportunity_marks SET lost = 1 WHERE uuid = 'ocds-1'`); err == nil { + t.Error("updating to won and lost together succeeded") + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..8328bb0 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,596 @@ +// Package store persists opportunities and their OCDS releases in SQLite using +// sqlx and hand-written SQL. The SQL is in queries.go; this file holds the +// logic. Requires building with -tags sqlite_fts5. +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + "unicode" + + "git.avsdev.uk/Go/job-search/internal/types" + "github.com/jmoiron/sqlx" + "github.com/mattn/go-sqlite3" +) + +// ErrNotFound is returned when a requested opportunity does not exist. +var ErrNotFound = errors.New("store: not found") + +// DefaultListLimit is used when OpportunityFilter.Limit is zero or negative. +const DefaultListLimit = 50 + +// Store is the opportunity database. Open it with Open; it is safe for +// concurrent use. +type Store struct { + db *sqlx.DB +} + +// SaveMultipleResult reports what SaveOpportunities changed. +type SaveMultipleResult struct { + // NewOpportunities is the number of opportunities that were not already stored. + NewOpportunities int + // NewReleases is the number of releases that were not already stored. + NewReleases int + // UpdatedOpportunities is incremented for each opportunity row and its classifications + // that are updated (when every release in the save was older than the + // one the stored opportunity was last updated from). It does not include new + // opportunities. + UpdatedOpportunities int + // OpportunityResults is a slice of the results of each opportunity saved. + OpportunityResults []SaveResult +} + +// SaveResult reports what SaveOpportunity changed. +type SaveResult struct { + // OpportunityUpdated is true if the opportunity row and its classifications were + // written. It is false when every release in the save was older than the + // one the stored opportunity was last updated from. + OpportunityUpdated bool + // NewReleases is the number of releases that were not already stored. + NewReleases int +} + +// MarkFilter filters ListOpportunities on one of an opportunity's marks. +type MarkFilter int + +const ( + // MarkAny does not filter. It is the zero value. + MarkAny MarkFilter = iota + // MarkSet matches opportunities with the mark set. + MarkSet + // MarkUnset matches opportunities with the mark unset, including those + // with no marks stored. + MarkUnset +) + +// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued fields do not +// filter. +type OpportunityFilter struct { + Status string + // ClassificationCode matches opportunities with a classification of this code + // in any scheme. + ClassificationCode string + // Title is free text matched against opportunity titles: every word must + // appear, and each word also matches as a prefix ("cloud migr" matches + // "Cloud migration services"). + Title string + // ReleasedFrom (inclusive) and ReleasedBefore (exclusive) bound + // Opportunity.LatestReleaseDate. + ReleasedFrom time.Time + ReleasedBefore time.Time + // Each mark filter matches Opportunity.Marks' field of the same name. + Hidden MarkFilter + Skipped MarkFilter + Interested MarkFilter + BidSubmitted MarkFilter + Won MarkFilter + Lost MarkFilter + // Limit defaults to DefaultListLimit. Results are ordered newest + // LatestReleaseDate first. + Limit int + Offset int +} + +func (f OpportunityFilter) limit() int { + if f.Limit <= 0 { + return DefaultListLimit + } + return f.Limit +} + +// prepareOpportunity validates t and normalises it in place before saving: child +// rows get the opportunity's UUID and their release id, times are converted to +// UTC (so stored timestamps compare correctly as text), and +// LatestReleaseDate becomes the newest release date in the save. +func prepareOpportunity(t *types.Opportunity) error { + if t == nil { + return errors.New("store: nil opportunity") + } + if t.UUID == "" { + return errors.New("store: opportunity has no UUID") + } + + t.Marks.UUID = t.UUID + if err := t.Marks.Validate(); err != nil { + return fmt.Errorf("store: opportunity %s: %w", t.UUID, err) + } + + latest := t.LatestReleaseDate + for i := range t.Releases { + r := &t.Releases[i] + if r.ReleaseID == "" { + return fmt.Errorf("store: opportunity %s: release %d has no release id", t.UUID, i) + } + r.UUID = t.UUID + r.ReleaseDate = r.ReleaseDate.UTC() + if r.ReleaseDate.After(latest) { + latest = r.ReleaseDate + } + for j := range r.Tags { + r.Tags[j].UUID = t.UUID + r.Tags[j].ReleaseID = r.ReleaseID + } + } + + if latest.IsZero() { + return fmt.Errorf("store: opportunity %s has no release date", t.UUID) + } + + t.LatestReleaseDate = latest.UTC() + t.StartDate = utcPtr(t.StartDate) + t.EndDate = utcPtr(t.EndDate) + + return nil +} + +func utcPtr(t *time.Time) *time.Time { + if t == nil { + return nil + } + u := t.UTC() + return &u +} + +// ftsQuery turns free text into an FTS5 query that requires every word, each +// as a prefix match. Words are split the same way the unicode61 tokenizer +// splits them and each is quoted, so user input can never be interpreted as +// FTS5 syntax (operators, column filters, unbalanced quotes). It returns "" +// when the text contains no words. +func ftsQuery(text string) string { + words := strings.FieldsFunc(text, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsNumber(r) + }) + for i, w := range words { + words[i] = `"` + w + `"*` + } + return strings.Join(words, " ") +} + +// Open opens the database at path. The schema must be exactly at this +// build's latest migration: Open returns ErrNeedsMigration if migrations are +// pending (run the migrate command first), or ErrSchemaTooNew if the database +// was migrated by a newer build. WithAutoMigrate applies pending migrations +// instead. +// +// A database file that doesn't exist is created, and then needs migrating. +func Open(ctx context.Context, path string, opts ...Option) (*Store, error) { + var o options + for _, opt := range opts { + opt(&o) + } + + sqldb, err := OpenDB(path) + if err != nil { + return nil, err + } + if err := sqldb.PingContext(ctx); err != nil { + return nil, errors.Join(err, sqldb.Close()) + } + if err := prepareSchema(ctx, sqldb, o); err != nil { + return nil, errors.Join(err, sqldb.Close()) + } + return &Store{db: sqlx.NewDb(sqldb, "sqlite3")}, nil +} + +// Close closes the database. +func (s *Store) Close() error { return s.db.Close() } + +// Reads select whole rows ("SELECT *", "t.*", "c.*"). sqlx rejects a result +// column with no matching field, so a column added by a migration makes the +// reads fail loudly ("missing destination name ...") until it is mapped, +// instead of being silently ignored. +// +// opportunityRow gives the opportunities table's internal id column a home, so it stays +// out of types.Opportunity. sqlx maps the embedded Opportunity's fields as if declared +// here. The other tables have no internal columns and read straight into +// internal/types. +type opportunityRow struct { + ID int64 `db:"id"` + types.Opportunity +} + +// SaveOpportunity stores t and its releases in one transaction. +// +// Releases (and their tags) are write-once: ones already stored are skipped. +// The opportunity row and its classifications are replaced only if t's newest +// release is at least as new as the release the stored opportunity came from, so +// a late-arriving older release never undoes a newer one. +// +// t.Marks are stored only for a new opportunity; existing marks are kept (use +// SetMarks to change them). +// +// t is normalised in place (see prepareOpportunity), and when the opportunity is +// written its classifications' IDs are filled in. +func (s *Store) SaveOpportunity(ctx context.Context, t *types.Opportunity) (SaveResult, error) { + if err := prepareOpportunity(t); err != nil { + return SaveResult{}, err + } + + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return SaveResult{}, err + } + defer tx.Rollback() //nolint:errcheck // no-op after Commit + + res, _, err := saveOpportunity(ctx, tx, t) + if err != nil { + return SaveResult{}, err + } + if err := tx.Commit(); err != nil { + return SaveResult{}, err + } + return res, nil +} + +// SaveOpportunities stores t and its releases in one transaction. +// +// Releases (and their tags) are write-once: ones already stored are skipped. +// The opportunity row and its classifications are replaced only if t's newest +// release is at least as new as the release the stored opportunity came from, so +// a late-arriving older release never undoes a newer one. +// +// t.Marks are stored only for a new opportunity; existing marks are kept (use +// SetMarks to change them). +// +// t is normalised in place (see prepareOpportunity), and when the opportunity is +// written its classifications' IDs are filled in. +func (s *Store) SaveOpportunities( + ctx context.Context, opportunities []*types.Opportunity, +) (SaveMultipleResult, error) { + for _, t := range opportunities { + if err := prepareOpportunity(t); err != nil { + return SaveMultipleResult{}, err + } + } + + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return SaveMultipleResult{}, err + } + defer tx.Rollback() //nolint:errcheck // no-op after Commit + + allRes := SaveMultipleResult{OpportunityResults: make([]SaveResult, len(opportunities))} + for i, t := range opportunities { + res, isNew, err := saveOpportunity(ctx, tx, t) + if err != nil { + return SaveMultipleResult{}, err + } + allRes.OpportunityResults[i] = res + allRes.NewReleases += res.NewReleases + switch { + case isNew: + allRes.NewOpportunities++ + case res.OpportunityUpdated: + allRes.UpdatedOpportunities++ + } + } + + if err := tx.Commit(); err != nil { + return SaveMultipleResult{}, err + } + return allRes, nil +} + +// saveOpportunity does the work of SaveOpportunity and SaveOpportunities for +// one prepared opportunity inside tx. isNew reports that the opportunity was +// not already stored. +func saveOpportunity( + ctx context.Context, tx *sqlx.Tx, t *types.Opportunity, +) (res SaveResult, isNew bool, err error) { + var stored time.Time + err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, t.UUID) + switch { + case errors.Is(err, sql.ErrNoRows): + isNew = true + res.OpportunityUpdated = true + case err != nil: + return SaveResult{}, false, fmt.Errorf("store: reading opportunity %s: %w", t.UUID, err) + default: + res.OpportunityUpdated = !t.LatestReleaseDate.Before(stored) + } + if res.OpportunityUpdated { + if err := writeOpportunity(ctx, tx, t); err != nil { + return SaveResult{}, false, fmt.Errorf("store: writing opportunity %s: %w", t.UUID, err) + } + } + + for _, r := range t.Releases { + inserted, err := insertReleaseWithTags(ctx, tx, r) + if err != nil { + return SaveResult{}, false, err + } + if inserted { + res.NewReleases++ + } + } + return res, isNew, nil +} + +// insertReleaseWithTags stores r and its tags unless r is already stored, and +// reports whether it was inserted. +func insertReleaseWithTags( + ctx context.Context, tx *sqlx.Tx, r types.OpportunityRelease, +) (bool, error) { + result, err := tx.NamedExecContext(ctx, insertRelease, r) + if err != nil { + return false, fmt.Errorf("store: inserting release %s/%s: %w", r.UUID, r.ReleaseID, err) + } + if n, err := result.RowsAffected(); err != nil || n == 0 { + return false, err // n == 0: already stored + } + for _, tag := range r.Tags { + if _, err := tx.NamedExecContext(ctx, insertReleaseTag, tag); err != nil { + return false, fmt.Errorf( + "store: inserting tag for %s/%s: %w", r.UUID, r.ReleaseID, err, + ) + } + } + return true, nil +} + +// writeOpportunity upserts t's row, gives it a marks row if it has none +// (existing marks are kept), and replaces its classifications, filling in +// their IDs. +func writeOpportunity(ctx context.Context, tx *sqlx.Tx, t *types.Opportunity) (err error) { + if _, err := tx.NamedExecContext(ctx, upsertOpportunity, t); err != nil { + return err + } + if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, t.Marks); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, t.UUID); err != nil { + return err + } + + stmt, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID) + if err != nil { + return err + } + defer func() { err = errors.Join(err, stmt.Close()) }() + for i := range t.Classifications { + c := &t.Classifications[i] + if err := stmt.GetContext(ctx, &c.ID, c); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, insertOpportunityClassification, t.UUID, c.ID); err != nil { + return err + } + } + return nil +} + +// SetMarks replaces the marks of the opportunity m.UUID, or returns ErrNotFound. +// Marks that fail m.Validate (e.g. types.ErrWonAndLost) are not stored. +func (s *Store) SetMarks(ctx context.Context, m types.OpportunityMarks) error { + if m.UUID == "" { + return errors.New("store: marks have no UUID") + } + if err := m.Validate(); err != nil { + return fmt.Errorf("store: opportunity %s: %w", m.UUID, err) + } + _, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, m) + if isForeignKeyError(err) { + return ErrNotFound + } + return err +} + +// isForeignKeyError reports whether err is SQLite rejecting a write that +// breaks a foreign key, e.g. one referring to a missing opportunity. +func isForeignKeyError(err error) bool { + var se sqlite3.Error + return errors.As(err, &se) && se.ExtendedCode == sqlite3.ErrConstraintForeignKey +} + +// GetOpportunity returns the opportunity with its classifications, marks and +// all of its releases (with tags), or ErrNotFound. +func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) { + var row opportunityRow + err := s.db.GetContext(ctx, &row, selectOpportunityByUUID, uuid) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + opportunities := []types.Opportunity{row.Opportunity} + if err := s.loadClassifications(ctx, opportunities); err != nil { + return nil, err + } + if err := s.loadMarks(ctx, opportunities); err != nil { + return nil, err + } + if opportunities[0].Releases, err = s.ListReleases(ctx, uuid); err != nil { + return nil, err + } + return &opportunities[0], nil +} + +// ListOpportunities returns opportunities matching f, with their classifications +// and marks but without releases (use ListReleases or GetOpportunity for those). +func (s *Store) ListOpportunities( + ctx context.Context, + f OpportunityFilter, +) ([]types.Opportunity, error) { + query, args := listOpportunitiesQuery(f) + var rows []opportunityRow + if err := s.db.SelectContext(ctx, &rows, query, args...); err != nil { + return nil, err + } + opportunities := make([]types.Opportunity, len(rows)) + for i, row := range rows { + opportunities[i] = row.Opportunity + } + if err := s.loadClassifications(ctx, opportunities); err != nil { + return nil, err + } + if err := s.loadMarks(ctx, opportunities); err != nil { + return nil, err + } + return opportunities, nil +} + +// listOpportunitiesQuery assembles the ListOpportunities query from the fragments in +// queries.go, with one argument per filter that is set (none for mark filters). +func listOpportunitiesQuery(f OpportunityFilter) (string, []any) { + var where []string + var args []any + add := func(clause string, arg any) { + where = append(where, clause) + args = append(args, arg) + } + if f.Status != "" { + add(filterOpportunityStatus, f.Status) + } + if f.ClassificationCode != "" { + add(filterOpportunityClassificationCode, f.ClassificationCode) + } + if q := ftsQuery(f.Title); q != "" { + add(filterOpportunityTitleMatch, q) + } + if !f.ReleasedFrom.IsZero() { + add(filterOpportunityReleasedFrom, f.ReleasedFrom.UTC()) + } + if !f.ReleasedBefore.IsZero() { + add(filterOpportunityReleasedBefore, f.ReleasedBefore.UTC()) + } + for _, m := range []struct { + filter MarkFilter + clause string + }{ + {f.Hidden, filterOpportunityMarkHidden}, + {f.Skipped, filterOpportunityMarkSkipped}, + {f.Interested, filterOpportunityMarkInterested}, + {f.BidSubmitted, filterOpportunityMarkBidSubmitted}, + {f.Won, filterOpportunityMarkWon}, + {f.Lost, filterOpportunityMarkLost}, + } { + switch m.filter { + case MarkSet: + where = append(where, m.clause) + case MarkUnset: + where = append(where, "NOT "+m.clause) + } + } + + query := listOpportunitiesSelect + if len(where) > 0 { + query += "\nWHERE " + strings.Join(where, "\n\tAND ") + } + query += listOpportunitiesOrderAndPage + return query, append(args, f.limit(), f.Offset) +} + +// loadClassifications fills in Classifications for all opportunities with one query. +func (s *Store) loadClassifications(ctx context.Context, opportunities []types.Opportunity) error { + if len(opportunities) == 0 { + return nil + } + uuids := make([]string, len(opportunities)) + byUUID := make(map[string]*types.Opportunity, len(opportunities)) + for i := range opportunities { + uuids[i] = opportunities[i].UUID + byUUID[opportunities[i].UUID] = &opportunities[i] + } + + query, args, err := sqlx.In(selectClassificationsForOpportunities, uuids) + if err != nil { + return err + } + // opportunity_uuid says which opportunity each row belongs to. + var rows []struct { + OpportunityUUID string `db:"opportunity_uuid"` + types.ClassificationCode + } + if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil { + return err + } + for _, r := range rows { + t := byUUID[r.OpportunityUUID] + t.Classifications = append(t.Classifications, r.ClassificationCode) + } + return nil +} + +// loadMarks fills in Marks for all opportunities with one query. An +// opportunity with no marks row gets zero marks carrying its UUID. +func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity) error { + if len(opportunities) == 0 { + return nil + } + uuids := make([]string, len(opportunities)) + byUUID := make(map[string]*types.Opportunity, len(opportunities)) + for i := range opportunities { + uuids[i] = opportunities[i].UUID + byUUID[opportunities[i].UUID] = &opportunities[i] + opportunities[i].Marks = types.OpportunityMarks{UUID: opportunities[i].UUID} + } + + query, args, err := sqlx.In(selectMarksForOpportunities, uuids) + if err != nil { + return err + } + var rows []types.OpportunityMarks + if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil { + return err + } + for _, m := range rows { + byUUID[m.UUID].Marks = m + } + return nil +} + +// ListReleases returns a opportunity's releases with their tags, oldest first. +func (s *Store) ListReleases(ctx context.Context, uuid string) ([]types.OpportunityRelease, error) { + releases := []types.OpportunityRelease{} + if err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid); err != nil { + return nil, err + } + byID := make(map[string]*types.OpportunityRelease, len(releases)) + for i := range releases { + byID[releases[i].ReleaseID] = &releases[i] + } + + var tags []types.OpportunityReleaseTag + if err := s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid); err != nil { + return nil, err + } + for _, tag := range tags { + if r := byID[tag.ReleaseID]; r != nil { + r.Tags = append(r.Tags, tag) + } + } + return releases, nil +} + +// ReleaseExists reports whether a release is already stored, so a fetcher +// can skip converting it. +func (s *Store) ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) { + var exists bool + err := s.db.GetContext(ctx, &exists, selectReleaseExists, uuid, releaseID) + return exists, err +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..24d732e --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,647 @@ +package store_test + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "git.avsdev.uk/Go/job-search/internal/store" + "git.avsdev.uk/Go/job-search/internal/types" +) + +// Run with: go test -tags sqlite_fts5 ./... + +func openStore(t *testing.T) *store.Store { + t.Helper() + path := filepath.Join(t.TempDir(), "test.db") + s, err := store.Open(context.Background(), path, store.WithAutoMigrate()) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func day(d int) time.Time { + return time.Date(2026, time.March, d, 9, 30, 0, 0, time.UTC) +} + +func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *types.Opportunity { + start, end := day(20), day(28) + t := &types.Opportunity{ + UUID: uuid, + UUIDType: "ocid", + Title: title, + Status: "active", + Description: "A description", + BuyerName: "Example Council", + BuyerId: "GB-EXAMPLE", + MainCategory: "services", + ProcurementMethod: "open", + MultiLot: true, + SMESuitable: true, + StartDate: &start, + EndDate: &end, + GrossValue: 125000.50, + HasDocuments: true, + Classifications: []types.ClassificationCode{ + {IsCPV: true, Code: "72000000", Classification: "IT services"}, + {IsCPV: true, Code: "03000000", Classification: "Agricultural products"}, + }, + // No UUID: SaveOpportunity must fill it in. + Marks: types.OpportunityMarks{Interested: true, BidSubmitted: true}, + } + for i, id := range releaseIDs { + t.Releases = append(t.Releases, types.OpportunityRelease{ + ReleaseID: id, + ReleaseDate: day(releaseDay).Add(time.Duration(i) * time.Minute), + JSON: `{"id":"` + id + `"}`, + Tags: []types.OpportunityReleaseTag{{Tag: "opportunity"}, {Tag: "planning"}}, + }) + } + return t +} + +func mustSave(t *testing.T, s *store.Store, td *types.Opportunity) store.SaveResult { + t.Helper() + res, err := s.SaveOpportunity(context.Background(), td) + if err != nil { + t.Fatalf("SaveOpportunity(%s): %v", td.UUID, err) + } + return res +} + +func mustGet(t *testing.T, s *store.Store, uuid string) *types.Opportunity { + t.Helper() + got, err := s.GetOpportunity(context.Background(), uuid) + if err != nil { + t.Fatalf("GetOpportunity(%s): %v", uuid, err) + } + return got +} + +func uuids(ts []types.Opportunity) []string { + out := []string{} + for _, t := range ts { + out = append(out, t.UUID) + } + return out +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestSaveAndGetRoundTrip(t *testing.T) { + s := openStore(t) + in := opportunity("ocds-1", "Cloud migration services", 1, "r1", "r2") + res := mustSave(t, s, in) + if !res.OpportunityUpdated || res.NewReleases != 2 { + t.Fatalf("first save: got %+v", res) + } + + got := mustGet(t, s, "ocds-1") + if got.Title != in.Title || got.BuyerName != in.BuyerName || got.BuyerId != in.BuyerId || + got.MainCategory != in.MainCategory || got.ProcurementMethod != in.ProcurementMethod || + got.UUIDType != "ocid" || got.Status != "active" || got.Description != in.Description || + !got.MultiLot || !got.SMESuitable || got.VCSESuitable || !got.HasDocuments || + got.GrossValue != in.GrossValue { + t.Errorf("opportunity fields differ:\n got %+v\nwant %+v", got, in) + } + if got.StartDate == nil || !got.StartDate.Equal(day(20)) || got.EndDate == nil || + !got.EndDate.Equal(day(28)) { + t.Errorf("dates: got %v / %v", got.StartDate, got.EndDate) + } + if want := day(1).Add(time.Minute); !got.LatestReleaseDate.Equal(want) { + t.Errorf("LatestReleaseDate = %v, want %v", got.LatestReleaseDate, want) + } + + if len(got.Classifications) != 2 || got.Classifications[0].Code != "03000000" || + got.Classifications[1].Code != "72000000" || got.Classifications[0].ID == 0 { + t.Errorf("classifications: got %+v (leading zero must survive)", got.Classifications) + } + + wantMarks := types.OpportunityMarks{UUID: "ocds-1", Interested: true, BidSubmitted: true} + if got.Marks != wantMarks { + t.Errorf("marks: got %+v, want %+v", got.Marks, wantMarks) + } + + if len(got.Releases) != 2 { + t.Fatalf("releases: got %d", len(got.Releases)) + } + r := got.Releases[0] + if r.ReleaseID != "r1" || r.UUID != "ocds-1" || r.JSON != `{"id":"r1"}` || + !r.ReleaseDate.Equal(day(1)) { + t.Errorf("release: got %+v", r) + } + if len(r.Tags) != 2 || + r.Tags[0].Tag != "opportunity" || r.Tags[1].Tag != "planning" || + r.Tags[0].ReleaseID != "r1" || + r.Tags[0].UUID != "ocds-1" { + t.Errorf("tags: got %+v", r.Tags) + } +} + +func TestStartAndEndDateMayBeNil(t *testing.T) { + s := openStore(t) + in := opportunity("ocds-1", "No dates", 1, "r1") + in.StartDate, in.EndDate = nil, nil + mustSave(t, s, in) + got := mustGet(t, s, "ocds-1") + if got.StartDate != nil || got.EndDate != nil { + t.Errorf("got %v / %v, want nil", got.StartDate, got.EndDate) + } +} + +func TestReleasesAreWriteOnce(t *testing.T) { + s := openStore(t) + mustSave(t, s, opportunity("ocds-1", "Title", 1, "r1")) + + again := opportunity("ocds-1", "Title", 1, "r1") + again.Releases[0].JSON = `{"changed":true}` + again.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}} + if res := mustSave(t, s, again); res.NewReleases != 0 { + t.Errorf("re-save: NewReleases = %d, want 0", res.NewReleases) + } + + got := mustGet(t, s, "ocds-1").Releases + if len(got) != 1 || got[0].JSON != `{"id":"r1"}` || len(got[0].Tags) != 2 { + t.Errorf("stored release changed: %+v", got) + } +} + +func TestOlderReleaseDoesNotOverwriteOpportunity(t *testing.T) { + s := openStore(t) + mustSave(t, s, opportunity("ocds-1", "Newer title", 10, "r2")) + + old := opportunity("ocds-1", "Older title", 5, "r1") + old.Classifications = old.Classifications[:1] + res := mustSave(t, s, old) + if res.OpportunityUpdated || res.NewReleases != 1 { + t.Fatalf("older save: got %+v", res) + } + + got := mustGet(t, s, "ocds-1") + if got.Title != "Newer title" || len(got.Classifications) != 2 || + !got.LatestReleaseDate.Equal(day(10)) { + t.Errorf("opportunity was overwritten by an older release: %+v", got) + } + if len(got.Releases) != 2 || got.Releases[0].ReleaseID != "r1" { + t.Errorf("older release should still be stored, oldest first: %+v", got.Releases) + } +} + +func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) { + s := openStore(t) + ctx := context.Background() + first := opportunity("ocds-1", "Cloud hosting", 1, "r1") + mustSave(t, s, first) + oldIT := first.Classifications[0].ID + + newer := opportunity("ocds-1", "Website redesign", 2, "r2") + newer.Status = "complete" + newer.Classifications = []types.ClassificationCode{ + {IsCPV: true, Code: "72000000", Classification: "IT services (amended)"}, + } + if res := mustSave(t, s, newer); !res.OpportunityUpdated { + t.Fatalf("newer save not applied: %+v", res) + } + + got := mustGet(t, s, "ocds-1") + if got.Title != "Website redesign" || got.Status != "complete" { + t.Errorf("opportunity not updated: %+v", got) + } + if len(got.Classifications) != 1 || + got.Classifications[0].Classification != "IT services (amended)" { + t.Fatalf("classifications not replaced: %+v", got.Classifications) + } + if got.Classifications[0].ID == oldIT { + t.Errorf("changed description should create a new classification row") + } + + for title, want := range map[string]int{"cloud": 0, "redesign": 1} { + ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{Title: title}) + if err != nil { + t.Fatal(err) + } + if len(ts) != want { + t.Errorf("search %q after title change: %d results, want %d", title, len(ts), want) + } + } +} + +func TestClassificationRowsAreShared(t *testing.T) { + s := openStore(t) + a := opportunity("ocds-a", "A", 1, "r1") + b := opportunity("ocds-b", "B", 1, "r1") + mustSave(t, s, a) + mustSave(t, s, b) + if a.Classifications[0].ID == 0 || a.Classifications[0].ID != b.Classifications[0].ID { + t.Errorf("identical classifications should share an ID: %d vs %d", + a.Classifications[0].ID, b.Classifications[0].ID) + } + + c := opportunity("ocds-c", "C", 1, "r1") + c.Classifications = []types.ClassificationCode{ + {IsCPV: false, Code: "72000000", Classification: "IT services"}, + } + mustSave(t, s, c) + if c.Classifications[0].ID == a.Classifications[0].ID { + t.Errorf("different is_cpv should not share an ID") + } +} + +func TestSameReleaseIDInDifferentOpportunities(t *testing.T) { + s := openStore(t) + a := opportunity("ocds-a", "A", 1, "001") + b := opportunity("ocds-b", "B", 1, "001") + b.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}} + if res := mustSave(t, s, a); res.NewReleases != 1 { + t.Fatal(res) + } + if res := mustSave(t, s, b); res.NewReleases != 1 { + t.Fatalf("release id reused under another OCID must be stored: %+v", res) + } + got := mustGet(t, s, "ocds-b").Releases + if len(got) != 1 || len(got[0].Tags) != 1 || got[0].Tags[0].Tag != "award" { + t.Errorf("tags leaked between opportunities: %+v", got) + } +} + +func TestListOpportunitiesFilters(t *testing.T) { + s := openStore(t) + ctx := context.Background() + + cloud := opportunity("ocds-1", "Cloud migration services", 1, "r1") + roads := opportunity("ocds-2", "Road resurfacing works", 2, "r1") + roads.Status = "complete" + roads.Classifications = []types.ClassificationCode{ + {IsCPV: true, Code: "45233142", Classification: "Road-repair works"}, + } + hosting := opportunity("ocds-3", "Managed cloud hosting", 3, "r1") + for _, td := range []*types.Opportunity{cloud, roads, hosting} { + mustSave(t, s, td) + } + + cases := []struct { + name string + f store.OpportunityFilter + want []string + }{ + {"all, newest first", store.OpportunityFilter{}, []string{"ocds-3", "ocds-2", "ocds-1"}}, + {"status", store.OpportunityFilter{Status: "complete"}, []string{"ocds-2"}}, + { + "classification", + store.OpportunityFilter{ClassificationCode: "72000000"}, + []string{"ocds-3", "ocds-1"}, + }, + {"title word", store.OpportunityFilter{Title: "cloud"}, []string{"ocds-3", "ocds-1"}}, + {"title prefixes", store.OpportunityFilter{Title: "clo migr"}, []string{"ocds-1"}}, + {"title stemming", store.OpportunityFilter{Title: "migrating"}, []string{"ocds-1"}}, + {"title case", store.OpportunityFilter{Title: "ROAD"}, []string{"ocds-2"}}, + { + "title syntax is not FTS syntax", + store.OpportunityFilter{Title: `cloud" OR title:road*`}, + []string{}, + }, + { + "title punctuation only", + store.OpportunityFilter{Title: `"*:()`}, + []string{"ocds-3", "ocds-2", "ocds-1"}, + }, + {"released from", store.OpportunityFilter{ReleasedFrom: day(2)}, []string{"ocds-3", "ocds-2"}}, + {"released before", store.OpportunityFilter{ReleasedBefore: day(2)}, []string{"ocds-1"}}, + {"combined", store.OpportunityFilter{Title: "cloud", ReleasedFrom: day(2)}, []string{"ocds-3"}}, + {"limit", store.OpportunityFilter{Limit: 2}, []string{"ocds-3", "ocds-2"}}, + {"offset", store.OpportunityFilter{Limit: 2, Offset: 2}, []string{"ocds-1"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := s.ListOpportunities(ctx, tc.f) + if err != nil { + t.Fatalf("ListOpportunities: %v", err) + } + if !equal(uuids(got), tc.want) { + t.Errorf("got %v, want %v", uuids(got), tc.want) + } + }) + } + + got, err := s.ListOpportunities(ctx, store.OpportunityFilter{Status: "complete"}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || len(got[0].Classifications) != 1 || got[0].Releases != nil || + got[0].Marks.UUID != "ocds-2" || !got[0].Marks.Interested { + t.Errorf("list should include classifications and marks and omit releases: %+v", got) + } +} + +func TestReleaseExistsAndListReleases(t *testing.T) { + s := openStore(t) + ctx := context.Background() + mustSave(t, s, opportunity("ocds-1", "T", 1, "r1", "r2")) + + for _, tc := range []struct { + uuid, id string + want bool + }{{"ocds-1", "r1", true}, {"ocds-1", "r3", false}, {"ocds-2", "r1", false}} { + got, err := s.ReleaseExists(ctx, tc.uuid, tc.id) + if err != nil || got != tc.want { + t.Errorf("ReleaseExists(%s, %s) = %v, %v; want %v", tc.uuid, tc.id, got, err, tc.want) + } + } + + rs, err := s.ListReleases(ctx, "ocds-1") + if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" || rs[1].ReleaseID != "r2" || + len(rs[1].Tags) != 2 { + t.Errorf("ListReleases: %+v, %v", rs, err) + } +} + +func TestGetOpportunityNotFound(t *testing.T) { + s := openStore(t) + if _, err := s.GetOpportunity( + context.Background(), + "missing", + ); !errors.Is( + err, + store.ErrNotFound, + ) { + t.Errorf("err = %v, want ErrNotFound", err) + } +} + +func TestSaveOpportunityValidation(t *testing.T) { + s := openStore(t) + ctx := context.Background() + for name, td := range map[string]*types.Opportunity{ + "nil": nil, + "no uuid": opportunity("", "T", 1, "r1"), + "no release date": {UUID: "ocds-1", Title: "T", Status: "active"}, + "no release id": opportunity("ocds-1", "T", 1, ""), + } { + if _, err := s.SaveOpportunity(ctx, td); err == nil { + t.Errorf("%s: expected an error", name) + } + } +} + +func TestReopenKeepsDataAndSkipsAppliedMigrations(t *testing.T) { + path := filepath.Join(t.TempDir(), "test.db") + ctx := context.Background() + s, err := store.Open(ctx, path, store.WithAutoMigrate()) + if err != nil { + t.Fatal(err) + } + mustSave(t, s, opportunity("ocds-1", "Cloud migration", 1, "r1")) + s.Close() + + // No WithAutoMigrate: an up-to-date database must pass the version check. + s, err = store.Open(ctx, path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s.Close() + got, err := s.ListOpportunities(ctx, store.OpportunityFilter{Title: "cloud"}) + if err != nil || len(got) != 1 { + t.Errorf("after reopen: %v, %v", uuids(got), err) + } +} + +func TestResaveLoadedOpportunity(t *testing.T) { + s := openStore(t) + mustSave(t, s, opportunity("ocds-1", "Original", 1, "r1")) + + loaded := mustGet(t, s, "ocds-1") // classifications carry database IDs + ids := []int64{loaded.Classifications[0].ID, loaded.Classifications[1].ID} + loaded.Title = "Amended" + loaded.Releases = append(loaded.Releases, types.OpportunityRelease{ + ReleaseID: "r2", ReleaseDate: day(2), JSON: `{}`, + Tags: []types.OpportunityReleaseTag{{Tag: "opportunityAmendment"}}, + }) + res := mustSave(t, s, loaded) + if !res.OpportunityUpdated || res.NewReleases != 1 { + t.Fatalf("resave: %+v", res) + } + + got := mustGet(t, s, "ocds-1") + if got.Title != "Amended" || len(got.Releases) != 2 || !got.LatestReleaseDate.Equal(day(2)) { + t.Errorf("resave not applied: %+v", got) + } + if got.Classifications[0].ID != ids[0] || got.Classifications[1].ID != ids[1] { + t.Errorf("classification IDs changed on resave: %v -> %d, %d", + ids, got.Classifications[0].ID, got.Classifications[1].ID) + } +} + +func mustSetMarks(t *testing.T, s *store.Store, m types.OpportunityMarks) { + t.Helper() + if err := s.SetMarks(context.Background(), m); err != nil { + t.Fatalf("SetMarks(%s): %v", m.UUID, err) + } +} + +func TestSetMarks(t *testing.T) { + s := openStore(t) + mustSave(t, s, opportunity("ocds-1", "T", 1, "r1")) + + for _, want := range []types.OpportunityMarks{ + {UUID: "ocds-1", Hidden: true, Skipped: true}, + {UUID: "ocds-1", Interested: true, BidSubmitted: true, Won: true}, + {UUID: "ocds-1"}, + } { + mustSetMarks(t, s, want) + if got := mustGet(t, s, "ocds-1").Marks; got != want { + t.Errorf("after SetMarks: got %+v, want %+v", got, want) + } + } +} + +func TestSetMarksErrors(t *testing.T) { + s := openStore(t) + ctx := context.Background() + if err := s.SetMarks( + ctx, + types.OpportunityMarks{UUID: "missing", Hidden: true}, + ); !errors.Is( + err, + store.ErrNotFound, + ) { + t.Errorf("unknown opportunity: err = %v, want ErrNotFound", err) + } + if err := s.SetMarks(ctx, types.OpportunityMarks{Hidden: true}); err == nil { + t.Errorf("no UUID: expected an error") + } +} + +// Fetched opportunities carry zero marks; saving them must not reset the +// marks the user has set. +func TestSaveKeepsExistingMarks(t *testing.T) { + s := openStore(t) + ctx := context.Background() + mustSave(t, s, opportunity("ocds-1", "T", 1, "r1")) + mustSave(t, s, opportunity("ocds-2", "T", 1, "r1")) + want := types.OpportunityMarks{UUID: "ocds-1", Hidden: true, Lost: true} + mustSetMarks(t, s, want) + + newer := opportunity("ocds-1", "Amended", 2, "r2") + newer.Marks = types.OpportunityMarks{} + if res := mustSave(t, s, newer); !res.OpportunityUpdated { + t.Fatalf("newer save not applied: %+v", res) + } + if got := mustGet(t, s, "ocds-1").Marks; got != want { + t.Errorf("SaveOpportunity changed marks: got %+v, want %+v", got, want) + } + + newest := opportunity("ocds-1", "Amended again", 3, "r3") + newest.Marks = types.OpportunityMarks{} + if _, err := s.SaveOpportunities(ctx, []*types.Opportunity{newest}); err != nil { + t.Fatal(err) + } + if got := mustGet(t, s, "ocds-1").Marks; got != want { + t.Errorf("SaveOpportunities changed marks: got %+v, want %+v", got, want) + } +} + +func TestSaveOpportunitiesStoresMarks(t *testing.T) { + s := openStore(t) + a := opportunity("ocds-a", "A", 1, "r1") + b := opportunity("ocds-b", "B", 1, "r1") + b.Marks = types.OpportunityMarks{Skipped: true} + if _, err := s.SaveOpportunities(context.Background(), []*types.Opportunity{a, b}); err != nil { + t.Fatal(err) + } + for uuid, want := range map[string]types.OpportunityMarks{ + "ocds-a": {UUID: "ocds-a", Interested: true, BidSubmitted: true}, + "ocds-b": {UUID: "ocds-b", Skipped: true}, + } { + if got := mustGet(t, s, uuid).Marks; got != want { + t.Errorf("%s marks: got %+v, want %+v", uuid, got, want) + } + } +} + +func TestListOpportunitiesMarkFilters(t *testing.T) { + s := openStore(t) + ctx := context.Background() + for i, uuid := range []string{"ocds-1", "ocds-2", "ocds-3", "ocds-4"} { + mustSave(t, s, opportunity(uuid, "Cloud hosting", i+1, "r1")) + } + // ocds-1 keeps the marks it was saved with: Interested, BidSubmitted. + mustSetMarks(t, s, types.OpportunityMarks{UUID: "ocds-2", Hidden: true}) + mustSetMarks(t, s, types.OpportunityMarks{UUID: "ocds-3", Skipped: true, Lost: true}) + mustSetMarks(t, s, types.OpportunityMarks{ + UUID: "ocds-4", Interested: true, BidSubmitted: true, Won: true, + }) + + set, unset := store.MarkSet, store.MarkUnset + cases := []struct { + name string + f store.OpportunityFilter + want []string + }{ + {"any", store.OpportunityFilter{}, []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}}, + {"hidden", store.OpportunityFilter{Hidden: set}, []string{"ocds-2"}}, + {"not hidden", store.OpportunityFilter{Hidden: unset}, []string{"ocds-4", "ocds-3", "ocds-1"}}, + {"skipped", store.OpportunityFilter{Skipped: set}, []string{"ocds-3"}}, + { + "not skipped", + store.OpportunityFilter{Skipped: unset}, + []string{"ocds-4", "ocds-2", "ocds-1"}, + }, + {"interested", store.OpportunityFilter{Interested: set}, []string{"ocds-4", "ocds-1"}}, + {"not interested", store.OpportunityFilter{Interested: unset}, []string{"ocds-3", "ocds-2"}}, + {"bid submitted", store.OpportunityFilter{BidSubmitted: set}, []string{"ocds-4", "ocds-1"}}, + { + "bid not submitted", + store.OpportunityFilter{BidSubmitted: unset}, + []string{"ocds-3", "ocds-2"}, + }, + {"won", store.OpportunityFilter{Won: set}, []string{"ocds-4"}}, + {"not won", store.OpportunityFilter{Won: unset}, []string{"ocds-3", "ocds-2", "ocds-1"}}, + {"lost", store.OpportunityFilter{Lost: set}, []string{"ocds-3"}}, + {"not lost", store.OpportunityFilter{Lost: unset}, []string{"ocds-4", "ocds-2", "ocds-1"}}, + {"two marks", store.OpportunityFilter{BidSubmitted: set, Won: unset}, []string{"ocds-1"}}, + {"no match", store.OpportunityFilter{Won: set, Lost: set}, []string{}}, + { + "with other filters", + store.OpportunityFilter{Hidden: unset, Title: "cloud", ReleasedFrom: day(3)}, + []string{"ocds-4", "ocds-3"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := s.ListOpportunities(ctx, tc.f) + if err != nil { + t.Fatalf("ListOpportunities: %v", err) + } + if !equal(uuids(got), tc.want) { + t.Errorf("got %v, want %v", uuids(got), tc.want) + } + }) + } + + got, err := s.ListOpportunities(ctx, store.OpportunityFilter{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost || !got[2].Marks.Hidden || + got[3].Marks.Hidden || !got[3].Marks.Interested { + t.Errorf("list marks: %+v", got) + } +} + +func TestWonAndLostAreExclusive(t *testing.T) { + s := openStore(t) + ctx := context.Background() + mustSave(t, s, opportunity("ocds-1", "T", 1, "r1")) + before := mustGet(t, s, "ocds-1").Marks + + both := types.OpportunityMarks{UUID: "ocds-1", Won: true, Lost: true} + if err := s.SetMarks(ctx, both); !errors.Is(err, types.ErrWonAndLost) { + t.Errorf("SetMarks(won and lost): err = %v, want ErrWonAndLost", err) + } + if got := mustGet(t, s, "ocds-1").Marks; got != before { + t.Errorf("rejected SetMarks changed marks: got %+v, want %+v", got, before) + } + + in := opportunity("ocds-2", "T", 1, "r1") + in.Marks = types.OpportunityMarks{Won: true, Lost: true} + if _, err := s.SaveOpportunity(ctx, in); !errors.Is(err, types.ErrWonAndLost) { + t.Errorf("SaveOpportunity(won and lost): err = %v, want ErrWonAndLost", err) + } + if _, err := s.SaveOpportunities( + ctx, + []*types.Opportunity{in}, + ); !errors.Is( + err, + types.ErrWonAndLost, + ) { + t.Errorf("SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err) + } + if _, err := s.GetOpportunity(ctx, "ocds-2"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("rejected opportunity was stored: err = %v", err) + } + + // Either one alone is fine, and switching between them works. + for _, m := range []types.OpportunityMarks{ + {UUID: "ocds-1", Won: true}, + {UUID: "ocds-1", Lost: true}, + } { + mustSetMarks(t, s, m) + if got := mustGet(t, s, "ocds-1").Marks; got != m { + t.Errorf("got %+v, want %+v", got, m) + } + } +} diff --git a/scripts/build.sh b/scripts/build.sh index 31c25bd..ec73670 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -43,7 +43,10 @@ fi go generate ./... # Build binary with version info +# sqlite_fts5 compiles in SQLite's FTS5 module, needed by the store's title +# search (migration 6 fails without it) go build \ + -tags sqlite_fts5 \ -ldflags="$LD_FLAGS" \ -trimpath \ -o $BUILD_DIR/job-search-$VERSION \