First pass implementation of an SQLite backing store for fetched opportunities
This commit is contained in:
bwilliams committed
1 parent
9b0e6dccfe
commit
045492a148
29 files changed
+3130
-7
No files matched your search
@@ -19,7 +19,9 @@ The build script runs `go generate ./...` and stamps the version from the
|
||||
latest git tag. A build from untagged commits is marked as a development build
|
||||
and isn't packaged unless `--force` is also given.
|
||||
|
||||
During development, `go run ./cmd/jobsearch` works as well.
|
||||
During development, `go run -tags sqlite_fts5 ./cmd/jobsearch` works as well.
|
||||
The `sqlite_fts5` tag is needed for the database's title search (see
|
||||
[Testing](#testing)).
|
||||
|
||||
|
||||
## Configuration
|
||||
@@ -37,6 +39,8 @@ Example `config.yaml`:
|
||||
logging:
|
||||
level: info
|
||||
file: /var/log/job-search.log
|
||||
database:
|
||||
file: /var/lib/job-search/jobsearch.db
|
||||
sources:
|
||||
find_a_tender:
|
||||
enabled: true
|
||||
@@ -60,6 +64,10 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and
|
||||
| `job-search config get <key>` | Print one value, e.g. `logging.level` |
|
||||
| `job-search config set <key> <value>` | Validate and write one value to the config file |
|
||||
| `job-search config set --unset <key>` | Remove a value from the config file |
|
||||
| `job-search migrate` | Apply every pending database migration (same as `migrate up`) |
|
||||
| `job-search migrate up` | Apply pending migrations (`--steps N` for only N; `--create` to create a missing database file without asking) |
|
||||
| `job-search migrate down --steps N` | Roll back the last N migrations (`--steps 0` rolls back all of them, deleting all data, and needs `--yes`) |
|
||||
| `job-search migrate status` | Show the database version and which migrations are applied or pending |
|
||||
| `job-search version` | Print version information |
|
||||
|
||||
`fetch service` logs to `logging.file` if it's set, or to the terminal with
|
||||
@@ -68,6 +76,10 @@ Global flags: `--config`/`-c` (config file), `--debug`/`-d` and
|
||||
`--awards`, and with `--ignore-rate-limit` it fails on a rate limit instead
|
||||
of waiting.
|
||||
|
||||
The `migrate` commands work on the database in `database.file`. The store
|
||||
refuses to open a database that isn't at this build's schema version, so run
|
||||
`job-search migrate` after installing a new version.
|
||||
|
||||
|
||||
## Mock Find a Tender server
|
||||
To develop without using the real API, run the mock and enable the `fts_mock`
|
||||
@@ -95,6 +107,7 @@ See [internal/mock](internal/mock/README.md) for its options and behaviour.
|
||||
| `internal/cli` | Cobra commands and shared CLI setup |
|
||||
| `internal/config` | Configuration loading, validation and editing |
|
||||
| `internal/fetcher` | Runs a worker per enabled source |
|
||||
| `internal/store` | SQLite storage of opportunities, and its migrations (see [internal/store](internal/store/README.md)) |
|
||||
| `internal/findATender` | FTS API client and conversion from OCDS to JobSearch types |
|
||||
| `internal/types` | JobSearch's own opportunity types |
|
||||
| `internal/util` | The periodic `Worker` used by the fetcher |
|
||||
@@ -104,9 +117,14 @@ See [internal/mock](internal/mock/README.md) for its options and behaviour.
|
||||
|
||||
|
||||
## Testing
|
||||
The store's title search needs SQLite's FTS5 module, which is only compiled in
|
||||
with the `sqlite_fts5` build tag, so tests (and lint) need it. Without it the
|
||||
store tests fail with `no such module: fts5`.
|
||||
|
||||
```bash
|
||||
go test -race ./internal/...
|
||||
go test -coverprofile=cover.out ./internal/... && go tool cover -func=cover.out
|
||||
go test -race -tags sqlite_fts5 ./internal/...
|
||||
go test -tags sqlite_fts5 -coverprofile=cover.out ./internal/... && go tool cover -func=cover.out
|
||||
golangci-lint run --build-tags sqlite_fts5 ./...
|
||||
```
|
||||
|
||||
Tests don't need network access; HTTP calls go to local `httptest` servers.
|
||||
@@ -168,7 +186,13 @@ To install the tools:
|
||||
```bash
|
||||
go install github.com/spf13/cobra-cli@latest
|
||||
go install git.avsdev.uk/Go/go-ocds/cmd/ocdsgen@latest
|
||||
go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
|
||||
```
|
||||
|
||||
`goose` creates new migration files (see `internal/store`); the migrations
|
||||
themselves are applied by `job-search migrate`. `golangci-lint` uses the
|
||||
project's `.golangci.json`.
|
||||
|
||||
Ensure that the GOBIN path ($HOME/go/bin) is added to the environment before
|
||||
running any of the tools.
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/cli/config"
|
||||
"git.avsdev.uk/Go/job-search/internal/cli/fetch"
|
||||
"git.avsdev.uk/Go/job-search/internal/cli/migrate"
|
||||
"git.avsdev.uk/Go/job-search/internal/cli/version"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -73,6 +74,7 @@ opportunities.
|
||||
version.NewCommand(),
|
||||
config.NewCommand(opts),
|
||||
fetch.NewCommand(opts),
|
||||
migrate.NewCommand(opts),
|
||||
)
|
||||
|
||||
return root
|
||||
|
||||
@@ -3,9 +3,11 @@ module git.avsdev.uk/Go/job-search
|
||||
go 1.26.8
|
||||
|
||||
require (
|
||||
git.avsdev.uk/Go/go-ocds v1.0.0
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/lmittmann/tint v1.2.0
|
||||
github.com/mattn/go-sqlite3 v1.14.52
|
||||
github.com/pressly/goose/v3 v3.28.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
go.yaml.in/yaml/v3 v3.0.5
|
||||
@@ -16,15 +18,18 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sethvargo/go-retry v0.4.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/stretchr/testify v1.12.1 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
)
|
||||
@@ -1,31 +1,58 @@
|
||||
git.avsdev.uk/Go/go-ocds v1.0.0 h1:X/CoCyDcVy8ObGZ+UK4t6viPGj2bqATTEYc4Q6/2DKA=
|
||||
git.avsdev.uk/Go/go-ocds v1.0.0/go.mod h1:YPnqdf7ft6vPHiHLzeL/R7RtWYNKZJF1NXu/YEBpEbA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
|
||||
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lmittmann/tint v1.2.0 h1:AogHRHy8HUJUnNJBHJlYa+fR4YY8mko2cnCp67xn9JY=
|
||||
github.com/lmittmann/tint v1.2.0/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U=
|
||||
github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pressly/goose/v3 v3.28.0 h1:D2M+iL31GmpZxSHOhX8mqyqAT3CXnokUmm0eKoSP+Vc=
|
||||
github.com/pressly/goose/v3 v3.28.0/go.mod h1:v26MOuB8bL3kzzrt3Vqhb3R0PRVsl8hFQKdrht/L6Rk=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw=
|
||||
github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
@@ -43,15 +70,27 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus=
|
||||
modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg=
|
||||
modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
|
||||
resty.dev/v3 v3.0.0-rc.4 h1:n3360yEcFiyrX5Vkc9XG3Y8kMM24alRBxJ9Ev1mLmqs=
|
||||
resty.dev/v3 v3.0.0-rc.4/go.mod h1:NTOerrC/4T7/FE6tXIZGIysXXBdgNqwMZuKtxpea9NM=
|
||||
@@ -0,0 +1,34 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Confirm asks the user the yes/no question s on stdin and reports whether
|
||||
// they answered yes. "y" and "yes" confirm, "n" and "no" refuse, in any case;
|
||||
// anything else asks again. It exits the program if stdin can't be read.
|
||||
func Confirm(s string) bool {
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
|
||||
for {
|
||||
fmt.Printf("%s [y/n]: ", s)
|
||||
|
||||
response, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
response = strings.ToLower(strings.TrimSpace(response))
|
||||
|
||||
switch response {
|
||||
case "y", "yes":
|
||||
return true
|
||||
case "n", "no":
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// openMigrator loads the configuration and returns a migrator over the
|
||||
// store's embedded migrations for the configured database, opened with the
|
||||
// store's connection settings. The caller must close the returned *sql.DB.
|
||||
//
|
||||
// If the database file doesn't exist, create decides between asking whether
|
||||
// to create it (up) and failing (down, status: nothing to work on).
|
||||
func openMigrator(
|
||||
cmd *cobra.Command, opts *cli.Options, create bool, createConfirmed bool,
|
||||
) (*goose.Provider, *sql.DB, error) {
|
||||
conf, err := opts.RunSetup(cmd, true, true)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("no configuration loaded (%w)", err)
|
||||
}
|
||||
|
||||
file := conf.Database.File
|
||||
if len(file) == 0 {
|
||||
return nil, nil, errors.New(
|
||||
"no database file specified. Update your configuration",
|
||||
)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) {
|
||||
if !create {
|
||||
return nil, nil, fmt.Errorf("database file '%s' does not exist", file)
|
||||
}
|
||||
if !createConfirmed {
|
||||
question := fmt.Sprintf(
|
||||
"Database file '%s' does not exist. Would you like to create it?", file,
|
||||
)
|
||||
if !cli.Confirm(question) {
|
||||
return nil, nil, errors.New("migrate: no database file")
|
||||
}
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
db, err := store.OpenDB(file)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
m, err := store.NewMigrator(db)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Join(err, db.Close())
|
||||
}
|
||||
|
||||
return m, db, nil
|
||||
}
|
||||
|
||||
// closeDB closes db when deferred, adding any error to *err.
|
||||
func closeDB(db *sql.DB, err *error) {
|
||||
*err = errors.Join(*err, db.Close())
|
||||
}
|
||||
|
||||
// errWriter writes until a write fails, then keeps the error, so a run of
|
||||
// prints needs checking only once.
|
||||
type errWriter struct {
|
||||
w io.Writer
|
||||
err error
|
||||
}
|
||||
|
||||
// printf writes like fmt.Fprintf, unless an earlier write failed.
|
||||
func (e *errWriter) printf(format string, a ...any) {
|
||||
if e.err == nil {
|
||||
_, e.err = fmt.Fprintf(e.w, format, a...)
|
||||
}
|
||||
}
|
||||
|
||||
// printResults lists migrations that ran, e.g.
|
||||
//
|
||||
// applied 00003_create_release_tags_table.sql (2ms)
|
||||
func printResults(w io.Writer, results []*goose.MigrationResult) error {
|
||||
ew := &errWriter{w: w}
|
||||
for _, r := range results {
|
||||
verb := "applied"
|
||||
if r.Direction == "down" {
|
||||
verb = "reverted"
|
||||
}
|
||||
ew.printf(" %-9s %s (%s)\n",
|
||||
verb, filepath.Base(r.Source.Path), r.Duration.Round(time.Millisecond))
|
||||
}
|
||||
return ew.err
|
||||
}
|
||||
|
||||
// failure prints what ran before a failed migration and describes the
|
||||
// failure. Each migration runs in a transaction, so the failed one has been
|
||||
// rolled back and the database is at the last version that succeeded.
|
||||
func failure(w io.Writer, err error) error {
|
||||
var pe *goose.PartialError
|
||||
if !errors.As(err, &pe) {
|
||||
return err
|
||||
}
|
||||
|
||||
printErr := printResults(w, pe.Applied)
|
||||
|
||||
name := "unknown migration"
|
||||
if pe.Failed != nil && pe.Failed.Source != nil {
|
||||
name = filepath.Base(pe.Failed.Source.Path)
|
||||
}
|
||||
|
||||
return errors.Join(fmt.Errorf(
|
||||
"migration %s failed and was rolled back; the database is at the last "+
|
||||
"version that succeeded.\nMigration error: %w",
|
||||
name, pe.Err), printErr)
|
||||
}
|
||||
|
||||
// printVersion reports where the database ended up.
|
||||
func printVersion(ctx context.Context, w io.Writer, m *goose.Provider) error {
|
||||
v, err := m.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if v == 0 {
|
||||
_, err = fmt.Fprintln(w, "Database cleared")
|
||||
} else {
|
||||
_, err = fmt.Fprintf(w, "Database migrated to version %d\n", v)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func commandDown(opts *cli.Options) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "down",
|
||||
Short: "Migrate SQLite database to a previous version",
|
||||
Long: `Roll back the most recent migrations.
|
||||
|
||||
--steps 0 rolls back every migration which deletes all data and must be
|
||||
confirmed by providing -yes.`,
|
||||
Args: cobra.NoArgs,
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.down.PersistentPreRun")
|
||||
},
|
||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.down.PersistentPostRun")
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) (err error) {
|
||||
slog.Debug("migrate.down.Run")
|
||||
|
||||
yes, _ := cmd.Flags().GetBool("yes")
|
||||
steps, _ := cmd.Flags().GetUint("steps")
|
||||
|
||||
m, db, err := openMigrator(cmd, opts, false, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB(db, &err)
|
||||
|
||||
results, err := store.StepDown(cmd.Context(), m, int(steps), yes)
|
||||
|
||||
if len(results) == 0 && err == nil {
|
||||
fmt.Println("No changes made to database (empty)")
|
||||
return nil
|
||||
}
|
||||
if err == store.ErrWouldEmptyDatabase {
|
||||
return fmt.Errorf(
|
||||
"%w\n\nTo confirm you wish to proceed, pass the --yes flag", err,
|
||||
)
|
||||
}
|
||||
|
||||
if printErr := printResults(cmd.OutOrStdout(), results); printErr != nil {
|
||||
return errors.Join(err, printErr)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return failure(cmd.OutOrStdout(), err)
|
||||
}
|
||||
|
||||
return printVersion(cmd.Context(), cmd.OutOrStdout(), m)
|
||||
},
|
||||
}
|
||||
cmd.Flags().Uint(
|
||||
"steps",
|
||||
0,
|
||||
"number of migrations to roll back. 0 = all (deletes all data)",
|
||||
)
|
||||
if err := cmd.MarkFlagRequired("steps"); err != nil {
|
||||
panic(err) // only fails if the flag isn't defined
|
||||
}
|
||||
cmd.Flags().BoolP(
|
||||
"yes", "y",
|
||||
false,
|
||||
"confirm you are intending to roll back every migration",
|
||||
)
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Package migrate provides the "migrate" command and its subcommands, which
|
||||
// apply, roll back and report the store's migrations (goose, embedded in the
|
||||
// store package) on the configured SQLite database.
|
||||
//
|
||||
// goose runs each migration in its own transaction, so a failed migration is
|
||||
// rolled back and leaves the database at the previous version.
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// NewCommand returns the "migrate" command, which migrates the database schema (see
|
||||
// internal/store).
|
||||
func NewCommand(opts *cli.Options) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "migrate",
|
||||
Short: "Migrate SQLite database to latest version",
|
||||
Long: `Migrate the SQLite database to the latest version.
|
||||
|
||||
Running without a subcommand is the same as running "migrate up": every pending
|
||||
migration is applied.`,
|
||||
Args: cobra.NoArgs,
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.PersistentPreRun")
|
||||
},
|
||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.PersistentPostRun")
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
slog.Debug("migrate.Run")
|
||||
return runStepUp(cmd, opts, 0, false)
|
||||
},
|
||||
}
|
||||
cmd.AddCommand(commandUp(opts))
|
||||
cmd.AddCommand(commandDown(opts))
|
||||
cmd.AddCommand(commandStatus(opts))
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"text/tabwriter"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// printStatus lists every migration as applied (with when) or pending,
|
||||
// after a summary line, e.g.
|
||||
//
|
||||
// Database version 4 of 6 (2 pending)
|
||||
// 1 applied 2026-10-07 10:12:01 00001_create_opportunities_table.sql
|
||||
// ...
|
||||
// 5 pending 00005_create_opportunity_classifications_table.sql
|
||||
func printStatus(ctx context.Context, w io.Writer, m *goose.Provider) error {
|
||||
statuses, err := m.Status(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
current, err := m.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
latest := store.LatestVersion(m)
|
||||
|
||||
pending := 0
|
||||
for _, s := range statuses {
|
||||
if s.State == goose.StatePending {
|
||||
pending++
|
||||
}
|
||||
}
|
||||
|
||||
ew := &errWriter{w: w}
|
||||
switch {
|
||||
case current > latest:
|
||||
ew.printf("Database version %d is newer than this build (latest %d)\n",
|
||||
current, latest)
|
||||
case pending == 0:
|
||||
ew.printf("Database version %d of %d (up-to-date)\n", current, latest)
|
||||
default:
|
||||
ew.printf("Database version %d of %d (%d pending)\n", current, latest, pending)
|
||||
}
|
||||
if ew.err != nil {
|
||||
return ew.err
|
||||
}
|
||||
|
||||
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
|
||||
ew = &errWriter{w: tw}
|
||||
for _, s := range statuses {
|
||||
applied := ""
|
||||
if s.State == goose.StateApplied {
|
||||
applied = s.AppliedAt.Local().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
ew.printf(" %d\t%s\t%s\t%s\n",
|
||||
s.Source.Version, s.State, applied, filepath.Base(s.Source.Path))
|
||||
}
|
||||
return errors.Join(ew.err, tw.Flush())
|
||||
}
|
||||
|
||||
func commandStatus(opts *cli.Options) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Show which migrations have been applied to the SQLite database",
|
||||
Args: cobra.NoArgs,
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.status.PersistentPreRun")
|
||||
},
|
||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.status.PersistentPostRun")
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) (err error) {
|
||||
slog.Debug("migrate.status.Run")
|
||||
|
||||
m, db, err := openMigrator(cmd, opts, false, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB(db, &err)
|
||||
|
||||
return printStatus(cmd.Context(), cmd.OutOrStdout(), m)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/cli"
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// runStepUp applies steps pending migrations (0 means all), printing each one
|
||||
// and the resulting version. create says the user already agreed to create a
|
||||
// missing database file; otherwise they are asked.
|
||||
func runStepUp(
|
||||
cmd *cobra.Command, opts *cli.Options, steps int, create bool,
|
||||
) (err error) {
|
||||
m, db, err := openMigrator(cmd, opts, true, create)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB(db, &err)
|
||||
|
||||
results, err := store.StepUp(cmd.Context(), m, int(steps))
|
||||
|
||||
if len(results) == 0 && err == nil {
|
||||
_, err := fmt.Fprintln(cmd.OutOrStdout(), "No changes made to database (up-to-date)")
|
||||
return err
|
||||
}
|
||||
|
||||
if err := printResults(cmd.OutOrStdout(), results); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return failure(cmd.OutOrStdout(), err)
|
||||
}
|
||||
|
||||
return printVersion(cmd.Context(), cmd.OutOrStdout(), m)
|
||||
}
|
||||
|
||||
func commandUp(opts *cli.Options) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "up",
|
||||
Short: "Migrate SQLite database to the latest version, or by a number of steps",
|
||||
Args: cobra.NoArgs,
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.up.PersistentPreRun")
|
||||
},
|
||||
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
||||
slog.Debug("migrate.up.PersistentPostRun")
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
slog.Debug("migrate.up.Run")
|
||||
|
||||
steps, _ := cmd.Flags().GetUint("steps")
|
||||
create, _ := cmd.Flags().GetBool("create")
|
||||
|
||||
return runStepUp(cmd, opts, int(steps), create)
|
||||
},
|
||||
}
|
||||
cmd.Flags().Uint(
|
||||
"steps",
|
||||
0,
|
||||
"number of migrations to apply. 0 = all pending",
|
||||
)
|
||||
cmd.Flags().Bool(
|
||||
"create",
|
||||
false,
|
||||
"create the database file if it does not already exist",
|
||||
)
|
||||
return cmd
|
||||
}
|
||||
@@ -34,6 +34,7 @@ is kept.
|
||||
|---|---|---|---|
|
||||
| `logging.level` | string | `warn` | One of `debug`, `info`, `warn`, `error` |
|
||||
| `logging.file` | string | | Log file for `fetch service` (unless `--foreground`); other commands log to the terminal |
|
||||
| `database.file` | string | | Path to the SQLite database file, used by the `migrate` commands |
|
||||
| `sources.<source>.enabled` | bool | `false` | For every source |
|
||||
| `sources.<source>.interval` | duration | `10m` | For every source; e.g. `30s`, `1h` |
|
||||
| `sources.find_a_tender.api_root` | string | `https://www.find-tender.service.gov.uk/api/1.0` | URL of the Find a Tender API |
|
||||
@@ -49,6 +50,8 @@ Example `config.yaml`:
|
||||
logging:
|
||||
level: info
|
||||
file: ""
|
||||
database:
|
||||
file: ./jobsearch.db
|
||||
sources:
|
||||
find_a_tender:
|
||||
enabled: true
|
||||
@@ -126,6 +129,7 @@ Nothing in `section.go` needs changing.
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `config.go` | `JobSearchConfig`, defaults, Viper loading, text output |
|
||||
| `conf_database.go` | `database` section |
|
||||
| `load.go` | `Init`, `.env` and config file loading |
|
||||
| `modify.go` | `StoreValue` and `PurgeValue` |
|
||||
| `section.go` | The `Section` interface and the struct tag reflection shared by every section |
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package config
|
||||
|
||||
import "reflect"
|
||||
|
||||
// JSCKDatabase is the database section's configuration key.
|
||||
const JSCKDatabase string = "database"
|
||||
|
||||
// JobSearchConfigDatabase configures the SQLite database. File is the path to
|
||||
// the database file.
|
||||
type JobSearchConfigDatabase struct {
|
||||
File string `cfg:"file"`
|
||||
}
|
||||
|
||||
// ---------- Section methods (see Section) ----------
|
||||
|
||||
// Keys returns the keys directly in the section.
|
||||
func (c *JobSearchConfigDatabase) Keys() []string { return keysOf(c) }
|
||||
|
||||
func (c JobSearchConfigDatabase) Map() map[string]any { return mapOf(c) }
|
||||
|
||||
func (c JobSearchConfigDatabase) Flatten() map[string]any {
|
||||
return flattenOf(JSCKDatabase, c)
|
||||
}
|
||||
|
||||
func (c JobSearchConfigDatabase) TypeOf(key string) (reflect.Kind, error) {
|
||||
return typeOf(c, key)
|
||||
}
|
||||
|
||||
func (c JobSearchConfigDatabase) ValueOf(key string) (any, error) {
|
||||
return valueOf(c, key)
|
||||
}
|
||||
|
||||
func (c *JobSearchConfigDatabase) SetValue(key string, value any) error {
|
||||
return setValue(c, key, value)
|
||||
}
|
||||
@@ -23,6 +23,7 @@ const EtcConfigPath string = "/etc/job-search"
|
||||
// section. Each field's cfg tag must match the section's JSCK* constant.
|
||||
type JobSearchConfig struct {
|
||||
Logging JobSearchConfigLogging `cfg:"logging"`
|
||||
Database JobSearchConfigDatabase `cfg:"database"`
|
||||
Sources JobSearchConfigSources `cfg:"sources"`
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# Store
|
||||
|
||||
A SQLite store for opportunities and their OCDS releases, built on sqlx + goose and mattn/go-sqlite3.
|
||||
|
||||
## Build and test
|
||||
|
||||
The title search uses SQLite FTS5, which go-sqlite3 only compiles in with a build tag, so **every build and test needs `-tags sqlite_fts5`**. Without it, opening the store fails with `no such module: fts5`.
|
||||
|
||||
```bash
|
||||
go test -tags sqlite_fts5 ./...
|
||||
go run -tags sqlite_fts5 ./cmd/jobsearch migrate status
|
||||
```
|
||||
|
||||
The tag can be made the default for every build with
|
||||
`go env -w GOFLAGS=-tags=sqlite_fts5`.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
internal/types/ Go struct definitions for tables (opportunities.go, classifications.go)
|
||||
internal/store/migrations/ NNNNN_name.sql (Up and Down in one file)
|
||||
internal/store/migrate.go OpenDB, NewMigrator (goose), schema version checks
|
||||
internal/store/migrate_test.go version checks, stepping up and down, failed-migration rollback
|
||||
internal/store/queries.go all SQL, as named constants grouped by table
|
||||
internal/store/queries_test.go prepares every query against the migrated schema
|
||||
internal/store/store.go Store: logic, using the queries via sqlx
|
||||
internal/store/store_test.go behavioural tests
|
||||
internal/store/common.go common for migrate, store and direct access
|
||||
```
|
||||
|
||||
## Schema
|
||||
|
||||
| Table | Key | Notes |
|
||||
|---|---|---|
|
||||
| `opportunities` | `uuid` (unique) | Updatable. `id INTEGER PRIMARY KEY` exists only to give the search index a stable rowid (fts5). |
|
||||
| `opportunity_marks` | `uuid` | `uuid` references `opportunities`. The user's own marks (`hidden`, `skipped`, `interested`, `bid_submitted`, `won`, `lost`), each `BOOLEAN NOT NULL DEFAULT 0`. A `CHECK` constraint stops `won` and `lost` both being set. |
|
||||
| `opportunity_releases` | (`uuid`, `release_id`) | Write-once. `uuid` references `opportunities`. `data` is TEXT, so `json_extract` works on it. |
|
||||
| `opportunity_release_tags` | (`uuid`, `release_id`, `tag`) | Write-once. (`uuid`, `release_id`) references `opportunity_releases`. |
|
||||
| `classification_codes` | `id` | Shared between opportunities. A row is reused only when every content column matches. The table grows over time to include codes not already seen in fetched opportunities. |
|
||||
| `opportunity_classifications` | (`opportunity_uuid`, `classification_id`) | Mapping table; replaced as a set when the opportunity is updated. |
|
||||
| `opportunities_fts` | rowid = `opportunities.id` | FTS5 index on `opportunities.title`, kept in sync by triggers. |
|
||||
|
||||
Foreign keys are enforced (`_foreign_keys=on` in the connection string) and cascade on delete from `opportunities`. The connection also enables WAL mode, a 5s busy timeout, and `_txlock=immediate`, so a save takes the write lock up front rather than failing halfway through.
|
||||
|
||||
**When adding a classification scheme flag** (e.g. `is_unspsc`), the migration adding the column must drop and recreate the `classification_codes_content` unique index with the new column included. Otherwise rows differing only by that flag would be merged.
|
||||
|
||||
**internal/types**: The structs contain `db` tags to allow sqlx to map table columns to struct members. Any struct members not mapped to a column (including child slices) are referenced by `db:"-"`.
|
||||
|
||||
|
||||
## Migrations and Goose
|
||||
|
||||
Migrations are embedded in the store package and applied explicitly, by the project's `migrate` command. `store.Open` doesn't migrate: it checks the schema version and fails with
|
||||
|
||||
- `ErrNeedsMigration` if migrations are pending (resolved by running `migrate`);
|
||||
- `ErrSchemaTooNew` if a newer build has migrated the database.
|
||||
|
||||
`store.WithAutoMigrate()` makes `Open` apply pending migrations instead, for tests and throwaway databases. For tooling, `store.OpenDB(path)` opens the database with the store's connection settings, and `store.NewMigrator(db)` returns a goose `Provider` over the embedded migrations.
|
||||
|
||||
Each migration runs in its own transaction, so a failed migration is rolled back and leaves the database at the previous version. goose records applied migrations in `goose_db_version`. Each file has `-- +goose Up` and `-- +goose Down` sections. Each section is one `StatementBegin`/`StatementEnd` block, which sends it to SQLite in a single call; that's what lets trigger bodies (which contain `;`) work. A new migration is created with:
|
||||
```bash
|
||||
goose -dir internal/store/migrations create add_something sql
|
||||
```
|
||||
|
||||
## Queries and sqlx
|
||||
|
||||
All SQL is in `queries.go` as constants, grouped by table in migration order; `store.go` holds only the logic. Inserts use sqlx named parameters (`:column`, matched to the `db` struct tags in `internal/types`). sqlx treats any colon followed by a name inside named SQL as a parameter, including inside SQL comments, so comments go on the Go constant, never inside the string.
|
||||
|
||||
`queries_test.go` checks the SQL without needing a test for every code path:
|
||||
|
||||
- **TestQueriesPrepare** prepares every query constant against a freshly migrated database, so a syntax error, an unknown table or unknown column fails the test and names the failing statement.
|
||||
- **TestQueriesAllListed** parses `queries.go` and fails if a constant isn't listed in that test's `checks` map. Adding a query therefore means adding one line there, and it can't be skipped by accident.
|
||||
- **TestListOpportunitiesQueryAllFilters** builds the `ListOpportunities` query with every filter set and runs it, checking that the fragments combine and that the arguments line up with the placeholders.
|
||||
- **TestMarksCheckConstraint** checks the database itself rejects `won` and `lost` set together, for writes that bypass the Go validation.
|
||||
|
||||
Adding a table involves adding its migration, putting its queries in a new section of `queries.go`, listing them in `checks`, and adding the store methods.
|
||||
|
||||
Reads select whole rows (`SELECT *`, `t.*`, `c.*`) which map to a Go struct. sqlx refuses to read a result column that has no matching struct field, so when a migration adds a column, reads fail with `missing destination name <column>` until it is mapped: the field is added to the type in `internal/types` if callers should see it, or to the store's row wrapper if it is internal (like `opportunities.id` in `opportunityRow`). This means that queries should only return data for a single struct type at a time (nested structs are not supported - subsequent queries must be run to populate the referenced structs).
|
||||
|
||||
`ListOpportunities` is assembled by `listOpportunitiesQuery` from fragments in `queries.go`: a base `SELECT`, one `filterOpportunity*` clause per filter that is set, and the ordering and paging tail. Each filter fragment takes exactly one argument, except the `filterOpportunityMark*` fragments, which take none (`MarkUnset` puts `NOT` in front of them). Adding a filter means one fragment and one line in `listOpportunitiesQuery`. The classifications and the marks for a page of opportunities are then loaded with one `IN (...)` query each.
|
||||
|
||||
## Store API
|
||||
|
||||
```go
|
||||
s, err := store.Open(ctx, "jobsearch.db") // fails with ErrNeedsMigration if not migrated
|
||||
s, err := store.Open(ctx, path, store.WithAutoMigrate()) // tests: migrate on open
|
||||
res, err := s.SaveOpportunity(ctx, &opportunity) // SaveResult{OpportunityUpdated, NewReleases}
|
||||
res, err := s.SaveOpportunities(ctx, opportunities) // all in one transaction; SaveMultipleResult
|
||||
err = s.SetMarks(ctx, types.OpportunityMarks{UUID: uuid, Interested: true}) // ErrNotFound
|
||||
t, err := s.GetOpportunity(ctx, uuid) // classifications + marks + releases + tags; ErrNotFound
|
||||
ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{
|
||||
Title: "cloud migr",
|
||||
Hidden: store.MarkUnset, // MarkAny (default), MarkSet or MarkUnset per mark
|
||||
Interested: store.MarkSet,
|
||||
Limit: 20,
|
||||
})
|
||||
rs, err := s.ListReleases(ctx, uuid) // oldest first, with tags
|
||||
ok, err := s.ReleaseExists(ctx, uuid, releaseID)
|
||||
```
|
||||
|
||||
**Saving.** `SaveOpportunity` writes everything in one transaction. `SaveOpportunities` does the same for several opportunities in a single transaction, reporting totals (`NewOpportunities`, `UpdatedOpportunities`, `NewReleases`) and each opportunity's `SaveResult`:
|
||||
|
||||
- **Opportunity normalisation.** The opportunity is normalised in place. Releases, Marks and tags get the opportunity's `UUID` and their `ReleaseID`, times become UTC, `LatestReleaseDate` becomes the newest release date, and classifications get their database `ID`s.
|
||||
- **Minimal writing.** Releases and tags are write-once. A release already stored (same `uuid` and `release_id`) is skipped, and `NewReleases` counts only the new ones. `ReleaseExists` lets a fetcher skip converting a release it already has.
|
||||
- **Opportunity row updates.** The opportunity row and its classifications are replaced only if the save's newest release date is at least the stored `latest_release_date`. An older release arriving late is still stored, but never overwrites newer opportunity data; `OpportunityUpdated` reports which happened.
|
||||
- **Marks are kept.** An opportunity's marks are stored when it is first saved; saving it again never changes them, so fetched data (which carries no marks) can't reset what the user has set.
|
||||
|
||||
**Marks.** `SetMarks` replaces an opportunity's marks, returning `ErrNotFound` if the opportunity isn't stored. `won` and `lost` exclude each other: `SetMarks`, `SaveOpportunity` and `SaveOpportunities` reject marks with both set with `types.ErrWonAndLost` (from `OpportunityMarks.Validate`), and the database's `CHECK` constraint backs this up.
|
||||
|
||||
|
||||
**Listing.** `ListOpportunities` filters by status, classification code, title text, a range on `LatestReleaseDate` and each mark (`MarkSet` or `MarkUnset`; an opportunity with no marks stored has every mark unset), newest first, 50 per page by default. It loads classifications and marks but not releases (which can be large); those are loaded by `ListReleases` or `GetOpportunity`.
|
||||
|
||||
|
||||
**Title search.** The words are matched against the title using FTS5: every word must appear, each also matches as a prefix, case and accents are ignored, and "migrating" finds "migration", etc. Input is sanitised, so FTS5 operators and quotes typed by a user are searched as plain words, never interpreted.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Option configures Open.
|
||||
type Option func(*options)
|
||||
|
||||
type options struct {
|
||||
autoMigrate bool
|
||||
}
|
||||
|
||||
// OpenDB opens the SQLite database at path with the store's connection
|
||||
// settings (foreign keys, WAL, busy timeout, immediate transactions). The
|
||||
// file is created on first use. Use it for anything that works on the store's
|
||||
// database directly, such as running migrations.
|
||||
func OpenDB(path string) (*sql.DB, error) {
|
||||
return sql.Open("sqlite3", dsn(path))
|
||||
}
|
||||
|
||||
// WithAutoMigrate makes Open apply pending migrations instead of returning
|
||||
// ErrNeedsMigration. Intended for tests and throwaway databases; applications
|
||||
// should migrate explicitly with the migrate command.
|
||||
func WithAutoMigrate() Option {
|
||||
return func(o *options) { o.autoMigrate = true }
|
||||
}
|
||||
|
||||
// dsn builds the mattn/go-sqlite3 connection string.
|
||||
//
|
||||
// - _foreign_keys: SQLite ignores REFERENCES clauses unless this is on.
|
||||
// - _journal_mode=WAL: readers don't block the writer or each other.
|
||||
// - _busy_timeout: wait for a lock rather than failing immediately.
|
||||
// - _txlock=immediate: transactions take the write lock at BEGIN, so a
|
||||
// read-then-write transaction (as in SaveOpportunity) can't fail with
|
||||
// SQLITE_BUSY halfway through.
|
||||
func dsn(path string) string {
|
||||
return fmt.Sprintf(
|
||||
"file:%s?_foreign_keys=on&_journal_mode=WAL&_busy_timeout=5000&_txlock=immediate",
|
||||
path,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
// Each file holds both directions, marked with -- +goose Up / -- +goose Down.
|
||||
// Create new ones with: goose -dir internal/store/migrations create NAME sql
|
||||
//
|
||||
//go:embed migrations/*.sql
|
||||
var migrationsFS embed.FS
|
||||
|
||||
// ErrNeedsMigration is returned by Open when the database has pending
|
||||
// migrations. Apply them with the migrate command, or pass
|
||||
// WithAutoMigrate.
|
||||
var ErrNeedsMigration = errors.New("store: database needs migrating")
|
||||
|
||||
// ErrSchemaTooNew is returned by Open when the database has migrations
|
||||
// this build doesn't know about, i.e. it was migrated by a newer build.
|
||||
var ErrSchemaTooNew = errors.New("store: database schema is newer than this build")
|
||||
|
||||
// ErrWouldEmptyDatabase is returned by StepDown when the migration would drop
|
||||
// all of the tables and thus empty the database.
|
||||
var ErrWouldEmptyDatabase = errors.New(
|
||||
"migration cancelled; database would be emptied by undoing all migrations",
|
||||
)
|
||||
|
||||
// NewMigrator returns a goose Provider over the store's embedded migrations,
|
||||
// for applying, rolling back or inspecting them. Each migration runs in its
|
||||
// own transaction, so a failed migration leaves the database at the previous
|
||||
// version (goose has no "dirty" state).
|
||||
func NewMigrator(db *sql.DB) (*goose.Provider, error) {
|
||||
fsys, err := fs.Sub(migrationsFS, "migrations")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return goose.NewProvider(goose.DialectSQLite3, db, fsys)
|
||||
}
|
||||
|
||||
// LatestVersion is the highest migration version in this build.
|
||||
func LatestVersion(p *goose.Provider) int64 {
|
||||
sources := p.ListSources() // sorted ascending
|
||||
if len(sources) == 0 {
|
||||
return 0
|
||||
}
|
||||
return sources[len(sources)-1].Version
|
||||
}
|
||||
|
||||
// checkVersion returns ErrNeedsMigration or ErrSchemaTooNew unless the
|
||||
// database is exactly at this build's schema.
|
||||
func checkVersion(ctx context.Context, p *goose.Provider) error {
|
||||
current, err := p.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
latest := LatestVersion(p)
|
||||
if current > latest {
|
||||
return fmt.Errorf("%w (database is at version %d, this build knows up to %d)",
|
||||
ErrSchemaTooNew, current, latest)
|
||||
}
|
||||
|
||||
pending, err := p.HasPending(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if pending {
|
||||
return fmt.Errorf("%w (database is at version %d, latest is %d)",
|
||||
ErrNeedsMigration, current, latest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkNotNewer refuses to work on a database migrated by a newer build:
|
||||
// this build has neither the newer migrations nor their down steps.
|
||||
func checkNotNewer(ctx context.Context, m *goose.Provider) error {
|
||||
current, err := m.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if latest := LatestVersion(m); current > latest {
|
||||
return fmt.Errorf(
|
||||
"database is at version %d but this build only knows up to %d; use a newer build",
|
||||
current, latest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareSchema migrates (with WithAutoMigrate) or verifies the schema
|
||||
// version.
|
||||
func prepareSchema(ctx context.Context, db *sql.DB, o options) error {
|
||||
p, err := NewMigrator(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if o.autoMigrate {
|
||||
if _, err := p.Up(ctx); err != nil {
|
||||
return fmt.Errorf("store: migrating database: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return checkVersion(ctx, p)
|
||||
}
|
||||
|
||||
// StepUp applies steps pending migrations, or all of them when steps is 0.
|
||||
func StepUp(
|
||||
ctx context.Context, m *goose.Provider, steps int,
|
||||
) ([]*goose.MigrationResult, error) {
|
||||
var results []*goose.MigrationResult
|
||||
|
||||
if err := checkNotNewer(ctx, m); err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
var err error
|
||||
if steps == 0 {
|
||||
results, err = m.Up(ctx)
|
||||
} else {
|
||||
for range steps {
|
||||
var r *goose.MigrationResult
|
||||
r, err = m.UpByOne(ctx)
|
||||
if errors.Is(err, goose.ErrNoNextVersion) {
|
||||
err = nil
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
}
|
||||
|
||||
return results, err
|
||||
}
|
||||
|
||||
// StepDown rolls back steps applied migrations, or all of them when steps
|
||||
// is 0. Rolling everything back deletes all data, so confirmDrop is checked
|
||||
// first in that case.
|
||||
func StepDown(
|
||||
ctx context.Context, m *goose.Provider, steps int, confirmDrop bool,
|
||||
) ([]*goose.MigrationResult, error) {
|
||||
var results []*goose.MigrationResult
|
||||
|
||||
if err := checkNotNewer(ctx, m); err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
current, err := m.GetDBVersion(ctx)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
if current == 0 {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
statuses, err := m.Status(ctx)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
applied := 0
|
||||
for _, s := range statuses {
|
||||
if s.State == goose.StateApplied {
|
||||
applied++
|
||||
}
|
||||
}
|
||||
all := steps == 0 || steps >= applied
|
||||
if all && !confirmDrop {
|
||||
return results, ErrWouldEmptyDatabase
|
||||
}
|
||||
|
||||
if all {
|
||||
results, err = m.DownTo(ctx, 0)
|
||||
} else {
|
||||
for range steps {
|
||||
var r *goose.MigrationResult
|
||||
r, err = m.Down(ctx)
|
||||
if errors.Is(err, goose.ErrNoNextVersion) {
|
||||
err = nil
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
}
|
||||
|
||||
return results, err
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
// Run with: go test -tags sqlite_fts5 ./...
|
||||
|
||||
func tempDBPath(t *testing.T) string {
|
||||
return filepath.Join(t.TempDir(), "migrate.db")
|
||||
}
|
||||
|
||||
func newMigrator(t *testing.T) (*goose.Provider, *sql.DB) {
|
||||
t.Helper()
|
||||
db, err := store.OpenDB(filepath.Join(t.TempDir(), "migrate.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
m, err := store.NewMigrator(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, db
|
||||
}
|
||||
|
||||
func version(t *testing.T, m *goose.Provider) int64 {
|
||||
t.Helper()
|
||||
v, err := m.GetDBVersion(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func TestOpenRequiresMigration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := tempDBPath(t)
|
||||
|
||||
if _, err := store.Open(ctx, path); !errors.Is(err, store.ErrNeedsMigration) {
|
||||
t.Fatalf("fresh database: err = %v, want ErrNeedsMigration", err)
|
||||
}
|
||||
|
||||
db, err := store.OpenDB(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
m, err := store.NewMigrator(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := m.UpByOne(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Open(ctx, path); !errors.Is(err, store.ErrNeedsMigration) {
|
||||
t.Fatalf("partly migrated: err = %v, want ErrNeedsMigration", err)
|
||||
}
|
||||
|
||||
if _, err := m.Up(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := store.Open(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatalf("fully migrated: %v", err)
|
||||
}
|
||||
s.Close()
|
||||
}
|
||||
|
||||
func TestOpenRejectsNewerSchema(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := tempDBPath(t)
|
||||
s, err := store.Open(ctx, path, store.WithAutoMigrate())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Close()
|
||||
|
||||
// Simulate a migration applied by a newer build.
|
||||
db, err := store.OpenDB(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for name, opts := range map[string][]store.Option{
|
||||
"check": nil,
|
||||
"auto-migrate": {store.WithAutoMigrate()},
|
||||
} {
|
||||
if _, err := store.Open(ctx, path, opts...); !errors.Is(err, store.ErrSchemaTooNew) {
|
||||
t.Errorf("%s: err = %v, want ErrSchemaTooNew", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateDownAllAndUpAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := tempDBPath(t)
|
||||
db, err := store.OpenDB(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
m, err := store.NewMigrator(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := m.Up(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := m.GetDBVersion(ctx); v != store.LatestVersion(m) {
|
||||
t.Fatalf("after up: version %d, want %d", v, store.LatestVersion(m))
|
||||
}
|
||||
|
||||
if _, err := m.DownTo(ctx, 0); err != nil {
|
||||
t.Fatalf("down to 0: %v", err)
|
||||
}
|
||||
var tables int
|
||||
if err := db.QueryRow(`
|
||||
SELECT count(*) FROM sqlite_master
|
||||
WHERE type IN ('table', 'index', 'trigger')
|
||||
AND name NOT LIKE 'sqlite_%' AND name != 'goose_db_version'`).Scan(&tables); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tables != 0 {
|
||||
t.Errorf("after down to 0: %d schema objects remain", tables)
|
||||
}
|
||||
|
||||
if _, err := m.Up(ctx); err != nil {
|
||||
t.Fatalf("up again: %v", err)
|
||||
}
|
||||
s, err := store.Open(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatalf("open after round trip: %v", err)
|
||||
}
|
||||
s.Close()
|
||||
}
|
||||
|
||||
func TestUpAllThenUpToDate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, _ := newMigrator(t)
|
||||
latest := store.LatestVersion(m)
|
||||
|
||||
results, err := store.StepUp(ctx, m, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(results); got != int(latest) {
|
||||
t.Fatalf("applied %d migrations, want %d.", got, latest)
|
||||
}
|
||||
|
||||
results, err = store.StepUp(ctx, m, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(results) != 0 {
|
||||
t.Fatalf("expected no further migrations, got %d.", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpBySteps(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, _ := newMigrator(t)
|
||||
|
||||
nToApply := 2
|
||||
|
||||
_, err := store.StepUp(ctx, m, nToApply)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := version(t, m); v != int64(nToApply) {
|
||||
t.Fatalf("after 2 steps: version %d", v)
|
||||
}
|
||||
// More steps than pending stops at the latest version.
|
||||
_, err = store.StepUp(ctx, m, 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := version(t, m); v != store.LatestVersion(m) {
|
||||
t.Fatalf("after 50 steps: version %d", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownBySteps(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, _ := newMigrator(t)
|
||||
|
||||
_, err := store.StepUp(ctx, m, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = store.StepDown(ctx, m, 2, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, want := version(t, m), store.LatestVersion(m)-2; v != want {
|
||||
t.Fatalf("after 2 down steps: version %d, want %d", v, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownAllRequiresConfirmation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, _ := newMigrator(t)
|
||||
|
||||
_, err := store.StepUp(ctx, m, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hasErrored := false
|
||||
for name, steps := range map[string]int{"steps 0": 0, "more steps than applied": 99} {
|
||||
_, err := store.StepDown(ctx, m, steps, false)
|
||||
if !errors.Is(err, store.ErrWouldEmptyDatabase) {
|
||||
t.Errorf("%s: %s", name, err)
|
||||
hasErrored = true
|
||||
}
|
||||
if v := version(t, m); v != store.LatestVersion(m) {
|
||||
t.Errorf("%s: version changed to %d", name, v)
|
||||
hasErrored = true
|
||||
}
|
||||
}
|
||||
if hasErrored {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
_, err = store.StepDown(ctx, m, 0, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := version(t, m); v != 0 {
|
||||
t.Fatalf("after confirmed down: version %d", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownOnEmptyDatabase(t *testing.T) {
|
||||
m, _ := newMigrator(t)
|
||||
results, err := store.StepDown(context.Background(), m, 0, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(results) != 0 {
|
||||
t.Fatalf("expected no further migrations, got %d.", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewerDatabaseIsRefused(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, db := newMigrator(t)
|
||||
|
||||
if _, err := store.StepUp(ctx, m, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO goose_db_version (version_id, is_applied) VALUES (9999, 1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := store.StepUp(
|
||||
ctx,
|
||||
m,
|
||||
0,
|
||||
); err == nil ||
|
||||
!strings.Contains(err.Error(), "newer build") {
|
||||
t.Errorf("up: err = %v, want a newer-build error", err)
|
||||
}
|
||||
if _, err := store.StepDown(
|
||||
ctx,
|
||||
m,
|
||||
1,
|
||||
false,
|
||||
); err == nil ||
|
||||
!strings.Contains(err.Error(), "newer build") {
|
||||
t.Errorf("down: err = %v, want a newer-build error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedMigrationIsRolledBack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m, db := newMigrator(t)
|
||||
|
||||
if _, err := store.StepUp(ctx, m, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Occupy migration 4's table name with a view. Its CREATE TABLE IF NOT
|
||||
// EXISTS then skips, and the CREATE INDEX that follows fails because
|
||||
// views can't be indexed, part-way through the migration.
|
||||
if _, err := db.Exec(`CREATE VIEW classification_codes AS SELECT 1 AS x`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err := store.StepUp(ctx, m, 0)
|
||||
if err == nil {
|
||||
t.Fatal("expected migration 4 to fail")
|
||||
}
|
||||
if v := version(t, m); v != 3 {
|
||||
t.Errorf("version after failure: %d, want 3", v)
|
||||
}
|
||||
var indexes int
|
||||
db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name LIKE 'classification_codes_%'`).
|
||||
Scan(&indexes)
|
||||
if indexes != 0 {
|
||||
t.Errorf("failed migration left %d objects behind", indexes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS "opportunities" (
|
||||
-- Surrogate key. "uuid" is the real identifier; "id" exists only to give
|
||||
-- the title search index (opportunities_fts) a stable rowid to point at.
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"uuid" TEXT NOT NULL UNIQUE,
|
||||
"uuid_type" TEXT NOT NULL,
|
||||
"title" TEXT NOT NULL,
|
||||
"status" TEXT NOT NULL,
|
||||
"description" TEXT NOT NULL DEFAULT '',
|
||||
"buyer_name" TEXT NOT NULL DEFAULT '',
|
||||
-- classifications: see tender_classifications
|
||||
"buyer_id" TEXT NOT NULL DEFAULT '',
|
||||
"main_category" TEXT NOT NULL DEFAULT '',
|
||||
"procurement_method" TEXT NOT NULL DEFAULT '',
|
||||
"multi_lot" BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
"sme_suitable" BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
"vcse_suitable" BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
"start_date" DATETIME,
|
||||
"end_date" DATETIME,
|
||||
"gross_value" REAL NOT NULL DEFAULT 0,
|
||||
"has_documents" BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
"source" TEXT NOT NULL DEFAULT '',
|
||||
-- Release date of the newest release applied to this row. A release
|
||||
-- older than this never overwrites the tender.
|
||||
"latest_release_date" DATETIME NOT NULL
|
||||
-- releases: see tender_ocds_releases
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "opportunities_status"
|
||||
ON "opportunities" ("status");
|
||||
CREATE INDEX IF NOT EXISTS "opportunities_latest_release_date"
|
||||
ON "opportunities" ("latest_release_date");
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS "opportunities";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- Write-once: a release is never modified after it is stored.
|
||||
-- "release_id" is the OCDS release id, unique only within its OCID ("uuid").
|
||||
CREATE TABLE IF NOT EXISTS "opportunity_releases" (
|
||||
"uuid" TEXT NOT NULL
|
||||
REFERENCES "opportunities" ("uuid") ON DELETE CASCADE,
|
||||
"release_id" TEXT NOT NULL,
|
||||
"release_date" DATETIME NOT NULL,
|
||||
-- tags: see release_tags
|
||||
"data" TEXT NOT NULL,
|
||||
PRIMARY KEY ("uuid", "release_id")
|
||||
) WITHOUT ROWID;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS "opportunity_releases";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,18 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- Write-once, inserted together with the release it belongs to.
|
||||
CREATE TABLE IF NOT EXISTS "opportunity_release_tags" (
|
||||
"uuid" TEXT NOT NULL,
|
||||
"release_id" TEXT NOT NULL,
|
||||
"tag" TEXT NOT NULL,
|
||||
PRIMARY KEY ("uuid", "release_id", "tag"),
|
||||
FOREIGN KEY ("uuid", "release_id")
|
||||
REFERENCES "opportunity_releases" ("uuid", "release_id")
|
||||
ON DELETE CASCADE
|
||||
) WITHOUT ROWID;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS "opportunity_release_tags";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,26 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- Shared, write-once lookup of classifications. A row is reused only when
|
||||
-- every content column matches; any difference (e.g. a changed description)
|
||||
-- creates a new row.
|
||||
--
|
||||
-- When adding a new scheme flag column (e.g. "is_unspsc"), the same migration
|
||||
-- must drop and recreate "classification_codes_content" with the new column
|
||||
-- included, otherwise rows differing only by that flag would be merged.
|
||||
CREATE TABLE IF NOT EXISTS "classification_codes" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"is_cpv" BOOLEAN NOT NULL,
|
||||
"code" TEXT NOT NULL,
|
||||
"classification" TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "classification_codes_content"
|
||||
ON "classification_codes" ("is_cpv", "code", "classification");
|
||||
CREATE INDEX IF NOT EXISTS "classification_codes_code"
|
||||
ON "classification_codes" ("code");
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS "classification_codes";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- Maps opportunities to classification codes. Replaced as a set whenever the
|
||||
-- opportunity row is updated.
|
||||
CREATE TABLE IF NOT EXISTS "opportunity_classifications" (
|
||||
"opportunity_uuid" TEXT NOT NULL
|
||||
REFERENCES "opportunities" ("uuid") ON DELETE CASCADE,
|
||||
"classification_id" INTEGER NOT NULL
|
||||
REFERENCES "classification_codes" ("id"),
|
||||
PRIMARY KEY ("opportunity_uuid", "classification_id")
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "opportunity_classifications_classification_id"
|
||||
ON "opportunity_classifications" ("classification_id");
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS "opportunity_classifications";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,40 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- Full-text index over opportunities.title. Requires mattn/go-sqlite3 to be
|
||||
-- built with -tags sqlite_fts5. Kept in sync by the triggers below.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS "opportunities_fts" USING fts5(
|
||||
"title",
|
||||
content = 'opportunities',
|
||||
content_rowid = 'id',
|
||||
tokenize = 'porter unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS "opportunities_fts_insert" AFTER INSERT ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS "opportunities_fts_delete" AFTER DELETE ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
|
||||
VALUES ('delete', old."id", old."title");
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS "opportunities_fts_update" AFTER UPDATE OF "title" ON "opportunities"
|
||||
BEGIN
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts", "rowid", "title")
|
||||
VALUES ('delete', old."id", old."title");
|
||||
INSERT INTO "opportunities_fts" ("rowid", "title") VALUES (new."id", new."title");
|
||||
END;
|
||||
|
||||
-- Index any opportunities that existed before this migration.
|
||||
INSERT INTO "opportunities_fts" ("opportunities_fts") VALUES ('rebuild');
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_update";
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_delete";
|
||||
DROP TRIGGER IF EXISTS "opportunities_fts_insert";
|
||||
DROP TABLE IF EXISTS "opportunities_fts";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,22 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS "opportunity_marks" (
|
||||
"uuid" TEXT NOT NULL UNIQUE,
|
||||
"hidden" BOOLEAN NOT NULL DEFAULT 0,
|
||||
"skipped" BOOLEAN NOT NULL DEFAULT 0,
|
||||
"interested" BOOLEAN NOT NULL DEFAULT 0,
|
||||
"bid_submitted" BOOLEAN NOT NULL DEFAULT 0,
|
||||
"lost" BOOLEAN NOT NULL DEFAULT 0,
|
||||
"won" BOOLEAN NOT NULL DEFAULT 0,
|
||||
CHECK (NOT ("won" AND "lost")),
|
||||
PRIMARY KEY ("uuid"),
|
||||
FOREIGN KEY ("uuid")
|
||||
REFERENCES "opportunities" ("uuid")
|
||||
ON DELETE CASCADE
|
||||
);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS "opportunity_marks";
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,234 @@
|
||||
package store
|
||||
|
||||
// All SQL used by the store, grouped by table in migration order.
|
||||
//
|
||||
// Conventions:
|
||||
// - Statements are named <verb><Subject>, e.g. insertRelease. Fragments
|
||||
// combined at run time are named after what they build (listOpportunities*,
|
||||
// filterOpportunity*).
|
||||
// - Named-parameter statements (":column", bound from a struct's db tags)
|
||||
// say so in their comment. Everything else uses "?" placeholders.
|
||||
// - Reads select whole rows (*, t.*, c.*) so a column added by a migration
|
||||
// but not mapped to a struct field fails loudly; see opportunityRow.
|
||||
// - Keep comments out of the SQL strings: sqlx reads a colon followed by a
|
||||
// name as a parameter, even inside an SQL comment.
|
||||
// - Every constant here is prepared against the migrated schema by
|
||||
// TestQueriesPrepare (queries_test.go), which also fails if a constant is
|
||||
// added without being listed there.
|
||||
|
||||
// ---------- opportunities ----------
|
||||
|
||||
// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares against.
|
||||
const selectOpportunityLatestReleaseDate = `
|
||||
SELECT latest_release_date FROM opportunities WHERE uuid = ?`
|
||||
|
||||
// upsertOpportunity inserts or updates a opportunity. Named parameters (types.Opportunity).
|
||||
//
|
||||
// An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first,
|
||||
// cascading to the opportunity's releases and changing its id.
|
||||
const upsertOpportunity = `
|
||||
INSERT INTO opportunities (
|
||||
uuid, uuid_type, title, status, description, buyer_name, buyer_id,
|
||||
main_category, procurement_method, multi_lot, sme_suitable, vcse_suitable,
|
||||
start_date, end_date, gross_value, has_documents, latest_release_date
|
||||
) VALUES (
|
||||
:uuid, :uuid_type, :title, :status, :description, :buyer_name, :buyer_id,
|
||||
:main_category, :procurement_method, :multi_lot, :sme_suitable, :vcse_suitable,
|
||||
:start_date, :end_date, :gross_value, :has_documents, :latest_release_date
|
||||
)
|
||||
ON CONFLICT (uuid) DO UPDATE SET
|
||||
uuid_type = excluded.uuid_type,
|
||||
title = excluded.title,
|
||||
status = excluded.status,
|
||||
description = excluded.description,
|
||||
buyer_name = excluded.buyer_name,
|
||||
buyer_id = excluded.buyer_id,
|
||||
main_category = excluded.main_category,
|
||||
procurement_method = excluded.procurement_method,
|
||||
multi_lot = excluded.multi_lot,
|
||||
sme_suitable = excluded.sme_suitable,
|
||||
vcse_suitable = excluded.vcse_suitable,
|
||||
start_date = excluded.start_date,
|
||||
end_date = excluded.end_date,
|
||||
gross_value = excluded.gross_value,
|
||||
has_documents = excluded.has_documents,
|
||||
latest_release_date = excluded.latest_release_date`
|
||||
|
||||
// selectOpportunityByUUID reads one opportunity into a opportunityRow.
|
||||
const selectOpportunityByUUID = `
|
||||
SELECT t.* FROM opportunities t WHERE t.uuid = ?`
|
||||
|
||||
// ListOpportunities is assembled by listOpportunitiesQuery as:
|
||||
//
|
||||
// listOpportunitiesSelect [WHERE filter AND filter ...] listOpportunitiesOrderAndPage
|
||||
//
|
||||
// Each filterOpportunity* fragment takes exactly one "?" argument, except
|
||||
// filterOpportunityMark* fragments, which take none. A mark fragment is true
|
||||
// when the mark is set and is negated with NOT to match it unset; an
|
||||
// opportunity with no marks row has every mark unset.
|
||||
const (
|
||||
listOpportunitiesSelect = `
|
||||
SELECT t.* FROM opportunities t`
|
||||
|
||||
listOpportunitiesOrderAndPage = `
|
||||
ORDER BY t.latest_release_date DESC, t.uuid
|
||||
LIMIT ? OFFSET ?`
|
||||
|
||||
filterOpportunityStatus = `t.status = ?`
|
||||
|
||||
filterOpportunityClassificationCode = `EXISTS (
|
||||
SELECT 1
|
||||
FROM opportunity_classifications tc
|
||||
JOIN classification_codes c ON c.id = tc.classification_id
|
||||
WHERE tc.opportunity_uuid = t.uuid AND c.code = ?
|
||||
)`
|
||||
|
||||
// The argument must come from ftsQuery, never raw user input.
|
||||
filterOpportunityTitleMatch = `t.id IN (
|
||||
SELECT rowid FROM opportunities_fts WHERE opportunities_fts MATCH ?
|
||||
)`
|
||||
|
||||
filterOpportunityReleasedFrom = `t.latest_release_date >= ?`
|
||||
|
||||
filterOpportunityReleasedBefore = `t.latest_release_date < ?`
|
||||
|
||||
filterOpportunityMarkHidden = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.hidden
|
||||
)`
|
||||
|
||||
filterOpportunityMarkSkipped = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.skipped
|
||||
)`
|
||||
|
||||
filterOpportunityMarkInterested = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.interested
|
||||
)`
|
||||
|
||||
filterOpportunityMarkBidSubmitted = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.bid_submitted
|
||||
)`
|
||||
|
||||
filterOpportunityMarkWon = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.won
|
||||
)`
|
||||
|
||||
filterOpportunityMarkLost = `EXISTS (
|
||||
SELECT 1 FROM opportunity_marks m WHERE m.uuid = t.uuid AND m.lost
|
||||
)`
|
||||
)
|
||||
|
||||
// ---------- opportunity_releases ----------
|
||||
|
||||
// insertRelease stores a release; 0 rows affected means it was already
|
||||
// stored. Named parameters (types.OCDSReleaseJSON).
|
||||
const insertRelease = `
|
||||
INSERT INTO opportunity_releases (uuid, release_id, release_date, data)
|
||||
VALUES (:uuid, :release_id, :release_date, :data)
|
||||
ON CONFLICT DO NOTHING`
|
||||
|
||||
// selectReleasesByOpportunity reads a opportunity's releases, oldest first.
|
||||
const selectReleasesByOpportunity = `
|
||||
SELECT *
|
||||
FROM opportunity_releases
|
||||
WHERE uuid = ?
|
||||
ORDER BY release_date, release_id`
|
||||
|
||||
// selectReleaseExists returns a single boolean.
|
||||
const selectReleaseExists = `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM opportunity_releases WHERE uuid = ? AND release_id = ?
|
||||
)`
|
||||
|
||||
// ---------- opportunity_release_tags ----------
|
||||
|
||||
// insertReleaseTag stores a tag. Named parameters (types.OCDSReleaseTags).
|
||||
const insertReleaseTag = `
|
||||
INSERT INTO opportunity_release_tags (uuid, release_id, tag)
|
||||
VALUES (:uuid, :release_id, :tag)
|
||||
ON CONFLICT DO NOTHING`
|
||||
|
||||
// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's releases.
|
||||
const selectReleaseTagsByOpportunity = `
|
||||
SELECT *
|
||||
FROM opportunity_release_tags
|
||||
WHERE uuid = ?
|
||||
ORDER BY release_id, tag`
|
||||
|
||||
// ---------- classification_codes ----------
|
||||
|
||||
// upsertClassificationCodeReturningID returns the id of the row matching
|
||||
// every content column, inserting it first if needed. Named parameters
|
||||
// (types.ClassificationCode).
|
||||
//
|
||||
// The no-op DO UPDATE (rather than DO NOTHING) makes RETURNING yield the
|
||||
// existing row's id on conflict.
|
||||
const upsertClassificationCodeReturningID = `
|
||||
INSERT INTO classification_codes (is_cpv, code, classification)
|
||||
VALUES (:is_cpv, :code, :classification)
|
||||
ON CONFLICT (is_cpv, code, classification) DO UPDATE SET is_cpv = excluded.is_cpv
|
||||
RETURNING id`
|
||||
|
||||
// ---------- opportunity_classifications ----------
|
||||
|
||||
// deleteOpportunityClassifications clears a opportunity's mappings before they are
|
||||
// rewritten.
|
||||
const deleteOpportunityClassifications = `
|
||||
DELETE FROM opportunity_classifications WHERE opportunity_uuid = ?`
|
||||
|
||||
// insertOpportunityClassification maps a opportunity to a classification code.
|
||||
const insertOpportunityClassification = `
|
||||
INSERT INTO opportunity_classifications (opportunity_uuid, classification_id)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT DO NOTHING`
|
||||
|
||||
// selectClassificationsForOpportunities reads the classifications of several
|
||||
// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by Rebind.
|
||||
// c.* must map entirely onto types.ClassificationCode.
|
||||
const selectClassificationsForOpportunities = `
|
||||
SELECT tc.opportunity_uuid, c.*
|
||||
FROM opportunity_classifications tc
|
||||
JOIN classification_codes c ON c.id = tc.classification_id
|
||||
WHERE tc.opportunity_uuid IN (?)
|
||||
ORDER BY tc.opportunity_uuid, c.code, c.id`
|
||||
|
||||
// ---------- opportunity_marks ----------
|
||||
|
||||
// insertOpportunityMarks gives a new opportunity its marks row; an existing
|
||||
// row is left alone, so saving fetched data never resets marks set by the
|
||||
// user. Named parameters (types.OpportunityMarks).
|
||||
const insertOpportunityMarks = `
|
||||
INSERT INTO opportunity_marks (
|
||||
uuid, hidden, interested, skipped, bid_submitted, won, lost
|
||||
)
|
||||
VALUES (
|
||||
:uuid, :hidden, :interested, :skipped, :bid_submitted, :won, :lost
|
||||
)
|
||||
ON CONFLICT (uuid) DO NOTHING`
|
||||
|
||||
// upsertOpportunityMarks inserts or updates an opportunity's marks. Named
|
||||
// parameters (types.OpportunityMarks).
|
||||
//
|
||||
// An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first
|
||||
// which is unnecessary.
|
||||
const upsertOpportunityMarks = `
|
||||
INSERT INTO opportunity_marks (
|
||||
uuid, hidden, interested, skipped, bid_submitted, won, lost
|
||||
)
|
||||
VALUES (
|
||||
:uuid, :hidden, :interested, :skipped, :bid_submitted, :won, :lost
|
||||
)
|
||||
ON CONFLICT (uuid) DO UPDATE SET
|
||||
hidden = excluded.hidden,
|
||||
interested = excluded.interested,
|
||||
skipped = excluded.skipped,
|
||||
bid_submitted = excluded.bid_submitted,
|
||||
won = excluded.won,
|
||||
lost = excluded.lost`
|
||||
|
||||
// selectMarksForOpportunities reads the marks of several opportunities.
|
||||
// "IN (?)" is expanded by sqlx.In, which must be followed by Rebind.
|
||||
const selectMarksForOpportunities = `
|
||||
SELECT *
|
||||
FROM opportunity_marks
|
||||
WHERE uuid IN (?)
|
||||
ORDER BY uuid`
|
||||
@@ -0,0 +1,211 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
// Run with: go test -tags sqlite_fts5 ./...
|
||||
|
||||
// queryCheck is one statement to prepare. Preparing it makes SQLite parse it
|
||||
// against the migrated schema, catching syntax errors and unknown tables or
|
||||
// columns without running it. (Whether result columns map onto struct
|
||||
// fields is covered by the behavioural tests in store_test.go.)
|
||||
type queryCheck struct {
|
||||
sql string
|
||||
named bool // ":column" parameters, prepared via sqlx's named support
|
||||
in bool // contains "IN (?)" for sqlx.In
|
||||
}
|
||||
|
||||
// checks lists every constant in queries.go by name. Fragments are checked
|
||||
// inside a complete statement. TestQueriesAllListed fails if a constant is
|
||||
// missing here.
|
||||
var checks = map[string]queryCheck{
|
||||
// opportunities
|
||||
"selectOpportunityLatestReleaseDate": {sql: selectOpportunityLatestReleaseDate},
|
||||
"upsertOpportunity": {sql: upsertOpportunity, named: true},
|
||||
"selectOpportunityByUUID": {sql: selectOpportunityByUUID},
|
||||
"listOpportunitiesSelect": {
|
||||
sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage,
|
||||
},
|
||||
"listOpportunitiesOrderAndPage": {
|
||||
sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage,
|
||||
},
|
||||
"filterOpportunityStatus": {sql: withFilter(filterOpportunityStatus)},
|
||||
"filterOpportunityClassificationCode": {sql: withFilter(filterOpportunityClassificationCode)},
|
||||
"filterOpportunityTitleMatch": {sql: withFilter(filterOpportunityTitleMatch)},
|
||||
"filterOpportunityReleasedFrom": {sql: withFilter(filterOpportunityReleasedFrom)},
|
||||
"filterOpportunityReleasedBefore": {sql: withFilter(filterOpportunityReleasedBefore)},
|
||||
"filterOpportunityMarkHidden": {sql: withFilter(filterOpportunityMarkHidden)},
|
||||
"filterOpportunityMarkSkipped": {sql: withFilter(filterOpportunityMarkSkipped)},
|
||||
"filterOpportunityMarkInterested": {sql: withFilter(filterOpportunityMarkInterested)},
|
||||
"filterOpportunityMarkBidSubmitted": {sql: withFilter(filterOpportunityMarkBidSubmitted)},
|
||||
"filterOpportunityMarkWon": {sql: withFilter(filterOpportunityMarkWon)},
|
||||
"filterOpportunityMarkLost": {sql: withFilter(filterOpportunityMarkLost)},
|
||||
|
||||
// opportunity_ocds_releases
|
||||
"insertRelease": {sql: insertRelease, named: true},
|
||||
"selectReleasesByOpportunity": {sql: selectReleasesByOpportunity},
|
||||
"selectReleaseExists": {sql: selectReleaseExists},
|
||||
|
||||
// release_tags
|
||||
"insertReleaseTag": {sql: insertReleaseTag, named: true},
|
||||
"selectReleaseTagsByOpportunity": {sql: selectReleaseTagsByOpportunity},
|
||||
|
||||
// classification_codes
|
||||
"upsertClassificationCodeReturningID": {sql: upsertClassificationCodeReturningID, named: true},
|
||||
|
||||
// opportunity_classifications
|
||||
"deleteOpportunityClassifications": {sql: deleteOpportunityClassifications},
|
||||
"insertOpportunityClassification": {sql: insertOpportunityClassification},
|
||||
"selectClassificationsForOpportunities": {sql: selectClassificationsForOpportunities, in: true},
|
||||
|
||||
// opportunity_marks
|
||||
"insertOpportunityMarks": {sql: insertOpportunityMarks, named: true},
|
||||
"upsertOpportunityMarks": {sql: upsertOpportunityMarks, named: true},
|
||||
"selectMarksForOpportunities": {sql: selectMarksForOpportunities, in: true},
|
||||
}
|
||||
|
||||
func withFilter(fragment string) string {
|
||||
return listOpportunitiesSelect + "\nWHERE " + fragment + listOpportunitiesOrderAndPage
|
||||
}
|
||||
|
||||
func openTestDB(t *testing.T) *sqlx.DB {
|
||||
t.Helper()
|
||||
s, err := Open(context.Background(), filepath.Join(t.TempDir(), "queries.db"), WithAutoMigrate())
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s.db
|
||||
}
|
||||
|
||||
func prepare(db *sqlx.DB, c queryCheck) error {
|
||||
var err error
|
||||
switch {
|
||||
case c.named:
|
||||
var stmt *sqlx.NamedStmt
|
||||
if stmt, err = db.PrepareNamed(c.sql); err == nil {
|
||||
stmt.Close()
|
||||
}
|
||||
default:
|
||||
query := c.sql
|
||||
if c.in {
|
||||
var expanded string
|
||||
if expanded, _, err = sqlx.In(query, []string{"a", "b"}); err != nil {
|
||||
return err
|
||||
}
|
||||
query = db.Rebind(expanded)
|
||||
}
|
||||
var stmt *sqlx.Stmt
|
||||
if stmt, err = db.Preparex(query); err == nil {
|
||||
stmt.Close()
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func TestQueriesPrepare(t *testing.T) {
|
||||
db := openTestDB(t)
|
||||
for name, c := range checks {
|
||||
if err := prepare(db, c); err != nil {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestListOpportunitiesQueryAllFilters prepares the assembled query with every
|
||||
// filter set, checking the fragments combine correctly and that there is
|
||||
// exactly one argument per placeholder.
|
||||
func TestListOpportunitiesQueryAllFilters(t *testing.T) {
|
||||
db := openTestDB(t)
|
||||
query, args := listOpportunitiesQuery(OpportunityFilter{
|
||||
Status: "active",
|
||||
ClassificationCode: "72000000",
|
||||
Title: "cloud migration",
|
||||
ReleasedFrom: time.Now().Add(-time.Hour),
|
||||
ReleasedBefore: time.Now(),
|
||||
Hidden: MarkUnset,
|
||||
Skipped: MarkUnset,
|
||||
Interested: MarkSet,
|
||||
BidSubmitted: MarkSet,
|
||||
Won: MarkUnset,
|
||||
Lost: MarkUnset,
|
||||
Limit: 10,
|
||||
Offset: 5,
|
||||
})
|
||||
stmt, err := db.Preparex(query)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare:\n%s\n%v", query, err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
var rows []opportunityRow
|
||||
if err := stmt.Select(&rows, args...); err != nil {
|
||||
t.Errorf("run with %d args:\n%s\n%v", len(args), query, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueriesAllListed fails if queries.go declares a constant that checks
|
||||
// doesn't list, so every new statement is prepared by TestQueriesPrepare.
|
||||
func TestQueriesAllListed(t *testing.T) {
|
||||
file, err := parser.ParseFile(token.NewFileSet(), "queries.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var declared []string
|
||||
for _, decl := range file.Decls {
|
||||
gen, ok := decl.(*ast.GenDecl)
|
||||
if !ok || gen.Tok != token.CONST {
|
||||
continue
|
||||
}
|
||||
for _, spec := range gen.Specs {
|
||||
for _, name := range spec.(*ast.ValueSpec).Names {
|
||||
declared = append(declared, name.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(declared)
|
||||
if len(declared) == 0 {
|
||||
t.Fatal("found no constants in queries.go")
|
||||
}
|
||||
for _, name := range declared {
|
||||
if _, ok := checks[name]; !ok {
|
||||
t.Errorf("queries.go constant %s is not listed in checks", name)
|
||||
}
|
||||
}
|
||||
for name := range checks {
|
||||
if i := sort.SearchStrings(declared, name); i == len(declared) || declared[i] != name {
|
||||
t.Errorf("checks lists %s, which queries.go does not declare", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarksCheckConstraint checks the database itself rejects won and lost
|
||||
// together, for writes that bypass OpportunityMarks.Validate.
|
||||
func TestMarksCheckConstraint(t *testing.T) {
|
||||
db := openTestDB(t)
|
||||
if _, err := db.Exec(`INSERT INTO opportunities
|
||||
(uuid, uuid_type, title, status, latest_release_date)
|
||||
VALUES ('ocds-1', 'ocid', 'T', 'active', '2026-01-01T00:00:00Z')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won, lost)
|
||||
VALUES ('ocds-1', 1, 1)`); err == nil {
|
||||
t.Error("inserting won and lost together succeeded")
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won)
|
||||
VALUES ('ocds-1', 1)`); err != nil {
|
||||
t.Fatalf("inserting won: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`UPDATE opportunity_marks SET lost = 1 WHERE uuid = 'ocds-1'`); err == nil {
|
||||
t.Error("updating to won and lost together succeeded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
// Package store persists opportunities and their OCDS releases in SQLite using
|
||||
// sqlx and hand-written SQL. The SQL is in queries.go; this file holds the
|
||||
// logic. Requires building with -tags sqlite_fts5.
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned when a requested opportunity does not exist.
|
||||
var ErrNotFound = errors.New("store: not found")
|
||||
|
||||
// DefaultListLimit is used when OpportunityFilter.Limit is zero or negative.
|
||||
const DefaultListLimit = 50
|
||||
|
||||
// Store is the opportunity database. Open it with Open; it is safe for
|
||||
// concurrent use.
|
||||
type Store struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
// SaveMultipleResult reports what SaveOpportunities changed.
|
||||
type SaveMultipleResult struct {
|
||||
// NewOpportunities is the number of opportunities that were not already stored.
|
||||
NewOpportunities int
|
||||
// NewReleases is the number of releases that were not already stored.
|
||||
NewReleases int
|
||||
// UpdatedOpportunities is incremented for each opportunity row and its classifications
|
||||
// that are updated (when every release in the save was older than the
|
||||
// one the stored opportunity was last updated from). It does not include new
|
||||
// opportunities.
|
||||
UpdatedOpportunities int
|
||||
// OpportunityResults is a slice of the results of each opportunity saved.
|
||||
OpportunityResults []SaveResult
|
||||
}
|
||||
|
||||
// SaveResult reports what SaveOpportunity changed.
|
||||
type SaveResult struct {
|
||||
// OpportunityUpdated is true if the opportunity row and its classifications were
|
||||
// written. It is false when every release in the save was older than the
|
||||
// one the stored opportunity was last updated from.
|
||||
OpportunityUpdated bool
|
||||
// NewReleases is the number of releases that were not already stored.
|
||||
NewReleases int
|
||||
}
|
||||
|
||||
// MarkFilter filters ListOpportunities on one of an opportunity's marks.
|
||||
type MarkFilter int
|
||||
|
||||
const (
|
||||
// MarkAny does not filter. It is the zero value.
|
||||
MarkAny MarkFilter = iota
|
||||
// MarkSet matches opportunities with the mark set.
|
||||
MarkSet
|
||||
// MarkUnset matches opportunities with the mark unset, including those
|
||||
// with no marks stored.
|
||||
MarkUnset
|
||||
)
|
||||
|
||||
// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued fields do not
|
||||
// filter.
|
||||
type OpportunityFilter struct {
|
||||
Status string
|
||||
// ClassificationCode matches opportunities with a classification of this code
|
||||
// in any scheme.
|
||||
ClassificationCode string
|
||||
// Title is free text matched against opportunity titles: every word must
|
||||
// appear, and each word also matches as a prefix ("cloud migr" matches
|
||||
// "Cloud migration services").
|
||||
Title string
|
||||
// ReleasedFrom (inclusive) and ReleasedBefore (exclusive) bound
|
||||
// Opportunity.LatestReleaseDate.
|
||||
ReleasedFrom time.Time
|
||||
ReleasedBefore time.Time
|
||||
// Each mark filter matches Opportunity.Marks' field of the same name.
|
||||
Hidden MarkFilter
|
||||
Skipped MarkFilter
|
||||
Interested MarkFilter
|
||||
BidSubmitted MarkFilter
|
||||
Won MarkFilter
|
||||
Lost MarkFilter
|
||||
// Limit defaults to DefaultListLimit. Results are ordered newest
|
||||
// LatestReleaseDate first.
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
func (f OpportunityFilter) limit() int {
|
||||
if f.Limit <= 0 {
|
||||
return DefaultListLimit
|
||||
}
|
||||
return f.Limit
|
||||
}
|
||||
|
||||
// prepareOpportunity validates t and normalises it in place before saving: child
|
||||
// rows get the opportunity's UUID and their release id, times are converted to
|
||||
// UTC (so stored timestamps compare correctly as text), and
|
||||
// LatestReleaseDate becomes the newest release date in the save.
|
||||
func prepareOpportunity(t *types.Opportunity) error {
|
||||
if t == nil {
|
||||
return errors.New("store: nil opportunity")
|
||||
}
|
||||
if t.UUID == "" {
|
||||
return errors.New("store: opportunity has no UUID")
|
||||
}
|
||||
|
||||
t.Marks.UUID = t.UUID
|
||||
if err := t.Marks.Validate(); err != nil {
|
||||
return fmt.Errorf("store: opportunity %s: %w", t.UUID, err)
|
||||
}
|
||||
|
||||
latest := t.LatestReleaseDate
|
||||
for i := range t.Releases {
|
||||
r := &t.Releases[i]
|
||||
if r.ReleaseID == "" {
|
||||
return fmt.Errorf("store: opportunity %s: release %d has no release id", t.UUID, i)
|
||||
}
|
||||
r.UUID = t.UUID
|
||||
r.ReleaseDate = r.ReleaseDate.UTC()
|
||||
if r.ReleaseDate.After(latest) {
|
||||
latest = r.ReleaseDate
|
||||
}
|
||||
for j := range r.Tags {
|
||||
r.Tags[j].UUID = t.UUID
|
||||
r.Tags[j].ReleaseID = r.ReleaseID
|
||||
}
|
||||
}
|
||||
|
||||
if latest.IsZero() {
|
||||
return fmt.Errorf("store: opportunity %s has no release date", t.UUID)
|
||||
}
|
||||
|
||||
t.LatestReleaseDate = latest.UTC()
|
||||
t.StartDate = utcPtr(t.StartDate)
|
||||
t.EndDate = utcPtr(t.EndDate)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func utcPtr(t *time.Time) *time.Time {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
u := t.UTC()
|
||||
return &u
|
||||
}
|
||||
|
||||
// ftsQuery turns free text into an FTS5 query that requires every word, each
|
||||
// as a prefix match. Words are split the same way the unicode61 tokenizer
|
||||
// splits them and each is quoted, so user input can never be interpreted as
|
||||
// FTS5 syntax (operators, column filters, unbalanced quotes). It returns ""
|
||||
// when the text contains no words.
|
||||
func ftsQuery(text string) string {
|
||||
words := strings.FieldsFunc(text, func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
|
||||
})
|
||||
for i, w := range words {
|
||||
words[i] = `"` + w + `"*`
|
||||
}
|
||||
return strings.Join(words, " ")
|
||||
}
|
||||
|
||||
// Open opens the database at path. The schema must be exactly at this
|
||||
// build's latest migration: Open returns ErrNeedsMigration if migrations are
|
||||
// pending (run the migrate command first), or ErrSchemaTooNew if the database
|
||||
// was migrated by a newer build. WithAutoMigrate applies pending migrations
|
||||
// instead.
|
||||
//
|
||||
// A database file that doesn't exist is created, and then needs migrating.
|
||||
func Open(ctx context.Context, path string, opts ...Option) (*Store, error) {
|
||||
var o options
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
|
||||
sqldb, err := OpenDB(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := sqldb.PingContext(ctx); err != nil {
|
||||
return nil, errors.Join(err, sqldb.Close())
|
||||
}
|
||||
if err := prepareSchema(ctx, sqldb, o); err != nil {
|
||||
return nil, errors.Join(err, sqldb.Close())
|
||||
}
|
||||
return &Store{db: sqlx.NewDb(sqldb, "sqlite3")}, nil
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// Reads select whole rows ("SELECT *", "t.*", "c.*"). sqlx rejects a result
|
||||
// column with no matching field, so a column added by a migration makes the
|
||||
// reads fail loudly ("missing destination name ...") until it is mapped,
|
||||
// instead of being silently ignored.
|
||||
//
|
||||
// opportunityRow gives the opportunities table's internal id column a home, so it stays
|
||||
// out of types.Opportunity. sqlx maps the embedded Opportunity's fields as if declared
|
||||
// here. The other tables have no internal columns and read straight into
|
||||
// internal/types.
|
||||
type opportunityRow struct {
|
||||
ID int64 `db:"id"`
|
||||
types.Opportunity
|
||||
}
|
||||
|
||||
// SaveOpportunity stores t and its releases in one transaction.
|
||||
//
|
||||
// Releases (and their tags) are write-once: ones already stored are skipped.
|
||||
// The opportunity row and its classifications are replaced only if t's newest
|
||||
// release is at least as new as the release the stored opportunity came from, so
|
||||
// a late-arriving older release never undoes a newer one.
|
||||
//
|
||||
// t.Marks are stored only for a new opportunity; existing marks are kept (use
|
||||
// SetMarks to change them).
|
||||
//
|
||||
// t is normalised in place (see prepareOpportunity), and when the opportunity is
|
||||
// written its classifications' IDs are filled in.
|
||||
func (s *Store) SaveOpportunity(ctx context.Context, t *types.Opportunity) (SaveResult, error) {
|
||||
if err := prepareOpportunity(t); err != nil {
|
||||
return SaveResult{}, err
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return SaveResult{}, err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op after Commit
|
||||
|
||||
res, _, err := saveOpportunity(ctx, tx, t)
|
||||
if err != nil {
|
||||
return SaveResult{}, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return SaveResult{}, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// SaveOpportunities stores t and its releases in one transaction.
|
||||
//
|
||||
// Releases (and their tags) are write-once: ones already stored are skipped.
|
||||
// The opportunity row and its classifications are replaced only if t's newest
|
||||
// release is at least as new as the release the stored opportunity came from, so
|
||||
// a late-arriving older release never undoes a newer one.
|
||||
//
|
||||
// t.Marks are stored only for a new opportunity; existing marks are kept (use
|
||||
// SetMarks to change them).
|
||||
//
|
||||
// t is normalised in place (see prepareOpportunity), and when the opportunity is
|
||||
// written its classifications' IDs are filled in.
|
||||
func (s *Store) SaveOpportunities(
|
||||
ctx context.Context, opportunities []*types.Opportunity,
|
||||
) (SaveMultipleResult, error) {
|
||||
for _, t := range opportunities {
|
||||
if err := prepareOpportunity(t); err != nil {
|
||||
return SaveMultipleResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return SaveMultipleResult{}, err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op after Commit
|
||||
|
||||
allRes := SaveMultipleResult{OpportunityResults: make([]SaveResult, len(opportunities))}
|
||||
for i, t := range opportunities {
|
||||
res, isNew, err := saveOpportunity(ctx, tx, t)
|
||||
if err != nil {
|
||||
return SaveMultipleResult{}, err
|
||||
}
|
||||
allRes.OpportunityResults[i] = res
|
||||
allRes.NewReleases += res.NewReleases
|
||||
switch {
|
||||
case isNew:
|
||||
allRes.NewOpportunities++
|
||||
case res.OpportunityUpdated:
|
||||
allRes.UpdatedOpportunities++
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return SaveMultipleResult{}, err
|
||||
}
|
||||
return allRes, nil
|
||||
}
|
||||
|
||||
// saveOpportunity does the work of SaveOpportunity and SaveOpportunities for
|
||||
// one prepared opportunity inside tx. isNew reports that the opportunity was
|
||||
// not already stored.
|
||||
func saveOpportunity(
|
||||
ctx context.Context, tx *sqlx.Tx, t *types.Opportunity,
|
||||
) (res SaveResult, isNew bool, err error) {
|
||||
var stored time.Time
|
||||
err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, t.UUID)
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
isNew = true
|
||||
res.OpportunityUpdated = true
|
||||
case err != nil:
|
||||
return SaveResult{}, false, fmt.Errorf("store: reading opportunity %s: %w", t.UUID, err)
|
||||
default:
|
||||
res.OpportunityUpdated = !t.LatestReleaseDate.Before(stored)
|
||||
}
|
||||
if res.OpportunityUpdated {
|
||||
if err := writeOpportunity(ctx, tx, t); err != nil {
|
||||
return SaveResult{}, false, fmt.Errorf("store: writing opportunity %s: %w", t.UUID, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range t.Releases {
|
||||
inserted, err := insertReleaseWithTags(ctx, tx, r)
|
||||
if err != nil {
|
||||
return SaveResult{}, false, err
|
||||
}
|
||||
if inserted {
|
||||
res.NewReleases++
|
||||
}
|
||||
}
|
||||
return res, isNew, nil
|
||||
}
|
||||
|
||||
// insertReleaseWithTags stores r and its tags unless r is already stored, and
|
||||
// reports whether it was inserted.
|
||||
func insertReleaseWithTags(
|
||||
ctx context.Context, tx *sqlx.Tx, r types.OpportunityRelease,
|
||||
) (bool, error) {
|
||||
result, err := tx.NamedExecContext(ctx, insertRelease, r)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("store: inserting release %s/%s: %w", r.UUID, r.ReleaseID, err)
|
||||
}
|
||||
if n, err := result.RowsAffected(); err != nil || n == 0 {
|
||||
return false, err // n == 0: already stored
|
||||
}
|
||||
for _, tag := range r.Tags {
|
||||
if _, err := tx.NamedExecContext(ctx, insertReleaseTag, tag); err != nil {
|
||||
return false, fmt.Errorf(
|
||||
"store: inserting tag for %s/%s: %w", r.UUID, r.ReleaseID, err,
|
||||
)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// writeOpportunity upserts t's row, gives it a marks row if it has none
|
||||
// (existing marks are kept), and replaces its classifications, filling in
|
||||
// their IDs.
|
||||
func writeOpportunity(ctx context.Context, tx *sqlx.Tx, t *types.Opportunity) (err error) {
|
||||
if _, err := tx.NamedExecContext(ctx, upsertOpportunity, t); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, t.Marks); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, t.UUID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { err = errors.Join(err, stmt.Close()) }()
|
||||
for i := range t.Classifications {
|
||||
c := &t.Classifications[i]
|
||||
if err := stmt.GetContext(ctx, &c.ID, c); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, insertOpportunityClassification, t.UUID, c.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMarks replaces the marks of the opportunity m.UUID, or returns ErrNotFound.
|
||||
// Marks that fail m.Validate (e.g. types.ErrWonAndLost) are not stored.
|
||||
func (s *Store) SetMarks(ctx context.Context, m types.OpportunityMarks) error {
|
||||
if m.UUID == "" {
|
||||
return errors.New("store: marks have no UUID")
|
||||
}
|
||||
if err := m.Validate(); err != nil {
|
||||
return fmt.Errorf("store: opportunity %s: %w", m.UUID, err)
|
||||
}
|
||||
_, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, m)
|
||||
if isForeignKeyError(err) {
|
||||
return ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// isForeignKeyError reports whether err is SQLite rejecting a write that
|
||||
// breaks a foreign key, e.g. one referring to a missing opportunity.
|
||||
func isForeignKeyError(err error) bool {
|
||||
var se sqlite3.Error
|
||||
return errors.As(err, &se) && se.ExtendedCode == sqlite3.ErrConstraintForeignKey
|
||||
}
|
||||
|
||||
// GetOpportunity returns the opportunity with its classifications, marks and
|
||||
// all of its releases (with tags), or ErrNotFound.
|
||||
func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) {
|
||||
var row opportunityRow
|
||||
err := s.db.GetContext(ctx, &row, selectOpportunityByUUID, uuid)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opportunities := []types.Opportunity{row.Opportunity}
|
||||
if err := s.loadClassifications(ctx, opportunities); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.loadMarks(ctx, opportunities); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opportunities[0].Releases, err = s.ListReleases(ctx, uuid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &opportunities[0], nil
|
||||
}
|
||||
|
||||
// ListOpportunities returns opportunities matching f, with their classifications
|
||||
// and marks but without releases (use ListReleases or GetOpportunity for those).
|
||||
func (s *Store) ListOpportunities(
|
||||
ctx context.Context,
|
||||
f OpportunityFilter,
|
||||
) ([]types.Opportunity, error) {
|
||||
query, args := listOpportunitiesQuery(f)
|
||||
var rows []opportunityRow
|
||||
if err := s.db.SelectContext(ctx, &rows, query, args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opportunities := make([]types.Opportunity, len(rows))
|
||||
for i, row := range rows {
|
||||
opportunities[i] = row.Opportunity
|
||||
}
|
||||
if err := s.loadClassifications(ctx, opportunities); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.loadMarks(ctx, opportunities); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return opportunities, nil
|
||||
}
|
||||
|
||||
// listOpportunitiesQuery assembles the ListOpportunities query from the fragments in
|
||||
// queries.go, with one argument per filter that is set (none for mark filters).
|
||||
func listOpportunitiesQuery(f OpportunityFilter) (string, []any) {
|
||||
var where []string
|
||||
var args []any
|
||||
add := func(clause string, arg any) {
|
||||
where = append(where, clause)
|
||||
args = append(args, arg)
|
||||
}
|
||||
if f.Status != "" {
|
||||
add(filterOpportunityStatus, f.Status)
|
||||
}
|
||||
if f.ClassificationCode != "" {
|
||||
add(filterOpportunityClassificationCode, f.ClassificationCode)
|
||||
}
|
||||
if q := ftsQuery(f.Title); q != "" {
|
||||
add(filterOpportunityTitleMatch, q)
|
||||
}
|
||||
if !f.ReleasedFrom.IsZero() {
|
||||
add(filterOpportunityReleasedFrom, f.ReleasedFrom.UTC())
|
||||
}
|
||||
if !f.ReleasedBefore.IsZero() {
|
||||
add(filterOpportunityReleasedBefore, f.ReleasedBefore.UTC())
|
||||
}
|
||||
for _, m := range []struct {
|
||||
filter MarkFilter
|
||||
clause string
|
||||
}{
|
||||
{f.Hidden, filterOpportunityMarkHidden},
|
||||
{f.Skipped, filterOpportunityMarkSkipped},
|
||||
{f.Interested, filterOpportunityMarkInterested},
|
||||
{f.BidSubmitted, filterOpportunityMarkBidSubmitted},
|
||||
{f.Won, filterOpportunityMarkWon},
|
||||
{f.Lost, filterOpportunityMarkLost},
|
||||
} {
|
||||
switch m.filter {
|
||||
case MarkSet:
|
||||
where = append(where, m.clause)
|
||||
case MarkUnset:
|
||||
where = append(where, "NOT "+m.clause)
|
||||
}
|
||||
}
|
||||
|
||||
query := listOpportunitiesSelect
|
||||
if len(where) > 0 {
|
||||
query += "\nWHERE " + strings.Join(where, "\n\tAND ")
|
||||
}
|
||||
query += listOpportunitiesOrderAndPage
|
||||
return query, append(args, f.limit(), f.Offset)
|
||||
}
|
||||
|
||||
// loadClassifications fills in Classifications for all opportunities with one query.
|
||||
func (s *Store) loadClassifications(ctx context.Context, opportunities []types.Opportunity) error {
|
||||
if len(opportunities) == 0 {
|
||||
return nil
|
||||
}
|
||||
uuids := make([]string, len(opportunities))
|
||||
byUUID := make(map[string]*types.Opportunity, len(opportunities))
|
||||
for i := range opportunities {
|
||||
uuids[i] = opportunities[i].UUID
|
||||
byUUID[opportunities[i].UUID] = &opportunities[i]
|
||||
}
|
||||
|
||||
query, args, err := sqlx.In(selectClassificationsForOpportunities, uuids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// opportunity_uuid says which opportunity each row belongs to.
|
||||
var rows []struct {
|
||||
OpportunityUUID string `db:"opportunity_uuid"`
|
||||
types.ClassificationCode
|
||||
}
|
||||
if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range rows {
|
||||
t := byUUID[r.OpportunityUUID]
|
||||
t.Classifications = append(t.Classifications, r.ClassificationCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadMarks fills in Marks for all opportunities with one query. An
|
||||
// opportunity with no marks row gets zero marks carrying its UUID.
|
||||
func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity) error {
|
||||
if len(opportunities) == 0 {
|
||||
return nil
|
||||
}
|
||||
uuids := make([]string, len(opportunities))
|
||||
byUUID := make(map[string]*types.Opportunity, len(opportunities))
|
||||
for i := range opportunities {
|
||||
uuids[i] = opportunities[i].UUID
|
||||
byUUID[opportunities[i].UUID] = &opportunities[i]
|
||||
opportunities[i].Marks = types.OpportunityMarks{UUID: opportunities[i].UUID}
|
||||
}
|
||||
|
||||
query, args, err := sqlx.In(selectMarksForOpportunities, uuids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rows []types.OpportunityMarks
|
||||
if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range rows {
|
||||
byUUID[m.UUID].Marks = m
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListReleases returns a opportunity's releases with their tags, oldest first.
|
||||
func (s *Store) ListReleases(ctx context.Context, uuid string) ([]types.OpportunityRelease, error) {
|
||||
releases := []types.OpportunityRelease{}
|
||||
if err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[string]*types.OpportunityRelease, len(releases))
|
||||
for i := range releases {
|
||||
byID[releases[i].ReleaseID] = &releases[i]
|
||||
}
|
||||
|
||||
var tags []types.OpportunityReleaseTag
|
||||
if err := s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if r := byID[tag.ReleaseID]; r != nil {
|
||||
r.Tags = append(r.Tags, tag)
|
||||
}
|
||||
}
|
||||
return releases, nil
|
||||
}
|
||||
|
||||
// ReleaseExists reports whether a release is already stored, so a fetcher
|
||||
// can skip converting it.
|
||||
func (s *Store) ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) {
|
||||
var exists bool
|
||||
err := s.db.GetContext(ctx, &exists, selectReleaseExists, uuid, releaseID)
|
||||
return exists, err
|
||||
}
|
||||
@@ -0,0 +1,647 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.avsdev.uk/Go/job-search/internal/store"
|
||||
"git.avsdev.uk/Go/job-search/internal/types"
|
||||
)
|
||||
|
||||
// Run with: go test -tags sqlite_fts5 ./...
|
||||
|
||||
func openStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "test.db")
|
||||
s, err := store.Open(context.Background(), path, store.WithAutoMigrate())
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func day(d int) time.Time {
|
||||
return time.Date(2026, time.March, d, 9, 30, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *types.Opportunity {
|
||||
start, end := day(20), day(28)
|
||||
t := &types.Opportunity{
|
||||
UUID: uuid,
|
||||
UUIDType: "ocid",
|
||||
Title: title,
|
||||
Status: "active",
|
||||
Description: "A description",
|
||||
BuyerName: "Example Council",
|
||||
BuyerId: "GB-EXAMPLE",
|
||||
MainCategory: "services",
|
||||
ProcurementMethod: "open",
|
||||
MultiLot: true,
|
||||
SMESuitable: true,
|
||||
StartDate: &start,
|
||||
EndDate: &end,
|
||||
GrossValue: 125000.50,
|
||||
HasDocuments: true,
|
||||
Classifications: []types.ClassificationCode{
|
||||
{IsCPV: true, Code: "72000000", Classification: "IT services"},
|
||||
{IsCPV: true, Code: "03000000", Classification: "Agricultural products"},
|
||||
},
|
||||
// No UUID: SaveOpportunity must fill it in.
|
||||
Marks: types.OpportunityMarks{Interested: true, BidSubmitted: true},
|
||||
}
|
||||
for i, id := range releaseIDs {
|
||||
t.Releases = append(t.Releases, types.OpportunityRelease{
|
||||
ReleaseID: id,
|
||||
ReleaseDate: day(releaseDay).Add(time.Duration(i) * time.Minute),
|
||||
JSON: `{"id":"` + id + `"}`,
|
||||
Tags: []types.OpportunityReleaseTag{{Tag: "opportunity"}, {Tag: "planning"}},
|
||||
})
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func mustSave(t *testing.T, s *store.Store, td *types.Opportunity) store.SaveResult {
|
||||
t.Helper()
|
||||
res, err := s.SaveOpportunity(context.Background(), td)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveOpportunity(%s): %v", td.UUID, err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func mustGet(t *testing.T, s *store.Store, uuid string) *types.Opportunity {
|
||||
t.Helper()
|
||||
got, err := s.GetOpportunity(context.Background(), uuid)
|
||||
if err != nil {
|
||||
t.Fatalf("GetOpportunity(%s): %v", uuid, err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func uuids(ts []types.Opportunity) []string {
|
||||
out := []string{}
|
||||
for _, t := range ts {
|
||||
out = append(out, t.UUID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func equal(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestSaveAndGetRoundTrip(t *testing.T) {
|
||||
s := openStore(t)
|
||||
in := opportunity("ocds-1", "Cloud migration services", 1, "r1", "r2")
|
||||
res := mustSave(t, s, in)
|
||||
if !res.OpportunityUpdated || res.NewReleases != 2 {
|
||||
t.Fatalf("first save: got %+v", res)
|
||||
}
|
||||
|
||||
got := mustGet(t, s, "ocds-1")
|
||||
if got.Title != in.Title || got.BuyerName != in.BuyerName || got.BuyerId != in.BuyerId ||
|
||||
got.MainCategory != in.MainCategory || got.ProcurementMethod != in.ProcurementMethod ||
|
||||
got.UUIDType != "ocid" || got.Status != "active" || got.Description != in.Description ||
|
||||
!got.MultiLot || !got.SMESuitable || got.VCSESuitable || !got.HasDocuments ||
|
||||
got.GrossValue != in.GrossValue {
|
||||
t.Errorf("opportunity fields differ:\n got %+v\nwant %+v", got, in)
|
||||
}
|
||||
if got.StartDate == nil || !got.StartDate.Equal(day(20)) || got.EndDate == nil ||
|
||||
!got.EndDate.Equal(day(28)) {
|
||||
t.Errorf("dates: got %v / %v", got.StartDate, got.EndDate)
|
||||
}
|
||||
if want := day(1).Add(time.Minute); !got.LatestReleaseDate.Equal(want) {
|
||||
t.Errorf("LatestReleaseDate = %v, want %v", got.LatestReleaseDate, want)
|
||||
}
|
||||
|
||||
if len(got.Classifications) != 2 || got.Classifications[0].Code != "03000000" ||
|
||||
got.Classifications[1].Code != "72000000" || got.Classifications[0].ID == 0 {
|
||||
t.Errorf("classifications: got %+v (leading zero must survive)", got.Classifications)
|
||||
}
|
||||
|
||||
wantMarks := types.OpportunityMarks{UUID: "ocds-1", Interested: true, BidSubmitted: true}
|
||||
if got.Marks != wantMarks {
|
||||
t.Errorf("marks: got %+v, want %+v", got.Marks, wantMarks)
|
||||
}
|
||||
|
||||
if len(got.Releases) != 2 {
|
||||
t.Fatalf("releases: got %d", len(got.Releases))
|
||||
}
|
||||
r := got.Releases[0]
|
||||
if r.ReleaseID != "r1" || r.UUID != "ocds-1" || r.JSON != `{"id":"r1"}` ||
|
||||
!r.ReleaseDate.Equal(day(1)) {
|
||||
t.Errorf("release: got %+v", r)
|
||||
}
|
||||
if len(r.Tags) != 2 ||
|
||||
r.Tags[0].Tag != "opportunity" || r.Tags[1].Tag != "planning" ||
|
||||
r.Tags[0].ReleaseID != "r1" ||
|
||||
r.Tags[0].UUID != "ocds-1" {
|
||||
t.Errorf("tags: got %+v", r.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartAndEndDateMayBeNil(t *testing.T) {
|
||||
s := openStore(t)
|
||||
in := opportunity("ocds-1", "No dates", 1, "r1")
|
||||
in.StartDate, in.EndDate = nil, nil
|
||||
mustSave(t, s, in)
|
||||
got := mustGet(t, s, "ocds-1")
|
||||
if got.StartDate != nil || got.EndDate != nil {
|
||||
t.Errorf("got %v / %v, want nil", got.StartDate, got.EndDate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleasesAreWriteOnce(t *testing.T) {
|
||||
s := openStore(t)
|
||||
mustSave(t, s, opportunity("ocds-1", "Title", 1, "r1"))
|
||||
|
||||
again := opportunity("ocds-1", "Title", 1, "r1")
|
||||
again.Releases[0].JSON = `{"changed":true}`
|
||||
again.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}}
|
||||
if res := mustSave(t, s, again); res.NewReleases != 0 {
|
||||
t.Errorf("re-save: NewReleases = %d, want 0", res.NewReleases)
|
||||
}
|
||||
|
||||
got := mustGet(t, s, "ocds-1").Releases
|
||||
if len(got) != 1 || got[0].JSON != `{"id":"r1"}` || len(got[0].Tags) != 2 {
|
||||
t.Errorf("stored release changed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOlderReleaseDoesNotOverwriteOpportunity(t *testing.T) {
|
||||
s := openStore(t)
|
||||
mustSave(t, s, opportunity("ocds-1", "Newer title", 10, "r2"))
|
||||
|
||||
old := opportunity("ocds-1", "Older title", 5, "r1")
|
||||
old.Classifications = old.Classifications[:1]
|
||||
res := mustSave(t, s, old)
|
||||
if res.OpportunityUpdated || res.NewReleases != 1 {
|
||||
t.Fatalf("older save: got %+v", res)
|
||||
}
|
||||
|
||||
got := mustGet(t, s, "ocds-1")
|
||||
if got.Title != "Newer title" || len(got.Classifications) != 2 ||
|
||||
!got.LatestReleaseDate.Equal(day(10)) {
|
||||
t.Errorf("opportunity was overwritten by an older release: %+v", got)
|
||||
}
|
||||
if len(got.Releases) != 2 || got.Releases[0].ReleaseID != "r1" {
|
||||
t.Errorf("older release should still be stored, oldest first: %+v", got.Releases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
first := opportunity("ocds-1", "Cloud hosting", 1, "r1")
|
||||
mustSave(t, s, first)
|
||||
oldIT := first.Classifications[0].ID
|
||||
|
||||
newer := opportunity("ocds-1", "Website redesign", 2, "r2")
|
||||
newer.Status = "complete"
|
||||
newer.Classifications = []types.ClassificationCode{
|
||||
{IsCPV: true, Code: "72000000", Classification: "IT services (amended)"},
|
||||
}
|
||||
if res := mustSave(t, s, newer); !res.OpportunityUpdated {
|
||||
t.Fatalf("newer save not applied: %+v", res)
|
||||
}
|
||||
|
||||
got := mustGet(t, s, "ocds-1")
|
||||
if got.Title != "Website redesign" || got.Status != "complete" {
|
||||
t.Errorf("opportunity not updated: %+v", got)
|
||||
}
|
||||
if len(got.Classifications) != 1 ||
|
||||
got.Classifications[0].Classification != "IT services (amended)" {
|
||||
t.Fatalf("classifications not replaced: %+v", got.Classifications)
|
||||
}
|
||||
if got.Classifications[0].ID == oldIT {
|
||||
t.Errorf("changed description should create a new classification row")
|
||||
}
|
||||
|
||||
for title, want := range map[string]int{"cloud": 0, "redesign": 1} {
|
||||
ts, err := s.ListOpportunities(ctx, store.OpportunityFilter{Title: title})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ts) != want {
|
||||
t.Errorf("search %q after title change: %d results, want %d", title, len(ts), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassificationRowsAreShared(t *testing.T) {
|
||||
s := openStore(t)
|
||||
a := opportunity("ocds-a", "A", 1, "r1")
|
||||
b := opportunity("ocds-b", "B", 1, "r1")
|
||||
mustSave(t, s, a)
|
||||
mustSave(t, s, b)
|
||||
if a.Classifications[0].ID == 0 || a.Classifications[0].ID != b.Classifications[0].ID {
|
||||
t.Errorf("identical classifications should share an ID: %d vs %d",
|
||||
a.Classifications[0].ID, b.Classifications[0].ID)
|
||||
}
|
||||
|
||||
c := opportunity("ocds-c", "C", 1, "r1")
|
||||
c.Classifications = []types.ClassificationCode{
|
||||
{IsCPV: false, Code: "72000000", Classification: "IT services"},
|
||||
}
|
||||
mustSave(t, s, c)
|
||||
if c.Classifications[0].ID == a.Classifications[0].ID {
|
||||
t.Errorf("different is_cpv should not share an ID")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameReleaseIDInDifferentOpportunities(t *testing.T) {
|
||||
s := openStore(t)
|
||||
a := opportunity("ocds-a", "A", 1, "001")
|
||||
b := opportunity("ocds-b", "B", 1, "001")
|
||||
b.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}}
|
||||
if res := mustSave(t, s, a); res.NewReleases != 1 {
|
||||
t.Fatal(res)
|
||||
}
|
||||
if res := mustSave(t, s, b); res.NewReleases != 1 {
|
||||
t.Fatalf("release id reused under another OCID must be stored: %+v", res)
|
||||
}
|
||||
got := mustGet(t, s, "ocds-b").Releases
|
||||
if len(got) != 1 || len(got[0].Tags) != 1 || got[0].Tags[0].Tag != "award" {
|
||||
t.Errorf("tags leaked between opportunities: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOpportunitiesFilters(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
cloud := opportunity("ocds-1", "Cloud migration services", 1, "r1")
|
||||
roads := opportunity("ocds-2", "Road resurfacing works", 2, "r1")
|
||||
roads.Status = "complete"
|
||||
roads.Classifications = []types.ClassificationCode{
|
||||
{IsCPV: true, Code: "45233142", Classification: "Road-repair works"},
|
||||
}
|
||||
hosting := opportunity("ocds-3", "Managed cloud hosting", 3, "r1")
|
||||
for _, td := range []*types.Opportunity{cloud, roads, hosting} {
|
||||
mustSave(t, s, td)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
f store.OpportunityFilter
|
||||
want []string
|
||||
}{
|
||||
{"all, newest first", store.OpportunityFilter{}, []string{"ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"status", store.OpportunityFilter{Status: "complete"}, []string{"ocds-2"}},
|
||||
{
|
||||
"classification",
|
||||
store.OpportunityFilter{ClassificationCode: "72000000"},
|
||||
[]string{"ocds-3", "ocds-1"},
|
||||
},
|
||||
{"title word", store.OpportunityFilter{Title: "cloud"}, []string{"ocds-3", "ocds-1"}},
|
||||
{"title prefixes", store.OpportunityFilter{Title: "clo migr"}, []string{"ocds-1"}},
|
||||
{"title stemming", store.OpportunityFilter{Title: "migrating"}, []string{"ocds-1"}},
|
||||
{"title case", store.OpportunityFilter{Title: "ROAD"}, []string{"ocds-2"}},
|
||||
{
|
||||
"title syntax is not FTS syntax",
|
||||
store.OpportunityFilter{Title: `cloud" OR title:road*`},
|
||||
[]string{},
|
||||
},
|
||||
{
|
||||
"title punctuation only",
|
||||
store.OpportunityFilter{Title: `"*:()`},
|
||||
[]string{"ocds-3", "ocds-2", "ocds-1"},
|
||||
},
|
||||
{"released from", store.OpportunityFilter{ReleasedFrom: day(2)}, []string{"ocds-3", "ocds-2"}},
|
||||
{"released before", store.OpportunityFilter{ReleasedBefore: day(2)}, []string{"ocds-1"}},
|
||||
{"combined", store.OpportunityFilter{Title: "cloud", ReleasedFrom: day(2)}, []string{"ocds-3"}},
|
||||
{"limit", store.OpportunityFilter{Limit: 2}, []string{"ocds-3", "ocds-2"}},
|
||||
{"offset", store.OpportunityFilter{Limit: 2, Offset: 2}, []string{"ocds-1"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := s.ListOpportunities(ctx, tc.f)
|
||||
if err != nil {
|
||||
t.Fatalf("ListOpportunities: %v", err)
|
||||
}
|
||||
if !equal(uuids(got), tc.want) {
|
||||
t.Errorf("got %v, want %v", uuids(got), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{Status: "complete"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || len(got[0].Classifications) != 1 || got[0].Releases != nil ||
|
||||
got[0].Marks.UUID != "ocds-2" || !got[0].Marks.Interested {
|
||||
t.Errorf("list should include classifications and marks and omit releases: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseExistsAndListReleases(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1", "r2"))
|
||||
|
||||
for _, tc := range []struct {
|
||||
uuid, id string
|
||||
want bool
|
||||
}{{"ocds-1", "r1", true}, {"ocds-1", "r3", false}, {"ocds-2", "r1", false}} {
|
||||
got, err := s.ReleaseExists(ctx, tc.uuid, tc.id)
|
||||
if err != nil || got != tc.want {
|
||||
t.Errorf("ReleaseExists(%s, %s) = %v, %v; want %v", tc.uuid, tc.id, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
rs, err := s.ListReleases(ctx, "ocds-1")
|
||||
if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" || rs[1].ReleaseID != "r2" ||
|
||||
len(rs[1].Tags) != 2 {
|
||||
t.Errorf("ListReleases: %+v, %v", rs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOpportunityNotFound(t *testing.T) {
|
||||
s := openStore(t)
|
||||
if _, err := s.GetOpportunity(
|
||||
context.Background(),
|
||||
"missing",
|
||||
); !errors.Is(
|
||||
err,
|
||||
store.ErrNotFound,
|
||||
) {
|
||||
t.Errorf("err = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveOpportunityValidation(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
for name, td := range map[string]*types.Opportunity{
|
||||
"nil": nil,
|
||||
"no uuid": opportunity("", "T", 1, "r1"),
|
||||
"no release date": {UUID: "ocds-1", Title: "T", Status: "active"},
|
||||
"no release id": opportunity("ocds-1", "T", 1, ""),
|
||||
} {
|
||||
if _, err := s.SaveOpportunity(ctx, td); err == nil {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReopenKeepsDataAndSkipsAppliedMigrations(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "test.db")
|
||||
ctx := context.Background()
|
||||
s, err := store.Open(ctx, path, store.WithAutoMigrate())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustSave(t, s, opportunity("ocds-1", "Cloud migration", 1, "r1"))
|
||||
s.Close()
|
||||
|
||||
// No WithAutoMigrate: an up-to-date database must pass the version check.
|
||||
s, err = store.Open(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{Title: "cloud"})
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Errorf("after reopen: %v, %v", uuids(got), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResaveLoadedOpportunity(t *testing.T) {
|
||||
s := openStore(t)
|
||||
mustSave(t, s, opportunity("ocds-1", "Original", 1, "r1"))
|
||||
|
||||
loaded := mustGet(t, s, "ocds-1") // classifications carry database IDs
|
||||
ids := []int64{loaded.Classifications[0].ID, loaded.Classifications[1].ID}
|
||||
loaded.Title = "Amended"
|
||||
loaded.Releases = append(loaded.Releases, types.OpportunityRelease{
|
||||
ReleaseID: "r2", ReleaseDate: day(2), JSON: `{}`,
|
||||
Tags: []types.OpportunityReleaseTag{{Tag: "opportunityAmendment"}},
|
||||
})
|
||||
res := mustSave(t, s, loaded)
|
||||
if !res.OpportunityUpdated || res.NewReleases != 1 {
|
||||
t.Fatalf("resave: %+v", res)
|
||||
}
|
||||
|
||||
got := mustGet(t, s, "ocds-1")
|
||||
if got.Title != "Amended" || len(got.Releases) != 2 || !got.LatestReleaseDate.Equal(day(2)) {
|
||||
t.Errorf("resave not applied: %+v", got)
|
||||
}
|
||||
if got.Classifications[0].ID != ids[0] || got.Classifications[1].ID != ids[1] {
|
||||
t.Errorf("classification IDs changed on resave: %v -> %d, %d",
|
||||
ids, got.Classifications[0].ID, got.Classifications[1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func mustSetMarks(t *testing.T, s *store.Store, m types.OpportunityMarks) {
|
||||
t.Helper()
|
||||
if err := s.SetMarks(context.Background(), m); err != nil {
|
||||
t.Fatalf("SetMarks(%s): %v", m.UUID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetMarks(t *testing.T) {
|
||||
s := openStore(t)
|
||||
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
||||
|
||||
for _, want := range []types.OpportunityMarks{
|
||||
{UUID: "ocds-1", Hidden: true, Skipped: true},
|
||||
{UUID: "ocds-1", Interested: true, BidSubmitted: true, Won: true},
|
||||
{UUID: "ocds-1"},
|
||||
} {
|
||||
mustSetMarks(t, s, want)
|
||||
if got := mustGet(t, s, "ocds-1").Marks; got != want {
|
||||
t.Errorf("after SetMarks: got %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetMarksErrors(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.SetMarks(
|
||||
ctx,
|
||||
types.OpportunityMarks{UUID: "missing", Hidden: true},
|
||||
); !errors.Is(
|
||||
err,
|
||||
store.ErrNotFound,
|
||||
) {
|
||||
t.Errorf("unknown opportunity: err = %v, want ErrNotFound", err)
|
||||
}
|
||||
if err := s.SetMarks(ctx, types.OpportunityMarks{Hidden: true}); err == nil {
|
||||
t.Errorf("no UUID: expected an error")
|
||||
}
|
||||
}
|
||||
|
||||
// Fetched opportunities carry zero marks; saving them must not reset the
|
||||
// marks the user has set.
|
||||
func TestSaveKeepsExistingMarks(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
||||
mustSave(t, s, opportunity("ocds-2", "T", 1, "r1"))
|
||||
want := types.OpportunityMarks{UUID: "ocds-1", Hidden: true, Lost: true}
|
||||
mustSetMarks(t, s, want)
|
||||
|
||||
newer := opportunity("ocds-1", "Amended", 2, "r2")
|
||||
newer.Marks = types.OpportunityMarks{}
|
||||
if res := mustSave(t, s, newer); !res.OpportunityUpdated {
|
||||
t.Fatalf("newer save not applied: %+v", res)
|
||||
}
|
||||
if got := mustGet(t, s, "ocds-1").Marks; got != want {
|
||||
t.Errorf("SaveOpportunity changed marks: got %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
newest := opportunity("ocds-1", "Amended again", 3, "r3")
|
||||
newest.Marks = types.OpportunityMarks{}
|
||||
if _, err := s.SaveOpportunities(ctx, []*types.Opportunity{newest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := mustGet(t, s, "ocds-1").Marks; got != want {
|
||||
t.Errorf("SaveOpportunities changed marks: got %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveOpportunitiesStoresMarks(t *testing.T) {
|
||||
s := openStore(t)
|
||||
a := opportunity("ocds-a", "A", 1, "r1")
|
||||
b := opportunity("ocds-b", "B", 1, "r1")
|
||||
b.Marks = types.OpportunityMarks{Skipped: true}
|
||||
if _, err := s.SaveOpportunities(context.Background(), []*types.Opportunity{a, b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for uuid, want := range map[string]types.OpportunityMarks{
|
||||
"ocds-a": {UUID: "ocds-a", Interested: true, BidSubmitted: true},
|
||||
"ocds-b": {UUID: "ocds-b", Skipped: true},
|
||||
} {
|
||||
if got := mustGet(t, s, uuid).Marks; got != want {
|
||||
t.Errorf("%s marks: got %+v, want %+v", uuid, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOpportunitiesMarkFilters(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
for i, uuid := range []string{"ocds-1", "ocds-2", "ocds-3", "ocds-4"} {
|
||||
mustSave(t, s, opportunity(uuid, "Cloud hosting", i+1, "r1"))
|
||||
}
|
||||
// ocds-1 keeps the marks it was saved with: Interested, BidSubmitted.
|
||||
mustSetMarks(t, s, types.OpportunityMarks{UUID: "ocds-2", Hidden: true})
|
||||
mustSetMarks(t, s, types.OpportunityMarks{UUID: "ocds-3", Skipped: true, Lost: true})
|
||||
mustSetMarks(t, s, types.OpportunityMarks{
|
||||
UUID: "ocds-4", Interested: true, BidSubmitted: true, Won: true,
|
||||
})
|
||||
|
||||
set, unset := store.MarkSet, store.MarkUnset
|
||||
cases := []struct {
|
||||
name string
|
||||
f store.OpportunityFilter
|
||||
want []string
|
||||
}{
|
||||
{"any", store.OpportunityFilter{}, []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"hidden", store.OpportunityFilter{Hidden: set}, []string{"ocds-2"}},
|
||||
{"not hidden", store.OpportunityFilter{Hidden: unset}, []string{"ocds-4", "ocds-3", "ocds-1"}},
|
||||
{"skipped", store.OpportunityFilter{Skipped: set}, []string{"ocds-3"}},
|
||||
{
|
||||
"not skipped",
|
||||
store.OpportunityFilter{Skipped: unset},
|
||||
[]string{"ocds-4", "ocds-2", "ocds-1"},
|
||||
},
|
||||
{"interested", store.OpportunityFilter{Interested: set}, []string{"ocds-4", "ocds-1"}},
|
||||
{"not interested", store.OpportunityFilter{Interested: unset}, []string{"ocds-3", "ocds-2"}},
|
||||
{"bid submitted", store.OpportunityFilter{BidSubmitted: set}, []string{"ocds-4", "ocds-1"}},
|
||||
{
|
||||
"bid not submitted",
|
||||
store.OpportunityFilter{BidSubmitted: unset},
|
||||
[]string{"ocds-3", "ocds-2"},
|
||||
},
|
||||
{"won", store.OpportunityFilter{Won: set}, []string{"ocds-4"}},
|
||||
{"not won", store.OpportunityFilter{Won: unset}, []string{"ocds-3", "ocds-2", "ocds-1"}},
|
||||
{"lost", store.OpportunityFilter{Lost: set}, []string{"ocds-3"}},
|
||||
{"not lost", store.OpportunityFilter{Lost: unset}, []string{"ocds-4", "ocds-2", "ocds-1"}},
|
||||
{"two marks", store.OpportunityFilter{BidSubmitted: set, Won: unset}, []string{"ocds-1"}},
|
||||
{"no match", store.OpportunityFilter{Won: set, Lost: set}, []string{}},
|
||||
{
|
||||
"with other filters",
|
||||
store.OpportunityFilter{Hidden: unset, Title: "cloud", ReleasedFrom: day(3)},
|
||||
[]string{"ocds-4", "ocds-3"},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := s.ListOpportunities(ctx, tc.f)
|
||||
if err != nil {
|
||||
t.Fatalf("ListOpportunities: %v", err)
|
||||
}
|
||||
if !equal(uuids(got), tc.want) {
|
||||
t.Errorf("got %v, want %v", uuids(got), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
got, err := s.ListOpportunities(ctx, store.OpportunityFilter{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost || !got[2].Marks.Hidden ||
|
||||
got[3].Marks.Hidden || !got[3].Marks.Interested {
|
||||
t.Errorf("list marks: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWonAndLostAreExclusive(t *testing.T) {
|
||||
s := openStore(t)
|
||||
ctx := context.Background()
|
||||
mustSave(t, s, opportunity("ocds-1", "T", 1, "r1"))
|
||||
before := mustGet(t, s, "ocds-1").Marks
|
||||
|
||||
both := types.OpportunityMarks{UUID: "ocds-1", Won: true, Lost: true}
|
||||
if err := s.SetMarks(ctx, both); !errors.Is(err, types.ErrWonAndLost) {
|
||||
t.Errorf("SetMarks(won and lost): err = %v, want ErrWonAndLost", err)
|
||||
}
|
||||
if got := mustGet(t, s, "ocds-1").Marks; got != before {
|
||||
t.Errorf("rejected SetMarks changed marks: got %+v, want %+v", got, before)
|
||||
}
|
||||
|
||||
in := opportunity("ocds-2", "T", 1, "r1")
|
||||
in.Marks = types.OpportunityMarks{Won: true, Lost: true}
|
||||
if _, err := s.SaveOpportunity(ctx, in); !errors.Is(err, types.ErrWonAndLost) {
|
||||
t.Errorf("SaveOpportunity(won and lost): err = %v, want ErrWonAndLost", err)
|
||||
}
|
||||
if _, err := s.SaveOpportunities(
|
||||
ctx,
|
||||
[]*types.Opportunity{in},
|
||||
); !errors.Is(
|
||||
err,
|
||||
types.ErrWonAndLost,
|
||||
) {
|
||||
t.Errorf("SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err)
|
||||
}
|
||||
if _, err := s.GetOpportunity(ctx, "ocds-2"); !errors.Is(err, store.ErrNotFound) {
|
||||
t.Errorf("rejected opportunity was stored: err = %v", err)
|
||||
}
|
||||
|
||||
// Either one alone is fine, and switching between them works.
|
||||
for _, m := range []types.OpportunityMarks{
|
||||
{UUID: "ocds-1", Won: true},
|
||||
{UUID: "ocds-1", Lost: true},
|
||||
} {
|
||||
mustSetMarks(t, s, m)
|
||||
if got := mustGet(t, s, "ocds-1").Marks; got != m {
|
||||
t.Errorf("got %+v, want %+v", got, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,7 +43,10 @@ fi
|
||||
go generate ./...
|
||||
|
||||
# Build binary with version info
|
||||
# sqlite_fts5 compiles in SQLite's FTS5 module, needed by the store's title
|
||||
# search (migration 6 fails without it)
|
||||
go build \
|
||||
-tags sqlite_fts5 \
|
||||
-ldflags="$LD_FLAGS" \
|
||||
-trimpath \
|
||||
-o $BUILD_DIR/job-search-$VERSION \
|
||||
|
||||
Reference in new issue
Block a user