From 67215a6aaaa5b04d2c0f08626d3b98c1c40be058 Mon Sep 17 00:00:00 2001 From: Craig Williams Date: Thu, 15 Oct 2026 16:26:42 +0100 Subject: [PATCH] Added CODE_CONVENTIONS.md and linters to enforce it, and applied it across the codebase CODE_CONVENTIONS.md sets out the conventions for Go code in this repository: naming, layout, comments, errors, logging, configuration and tests. .golangci.json enables the linters that check the parts that can be checked (each limited to the rules the conventions use); the rest is applied by hand. Applied across the codebase: - Naming: full-word names in application code, initialisms in one case (BuyerID, PrintJSON, PrintYAML, ocdsGetBuyerID), no names shadowing built-ins (min/max are minGap/maxGap), mustRaw renamed rawJSON as it returns nil rather than panicking, and config.ConfigLevelToLogLevel renamed LevelToLogLevel. - Layout: lines kept to 80 characters where possible, assignments taken out of if and for statements that don't fit, wrapped arguments all on one line or one per line, steps separated by blank lines, no else after a return, function and struct literals wrapped. - Comments: doc comments on every exported name (including each section's Map, Flatten, TypeOf, ValueOf and SetValue), section banners, British spelling. - Errors: a created error starts with its package name, a wrapping error adds only what was being done, sentinels in one commented var block, errors.Is and errors.As in place of == and type assertions, and no err := shadowing of a named result. That fixes the mock server losing its access log's close error. - Logging: the fetcher passes its workers an untagged logger, so findatender's lines are tagged with their own component rather than with fetcher's as well; cmd/mock-fts logs "stopped" with slog. - Configuration: util.WithTimeout (moved from the websocket package) applies a timeout of 0 or less as no limit, used for the websocket write and ping, TCP close and tcpserver shutdown timeouts; every setting documents what 0 or empty means, including the rate limit rule's keys in internal/config/README.md. - Tests: related tests merged into table-driven ones with named cases. Separately from CODE_CONVENTIONS.md, internal/store/README.md gains a soft SQL style convention for the queries in internal/store/queries.go, and the queries are reformatted to follow it. --- .golangci.json | 102 ++- CODE_CONVENTIONS.md | 786 ++++++++++++++++++++ cmd/jobsearch/main_test.go | 6 +- cmd/mock-fts/main.go | 82 +- internal/cli/cli_test.go | 20 +- internal/cli/cmdtest/cmdtest.go | 21 +- internal/cli/config/command.go | 4 +- internal/cli/config/config_test.go | 46 +- internal/cli/config/get.go | 2 +- internal/cli/config/init.go | 18 +- internal/cli/config/set.go | 3 +- internal/cli/config/show.go | 7 +- internal/cli/confirm_action.go | 14 +- internal/cli/fetch/command.go | 4 +- internal/cli/fetch/once.go | 25 +- internal/cli/fetch/once_test.go | 5 +- internal/cli/fetch/service.go | 3 +- internal/cli/flags_test.go | 7 +- internal/cli/migrate/common.go | 20 +- internal/cli/migrate/common_test.go | 18 +- internal/cli/migrate/down.go | 12 +- internal/cli/migrate/migrate.go | 5 +- internal/cli/migrate/migrate_test.go | 28 +- internal/cli/migrate/status.go | 13 +- internal/cli/migrate/up.go | 21 +- internal/cli/options.go | 6 +- internal/cli/wsserve/command.go | 18 +- internal/config/README.md | 7 +- internal/config/conf_database.go | 5 + internal/config/conf_logging.go | 13 +- internal/config/conf_ratelimitrule.go | 5 + internal/config/conf_sections_test.go | 145 ++-- internal/config/conf_server.go | 5 + internal/config/conf_server_ratelimits.go | 5 + internal/config/conf_server_timeouts.go | 5 + internal/config/conf_server_websocket.go | 10 +- internal/config/conf_source_findatender.go | 5 + internal/config/conf_source_ftsmock.go | 5 + internal/config/conf_source_sleeper.go | 5 + internal/config/conf_sources.go | 14 +- internal/config/config.go | 25 +- internal/config/config_test.go | 42 +- internal/config/load.go | 34 +- internal/config/load_test.go | 43 +- internal/config/modify.go | 20 +- internal/config/modify_test.go | 48 +- internal/config/print.go | 14 +- internal/config/print_test.go | 54 +- internal/config/section.go | 133 ++-- internal/config/section_test.go | 28 +- internal/fetcher/fetcher.go | 28 +- internal/fetcher/fetcher_test.go | 59 +- internal/fetcher/worker_findatender.go | 23 +- internal/fetcher/worker_ftsmock.go | 23 +- internal/fetcher/worker_sleeper.go | 7 +- internal/findatender/api.go | 83 ++- internal/findatender/api_test.go | 249 +++++-- internal/findatender/findatender.go | 14 +- internal/findatender/findatender_test.go | 19 +- internal/findatender/type_convert.go | 29 +- internal/findatender/type_convert_test.go | 170 +++-- internal/httpx/httpx_test.go | 8 +- internal/httpx/problem.go | 5 +- internal/logging/accesslog.go | 42 +- internal/logging/accesslog_test.go | 1 + internal/logging/logging_test.go | 22 +- internal/middleware/ratelimit.go | 20 +- internal/middleware/ratelimit_test.go | 10 +- internal/middleware/real_ip.go | 3 +- internal/middleware/real_ip_test.go | 28 +- internal/mock/findatender/catalog.go | 58 +- internal/mock/findatender/catalog_test.go | 19 +- internal/mock/findatender/generator.go | 113 +-- internal/mock/findatender/generator_test.go | 49 +- internal/mock/findatender/handler.go | 52 +- internal/mock/findatender/handler_test.go | 10 +- internal/mock/findatender/ratelimit.go | 23 +- internal/mock/findatender/ratelimit_test.go | 3 +- internal/mock/findatender/store.go | 102 ++- internal/mock/findatender/store_test.go | 29 +- internal/store/README.md | 12 + internal/store/migrate.go | 22 +- internal/store/migrate_test.go | 38 +- internal/store/queries.go | 30 +- internal/store/queries_test.go | 109 ++- internal/store/store.go | 300 +++++--- internal/store/store_test.go | 211 ++++-- internal/types/opportunities.go | 4 +- internal/util/bytesize.go | 8 +- internal/util/bytesize_test.go | 11 +- internal/util/context.go | 19 + internal/util/context_test.go | 20 + internal/util/worker.go | 45 +- internal/util/worker_test.go | 55 +- internal/websocket/client.go | 41 +- internal/websocket/client_test.go | 39 +- internal/websocket/http_test.go | 13 +- internal/websocket/hub.go | 9 +- internal/websocket/hub_test.go | 10 +- internal/websocket/messages.go | 5 +- internal/websocket/tcp.go | 36 +- internal/websocket/tcp_test.go | 44 +- internal/websocket/tcpserver/server.go | 17 +- internal/websocket/tcpserver/server_test.go | 30 +- 104 files changed, 3189 insertions(+), 1206 deletions(-) create mode 100644 CODE_CONVENTIONS.md create mode 100644 internal/util/context.go create mode 100644 internal/util/context_test.go diff --git a/.golangci.json b/.golangci.json index 1100652..b5a9593 100644 --- a/.golangci.json +++ b/.golangci.json @@ -9,19 +9,111 @@ "cyclop", "dogsled", "errcheck", + "errorlint", + "gocritic", "godox", - "govet", "gosec", + "govet", "ineffassign", + "misspell", "nilerr", "nilnil", "nolintlint", + "revive", + "sloglint", + "thelper", "unused", - "whitespace" + "varnamelen", + "whitespace", + "wsl_v5" ], "settings": { "cyclop": { "max-complexity": 15 + }, + "misspell": { + "locale": "UK", + "ignore-rules": [ + "organization", + "humanize" + ] + }, + "sloglint": { + "no-mixed-args": true, + "key-naming-case": "snake", + "static-msg": true + }, + "revive": { + "rules": [ + { + "name": "context-as-argument", + "arguments": [ + { + "allowTypesBefore": "*testing.T" + } + ] + }, + { + "name": "error-naming" + }, + { + "name": "error-strings" + }, + { + "name": "errorf" + }, + { + "name": "exported" + }, + { + "name": "indent-error-flow" + }, + { + "name": "receiver-naming" + }, + { + "name": "superfluous-else" + }, + { + "name": "var-naming" + } + ] + }, + "varnamelen": { + "ignore-names": [ + "err", + "ctx", + "i", + "idx", + "msg", + "w", + "r", + "req", + "res", + "db", + "tx", + "wg", + "mu", + "t", + "ok", + "id", + "ip" + ], + "ignore-decls": [] + }, + "gocritic": { + "disable-all": true, + "enabled-checks": [ + "builtinShadow", + "emptyStringTest", + "paramTypeCombine" + ] + }, + "wsl_v5": { + "default": "none", + "enable": [ + "err" + ] } }, "exclusions": { @@ -39,6 +131,12 @@ "gosec" ], "text": "G104" + }, + { + "path": "_test\\.go", + "linters": [ + "varnamelen" + ] } ] } diff --git a/CODE_CONVENTIONS.md b/CODE_CONVENTIONS.md new file mode 100644 index 0000000..9858de8 --- /dev/null +++ b/CODE_CONVENTIONS.md @@ -0,0 +1,786 @@ +# Code conventions + +These are the conventions for Go code in this repository. Most formatting +is enforced by `golangci-lint` (see [Tooling](#tooling)); the rest are +applied by hand and checked in review. + +Generated code is never edited to follow these conventions. It is +recognised by Go's standard marker, a comment before the package clause: + +```go +// Code generated by ocdsgen from extensions.json. DO NOT EDIT. +``` + + +## Tooling + +```bash +golangci-lint run --build-tags sqlite_fts5 ./... # lint +golangci-lint fmt ./... # format (gofmt, goimports, gci, golines) +``` + +- Anything that touches the store needs the `sqlite_fts5` build tag (the + title search uses SQLite FTS5); `scripts/build.sh` passes it. +- `golines` limits lines to 100 characters, counting a tab as 2. See + [Line length](#line-length) for the length to aim for. +- In `_test.go` files, `.golangci.json` turns off `errcheck` and + `cyclop`, and gosec's G104 only (it duplicates errcheck). Every other + linter still runs on tests. + +### Linters that check these conventions + +Several of the conventions below are checked by linters enabled in +`.golangci.json`, so `golangci-lint run` reports them rather than review: + +| Linter | Checks | Settings | +|---|---|---| +| `revive` | doc comments on exported names ([Doc comments](#doc-comments)), consistent receiver names, `ctx` first ([Functions and types](#functions-and-types)), [initialisms](#initialisms), lowercase error strings and `Err…` names ([Error messages](#error-messages)), [no `else` after a `return`](#no-else-after-a-return) | Only these rules are enabled. `context-as-argument` allows a `*testing.T` before `ctx`, for test helpers. | +| `errorlint` | errors are compared with `errors.Is` and `errors.As`, not `==` or a type assertion, so wrapped errors still match | | +| `misspell` | British spelling ([Spelling](#spelling)) in comments, strings and names | `locale: UK`. `organization` and `humanize` are ignored: they are external names (OCDS's `Organization`, the `go-humanize` package). | +| `sloglint` | snake_case keys, no mixing of key/value pairs with `slog.Attr`, constant messages ([Logging](#logging)); `slog.DiscardHandler` for discarded output | | +| `thelper` | test helpers call `t.Helper()` first ([Tests](#tests)) | | +| `gocritic` | neighbouring parameters and results of the same type share one type name, and strings are tested against `""` ([Functions and types](#functions-and-types)); no names shadowing built-ins ([Naming](#naming)) | Only `paramTypeCombine`, `emptyStringTest` and `builtinShadow` are enabled. | +| `wsl_v5` | no blank line between a call and the `if err != nil` that checks it ([Blank lines](#blank-lines)) | Only the `err` check is enabled; the rest of wsl's rules conflict with the Blank lines convention. | +| `varnamelen` | short names used over a wide scope ([Naming](#naming)) | Off in `_test.go` files. The short names in the Naming table are allowed. | + +A finding that is a false positive is silenced with a `//nolint` +directive and a reason, as for any linter (see +[Other `//nolint` directives](#other-nolint-directives)). + + +## Files and packages + +- File names are lowercase. Compound names run together (`findatender`, + `ftsmock`, `ratelimits`); underscores separate parts + (`conf_source_ftsmock.go`, `confirm_action.go`). +- Never end a file name in `_` or `_` (e.g. `_js`, `_arm`): + Go treats it as a build constraint. +- Package directories are lowercase, with no underscores. + + +## Imports + +Three groups, each separated by a blank line: the standard library, then +third-party packages, then this module's own packages. + +```go +import ( + "context" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.avsdev.uk/Go/job-search/internal/config" +) +``` + +`golangci-lint fmt` applies this (gci sections `standard`, `default`, +`localmodule`; goimports' local prefix is the module path). + + +## Naming + +Variables and parameters in application code have descriptive full-word +names, not single letters or abbreviations. + +| Use | Not | +|---|---| +| `listen`, `newConn`, `client`, `limiter` | `ln`, `nc`, `c`, `lim` | +| `shutdownCtx`, `writeCtx` | `shCtx`, `wctx` | +| `release`, `opportunity`, `entry`, `value` | `r`, `t`, `e`, `v` | + +These short names are conventional enough to keep: + +| Name | For | +|---|---| +| a short method receiver | e.g. `h *Hub`, `t *tcpConn`, `s *Store` | +| `err`, `ctx` | errors and contexts | +| `i`, `idx` | a plain index loop | +| `msg` | a message | +| `w`, `r` | `http.ResponseWriter` and `*http.Request` in HTTP handlers and middleware, as in the standard library | +| `req`, `res` | a request and a response elsewhere, e.g. building an `*http.Request` or receiving an `*http.Response` in client code | +| `db`, `tx` | `*sql.DB` and a database transaction (`*sql.Tx`, `*sqlx.Tx`) | +| `wg`, `mu` | `sync.WaitGroup` and `sync.Mutex` | +| `t` | `*testing.T`, including in test helper packages such as `internal/cli/cmdtest` | + +`w` and `r` are used for nothing else: a release is `release`, an +`io.Writer` is `out`, a `*rand.Rand` is `rng`. + +Names never shadow Go's predeclared identifiers: not `min`, `max`, +`len`, `cap`, `new`, `copy`, `clear`, `close`, `error`, `string` and so +on. A gap's bounds are `minGap` and `maxGap`, not `min` and `max`, so the +built-in `min` and `max` still work inside the function. + +`tx` means only a database transaction. It is not used for transmit (or +`rx` for receive), as in tx/rx: name those `transmit` and `receive`, or +more specifically (e.g. `sent`, `received`). + +Test files are exempt from this rule: short names (`h`, `c`, `e`, `ln`, +`tc`) are fine there, and existing ones shouldn't be renamed. + +### Initialisms + +Initialisms and acronyms keep one case throughout a name, as in the +standard library: `ID`, `URL`, `URI`, `JSON`, `HTTP`, `API`, `UUID`, +`SQL`. So `ReleaseID`, `APIRoot`, `UUIDType` and `baseURL`, not +`ReleaseId`, `ApiRoot` or `baseUrl`. At the start of an unexported name +the whole initialism is lowercase: `uuid`, `apiRoot`, `jsonHandler`. + +`revive` checks this (`var-naming`). + +### `must` functions + +A name starting with `must` (or `Must`) promises that the function never +returns an error to its caller, as in the standard library's +`regexp.MustCompile` and `template.Must`: + +- In tests, a `must…` helper takes `t` first, calls `t.Helper()`, and + stops the test with `t.Fatal` on error: `mustSave(t, s, opportunity)`. +- In application code, a `must…` function panics on error, and is only + used where the input is known to be valid, such as constants or + package-level setup. +- A function that hides an error some other way, such as returning a zero + value or `nil`, isn't named `must…`: `rawJSON` returns `nil` for a value + it can't marshal. + + +## Functions and types + +- A function that takes a context takes it first, named `ctx`: + `func (s *Store) Get(ctx context.Context, id string)`. +- Constructors are `New` when a package has one main type + (`tcpserver.New`), otherwise `NewX` (`NewHub`, `NewKeyedLimiter`). +- All of a type's methods use the same receiver name, usually its first + letter (`h *Hub`, `t *tcpConn`), for value and pointer receivers alike. +- A string is tested for emptiness against `""`, not with `len`: + `if file == ""`, not `if len(file) == 0`. It says what is meant, and + reads the same as any other string comparison. +- Neighbouring parameters or results of the same type share one type + name, as in the standard library: `func f(ocid, id string)`, not + `func f(ocid string, id string)`. Parameters are never reordered just + to share a type. + +### No `else` after a `return` + +When an `if` block ends in a `return` (or `break`, `continue`, `panic`), +there is no `else`: the code that would be in it follows the `if`, +un-indented. The usual path then reads straight down the left edge of +the function, with the early exits indented, which is how Go code is +normally read (see Effective Go and the Go Code Review Comments, "Indent +Error Flow"). + +```go +if blockShutdown { + return sleep(taskCtx, true, sleepDuration) +} +return sleep(ctx, false, sleepDuration) +``` + +Not: + +```go +if blockShutdown { + return sleep(taskCtx, true, sleepDuration) +} else { + return sleep(ctx, false, sleepDuration) +} +``` + +Where there are several outcomes, none of which is the usual path, a +`switch` with no condition reads better than nested `if`/`else`: + +```go +switch { +case errors.As(err, ¬Found): + slog.Warn("no config file found; using defaults") +case err != nil: + return err +default: + slog.Info("using config file", "file", viper.ConfigFileUsed()) +} +``` + +`revive` checks this (`indent-error-flow`, `superfluous-else`). + + +## Layout + +These rules are applied by hand and checked in review; `golines` only +enforces the 100-character limit. They apply to tests as well as +application code, except where a rule says otherwise. + +### Line length + +Lines should be at most 80 characters (counting a tab as 2), so code +reads well side by side and in diffs. 100 is the absolute maximum, +enforced by `golines`. + +Go past 80 only when breaking the line would make it harder to read: + +- a line ending in a `//nolint` directive, which must stay on the line it + applies to; +- struct tags; +- string literals that would lose meaning or searchability if split: + URLs, error messages, regular expressions, JSON fixtures, help text; +- rows of table-driven tests. + +Don't run `golines` at 80 to do this: it splits method chains, separates +slog keys from their values and moves `//nolint` directives off their +lines. + +### Assignments in `if` and `for` + +When an `if` statement's assignment and check don't fit within the line +length together, the assignment is taken out of the `if`, rather than the +call being wrapped inside it. That keeps the condition readable, +particularly an `if err != nil`, which would otherwise be left on the +call's closing line, after its arguments: + +```go +_, err = tx.ExecContext( + ctx, + deleteOpportunityClassifications, + opportunity.UUID, +) +if err != nil { + return err +} +``` + +Not: + +```go +if _, err := tx.ExecContext( + ctx, + deleteOpportunityClassifications, + opportunity.UUID, +); err != nil { + return err +} +``` + +The same applies to a `for` statement's init statement, and to a range +expression that doesn't fit on the `for` line: it is assigned to a +variable first, so the loop's header stays on one line, next to its body. + +```go +marks := []struct { + filter MarkFilter + clause string +}{ + {filter: filter.Hidden, clause: filterOpportunityMarkHidden}, + {filter: filter.Skipped, clause: filterOpportunityMarkSkipped}, +} +for _, mark := range marks { +``` + +Tests are exempt from this rule. + +### Wrapped calls and declarations + +When a call doesn't fit on one line, it breaks straight after the opening +bracket and the closing bracket goes on a line of its own. If the +arguments all fit on one line between them, they may stay together on +that line, followed by a comma: + +```go +ctx, stop := signal.NotifyContext( + context.Background(), os.Interrupt, syscall.SIGTERM, +) +``` + +Otherwise each argument goes on its own line, followed by a comma. The +arguments are either all on one line or one per line, never grouped two +or three to a line: + +```go +server := tcpserver.New( + &conf.Server, + websocket.NewHub(conf.Server.WebSocket, websocket.WithLogger(log)), + tcpserver.WithLogger(log), +) +``` + +Not: + +```go +server := tcpserver.New(&conf.Server, + websocket.NewHub(conf.Server.WebSocket, websocket.WithLogger(log)), + tcpserver.WithLogger(log)) +``` + +A function declaration whose parameters don't fit on one line wraps the +same way: + +```go +func insertReleaseWithTags( + ctx context.Context, tx *sqlx.Tx, release types.OpportunityRelease, +) (bool, error) { +``` + +A `log/slog` call that fits on one line stays on one line, like any other +call. When it doesn't, its attributes are the exception to the rule above: +the message goes on its own line, then each field name stays on the same +line as its value, one field per line, even when the fields would all fit +together on one line. + +```go +slog.Info( + "fetched opportunities", + "source", "find_a_tender", + "count", len(opportunities), + "since", since, +) +``` + +A `//nolint` directive on a wrapped call goes on its first line, after the +opening bracket (see [Ignoring an error](#ignoring-an-error)). + +### Method chains + +A chain of calls that fits on one line stays on one line. One that +doesn't breaks after each `.`, one call per line, indented once, with +the dot at the end of the line: + +```go +findATender.httpClient = resty.New(). + SetTimeout(60 * time.Second). + SetRetryCount(1) +``` + +A short accessor at the start of a chain, such as `h.logger()` or +`cmd.Flags()`, stays with the call after it; if that call doesn't fit, +its arguments are wrapped instead (see +[Wrapped calls and declarations](#wrapped-calls-and-declarations)): + +```go +h.logger().Info( + "client connected", + "client", client.id, + "ip", ip, +) +``` + +Not: + +```go +h.logger(). + Info("client connected", "client", client.id, "ip", ip) +``` + +### Long conditions + +A condition that doesn't fit on one line breaks after a `||` or `&&`, +which ends the line, and continues indented once. Each line is filled up +to the line length, keeping related comparisons together, rather than +one comparison per line: + +```go +if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || + len(cat.Subjects) == 0 || len(cat.Methods) == 0 { +``` + +A condition is never broken inside a call's brackets; if a call in it +doesn't fit, its result is assigned to a variable first (see +[Assignments in `if` and `for`](#assignments-in-if-and-for)). A +condition that runs to more than about three lines is better given a +name, as a boolean variable or a small function. + +### Blank lines + +A function body is split into steps, with one blank line between them. A +step is a statement, or a few closely related ones, together with its +error handling. Go itself has no rule for this (`gofmt` only collapses +several blank lines into one, and the `whitespace` linter only rejects +blank lines at the start or end of a block), so it is applied by hand. + +- A call and the `if err != nil` that checks it are never separated by a + blank line. `wsl_v5` checks this; the rest of the rule is applied + by hand. +- A `defer` that releases something goes straight after the check of the + call that acquired it, with no blank line, so acquiring and releasing + read as one step. +- A function or block of only one or two steps needs no blank lines. +- A final `return` after several (three or more) steps is set off by a + blank line. The `return` itself isn't counted as a step. + +```go +statement, err := tx.PrepareNamedContext( + ctx, upsertClassificationCodeReturningID, +) +if err != nil { + return err +} +defer func() { + err = errors.Join(err, statement.Close()) +}() + +for i := range opportunity.Classifications { + ... +} + +return nil +``` + +### Function literals + +A function literal (`func(...) { ... }` written inline) is never written on +one line, however short its body: + +```go +t.Cleanup(func() { + server.Close() +}) +``` + +Not `t.Cleanup(func() { server.Close() })`. + +This is about function literals only. Named one-line functions and methods, +such as a config section's `Map() map[string]any { return mapOf(c) }`, are +unaffected. + +### Struct literals + +A struct literal is always wrapped, one field per line, and names its +fields: + +```go +envelope := websocket.Envelope{ + Type: websocket.TypePing, + ID: "1", +} +``` + +Not `websocket.Envelope{Type: websocket.TypePing, ID: "1"}`, and never +positional (`websocket.Envelope{"ping", "1", nil, ""}`). + +- Empty struct literals (`netip.Addr{}`, `time.Time{}`, `struct{}{}`, + `&Server{}`) have nothing to wrap and stay on one line. +- Rows of a table-driven test (`{"size", "4096", "4KiB"},`) stay one row per + line, so the table stays easy to scan. A struct literal used as a value + inside a row is still wrapped. +- Rows of a short lookup table in application code (a slice or map + literal of small structs, used as data) may each stay on one line if + they fit within the line length, so the table is easy to scan. They + still name their fields: + `{filter: filter.Hidden, clause: filterOpportunityMarkHidden},`. +- This is about struct literals only; slice and map literals + (`[]string{"a", "b"}`) may stay on one line. + + +## Comments + +### Doc comments + +Every exported identifier has a doc comment, and it starts with the name +it documents. + +```go +// NewHub creates a hub. +func NewHub(conf config.JSCServerWebSocket, opts ...Option) *Hub { +``` + +### Grouped declarations + +Related package-level constants and variables go in one block, with a +comment for the group and one for each member. + +```go +// Errors returned by CSRF.Validate for a token that is rejected. +var ( + // ErrCSRFMissing is returned when no token is presented. + ErrCSRFMissing = errors.New("csrf token missing") + // ErrCSRFExpired is returned for a token older than the token TTL. + ErrCSRFExpired = errors.New("csrf token expired") +) +``` + +### Section banners + +A file is divided into sections with a banner, followed by a blank line +so godoc doesn't attach the banner to the next declaration: + +```go +// ---------- tcpConn ---------- + +// tcpConn adapts a net.Conn ... +type tcpConn struct { +``` + +In a file with more than one type (such as a config section and its +nested section), each type gets a `==========` sub-section, which then +has its own `----------` banners: + +```go +// ========== HSTS Headers ========== + +// JSCServerHeadersHSTS configures the Strict-Transport-Security header. +type JSCServerHeadersHSTS struct { + ... +} + +// ---------- Section methods (see Section) ---------- +``` + +`internal/store/queries.go` uses `----------` banners for its per-table +groups of queries. + +### Trailing comments + +A comment after code on the same line is for a short note, and only when +the line still fits within the [line length](#line-length). Otherwise +the comment goes on its own line above the code it describes: + +```go + // location is the zone for updatedFrom/updatedTo without an offset; + // nil means UTC. + location *time.Location +``` + +Not a trailing comment that runs the line past 80, and never a comment +after the closing bracket of a wrapped call. + +A comment on a row of a table-driven test stays on the row, like the row +itself (see [Line length](#line-length)). + +`//nolint` directives are exempt: a directive must be on the line the +linter reports, so it stays a trailing comment even when that takes the +line past 80 (see [Ignoring an error](#ignoring-an-error)). + +### Spelling + +Comments and documentation use British English ("normalised", "honoured", +"recognise", "behaviour"). Identifiers, and anything that must match an +external name, keep their own spelling (`Authorization`, `Color` in a +third-party API). + + +## Errors + +### Error messages + +- Error strings are lowercase and don't end with punctuation, as they are + usually wrapped into a longer message (`revive`'s `error-strings` + reports capitalised ones): `errors.New("store: not found")`. +- An error from another call is wrapped with `%w`, adding what was being + done, so the chain reads as a sentence: + `fmt.Errorf("listen %s: %w", addr, err)`. + +### Package prefixes + +An error names the package it came from once, where it is created, as +the standard library does (`sql: no rows in result set`, `http: server +closed`) and Effective Go recommends: + +- An error a package creates itself, a sentinel, `errors.New`, or + `fmt.Errorf` without `%w`, starts with the package name: + `errors.New("store: not found")`, + `fmt.Errorf("util: byte size %q is too large", text)`. +- Wrapping an error adds what was being done, never a package name: + `fmt.Errorf("inserting release %s/%s: %w", uuid, id, err)`, not + `"store: inserting release …"`. The wrapped error already says where it + came from, so a chain reads `writing opportunity ocds-1: store: …` + rather than repeating `store:` at every level. + +Two kinds of error are exempt and have no prefix: + +- Errors from `package main` and the command packages in `internal/cli`. + Their text is the final message shown to the user, who has no use for + an internal package name. +- Error text that is part of a protocol sent to a client, such as the + mock Find a Tender API's error responses, which match the real API's. +- Errors that callers test for are sentinel values named `Err…` + (exported) or `err…` (unexported), checked with `errors.Is`. A + package's sentinels go in one commented `var` block (see + [Grouped declarations](#grouped-declarations)). + +### Ignoring an error + +An error is never ignored with `_ = f()`. Instead, the call says why +with a `//nolint` directive naming only the linters that actually report +the line (`nolintlint` reports any that don't): + +```go +json.NewEncoder(w).Encode(problem) //nolint:errcheck,gosec // the client may have gone +``` + +- Most calls need `errcheck,gosec`. `hash.Write` needs only `gosec` + (errcheck excludes it); `fmt.Fprint*` needs only `errcheck` (gosec + excludes it). +- A deferred call needs only `errcheck`, since gosec doesn't check + deferred calls: + + ```go + defer rows.Close() //nolint:errcheck // read only + ``` + +- For a call spanning several lines, the directive goes on the first + line, after the opening parenthesis: + + ```go + tcpSocket.SetKeepAliveConfig(net.KeepAliveConfig{ //nolint:errcheck,gosec // best effort + ``` + + A linter that reports one argument rather than the call (such as + `sloglint` on a message that isn't a constant) needs the directive on + that argument's line instead. One that reports several lines of a + statement, such as a table of test cases, takes the directive on a line + of its own directly above the statement. + +- A call whose error both linters already exclude (such as + `crypto/rand.Read`, which never returns one) gets a plain comment + rather than a directive, which would be reported as unused: + + ```go + rand.Read(key) // crypto/rand.Read never returns an error + ``` + +- A multi-value assignment such as `value, _ := f()` can't take a + directive (errcheck doesn't check blank assignments, so the directive + would be unused) and stays as it is. These are mostly cobra flag + getters, which only fail for an undefined flag. Where ignoring the + error is ambiguous or potentially dangerous in future, an explanation + comment can be added. + +### Errors that matter + +When closing something can lose data (a file being written, a database +handle), its error is joined into the function's returned error with +`errors.Join`, through a named return. + +### Named results + +A function names its results only when it needs to: + +- **A deferred function changes the error.** This is the only reason to + name `err` for its own sake (see [Errors that matter](#errors-that-matter)). +- **The names say which result is which.** When several results share a + type, naming them documents the order, as the Go Code Review Comments + suggest: `func releaseIDs(raw json.RawMessage) (ocid, id string, err error)`. + Go names all of a function's results or none, so `err` is named here too. + +Otherwise results are unnamed. A function with named results: + +- never uses a bare `return`: each `return` lists what it returns, so a + reader doesn't have to track the named variables to know; +- where a deferred function changes `err`, assigns to it with `=`, never + `:=`, in nested blocks. `err :=` there declares a new variable that + shadows the result, and the error assigned to it never reaches the + deferred `errors.Join`: + + ```go + for i := range opportunity.Classifications { + err = statement.GetContext(ctx, &classification.ID, classification) + if err != nil { + return err + } + } + ``` + +### Other `//nolint` directives + +Otherwise `//nolint` is only for false positives that the code can't +express, always with a reason. For example, a seeded `math/rand` in the +mock, or a gosec finding explained in the comment above it: + +```go +// Secure is false only on plain HTTP (local development), where +// browsers refuse Secure cookies; HttpOnly and SameSite are always set. +http.SetCookie(w, &http.Cookie{ //nolint:gosec // G124: see above +``` + + +## Logging + +- Logging uses `log/slog`. A component (a type such as `Hub`, or a + package-level entry point such as `fetcher.Run`) logs to `slog.Default()` + unless it is given a logger with a `WithLogger` option, so callers that + are happy with the default pass nothing, and tests can still capture or + discard its output. +- The logger is given to the constructor (or entry point) as an option, + never set afterwards: a setter called once goroutines are running would + be a data race. Options follow the functional options pattern used by + `store.Open`. +- With no logger given, the component looks up `slog.Default()` when it + logs, not when it is created, so it follows `logging.Setup` and a test's + `slog.SetDefault` even if it was created first. + + ```go + // WithLogger makes the hub log to logger instead of slog.Default(). + func WithLogger(logger *slog.Logger) Option { + return func(h *Hub) { + h.log = logger + } + } + + // logger returns the logger the hub logs to, tagged with its component. + func (h *Hub) logger() *slog.Logger { + log := h.log + if log == nil { + log = slog.Default() + } + return log.With("component", "ws") + } + ``` + + ```go + hub := websocket.NewHub(conf.Server.WebSocket) // slog.Default() + hub := websocket.NewHub(conf.Server.WebSocket, websocket.WithLogger(log)) // given logger + ``` + +- Field keys are snake_case and name one thing each: `client`, `ip`, + `request_id`, `retry_in`. Durations, sizes and counts are logged with + the matching `slog` type, not as pre-formatted strings. +- A component tags its logger with `"component"` in one place (its + `logger()` method, as above), whether the logger was given or is the + default, rather than on every line. +- Messages are short lowercase phrases ("client connected", "tcp accept + failed"); the details go in fields. + + +## Configuration + +### Keys + +Configuration keys (the `cfg` struct tags) are snake_case: +`max_message_size`, `connection_limit`. Viper lowercases every key it +reads, so a camelCase key such as `minVersion` is never loaded from a +config file or environment variable. + +### Zero values + +Every setting's doc comment (and its row in `internal/config/README.md`) +says what 0 or empty means, and the code treats it consistently: + +- A timeout of 0 or less means **no limit**, and an interval of 0 or less + means **off** (e.g. `ping_interval`), never "already expired". Where off + makes no sense the setting rejects 0 instead (a source's `interval`: a + source is turned off with `enabled`). Code applies a timeout with a helper + rather than `context.WithTimeout(ctx, 0)`: + + ```go + writeCtx, cancel := util.WithTimeout(ctx, conf.WriteTimeout) // no deadline if 0 or less + ``` + +- A size of 0 means **the default** (`max_message_size` of `0B` reads + 32KiB), and -1 means **no limit** (`util.ByteSizeNoLimit`); no other size + may be negative. A setting for which no limit is unsafe rejects -1 in its + validation and says why (`max_message_size` does: a client could exhaust + the server's memory). +- An empty list means **none** (no trusted origins, no exempt paths), and + is stored as `nil`. + + +## Tests + +- Tests are table-driven where several cases share a check, with a name + or input field identifying each case in failure messages. +- Helpers call `t.Helper()` first, and register their own teardown with + `t.Cleanup` rather than returning a cleanup function. +- Tests that touch global state (Viper, `slog.Default`, the working + directory, environment variables) don't call `t.Parallel()`, and reset + what they change: `resetViper(t)`, `t.Chdir`, `t.Setenv`. +- Network tests listen on `127.0.0.1:0` and use the address they were + given, never a fixed port. +- Short names are fine in tests (see [Naming](#naming)), and setup calls + may ignore errors with a bare call (errcheck is off for tests). diff --git a/cmd/jobsearch/main_test.go b/cmd/jobsearch/main_test.go index b7d83d1..b0c29a2 100644 --- a/cmd/jobsearch/main_test.go +++ b/cmd/jobsearch/main_test.go @@ -12,15 +12,19 @@ import ( func TestCommandTree(t *testing.T) { root := newRootCmd() + var names []string for _, c := range root.Commands() { names = append(names, c.Name()) } - for _, want := range []string{"config", "fetch", "migrate", "version", "wsserve"} { + + commands := []string{"config", "fetch", "migrate", "version", "wsserve"} + for _, want := range commands { if !slices.Contains(names, want) { t.Errorf("job-search has no %q command; has %v", want, names) } } + for _, flag := range []string{"config", "debug", "quiet"} { if root.PersistentFlags().Lookup(flag) == nil { t.Errorf("job-search has no --%s flag", flag) diff --git a/cmd/mock-fts/main.go b/cmd/mock-fts/main.go index 8521fd1..ef5408e 100644 --- a/cmd/mock-fts/main.go +++ b/cmd/mock-fts/main.go @@ -52,7 +52,7 @@ type options struct { rateWindow time.Duration accessLogPath string logFormat string - tz string + timeZone string } const longHelp = `Mock of the Find a Tender (FTS) OCDS release package endpoint: @@ -170,10 +170,19 @@ func newRootCmd() *cobra.Command { "maximum time between generated releases", ) - flags.IntVar(&opts.rateLimit, "rate-limit", 12, "requests allowed per window per client") - flags.DurationVar(&opts.rateWindow, "rate-window", 5*time.Minute, "rate limit window length") + flags.IntVar( + &opts.rateLimit, "rate-limit", 12, "requests allowed per window per client", + ) + flags.DurationVar( + &opts.rateWindow, "rate-window", 5*time.Minute, "rate limit window length", + ) - flags.StringVar(&opts.accessLogPath, "access-log", "-", `access log file, or "-" for stdout`) + flags.StringVar( + &opts.accessLogPath, + "access-log", + "-", + `access log file, or "-" for stdout`, + ) flags.StringVar( &opts.logFormat, "log-format", @@ -181,7 +190,7 @@ func newRootCmd() *cobra.Command { `access log format: "combined" or "common"`, ) flags.StringVar( - &opts.tz, + &opts.timeZone, "tz", "Europe/London", `time zone for updatedFrom/updatedTo without an offset ("Local" for this machine's)`, @@ -193,34 +202,40 @@ func newRootCmd() *cobra.Command { // validate checks flag values before anything is started. func (o *options) validate() error { if o.logFormat != "combined" && o.logFormat != "common" { - return fmt.Errorf(`--log-format must be "combined" or "common", got %q`, o.logFormat) + return fmt.Errorf( + `--log-format must be "combined" or "common", got %q`, o.logFormat, + ) } if o.minGap <= 0 || o.maxGap < o.minGap { return errors.New("--min-gap must be positive and no larger than --max-gap") } if o.rateLimit < 1 || o.rateWindow <= 0 { - return errors.New("--rate-limit must be at least 1 and --rate-window positive") + return errors.New( + "--rate-limit must be at least 1 and --rate-window positive", + ) } - if _, err := time.LoadLocation(o.tz); err != nil { + if _, err := time.LoadLocation(o.timeZone); err != nil { return fmt.Errorf("--tz: %w", err) } + return nil } // run starts the mock and blocks until ctx is cancelled or the server fails. func run(ctx context.Context, opts *options) (err error) { - location, err := time.LoadLocation(opts.tz) // already validated + location, err := time.LoadLocation(opts.timeZone) // already validated if err != nil { return fmt.Errorf("--tz: %w", err) } - if err := os.MkdirAll(opts.dataDir, 0o750); err != nil { - return err + + if err = os.MkdirAll(opts.dataDir, 0o750); err != nil { + return fmt.Errorf("creating data directory: %w", err) } - // ---------- catalog ---------- + // Load the catalog. catalog, err := mock.LoadCatalog() if err != nil { - return err + return fmt.Errorf("loading catalog: %w", err) } slog.Info( "catalog loaded", @@ -231,27 +246,30 @@ func run(ctx context.Context, opts *options) (err error) { "classifications", len(catalog.Classifications), ) - // ---------- store ---------- + // Open (or reset) the store. storePath := filepath.Join(opts.dataDir, mock.ReleasesFile) if opts.reset { - if err := os.Remove(storePath); err != nil && !errors.Is(err, os.ErrNotExist) { - return err + if err = os.Remove(storePath); err != nil && + !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resetting store: %w", err) } slog.Info("store reset", "path", storePath) } store, err := mock.OpenStore(storePath) if err != nil { - return err + return fmt.Errorf("opening store: %w", err) } defer func() { err = errors.Join(err, store.Close()) }() - // ---------- generator ---------- + // Backfill releases, then keep generating them. generator := mock.NewGenerator(catalog, store, time.Now().UnixNano()) - generated, err := generator.CatchUp(time.Now(), opts.backfill, opts.minGap, opts.maxGap) + generated, err := generator.CatchUp( + time.Now(), opts.backfill, opts.minGap, opts.maxGap, + ) if err != nil { - return err + return fmt.Errorf("backfilling releases: %w", err) } slog.Info("store ready", "releases", store.Len(), "generated", generated) @@ -267,10 +285,11 @@ func run(ctx context.Context, opts *options) (err error) { generator.Run(ctx, opts.minGap, opts.maxGap) }() - // ---------- HTTP ---------- + // Serve the API. out := os.Stdout if opts.accessLogPath != "-" { - logFile, err := os.OpenFile( + var logFile *os.File + logFile, err = os.OpenFile( opts.accessLogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600, ) if err != nil { @@ -288,12 +307,15 @@ func run(ctx context.Context, opts *options) (err error) { rateLimiter := mock.NewRateLimiter(opts.rateLimit, opts.rateWindow) mux := http.NewServeMux() mux.HandleFunc( - mock.BasePath, rateLimiter.Middleware(mock.NewHandler(store, opts.baseURL, location)), + mock.BasePath, + rateLimiter.Middleware(mock.NewHandler(store, opts.baseURL, location)), ) - httpSrv := &http.Server{ - Addr: opts.addr, - Handler: logging.AccessLog(accessLogger, opts.logFormat == "combined", mux), + httpServer := &http.Server{ + Addr: opts.addr, + Handler: logging.AccessLog( + accessLogger, opts.logFormat == "combined", mux, + ), ReadHeaderTimeout: 10 * time.Second, } go func() { @@ -303,7 +325,7 @@ func run(ctx context.Context, opts *options) (err error) { context.WithoutCancel(ctx), 5*time.Second, ) defer cancelShutdown() - if err := httpSrv.Shutdown(shutdownCtx); err != nil { + if err := httpServer.Shutdown(shutdownCtx); err != nil { slog.Error("shutdown failed", "error", err) } }() @@ -314,15 +336,15 @@ func run(ctx context.Context, opts *options) (err error) { "path", mock.BasePath, "time_zone", location.String(), ) - serveErr := httpSrv.ListenAndServe() + serveErr := httpServer.ListenAndServe() cancel() // stop the worker (and the shutdown goroutine) if the server failed <-workerDone if serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) { - return serveErr + return fmt.Errorf("serving: %w", serveErr) } - log.Print("stopped") + slog.Info("stopped") return nil } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 66a18eb..931f065 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -52,9 +52,11 @@ func newCommand(t *testing.T, opts *Options, args ...string) *cobra.Command { Use: "test", } opts.Bind(cmd) + if err := cmd.ParseFlags(args); err != nil { t.Fatal(err) } + return cmd } @@ -144,12 +146,16 @@ func TestRunSetupWithoutLoadingConfig(t *testing.T) { "-d -q": "debug", // --debug wins } { opts := &Options{} - conf, err := opts.RunSetup(newCommand(t, opts, strings.Fields(args)...), false, true) + conf, err := opts.RunSetup( + newCommand(t, opts, strings.Fields(args)...), false, true, + ) if err != nil { t.Fatal(err) } if conf.Logging.Level != want { - t.Errorf("%q: Logging.Level = %q, want %q", args, conf.Logging.Level, want) + t.Errorf( + "%q: Logging.Level = %q, want %q", args, conf.Logging.Level, want, + ) } } } @@ -192,22 +198,18 @@ func confirmWithInput(t *testing.T, input string) bool { dir := t.TempDir() in := filepath.Join(dir, "stdin") writeFile(t, in, input) + stdin, err := os.Open(filepath.Clean(in)) if err != nil { t.Fatal(err) } defer stdin.Close() + stdout, err := os.Create(filepath.Clean(filepath.Join(dir, "stdout"))) if err != nil { t.Fatal(err) } defer stdout.Close() - oldIn, oldOut := os.Stdin, os.Stdout - os.Stdin, os.Stdout = stdin, stdout - defer func() { - os.Stdin, os.Stdout = oldIn, oldOut - }() - - return Confirm("Proceed?") + return Confirm(stdout, stdin, "Proceed?") } diff --git a/internal/cli/cmdtest/cmdtest.go b/internal/cli/cmdtest/cmdtest.go index a3814e9..03ba651 100644 --- a/internal/cli/cmdtest/cmdtest.go +++ b/internal/cli/cmdtest/cmdtest.go @@ -28,10 +28,12 @@ func Setup(t *testing.T, config string) string { t.Helper() dir := t.TempDir() t.Chdir(dir) + WriteFile(t, ".env", "") if config != "" { WriteFile(t, "config.yaml", config) } + return dir } @@ -78,6 +80,7 @@ func NewRoot(opts *cli.Options, cmds ...*cobra.Command) *cobra.Command { cobra.EnableTraverseRunHooks = true opts.Bind(root) root.AddCommand(cmds...) + return root } @@ -95,18 +98,19 @@ func Run( dir := t.TempDir() inPath, outPath := filepath.Join(dir, "stdin"), filepath.Join(dir, "stdout") WriteFile(t, inPath, stdin) - in, err := os.Open(filepath.Clean(inPath)) + input, err := os.Open(filepath.Clean(inPath)) if err != nil { t.Fatal(err) } - defer in.Close() //nolint:errcheck // read only + defer input.Close() //nolint:errcheck // read only + out, err := os.Create(filepath.Clean(outPath)) if err != nil { t.Fatal(err) } oldIn, oldOut, oldLog := os.Stdin, os.Stdout, slog.Default() - os.Stdin, os.Stdout = in, out + os.Stdin, os.Stdout = input, out defer func() { os.Stdin, os.Stdout = oldIn, oldOut slog.SetDefault(oldLog) @@ -120,11 +124,14 @@ func Run( if err := out.Close(); err != nil { t.Fatal(err) } + return ReadFile(t, outPath), runErr } // MustRun is Run with no input, for commands that must succeed. -func MustRun(t *testing.T, newRoot func() *cobra.Command, args ...string) string { +func MustRun( + t *testing.T, newRoot func() *cobra.Command, args ...string, +) string { t.Helper() out, err := Run(t, newRoot, "", args...) if err != nil { @@ -136,9 +143,9 @@ func MustRun(t *testing.T, newRoot func() *cobra.Command, args ...string) string // Contains fails the test unless out contains every string in want. func Contains(t *testing.T, out string, want ...string) { t.Helper() - for _, w := range want { - if !strings.Contains(out, w) { - t.Errorf("output does not contain %q:\n%s", w, out) + for _, wanted := range want { + if !strings.Contains(out, wanted) { + t.Errorf("output does not contain %q:\n%s", wanted, out) } } } diff --git a/internal/cli/config/command.go b/internal/cli/config/command.go index c898930..4b7944c 100644 --- a/internal/cli/config/command.go +++ b/internal/cli/config/command.go @@ -8,7 +8,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "config" command, which shows and edits the configuration. +// NewCommand returns the "config" command, which shows and edits the +// configuration. func NewCommand(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "config", @@ -24,5 +25,6 @@ func NewCommand(opts *cli.Options) *cobra.Command { cmd.AddCommand(commandGet(opts)) cmd.AddCommand(commandSet(opts)) cmd.AddCommand(commandInit()) + return cmd } diff --git a/internal/cli/config/config_test.go b/internal/cli/config/config_test.go index ec11cee..de28a19 100644 --- a/internal/cli/config/config_test.go +++ b/internal/cli/config/config_test.go @@ -41,9 +41,12 @@ func mustRun(t *testing.T, args ...string) string { func fileSettings(t *testing.T, path string) map[string]any { t.Helper() var nested map[string]any - if err := yaml.Unmarshal([]byte(cmdtest.ReadFile(t, path)), &nested); err != nil { + if err := yaml.Unmarshal( + []byte(cmdtest.ReadFile(t, path)), &nested, + ); err != nil { t.Fatal(err) } + flat := map[string]any{} var walk func(prefix string, m map[string]any) walk = func(prefix string, m map[string]any) { @@ -56,6 +59,7 @@ func fileSettings(t *testing.T, path string) map[string]any { } } walk("", nested) + return flat } @@ -69,11 +73,17 @@ func TestConfigShow(t *testing.T) { "logging.level = info ", "sources.sleeper.interval = 2m0s ", ) + // Keys are listed in order, the same every time. if again := mustRun(t, "config", "show"); again != out { - t.Errorf("config show output changed between runs:\n%s\n---\n%s", out, again) + t.Errorf( + "config show output changed between runs:\n%s\n---\n%s", out, again, + ) } - if i, j := strings.Index(out, "database.file"), strings.Index(out, "logging.level"); i > j { + + database := strings.Index(out, "database.file") + logging := strings.Index(out, "logging.level") + if database > logging { t.Errorf("keys not sorted:\n%s", out) } @@ -87,9 +97,17 @@ func TestConfigShow(t *testing.T) { func TestConfigGet(t *testing.T) { cmdtest.Setup(t, appConfig) - cmdtest.Contains(t, mustRun(t, "config", "get", "logging.level"), "info ") - cmdtest.Contains(t, mustRun(t, "config", "get", "sources.sleeper.interval"), "2m0s ") - cmdtest.Contains(t, mustRun(t, "config", "get", "sources.sleeper.enabled"), "false ") + for _, tt := range []struct { + key, want string + }{ + {"logging.level", "info "}, + {"sources.sleeper.interval", "2m0s "}, + {"sources.sleeper.enabled", "false "}, + } { + t.Run(tt.key, func(t *testing.T) { + cmdtest.Contains(t, mustRun(t, "config", "get", tt.key), tt.want) + }) + } if _, err := run(t, "config", "get", "no.such.key"); err == nil { t.Error("get of an unknown key succeeded") @@ -104,7 +122,9 @@ func TestConfigSet(t *testing.T) { out := mustRun(t, "config", "set", "logging.level", "debug") cmdtest.Contains(t, out, "logging.level = debug ") - cmdtest.Contains(t, mustRun(t, "config", "get", "logging.level"), "debug ") + cmdtest.Contains( + t, mustRun(t, "config", "get", "logging.level"), "debug ", + ) out = mustRun(t, "config", "set", "sources.sleeper.enabled", "true") cmdtest.Contains(t, out, "sources.sleeper.enabled = true ") @@ -156,11 +176,13 @@ func TestConfigSetUnset(t *testing.T) { cmdtest.Setup(t, appConfig) mustRun(t, "config", "set", "--unset", "sources.sleeper.interval") + // Only the unset key is gone: no defaults are written in its place. want := map[string]any{"logging.level": "info"} if got := fileSettings(t, "config.yaml"); !equalSettings(got, want) { t.Errorf("config.yaml settings = %v; want %v", got, want) } + // Back to the default. cmdtest.Contains( t, @@ -180,7 +202,9 @@ func TestConfigInit(t *testing.T) { out := mustRun(t, "config", "init", "./new.yaml") cmdtest.Contains(t, out, "writing default configuration to file: ./new.yaml") - cmdtest.Contains(t, cmdtest.ReadFile(t, "new.yaml"), "level: warn", "interval: 10m0s") + cmdtest.Contains( + t, cmdtest.ReadFile(t, "new.yaml"), "level: warn", "interval: 10m0s", + ) _, err := run(t, "config", "init", "./new.yaml") cmdtest.ErrContains(t, err, "already exists") @@ -199,8 +223,10 @@ func TestConfigInitDefaultLocations(t *testing.T) { {"config", "init"}, {"config", "init", " "}, } { - _, err := run(t, args...) - cmdtest.ErrContains(t, err, "already exists") + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := run(t, args...) + cmdtest.ErrContains(t, err, "already exists") + }) } } diff --git a/internal/cli/config/get.go b/internal/cli/config/get.go index a636c14..86aff14 100644 --- a/internal/cli/config/get.go +++ b/internal/cli/config/get.go @@ -29,7 +29,7 @@ func commandGet(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } value, err := conf.ValueOf(args[0]) diff --git a/internal/cli/config/init.go b/internal/cli/config/init.go index 5b89e69..67164cf 100644 --- a/internal/cli/config/init.go +++ b/internal/cli/config/init.go @@ -39,13 +39,13 @@ func commandInit() *cobra.Command { cmd.SilenceUsage = true // With no path, the default locations are used. - cfgFile := "" + configFile := "" if len(args) > 0 { - cfgFile = strings.TrimSpace(args[0]) + configFile = strings.TrimSpace(args[0]) } - if cfgFile != "" { + if configFile != "" { // Use the config file given. - viper.SetConfigFile(cfgFile) + viper.SetConfigFile(configFile) } else { // Search config in /etc or local directory. viper.AddConfigPath(config.EtcConfigPath) @@ -58,8 +58,8 @@ func commandInit() *cobra.Command { return errors.New("configuration file already exists") } - _, ok := err.(viper.ConfigFileNotFoundError) - if !errors.Is(err, os.ErrNotExist) && !ok { + var notFound viper.ConfigFileNotFoundError + if !errors.Is(err, os.ErrNotExist) && !errors.As(err, ¬Found) { // Config file was found but another error was produced return err } @@ -80,13 +80,15 @@ func commandInit() *cobra.Command { if err := viper.WriteConfig(); err != nil { return err } + return nil }, } // Config flag should be hidden as the config path should be provided - cmd.SetHelpFunc(func(command *cobra.Command, strings []string) { + cmd.SetHelpFunc(func(command *cobra.Command, args []string) { command.Flags().MarkHidden("config") //nolint:errcheck,gosec // a root flag, so it always exists - command.Parent().HelpFunc()(command, strings) + command.Parent().HelpFunc()(command, args) }) + return cmd } diff --git a/internal/cli/config/set.go b/internal/cli/config/set.go index 7a69012..35078a8 100644 --- a/internal/cli/config/set.go +++ b/internal/cli/config/set.go @@ -43,7 +43,7 @@ func commandSet(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } if viper.ConfigFileUsed() == "" { @@ -65,5 +65,6 @@ func commandSet(opts *cli.Options) *cobra.Command { }, } cmd.Flags().BoolP("unset", "x", false, "clear configuration setting") + return cmd } diff --git a/internal/cli/config/show.go b/internal/cli/config/show.go index 1a89822..9946a61 100644 --- a/internal/cli/config/show.go +++ b/internal/cli/config/show.go @@ -24,13 +24,13 @@ func commandShow(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } if yaml, _ := cmd.Flags().GetBool("yaml"); yaml { - conf.PrintYaml() + conf.PrintYAML() } else if json, _ := cmd.Flags().GetBool("json"); json { - conf.PrintJson() + conf.PrintJSON() } else { conf.PrintTerminal() } @@ -40,5 +40,6 @@ func commandShow(opts *cli.Options) *cobra.Command { } cmd.Flags().Bool("json", false, "format configuration as json") cmd.Flags().Bool("yaml", false, "format configuration as yaml") + return cmd } diff --git a/internal/cli/confirm_action.go b/internal/cli/confirm_action.go index a282625..a0fcc14 100644 --- a/internal/cli/confirm_action.go +++ b/internal/cli/confirm_action.go @@ -3,7 +3,7 @@ package cli import ( "bufio" "fmt" - "log/slog" + "io" "os" "strings" ) @@ -11,15 +11,19 @@ import ( // Confirm asks the user the yes/no question prompt on stdin and reports whether // they answered yes. "y" and "yes" confirm, "n" and "no" refuse, in any case; // anything else asks again. It exits the program if stdin can't be read. -func Confirm(prompt string) bool { - reader := bufio.NewReader(os.Stdin) +func Confirm(out io.Writer, in io.Reader, prompt string) bool { + reader := bufio.NewReader(in) for { - fmt.Printf("%s [y/n]: ", prompt) + _, err := fmt.Fprintf(out, "%s [y/n]: ", prompt) + if err != nil { + fmt.Fprintf(os.Stderr, "error asking question: %s\n", err) + os.Exit(1) + } response, err := reader.ReadString('\n') if err != nil { - slog.Error("reading the answer", "error", err) + fmt.Fprintf(os.Stderr, "error reading answer: %s\n", err) os.Exit(1) } diff --git a/internal/cli/fetch/command.go b/internal/cli/fetch/command.go index cdc1709..0e0cccf 100644 --- a/internal/cli/fetch/command.go +++ b/internal/cli/fetch/command.go @@ -8,7 +8,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "fetch" command, which fetches opportunities once or as a service. +// NewCommand returns the "fetch" command, which fetches opportunities once or +// as a service. func NewCommand(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "fetch", @@ -22,5 +23,6 @@ func NewCommand(opts *cli.Options) *cobra.Command { } cmd.AddCommand(commandOnce(opts)) cmd.AddCommand(commandService(opts)) + return cmd } diff --git a/internal/cli/fetch/once.go b/internal/cli/fetch/once.go index b111887..d21e14f 100644 --- a/internal/cli/fetch/once.go +++ b/internal/cli/fetch/once.go @@ -29,7 +29,9 @@ var stageTags = map[string][]string{ // filter keeps the opportunities whose latest release is at stage. Releases // are oldest first, so the last one is the procedure's current stage. -func filter(opportunities []*types.Opportunity, stage string) []*types.Opportunity { +func filter( + opportunities []*types.Opportunity, stage string, +) []*types.Opportunity { var matching []*types.Opportunity for _, opportunity := range opportunities { @@ -63,7 +65,7 @@ func commandOnce(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } since, sinceErr := cmd.Flags().GetTime("since") @@ -96,15 +98,15 @@ func commandOnce(opts *cli.Options) *cobra.Command { filterTag = "award" } - if len(filterTag) > 0 { + if filterTag != "" { results = filter(results, filterTag) } - jbytes, err := json.MarshalIndent(results, "", " ") + encoded, err := json.MarshalIndent(results, "", " ") if err != nil { return err } - fmt.Println(string(jbytes)) + fmt.Println(string(encoded)) return nil }, @@ -118,17 +120,24 @@ func commandOnce(opts *cli.Options) *cobra.Command { "contract updates published since YYYY-MM-DD[ hh:mm:ss]", ) cmd.Flags().Bool( - "planning", false, "only return planning (i.e. exclude opportunities and awards)", + "planning", + false, + "only return planning (i.e. exclude opportunities and awards)", ) cmd.Flags().Bool( - "opportunities", false, "only return opportunities (i.e. exclude awards and planning)", + "opportunities", + false, + "only return opportunities (i.e. exclude awards and planning)", ) cmd.Flags().Bool( - "awards", false, "only return awards (i.e. exclude planning and opportunities)", + "awards", + false, + "only return awards (i.e. exclude planning and opportunities)", ) cmd.Flags().Bool( "ignore-rate-limit", false, "ignore the rate limit and fail if exceeded", ) cmd.MarkFlagsMutuallyExclusive("planning", "opportunities", "awards") + return cmd } diff --git a/internal/cli/fetch/once_test.go b/internal/cli/fetch/once_test.go index d078bd7..ad0c1b8 100644 --- a/internal/cli/fetch/once_test.go +++ b/internal/cli/fetch/once_test.go @@ -17,7 +17,9 @@ func opp(uuid string, tags ...string) *types.Opportunity { } for _, tag := range tags { o.Releases = append(o.Releases, types.OpportunityRelease{ - Tags: []types.OpportunityReleaseTag{{Tag: tag}}, + Tags: []types.OpportunityReleaseTag{{ + Tag: tag, + }}, }) } return o @@ -56,6 +58,7 @@ func TestOnceFiltersExclusive(t *testing.T) { cmd.SetArgs([]string{"--planning", "--awards"}) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) + err := cmd.Execute() if err == nil || !strings.Contains(err.Error(), "none of the others can be") { t.Fatalf("err = %v; want a mutually exclusive flags error", err) diff --git a/internal/cli/fetch/service.go b/internal/cli/fetch/service.go index 9fbd54d..40c7d2e 100644 --- a/internal/cli/fetch/service.go +++ b/internal/cli/fetch/service.go @@ -32,7 +32,7 @@ func commandService(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, foreground) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } since, sinceErr := cmd.Flags().GetTime("since") @@ -65,5 +65,6 @@ func commandService(opts *cli.Options) *cobra.Command { "contract updates published since YYYY-MM-DD[ hh:mm:ss]", ) cmd.Flags().Bool("foreground", false, "run in the foreground") + return cmd } diff --git a/internal/cli/flags_test.go b/internal/cli/flags_test.go index 3a69eac..22c8b2d 100644 --- a/internal/cli/flags_test.go +++ b/internal/cli/flags_test.go @@ -3,6 +3,7 @@ package cli_test import ( "fmt" "os" + "strings" "testing" "github.com/spf13/cobra" @@ -49,8 +50,10 @@ func TestLogLevelFlags(t *testing.T) { {[]string{"-q"}, "level: error"}, {[]string{"-q", "-d"}, "level: debug"}, } { - out := cmdtest.MustRun(t, newRoot, append([]string{"level"}, tt.args...)...) - cmdtest.Contains(t, out, tt.want) + args := append([]string{"level"}, tt.args...) + t.Run(strings.Join(args, " "), func(t *testing.T) { + cmdtest.Contains(t, cmdtest.MustRun(t, newRoot, args...), tt.want) + }) } } diff --git a/internal/cli/migrate/common.go b/internal/cli/migrate/common.go index e2b4c84..25488d7 100644 --- a/internal/cli/migrate/common.go +++ b/internal/cli/migrate/common.go @@ -24,15 +24,15 @@ import ( // If the database file doesn't exist, create decides between asking whether // to create it (up) and failing (down, status: nothing to work on). func openMigrator( - cmd *cobra.Command, opts *cli.Options, create bool, createConfirmed bool, + cmd *cobra.Command, opts *cli.Options, create, createConfirmed bool, ) (*goose.Provider, *sql.DB, error) { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return nil, nil, fmt.Errorf("no configuration loaded (%w)", err) + return nil, nil, fmt.Errorf("load configuration: %w", err) } file := conf.Database.File - if len(file) == 0 { + if file == "" { return nil, nil, errors.New( "no database file specified. Update your configuration", ) @@ -46,8 +46,8 @@ func openMigrator( question := fmt.Sprintf( "Database file '%s' does not exist. Would you like to create it?", file, ) - if !cli.Confirm(question) { - return nil, nil, errors.New("migrate: no database file") + if !cli.Confirm(cmd.OutOrStdout(), cmd.InOrStdin(), question) { + return nil, nil, errors.New("no database file") } } } else if err != nil { @@ -75,14 +75,14 @@ func closeDB(db *sql.DB, err *error) { // errWriter writes until a write fails, then keeps the error, so a run of // prints needs checking only once. type errWriter struct { - w io.Writer + out io.Writer err error } // printf writes like fmt.Fprintf, unless an earlier write failed. func (e *errWriter) printf(format string, args ...any) { if e.err == nil { - _, e.err = fmt.Fprintf(e.w, format, args...) + _, e.err = fmt.Fprintf(e.out, format, args...) } } @@ -91,7 +91,7 @@ func (e *errWriter) printf(format string, args ...any) { // applied 00003_create_release_tags_table.sql (2ms) func printResults(out io.Writer, results []*goose.MigrationResult) error { output := &errWriter{ - w: out, + out: out, } for _, result := range results { verb := "applied" @@ -136,7 +136,9 @@ func failure(out io.Writer, err error) error { } // printVersion reports where the database ended up. -func printVersion(ctx context.Context, out io.Writer, migrator *goose.Provider) error { +func printVersion( + ctx context.Context, out io.Writer, migrator *goose.Provider, +) error { dbVersion, err := migrator.GetDBVersion(ctx) if err != nil { return err diff --git a/internal/cli/migrate/common_test.go b/internal/cli/migrate/common_test.go index ac53364..5c6d28d 100644 --- a/internal/cli/migrate/common_test.go +++ b/internal/cli/migrate/common_test.go @@ -39,7 +39,11 @@ func TestFailurePartial(t *testing.T) { if !errors.Is(err, cause) { t.Errorf("err = %v, want it to wrap the migration error", err) } - for _, want := range []string{"00002_second.sql", "rolled back", "last version that succeeded"} { + for _, want := range []string{ + "00002_second.sql", + "rolled back", + "last version that succeeded", + } { if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("err = %v, want it to mention %q", err, want) } @@ -58,7 +62,10 @@ func TestFailureWithoutFailedSource(t *testing.T) { func TestFailureOtherError(t *testing.T) { cause := errors.New("database is locked") var out bytes.Buffer - if err := failure(&out, cause); err != cause { + if err := failure( + &out, + cause, + ); err != cause { //nolint:errorlint // checks the error is returned unchanged t.Errorf("err = %v, want the error unchanged", err) } if out.Len() != 0 { @@ -75,7 +82,8 @@ func TestPrintResults(t *testing.T) { if err != nil { t.Fatal(err) } - want := " applied 00001_first.sql (2ms)\n reverted 00002_second.sql (2ms)\n" + want := " applied 00001_first.sql (2ms)\n" + + " reverted 00002_second.sql (2ms)\n" if out.String() != want { t.Errorf("output = %q, want %q", out.String(), want) } @@ -97,7 +105,9 @@ func TestWriteErrors(t *testing.T) { cause := errors.New("boom") err := failure(failingWriter{}, &goose.PartialError{ - Applied: results, Failed: result("00002_second.sql", "up"), Err: cause, + Applied: results, + Failed: result("00002_second.sql", "up"), + Err: cause, }) if !errors.Is(err, cause) || !errors.Is(err, errWrite) { t.Errorf("failure: err = %v, want both the migration and write errors", err) diff --git a/internal/cli/migrate/down.go b/internal/cli/migrate/down.go index 11387ad..81081e0 100644 --- a/internal/cli/migrate/down.go +++ b/internal/cli/migrate/down.go @@ -39,12 +39,11 @@ confirmed by providing -yes.`, defer closeDB(db, &err) results, err := store.StepDown(cmd.Context(), migrator, int(steps), yes) - if len(results) == 0 && err == nil { fmt.Println("No changes made to database (empty)") return nil } - if err == store.ErrWouldEmptyDatabase { + if errors.Is(err, store.ErrWouldEmptyDatabase) { return fmt.Errorf( "%w\n\nTo confirm you wish to proceed, pass the --yes flag", err, ) @@ -62,17 +61,14 @@ confirmed by providing -yes.`, }, } cmd.Flags().Uint( - "steps", - 0, - "number of migrations to roll back. 0 = all (deletes all data)", + "steps", 0, "number of migrations to roll back. 0 = all (deletes all data)", ) if err := cmd.MarkFlagRequired("steps"); err != nil { panic(err) // only fails if the flag isn't defined } cmd.Flags().BoolP( - "yes", "y", - false, - "confirm you are intending to roll back every migration", + "yes", "y", false, "confirm you are intending to roll back every migration", ) + return cmd } diff --git a/internal/cli/migrate/migrate.go b/internal/cli/migrate/migrate.go index d04a9bd..8ac457a 100644 --- a/internal/cli/migrate/migrate.go +++ b/internal/cli/migrate/migrate.go @@ -14,8 +14,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "migrate" command, which migrates the database schema (see -// internal/store). +// NewCommand returns the "migrate" command, which migrates the database schema +// (see internal/store). func NewCommand(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "migrate", @@ -39,5 +39,6 @@ migration is applied.`, cmd.AddCommand(commandUp(opts)) cmd.AddCommand(commandDown(opts)) cmd.AddCommand(commandStatus(opts)) + return cmd } diff --git a/internal/cli/migrate/migrate_test.go b/internal/cli/migrate/migrate_test.go index d936864..60cbe4d 100644 --- a/internal/cli/migrate/migrate_test.go +++ b/internal/cli/migrate/migrate_test.go @@ -3,6 +3,7 @@ package migrate import ( "fmt" "path/filepath" + "strings" "testing" "github.com/spf13/cobra" @@ -43,10 +44,12 @@ func latestVersion(t *testing.T) int64 { t.Fatal(err) } defer db.Close() + m, err := store.NewMigrator(db) if err != nil { t.Fatal(err) } + return store.LatestVersion(m) } @@ -83,7 +86,9 @@ func TestMigrateWithoutSubcommand(t *testing.T) { cmdtest.WriteFile(t, "test.db", "") // exists, so no question is asked out := mustRun(t, "migrate") - cmdtest.Contains(t, out, fmt.Sprintf("Database migrated to version %d", latestVersion(t))) + cmdtest.Contains( + t, out, fmt.Sprintf("Database migrated to version %d", latestVersion(t)), + ) } func TestMigrateUpAsksBeforeCreating(t *testing.T) { @@ -130,16 +135,24 @@ func TestMigrateDown(t *testing.T) { mustRun(t, "migrate", "up", "--create") out := mustRun(t, "migrate", "down", "--steps", "1") - cmdtest.Contains(t, out, "reverted", fmt.Sprintf("Database migrated to version %d", latest-1)) + cmdtest.Contains( + t, + out, + "reverted", + fmt.Sprintf("Database migrated to version %d", latest-1), + ) // Rolling back everything deletes all data, so it must be confirmed. for _, args := range [][]string{ {"migrate", "down", "--steps", "0"}, {"migrate", "down", "--steps", "99"}, } { - _, err := run(t, "", args...) - cmdtest.ErrContains(t, err, "pass the --yes flag") + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := run(t, "", args...) + cmdtest.ErrContains(t, err, "pass the --yes flag") + }) } + cmdtest.Contains( t, mustRun(t, "migrate", "status"), @@ -166,9 +179,12 @@ func TestMigrateMissingDatabase(t *testing.T) { {"migrate", "status"}, {"migrate", "down", "--steps", "1"}, } { - _, err := run(t, "", args...) - cmdtest.ErrContains(t, err, "does not exist") + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := run(t, "", args...) + cmdtest.ErrContains(t, err, "does not exist") + }) } + if cmdtest.Exists("test.db") { t.Error("status or down created the database") } diff --git a/internal/cli/migrate/status.go b/internal/cli/migrate/status.go index e155424..2855829 100644 --- a/internal/cli/migrate/status.go +++ b/internal/cli/migrate/status.go @@ -22,7 +22,9 @@ import ( // 1 applied 2026-10-07 10:12:01 00001_create_opportunities_table.sql // ... // 5 pending 00005_create_opportunity_classifications_table.sql -func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) error { +func printStatus( + ctx context.Context, out io.Writer, migrator *goose.Provider, +) error { statuses, err := migrator.Status(ctx) if err != nil { return err @@ -43,7 +45,7 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e } output := &errWriter{ - w: out, + out: out, } switch { case current > latest: @@ -55,7 +57,9 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e case pending == 0: output.printf("Database version %d of %d (up-to-date)\n", current, latest) default: - output.printf("Database version %d of %d (%d pending)\n", current, latest, pending) + output.printf( + "Database version %d of %d (%d pending)\n", current, latest, pending, + ) } if output.err != nil { return output.err @@ -63,7 +67,7 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e table := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) output = &errWriter{ - w: table, + out: table, } for _, status := range statuses { applied := "" @@ -78,6 +82,7 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e filepath.Base(status.Source.Path), ) } + return errors.Join(output.err, table.Flush()) } diff --git a/internal/cli/migrate/up.go b/internal/cli/migrate/up.go index 71961b9..82bf663 100644 --- a/internal/cli/migrate/up.go +++ b/internal/cli/migrate/up.go @@ -1,6 +1,7 @@ package migrate import ( + "errors" "fmt" "log/slog" @@ -23,14 +24,15 @@ func runStepUp( defer closeDB(db, &err) results, err := store.StepUp(cmd.Context(), migrator, int(steps)) - if len(results) == 0 && err == nil { - _, err := fmt.Fprintln(cmd.OutOrStdout(), "No changes made to database (up-to-date)") + _, err = fmt.Fprintln( + cmd.OutOrStdout(), "No changes made to database (up-to-date)", + ) return err } - if err := printResults(cmd.OutOrStdout(), results); err != nil { - return err + if printErr := printResults(cmd.OutOrStdout(), results); printErr != nil { + return errors.Join(err, printErr) } if err != nil { @@ -60,15 +62,10 @@ func commandUp(opts *cli.Options) *cobra.Command { return runStepUp(cmd, opts, int(steps), create) }, } - cmd.Flags().Uint( - "steps", - 0, - "number of migrations to apply. 0 = all pending", - ) + cmd.Flags().Uint("steps", 0, "number of migrations to apply. 0 = all pending") cmd.Flags().Bool( - "create", - false, - "create the database file if it does not already exist", + "create", false, "create the database file if it does not already exist", ) + return cmd } diff --git a/internal/cli/options.go b/internal/cli/options.go index 1c76243..1214b86 100644 --- a/internal/cli/options.go +++ b/internal/cli/options.go @@ -34,13 +34,13 @@ func (o *Options) Bind(cmd *cobra.Command) { // terminalLog is true, switches logging to the configured log file (staying // on the terminal, with a warning, if none is configured). func (o *Options) RunSetup( - cmd *cobra.Command, loadConfig bool, terminalLog bool, + cmd *cobra.Command, loadConfig, terminalLog bool, ) (*config.JobSearchConfig, error) { // Flags and arguments are valid from here on, so errors are runtime // failures: report them without cobra's usage text. cmd.SilenceUsage = true - // Check for debug level or quiet supression via args + // Check for debug level or quiet suppression via args conf := &config.JobSearchConfig{ Logging: config.JSCLogging{}, } @@ -64,7 +64,7 @@ func (o *Options) RunSetup( // Check if the logging should be switched to file if !terminalLog { - if len(conf.Logging.File) == 0 { + if conf.Logging.File == "" { slog.Warn("no log file configured; logging to the terminal") } else if err := logging.LogToFile(conf.Logging.File); err != nil { return nil, fmt.Errorf("opening log file: %w", err) diff --git a/internal/cli/wsserve/command.go b/internal/cli/wsserve/command.go index 237c30f..f5c446f 100644 --- a/internal/cli/wsserve/command.go +++ b/internal/cli/wsserve/command.go @@ -2,6 +2,7 @@ package wsserve import ( "context" + "errors" "fmt" "log/slog" "os" @@ -39,21 +40,21 @@ func NewCommand(opts *cli.Options) *cobra.Command { if addrOverride != "" { listenParts := strings.Split(addrOverride, ":") if len(listenParts) != 2 { - return fmt.Errorf( + return errors.New( "--listen should be in the format of 127.0.0.1:9393", ) } - i, err := strconv.Atoi(listenParts[1]) + port, err := strconv.Atoi(listenParts[1]) if err != nil { return fmt.Errorf("failed to parse --listen port: %w", err) } hostOverride = listenParts[0] - portOverride = i + portOverride = port } conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } if portOverride != 0 { @@ -82,11 +83,16 @@ func NewCommand(opts *cli.Options) *cobra.Command { return server.Run(ctx) }, } - cmd.Flags().String("listen", "", "listen address (default from config, 127.0.0.1:9393)") - cmd.Flags().String("host", "", "listen address (default from config, 127.0.0.1)") + cmd.Flags().String( + "listen", "", "listen address (default from config, 127.0.0.1:9393)", + ) + cmd.Flags().String( + "host", "", "listen address (default from config, 127.0.0.1)", + ) cmd.Flags().Int("port", 0, "listen address (default from config, 9393)") cmd.Flags().Bool("foreground", false, "run in the foreground") cmd.MarkFlagsMutuallyExclusive("listen", "host") cmd.MarkFlagsMutuallyExclusive("listen", "port") + return cmd } diff --git a/internal/config/README.md b/internal/config/README.md index 3cc47ac..c55185f 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -48,6 +48,9 @@ is kept. | `server.websocket.max_message_size` | byte size | `32KiB` | Largest message read from a client; `0B` for 32KiB. `-1` (no limit) is rejected, as a client could then exhaust the server's memory | | `server.websocket.connection_limit.*` | | | A rate limit rule for new connections from each client IP, WebSocket and plain TCP alike; limiters are kept for `server.rate_limits.idle_ttl` | | `server.websocket.message_rate_limit.*` | | | A rate limit rule for messages from each connection | +| `server.websocket.*.enabled` | bool | `false` | In a rate limit rule: whether it applies | +| `server.websocket.*.per_second` | int | `50` | In a rate limit rule: sustained rate. `0` or less: the rule is off | +| `server.websocket.*.burst` | int | `100` | In a rate limit rule: largest burst. Below `1`: 1 | | `server.websocket.write_timeout` | duration | `10s` | `0s` for none | | `server.websocket.ping_interval` | duration | `30s` | How long a connection may be idle before it is pinged; `0s`: no pings | | `server.websocket.pong_timeout` | duration | `10s` | `0s` for none | @@ -91,7 +94,7 @@ per key: passed to `SetValue` (e.g. `"10m"`, `"true"`). - A tagged field whose type is a struct is a nested section, addressed as `section.key`. An embedded struct has its fields promoted, as Go does; - every source embeds `JobSearchConfigSource` for `enabled` and `interval`. + every source embeds `JSCSourcesSource` for `enabled` and `interval`. - `time.Duration` fields report `reflect.String` from `TypeOf`, accept either a `time.Duration` or a duration string in `SetValue`, and are given out as strings (`"10m0s"`) by `Map`, `Flatten` and `ValueOf`, and so by `JSON`, @@ -125,7 +128,7 @@ To add a new section: its key, and an implementation of `Section` (copy an existing section's methods). 2. Add the struct as a tagged field of its parent: `JobSearchConfig` in - `config.go`, or `JobSearchConfigSources` in `conf_sources.go` for a source, + `config.go`, or `JSCSources` in `conf_sources.go` for a source, etc. The field's `cfg` tag must match the `JSCK*` constant. Nothing in `section.go` needs changing. diff --git a/internal/config/conf_database.go b/internal/config/conf_database.go index a8a3fe7..c644797 100644 --- a/internal/config/conf_database.go +++ b/internal/config/conf_database.go @@ -17,20 +17,25 @@ type JSCDatabase struct { // Keys returns the keys directly in the section. func (c *JSCDatabase) Keys() []string { return keysOf(c) } +// Map implements Section. func (c JSCDatabase) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCDatabase) Flatten() map[string]any { return flattenOf(JSCKDatabase, c) } +// TypeOf implements Section. func (c JSCDatabase) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCDatabase) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCDatabase) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_logging.go b/internal/config/conf_logging.go index 2cb4f80..97f3de3 100644 --- a/internal/config/conf_logging.go +++ b/internal/config/conf_logging.go @@ -19,20 +19,25 @@ type JSCLogging struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCLogging) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCLogging) Flatten() map[string]any { return flattenOf(JSCKLogging, c) } +// TypeOf implements Section. func (c JSCLogging) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCLogging) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCLogging) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -50,16 +55,16 @@ func (c *JSCLogging) validate(key string, value any) error { case "debug", "info", "warn", "error": default: return errors.New( - "invalid log level. Must be one of: debug, info, warn, error", + "config: invalid log level: must be one of debug, info, warn, error", ) } } return nil } -// ConfigLevelToLogLevel converts a configured level name to an slog level. An +// LevelToLogLevel converts a configured level name to an slog level. An // unknown name gives 99, above every real level, so nothing is logged. -func ConfigLevelToLogLevel(level string) slog.Level { +func LevelToLogLevel(level string) slog.Level { switch level { case "debug": return slog.LevelDebug @@ -77,5 +82,5 @@ func ConfigLevelToLogLevel(level string) slog.Level { // LogLevel returns the configured level as an slog level. An unknown level // gives 99, above every real level, so nothing is logged. func (c *JSCLogging) LogLevel() slog.Level { - return ConfigLevelToLogLevel(c.Level) + return LevelToLogLevel(c.Level) } diff --git a/internal/config/conf_ratelimitrule.go b/internal/config/conf_ratelimitrule.go index 3e7bf0c..42aaebd 100644 --- a/internal/config/conf_ratelimitrule.go +++ b/internal/config/conf_ratelimitrule.go @@ -14,18 +14,23 @@ type JSCRateLimitRule struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCRateLimitRule) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCRateLimitRule) Flatten() map[string]any { return flattenOf("", c) } +// TypeOf implements Section. func (c JSCRateLimitRule) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCRateLimitRule) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCRateLimitRule) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_sections_test.go b/internal/config/conf_sections_test.go index 0424e4d..0fe2d12 100644 --- a/internal/config/conf_sections_test.go +++ b/internal/config/conf_sections_test.go @@ -25,9 +25,10 @@ func captureLogs(t *testing.T, level slog.Level) *bytes.Buffer { t.Helper() var buf bytes.Buffer def := slog.Default() - slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{ + handler := slog.NewTextHandler(&buf, &slog.HandlerOptions{ Level: level, - }))) + }) + slog.SetDefault(slog.New(handler)) t.Cleanup(func() { slog.SetDefault(def) }) @@ -48,7 +49,8 @@ func TestDefaults(t *testing.T) { if conf.Logging.Level != "warn" { t.Errorf("Logging.Level = %q; want warn", conf.Logging.Level) } - if conf.Sources.FindATender.APIRoot != "https://www.find-tender.service.gov.uk/api/1.0" { + const apiRoot = "https://www.find-tender.service.gov.uk/api/1.0" + if conf.Sources.FindATender.APIRoot != apiRoot { t.Errorf("FindATender.APIRoot = %q", conf.Sources.FindATender.APIRoot) } if conf.Sources.FTSMock.APIRoot != "" { @@ -71,25 +73,22 @@ func TestLoggingSection(t *testing.T) { File: "/tmp/x.log", } - if got, want := c.Keys(), []string{"level", "file"}; !reflect.DeepEqual(got, want) { + if got, want := c.Keys(), []string{ + "level", + "file", + }; !reflect.DeepEqual(got, want) { t.Errorf("Keys = %v; want %v", got, want) } if got, want := c.Map(), map[string]any{ "level": "info", "file": "/tmp/x.log", - }; !reflect.DeepEqual( - got, - want, - ) { + }; !reflect.DeepEqual(got, want) { t.Errorf("Map = %v; want %v", got, want) } if got, want := c.Flatten(), map[string]any{ "logging.level": "info", "logging.file": "/tmp/x.log", - }; !reflect.DeepEqual( - got, - want, - ) { + }; !reflect.DeepEqual(got, want) { t.Errorf("Flatten = %v; want %v", got, want) } if k, err := c.TypeOf("file"); err != nil || k != reflect.String { @@ -101,7 +100,8 @@ func TestLoggingSection(t *testing.T) { if _, err := c.TypeOf("nope"); err == nil { t.Error("TypeOf(nope) succeeded") } - if err := c.SetValue("file", "/var/log/js.log"); err != nil || c.File != "/var/log/js.log" { + if err := c.SetValue("file", "/var/log/js.log"); err != nil || + c.File != "/var/log/js.log" { t.Errorf("SetValue(file) = %v, File = %q", err, c.File) } } @@ -114,10 +114,7 @@ func TestLoggingLevelValidation(t *testing.T) { } } for _, lvl := range []string{"", "trace", "WARN", "fatal"} { - if err := c.SetValue( - "level", - lvl, - ); err == nil || + if err := c.SetValue("level", lvl); err == nil || !strings.Contains(err.Error(), "invalid log level") { t.Errorf("SetValue(level, %q) = %v; want invalid log level", lvl, err) } @@ -136,7 +133,7 @@ func TestLoggingLevelValidation(t *testing.T) { } } -func TestConfigLevelToLogLevel(t *testing.T) { +func TestLevelToLogLevel(t *testing.T) { tests := map[string]slog.Level{ "debug": slog.LevelDebug, "info": slog.LevelInfo, @@ -146,8 +143,8 @@ func TestConfigLevelToLogLevel(t *testing.T) { "bogus": 99, } for in, want := range tests { - if got := ConfigLevelToLogLevel(in); got != want { - t.Errorf("ConfigLevelToLogLevel(%q) = %v; want %v", in, got, want) + if got := LevelToLogLevel(in); got != want { + t.Errorf("LevelToLogLevel(%q) = %v; want %v", in, got, want) } c := JSCLogging{ Level: in, @@ -164,7 +161,10 @@ func TestSourceSection(t *testing.T) { Interval: time.Minute, } - if got, want := c.Keys(), []string{"enabled", "interval"}; !reflect.DeepEqual(got, want) { + if got, want := c.Keys(), []string{ + "enabled", + "interval", + }; !reflect.DeepEqual(got, want) { t.Errorf("Keys = %v; want %v", got, want) } want := map[string]any{"enabled": true, "interval": "1m0s"} @@ -184,7 +184,8 @@ func TestSourceSection(t *testing.T) { if v, err := c.ValueOf("interval"); err != nil || v != "1m0s" { t.Errorf("ValueOf(interval) = %v, %v", v, err) } - if err := c.SetValue("interval", "5s"); err != nil || c.Interval != 5*time.Second { + if err := c.SetValue("interval", "5s"); err != nil || + c.Interval != 5*time.Second { t.Errorf("SetValue(interval, 5s) = %v, Interval = %v", err, c.Interval) } if err := c.SetValue("interval", "never"); err == nil { @@ -229,7 +230,8 @@ func TestConcreteSourceSections(t *testing.T) { if err := s.SetValue("api_root", "http://localhost:8080"); err != nil { t.Fatalf("SetValue(api_root): %v", err) } - if v, err := s.ValueOf("api_root"); err != nil || v != "http://localhost:8080" { + if v, err := s.ValueOf("api_root"); err != nil || + v != "http://localhost:8080" { t.Errorf("ValueOf(api_root) = %v, %v", v, err) } wantMap["api_root"] = "http://localhost:8080" @@ -241,7 +243,8 @@ func TestConcreteSourceSections(t *testing.T) { if err := s.SetValue("block_shutdown", "true"); err != nil { t.Fatalf("SetValue(block_shutdown): %v", err) } - if k, err := s.TypeOf("block_shutdown"); err != nil || k != reflect.Bool { + if k, err := s.TypeOf("block_shutdown"); err != nil || + k != reflect.Bool { t.Errorf("TypeOf(block_shutdown) = %v, %v", k, err) } wantMap["block_shutdown"] = true @@ -271,7 +274,8 @@ func TestSourcesSection(t *testing.T) { if c.FTSMock.APIRoot != "http://mock" { t.Errorf("FTSMock.APIRoot = %q", c.FTSMock.APIRoot) } - if v, err := c.ValueOf("fts_mock.api_root"); err != nil || v != "http://mock" { + if v, err := c.ValueOf("fts_mock.api_root"); err != nil || + v != "http://mock" { t.Errorf("ValueOf = %v, %v", v, err) } if k, err := c.TypeOf("sleeper.enabled"); err != nil || k != reflect.Bool { @@ -299,6 +303,7 @@ func TestSourcesSection(t *testing.T) { func TestJobSearchConfigMap(t *testing.T) { conf := defaultConfig(t) m := conf.Map() + logging, ok := m["logging"].(map[string]any) if !ok || logging["level"] != "warn" { t.Errorf("Map()[logging] = %#v", m["logging"]) @@ -330,31 +335,33 @@ func TestString(t *testing.T) { } } -func TestJSON(t *testing.T) { +// JSON and YAML give the configuration as nested sections. +func TestSerialised(t *testing.T) { conf := defaultConfig(t) - var m map[string]map[string]any - if err := json.Unmarshal([]byte(conf.JSON()), &m); err != nil { - t.Fatalf("JSON() is not valid JSON: %v\n%s", err, conf.JSON()) + tests := []struct { + name string + output string + unmarshal func([]byte, any) error + }{ + {"JSON", conf.JSON(), json.Unmarshal}, + {"YAML", conf.YAML(), yaml.Unmarshal}, } - if m["logging"]["level"] != "warn" { - t.Errorf("JSON logging.level = %#v", m["logging"]["level"]) - } - if _, ok := m["sources"]["fts_mock"]; !ok { - t.Errorf("JSON missing sources.fts_mock: %v", m["sources"]) - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m map[string]map[string]any + if err := tt.unmarshal([]byte(tt.output), &m); err != nil { + t.Fatalf("%s() doesn't parse: %v\n%s", tt.name, err, tt.output) + } -func TestYAML(t *testing.T) { - conf := defaultConfig(t) - var m map[string]map[string]any - if err := yaml.Unmarshal([]byte(conf.YAML()), &m); err != nil { - t.Fatalf("YAML() is not valid YAML: %v\n%s", err, conf.YAML()) - } - if m["logging"]["level"] != "warn" { - t.Errorf("YAML logging.level = %#v", m["logging"]["level"]) - } - if _, ok := m["sources"]["sleeper"]; !ok { - t.Errorf("YAML missing sources.sleeper: %v", m["sources"]) + if m["logging"]["level"] != "warn" { + t.Errorf("logging.level = %#v", m["logging"]["level"]) + } + for _, source := range []string{"fts_mock", "sleeper"} { + if _, ok := m["sources"][source]; !ok { + t.Errorf("missing sources.%s: %v", source, m["sources"]) + } + } + }) } } @@ -365,6 +372,7 @@ func TestSerialisedRoundTrip(t *testing.T) { orig.Sources.FTSMock.Enabled = true orig.Sources.FTSMock.APIRoot = "http://mock" orig.Server.WebSocket.MaxMessageSize = 64 << 10 + for _, format := range []string{"json", "yaml"} { t.Run(format, func(t *testing.T) { resetViper(t) @@ -372,10 +380,12 @@ func TestSerialisedRoundTrip(t *testing.T) { if format == "yaml" { out = orig.YAML() } + viper.SetConfigType(format) if err := viper.ReadConfig(strings.NewReader(out)); err != nil { t.Fatalf("ReadConfig: %v", err) } + var got JobSearchConfig LoadFromViper(&got) if !reflect.DeepEqual(got, orig) { @@ -402,7 +412,9 @@ func TestLoadDefaultConfig(t *testing.T) { if got := viper.GetString("logging.level"); got != "warn" { t.Errorf("logging.level = %q; want warn", got) } - if got := viper.GetDuration("sources.sleeper.interval"); got != 10*time.Minute { + if got := viper.GetDuration( + "sources.sleeper.interval", + ); got != 10*time.Minute { t.Errorf("sources.sleeper.interval = %v; want 10m", got) } } @@ -460,7 +472,9 @@ func TestLoadFromViperInvalidValues(t *testing.T) { t.Errorf("Logging.Level = %q; want warn kept", conf.Logging.Level) } if conf.Sources.Sleeper.Interval != 10*time.Minute { - t.Errorf("Sleeper.Interval = %v; want 10m kept", conf.Sources.Sleeper.Interval) + t.Errorf( + "Sleeper.Interval = %v; want 10m kept", conf.Sources.Sleeper.Interval, + ) } for _, key := range []string{"logging.level", "sources.sleeper.interval"} { if !strings.Contains(logs.String(), "key="+key) { @@ -477,7 +491,9 @@ func TestDatabaseSection(t *testing.T) { if got, want := c.Keys(), []string{"file"}; !reflect.DeepEqual(got, want) { t.Errorf("Keys = %v; want %v", got, want) } - if got, want := c.Map(), map[string]any{"file": "./jobsearch.db"}; !reflect.DeepEqual(got, want) { + if got, want := c.Map(), map[string]any{ + "file": "./jobsearch.db", + }; !reflect.DeepEqual(got, want) { t.Errorf("Map = %v; want %v", got, want) } want := map[string]any{"database.file": "./jobsearch.db"} @@ -496,7 +512,8 @@ func TestDatabaseSection(t *testing.T) { if _, err := c.ValueOf("nope"); err == nil { t.Error("ValueOf(nope) succeeded") } - if err := c.SetValue("file", "/var/lib/js.db"); err != nil || c.File != "/var/lib/js.db" { + if err := c.SetValue("file", "/var/lib/js.db"); err != nil || + c.File != "/var/lib/js.db" { t.Errorf("SetValue(file) = %v, File = %q", err, c.File) } if err := c.SetValue("file", 42); err == nil { @@ -532,10 +549,18 @@ func TestViperValue(t *testing.T) { } { got, ok := viperValue(tt.key, tt.kind) if !ok || !reflect.DeepEqual(got, tt.want) { - t.Errorf("viperValue(%q, %v) = %#v, %v; want %#v", tt.key, tt.kind, got, ok, tt.want) + t.Errorf( + "viperValue(%q, %v) = %#v, %v; want %#v", + tt.key, + tt.kind, + got, + ok, + tt.want, + ) } } - for _, kind := range []reflect.Kind{reflect.Float64, reflect.Int8, reflect.Map} { + unhandled := []reflect.Kind{reflect.Float64, reflect.Int8, reflect.Map} + for _, kind := range unhandled { if got, ok := viperValue("i", kind); ok { t.Errorf("viperValue(i, %v) = %#v, true; want unhandled", kind, got) } @@ -553,14 +578,20 @@ func TestWebSocketMaxMessageSizeNoLimit(t *testing.T) { if c.MaxMessageSize != 0 { t.Errorf("rejected size was stored: %v", c.MaxMessageSize) } + for _, text := range []string{"0B", "64KiB"} { if err := c.SetValue("max_message_size", text); err != nil { t.Errorf("SetValue(max_message_size, %s): %v", text, err) } } + conf := defaultConfig(t) - if err := conf.SetValue("server.websocket.max_message_size", "-1"); err == nil { - t.Error("JobSearchConfig.SetValue(server.websocket.max_message_size, -1) succeeded") + if err := conf.SetValue( + "server.websocket.max_message_size", "-1", + ); err == nil { + t.Error( + "JobSearchConfig.SetValue(server.websocket.max_message_size, -1) succeeded", + ) } } @@ -584,6 +615,8 @@ func TestSourceIntervalMustBePositive(t *testing.T) { } } if conf.Sources.FindATender.Interval.Minutes() != 1 { - t.Errorf("FindATender.Interval = %v; want 1m", conf.Sources.FindATender.Interval) + t.Errorf( + "FindATender.Interval = %v; want 1m", conf.Sources.FindATender.Interval, + ) } } diff --git a/internal/config/conf_server.go b/internal/config/conf_server.go index ea08ce4..dfeeb13 100644 --- a/internal/config/conf_server.go +++ b/internal/config/conf_server.go @@ -20,20 +20,25 @@ type JSCServer struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServer) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServer) Flatten() map[string]any { return flattenOf(JSCKServer, c) } +// TypeOf implements Section. func (c JSCServer) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServer) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServer) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_server_ratelimits.go b/internal/config/conf_server_ratelimits.go index ebc0e46..8341468 100644 --- a/internal/config/conf_server_ratelimits.go +++ b/internal/config/conf_server_ratelimits.go @@ -17,20 +17,25 @@ type JSCServerRateLimits struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServerRateLimits) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServerRateLimits) Flatten() map[string]any { return flattenOf(JSCKServerRateLimits, c) } +// TypeOf implements Section. func (c JSCServerRateLimits) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServerRateLimits) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServerRateLimits) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_server_timeouts.go b/internal/config/conf_server_timeouts.go index ccf60e4..ee70f9f 100644 --- a/internal/config/conf_server_timeouts.go +++ b/internal/config/conf_server_timeouts.go @@ -17,20 +17,25 @@ type JSCServerTimeouts struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServerTimeouts) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServerTimeouts) Flatten() map[string]any { return flattenOf(JSCKServerTimeouts, c) } +// TypeOf implements Section. func (c JSCServerTimeouts) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServerTimeouts) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServerTimeouts) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_server_websocket.go b/internal/config/conf_server_websocket.go index cef9239..92e3b02 100644 --- a/internal/config/conf_server_websocket.go +++ b/internal/config/conf_server_websocket.go @@ -28,20 +28,25 @@ type JSCServerWebSocket struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServerWebSocket) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServerWebSocket) Flatten() map[string]any { return flattenOf(JSCKServerWebSocket, c) } +// TypeOf implements Section. func (c JSCServerWebSocket) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServerWebSocket) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServerWebSocket) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -52,9 +57,10 @@ func (c *JSCServerWebSocket) SetValue(key string, value any) error { // max_message_size can't be -1 (no limit): a client could then send a // message of any size, and the server holds each message in memory. func (c *JSCServerWebSocket) validate(key string, value any) error { - if key == "max_message_size" && value.(util.ByteSize) == util.ByteSizeNoLimit { + if key == "max_message_size" && + value.(util.ByteSize) == util.ByteSizeNoLimit { return errors.New( - "max_message_size can't be -1 (no limit): set an explicit size, or 0 for the default", + "config: max_message_size can't be -1 (no limit): set an explicit size, or 0 for the default", ) } return nil diff --git a/internal/config/conf_source_findatender.go b/internal/config/conf_source_findatender.go index 50bb9c5..a871f98 100644 --- a/internal/config/conf_source_findatender.go +++ b/internal/config/conf_source_findatender.go @@ -14,20 +14,25 @@ type JSCSourceFindATender struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourceFindATender) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourceFindATender) Flatten() map[string]any { return flattenOf(JSCKSourceFindATender, c) } +// TypeOf implements Section. func (c JSCSourceFindATender) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourceFindATender) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourceFindATender) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_source_ftsmock.go b/internal/config/conf_source_ftsmock.go index 45a618e..98b8a30 100644 --- a/internal/config/conf_source_ftsmock.go +++ b/internal/config/conf_source_ftsmock.go @@ -15,20 +15,25 @@ type JSCSourceFTSMock struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourceFTSMock) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourceFTSMock) Flatten() map[string]any { return flattenOf(JSCKSourceFTSMock, c) } +// TypeOf implements Section. func (c JSCSourceFTSMock) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourceFTSMock) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourceFTSMock) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_source_sleeper.go b/internal/config/conf_source_sleeper.go index 2f9b308..1e09fb2 100644 --- a/internal/config/conf_source_sleeper.go +++ b/internal/config/conf_source_sleeper.go @@ -15,20 +15,25 @@ type JSCSourceSleeper struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourceSleeper) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourceSleeper) Flatten() map[string]any { return flattenOf(JSCKSourceSleeper, c) } +// TypeOf implements Section. func (c JSCSourceSleeper) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourceSleeper) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourceSleeper) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_sources.go b/internal/config/conf_sources.go index d1132f9..da542d4 100644 --- a/internal/config/conf_sources.go +++ b/internal/config/conf_sources.go @@ -6,6 +6,8 @@ import ( "time" ) +// ========== Sources ========== + // JSCKSources is the sources section's configuration key. const JSCKSources string = "sources" @@ -19,20 +21,25 @@ type JSCSources struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSources) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSources) Flatten() map[string]any { return flattenOf(JSCKSources, c) } +// TypeOf implements Section. func (c JSCSources) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSources) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSources) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -50,20 +57,25 @@ type JSCSourcesSource struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourcesSource) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourcesSource) Flatten() map[string]any { return flattenOf("", c) } +// TypeOf implements Section. func (c JSCSourcesSource) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourcesSource) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourcesSource) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -76,7 +88,7 @@ func (c *JSCSourcesSource) Keys() []string { return keysOf(c) } func (c *JSCSourcesSource) validate(key string, value any) error { if key == "interval" && value.(time.Duration) <= 0 { return errors.New( - "interval must be greater than 0; set enabled to false to turn the source off", + "config: interval must be greater than 0; set enabled to false to turn the source off", ) } return nil diff --git a/internal/config/config.go b/internal/config/config.go index 731b92b..a6c84fe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -42,18 +42,23 @@ func LoadDefaultConfig() { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JobSearchConfig) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JobSearchConfig) Flatten() map[string]any { return flattenOf("", c) } +// TypeOf implements Section. func (c JobSearchConfig) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JobSearchConfig) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JobSearchConfig) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -70,7 +75,11 @@ func LoadFromViper(conf *JobSearchConfig) { } value, ok := viperValue(key, keyType) if !ok { - slog.Error("unknown config key type", "key", key, "kind", keyType.String()) + slog.Error( + "unknown config key type", + "key", key, + "kind", keyType.String(), + ) continue } if err := conf.SetValue(key, value); err != nil { @@ -79,9 +88,9 @@ func LoadFromViper(conf *JobSearchConfig) { } } -// viperValue reads key from Viper as a value of kind. ok is false for a kind -// LoadFromViper doesn't handle. -func viperValue(key string, kind reflect.Kind) (value any, ok bool) { +// viperValue reads key from Viper as a value of kind. Its bool is false for a +// kind LoadFromViper doesn't handle. +func viperValue(key string, kind reflect.Kind) (any, bool) { switch kind { case reflect.String: return viper.GetString(key), true @@ -134,13 +143,13 @@ func (c JobSearchConfig) String() string { // JSON formats the configuration as indented JSON. func (c JobSearchConfig) JSON() string { confMap := c.Map() - jbytes, _ := json.MarshalIndent(confMap, "", " ") - return string(jbytes) + encoded, _ := json.MarshalIndent(confMap, "", " ") + return string(encoded) } // YAML formats the configuration as YAML. func (c JobSearchConfig) YAML() string { confMap := c.Map() - ybytes, _ := yaml.Marshal(confMap) - return string(ybytes) + encoded, _ := yaml.Marshal(confMap) + return string(encoded) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 53b3316..8c1f785 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -10,7 +10,10 @@ import ( // Kinds that LoadFromViper knows how to read. var viperKinds = map[reflect.Kind]bool{ - reflect.String: true, reflect.Int: true, reflect.Bool: true, reflect.Slice: true, + reflect.String: true, + reflect.Int: true, + reflect.Bool: true, + reflect.Slice: true, } // Every key round-trips through TypeOf/ValueOf/SetValue and has a type that @@ -25,7 +28,9 @@ func TestKeysRoundTrip(t *testing.T) { if err != nil { t.Errorf("TypeOf(%q): %v", key, err) } else if !viperKinds[kind] { - t.Errorf("TypeOf(%q) = %v, which LoadFromViper does not handle", key, kind) + t.Errorf( + "TypeOf(%q) = %v, which LoadFromViper does not handle", key, kind, + ) } got, err := conf.ValueOf(key) if err != nil || !reflect.DeepEqual(got, want) { @@ -53,6 +58,7 @@ func TestSections(t *testing.T) { if !reflect.PointerTo(reflect.TypeOf(conf)).Implements(sectionType) { t.Error("*JobSearchConfig does not implement Section") } + all := conf.Flatten() uses := map[reflect.Type]int{} countSections(uses, reflect.TypeOf(conf)) @@ -64,8 +70,8 @@ func TestSections(t *testing.T) { func countSections(uses map[reflect.Type]int, section reflect.Type) { for _, f := range fieldsOf(section) { if f.section { - uses[f.typ]++ - countSections(uses, f.typ) + uses[f.fieldType]++ + countSections(uses, f.fieldType) } } } @@ -78,14 +84,16 @@ func checkSections( prefix string, section reflect.Value, ) { + t.Helper() for _, f := range fieldsOf(section.Type()) { key := joinKey(prefix, f.key) - if isSectionList(f.typ) { + if isSectionList(f.fieldType) { checkSectionList(t, all, key, section.FieldByIndex(f.index)) } if !f.section { continue } + fv := section.FieldByIndex(f.index) name := fv.Type().Name() if !fv.Addr().Type().Implements(sectionType) { @@ -93,18 +101,16 @@ func checkSections( continue } checkEmbedded(t, fv.Type()) + keys := fv.Addr().Interface().(Section).Flatten() keyPrefix := prefix - if uses[f.typ] > 1 { + if uses[f.fieldType] > 1 { keyPrefix = key } for k := range keys { if _, ok := all[joinKey(keyPrefix, k)]; !ok { t.Errorf( - "%s.Flatten() key %q does not match its cfg tag %q", - name, - k, - key, + "%s.Flatten() key %q does not match its cfg tag %q", name, k, key, ) } } @@ -126,29 +132,36 @@ func checkSections( // checkSectionList checks the items of the list of sections at key implement // Section and that their methods agree with the list's value in all. -func checkSectionList(t *testing.T, all map[string]any, key string, list reflect.Value) { +func checkSectionList( + t *testing.T, all map[string]any, key string, list reflect.Value, +) { t.Helper() name := list.Type().Elem().Name() if !reflect.PointerTo(list.Type().Elem()).Implements(sectionType) { t.Errorf("*%s (items of %s) does not implement Section", name, key) return } + items, _ := all[key].([]map[string]any) if len(items) != list.Len() { t.Errorf("%s has %d items; Flatten gives %d", key, list.Len(), len(items)) return } + for i := range list.Len() { item, itemName := list.Index(i), fmt.Sprintf("%s[%d]", key, i) want := flattenMap("", items[i]) + // Items are not prefixed, and their kinds are their fields' kinds. - if got := item.Addr().Interface().(Section).Flatten(); !reflect.DeepEqual(got, want) { + got := item.Addr().Interface().(Section).Flatten() + if !reflect.DeepEqual(got, want) { t.Errorf("%s.Flatten() =\n%v\nwant\n%v", itemName, got, want) } + kinds := map[string]reflect.Kind{} for k := range want { _, f, _ := lookup(item, k) - kinds[k] = kindOf(f.typ) + kinds[k] = kindOf(f.fieldType) } checkSectionMethods(t, itemName, item, want, kinds) } @@ -164,10 +177,12 @@ func checkSectionMethods( want map[string]any, kinds map[string]reflect.Kind, ) { + t.Helper() s := section.Addr().Interface().(Section) if got := flattenMap("", s.Map()); !reflect.DeepEqual(got, want) { t.Errorf("%s.Map() =\n%v\nwant\n%v", name, got, want) } + before := section.Interface() for k, v := range want { if kind, err := s.TypeOf(k); err != nil || kind != kinds[k] { @@ -180,6 +195,7 @@ func checkSectionMethods( t.Errorf("%s.SetValue(%q, %#v): %v", name, k, v, err) } } + if !reflect.DeepEqual(section.Interface(), before) { t.Errorf("%s changed by setting its keys to their own values", name) } diff --git a/internal/config/load.go b/internal/config/load.go index 62afb56..782e4d9 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "log/slog" "os" @@ -41,13 +42,13 @@ func loadEnv() error { return nil } -// loadConfig sets up Viper with the defaults, cfgFile (or config.* found in +// loadConfig sets up Viper with the defaults, configFile (or config.* found in // EtcConfigPath or the working directory) and JOBSEARCH_* environment // variables. A missing config file is only a warning. -func loadConfig(cfgFile string) error { - if cfgFile != "" { +func loadConfig(configFile string) error { + if configFile != "" { // Use config file from the flag. - viper.SetConfigFile(cfgFile) + viper.SetConfigFile(configFile) } else { // Search config in /etc or local directory. viper.AddConfigPath(EtcConfigPath) @@ -64,26 +65,25 @@ func loadConfig(cfgFile string) error { viper.AutomaticEnv() // read in environment variables that match err := viper.ReadInConfig() - if err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - // Indicates an explicitly set config file is not found or - // that no config file was found in any search path - slog.Warn("no config file found; using defaults") - } else { - // Config file was found but another error was produced - return err - } - } else { - // Config file found and successfully parsed + var notFound viper.ConfigFileNotFoundError + switch { + case errors.As(err, ¬Found): + // No config file was found in any search path. + slog.Warn("no config file found; using defaults") + case err != nil: + // A config file was found but couldn't be read or parsed. + return err + default: slog.Info("using config file", "file", viper.ConfigFileUsed()) } + return nil } // Init loads the .env file and the configuration (see loadConfig) into conf. // A debug or error log level already set in conf, from --debug or --quiet, // is kept over the loaded one. -func Init(cfgFile string, conf *JobSearchConfig) error { +func Init(configFile string, conf *JobSearchConfig) error { if err := loadEnv(); err != nil { return err } @@ -91,7 +91,7 @@ func Init(cfgFile string, conf *JobSearchConfig) error { printEnv() } - if err := loadConfig(cfgFile); err != nil { + if err := loadConfig(configFile); err != nil { return err } diff --git a/internal/config/load_test.go b/internal/config/load_test.go index 757efd9..cf938ca 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -18,6 +18,7 @@ func initDir(t *testing.T, env []string, config string) string { resetViper(t) dir := t.TempDir() t.Chdir(dir) + if env != nil { // godotenv sets variables without cleaning up after the test. for _, kv := range env { @@ -28,9 +29,11 @@ func initDir(t *testing.T, env []string, config string) string { } writeFile(t, filepath.Join(dir, ".env"), strings.Join(env, "\n")+"\n") } + if config != "" { writeFile(t, filepath.Join(dir, "config.yaml"), config) } + return dir } @@ -63,12 +66,16 @@ func TestInitLoadsEnvAndConfigFile(t *testing.T) { if conf.Logging.Level != "info" { t.Errorf("Logging.Level = %q; want info", conf.Logging.Level) } - if !conf.Sources.FTSMock.Enabled || conf.Sources.FTSMock.Interval != 30*time.Second { + if !conf.Sources.FTSMock.Enabled || + conf.Sources.FTSMock.Interval != 30*time.Second { t.Errorf("FTSMock = %+v; want enabled every 30s", conf.Sources.FTSMock) } // Keys missing from the file keep their defaults. if conf.Sources.FindATender.Interval != 10*time.Minute { - t.Errorf("FindATender.Interval = %v; want default 10m", conf.Sources.FindATender.Interval) + t.Errorf( + "FindATender.Interval = %v; want default 10m", + conf.Sources.FindATender.Interval, + ) } } @@ -83,7 +90,9 @@ func TestInitExplicitConfigFile(t *testing.T) { t.Fatal(err) } if conf.Logging.Level != "error" { - t.Errorf("Logging.Level = %q; want error from %s", conf.Logging.Level, other) + t.Errorf( + "Logging.Level = %q; want error from %s", conf.Logging.Level, other, + ) } } @@ -124,7 +133,9 @@ func TestInitKeepsCommandLineLevel(t *testing.T) { // At debug, the environment and loaded config are logged. gotDebug := strings.Contains(logs.String(), "loaded config value") if gotDebug != (level == "debug") { - t.Errorf("debug output logged = %v at level %s: %s", gotDebug, level, logs) + t.Errorf( + "debug output logged = %v at level %s: %s", gotDebug, level, logs, + ) } }) } @@ -134,7 +145,8 @@ func TestInitErrors(t *testing.T) { t.Run("no .env file", func(t *testing.T) { initDir(t, nil, testConfigYAML) var conf JobSearchConfig - if err := Init("", &conf); err == nil || !strings.Contains(err.Error(), ".env") { + if err := Init("", &conf); err == nil || + !strings.Contains(err.Error(), ".env") { t.Fatalf("err = %v; want an invalid .env file error", err) } }) @@ -169,10 +181,15 @@ func TestInitEnvOverride(t *testing.T) { t.Fatal(err) } if conf.Logging.Level != "error" { - t.Errorf("Logging.Level = %q; want error from the environment", conf.Logging.Level) + t.Errorf( + "Logging.Level = %q; want error from the environment", conf.Logging.Level, + ) } if conf.Sources.FTSMock.Interval != 45*time.Second { - t.Errorf("FTSMock.Interval = %v; want 45s from the environment", conf.Sources.FTSMock.Interval) + t.Errorf( + "FTSMock.Interval = %v; want 45s from the environment", + conf.Sources.FTSMock.Interval, + ) } } @@ -204,11 +221,15 @@ func loadExplicit(t *testing.T) { t.Helper() resetViper(t) LoadDefaultConfig() + viper.SetEnvPrefix(EnvPrefix) viper.SetEnvKeyReplacer(envKeyReplacer) viper.AutomaticEnv() + viper.SetConfigType("yaml") - if err := viper.ReadConfig(strings.NewReader(explicitConfigYAML)); err != nil { + if err := viper.ReadConfig( + strings.NewReader(explicitConfigYAML), + ); err != nil { t.Fatal(err) } } @@ -255,7 +276,11 @@ func TestPrintLoadedConfig(t *testing.T) { t.Errorf("missing %q in:\n%s", want, out) } } - for _, unwanted := range []string{"sources.", "logging.file", "connection_limit.burst"} { + for _, unwanted := range []string{ + "sources.", + "logging.file", + "connection_limit.burst", + } { if strings.Contains(out, unwanted) { t.Errorf("logged default key %q:\n%s", unwanted, out) } diff --git a/internal/config/modify.go b/internal/config/modify.go index 7ae3212..7638b2b 100644 --- a/internal/config/modify.go +++ b/internal/config/modify.go @@ -14,14 +14,16 @@ import ( // file's own settings are written back, not defaults or environment // variables. It returns the stored value (as ValueOf gives it) and its kind. func StoreValue( - conf *JobSearchConfig, key string, value string, + conf *JobSearchConfig, key, value string, ) (any, reflect.Kind, error) { return storeValue(conf, key, value) } // storeValue is StoreValue for any section, so key types that // JobSearchConfig doesn't have yet can be tested. -func storeValue(section Section, key string, value string) (any, reflect.Kind, error) { +func storeValue( + section Section, key, value string, +) (any, reflect.Kind, error) { confType, err := section.TypeOf(key) if err != nil { return nil, confType, err @@ -31,7 +33,7 @@ func storeValue(section Section, key string, value string) (any, reflect.Kind, e // SetValue gives. if _, err := parseValue(confType, value); err != nil { return nil, confType, fmt.Errorf( - "config value type not accepted, expected <%s>", confType, + "config: value type not accepted, expected <%s>", confType, ) } @@ -69,7 +71,11 @@ func parseValue(kind reflect.Kind, value string) (any, error) { return value, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return strconv.ParseInt(value, 10, 64) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: return strconv.ParseUint(value, 10, 64) case reflect.Float32, reflect.Float64: return strconv.ParseFloat(value, 64) @@ -91,6 +97,7 @@ func PurgeValue(conf *JobSearchConfig, key string) error { if err != nil { return err } + out := viper.New() out.SetConfigFile(file.ConfigFileUsed()) for _, fileKey := range file.AllKeys() { @@ -98,6 +105,7 @@ func PurgeValue(conf *JobSearchConfig, key string) error { out.Set(fileKey, file.Get(fileKey)) } } + return out.WriteConfig() } @@ -106,12 +114,14 @@ func PurgeValue(conf *JobSearchConfig, key string) error { func readUsedConfigFile() (*viper.Viper, error) { path := viper.ConfigFileUsed() if path == "" { - return nil, errors.New("no configuration file in use") + return nil, errors.New("config: no configuration file in use") } + file := viper.New() file.SetConfigFile(path) if err := file.ReadInConfig(); err != nil { return nil, err } + return file, nil } diff --git a/internal/config/modify_test.go b/internal/config/modify_test.go index 1c264f1..c7591e2 100644 --- a/internal/config/modify_test.go +++ b/internal/config/modify_test.go @@ -20,10 +20,12 @@ func useConfigFile(t *testing.T, contents string) string { if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { t.Fatal(err) } + viper.SetConfigFile(path) if err := viper.ReadInConfig(); err != nil { t.Fatal(err) } + return path } @@ -74,6 +76,7 @@ func TestStoreValueKeepsOtherKeys(t *testing.T) { if _, _, err := StoreValue(&conf, "logging.level", "error"); err != nil { t.Fatal(err) } + v := readConfigFile(t, path) if got := v.GetString("logging.file"); got != "/tmp/js.log" { t.Errorf("logging.file = %q; want it kept", got) @@ -159,7 +162,9 @@ func TestPurgeValueErrors(t *testing.T) { func TestStoreValueUpdatesConf(t *testing.T) { useConfigFile(t, "logging:\n level: warn\n") conf := defaultConfig(t) - if _, _, err := StoreValue(&conf, "sources.fts_mock.interval", "30s"); err != nil { + if _, _, err := StoreValue( + &conf, "sources.fts_mock.interval", "30s", + ); err != nil { t.Fatal(err) } if conf.Sources.FTSMock.Interval != 30*time.Second { @@ -185,7 +190,14 @@ func TestParseValue(t *testing.T) { } { got, err := parseValue(tt.kind, tt.value) if err != nil || !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseValue(%v, %q) = %#v, %v; want %#v", tt.kind, tt.value, got, err, tt.want) + t.Errorf( + "parseValue(%v, %q) = %#v, %v; want %#v", + tt.kind, + tt.value, + got, + err, + tt.want, + ) } } @@ -199,11 +211,14 @@ func TestParseValue(t *testing.T) { {reflect.Float64, "x"}, } { if got, err := parseValue(tt.kind, tt.value); err == nil { - t.Errorf("parseValue(%v, %q) = %#v; want an error", tt.kind, tt.value, got) + t.Errorf( + "parseValue(%v, %q) = %#v; want an error", tt.kind, tt.value, got, + ) } } - if _, err := parseValue(reflect.Map, "a"); !errors.Is(err, errors.ErrUnsupported) { + _, err := parseValue(reflect.Map, "a") + if !errors.Is(err, errors.ErrUnsupported) { t.Errorf("parseValue(map) err = %v; want ErrUnsupported", err) } } @@ -217,11 +232,21 @@ type numberSection struct { Ratio float32 `cfg:"ratio"` } -func (c numberSection) Map() map[string]any { return mapOf(c) } -func (c numberSection) Flatten() map[string]any { return flattenOf("", c) } -func (c numberSection) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } -func (c numberSection) ValueOf(key string) (any, error) { return valueOf(c, key) } -func (c *numberSection) SetValue(key string, value any) error { return setValue(c, key, value) } +func (c numberSection) Map() map[string]any { return mapOf(c) } + +func (c numberSection) Flatten() map[string]any { return flattenOf("", c) } + +func (c numberSection) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +func (c numberSection) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +func (c *numberSection) SetValue(key string, value any) error { + return setValue(c, key, value) +} // Numbers are stored as the field's own type, not as the int64, uint64 or // float64 that parsing produces. @@ -276,7 +301,9 @@ func TestStoreAndPurgeWriteOnlyFileSettings(t *testing.T) { LoadDefaultConfig() conf := defaultConfig(t) - if _, _, err := StoreValue(&conf, "sources.sleeper.enabled", "true"); err != nil { + if _, _, err := StoreValue( + &conf, "sources.sleeper.enabled", "true", + ); err != nil { t.Fatal(err) } want := map[string]any{ @@ -301,6 +328,7 @@ func checkFileSettings(t *testing.T, path string, want map[string]any) { for _, k := range v.AllKeys() { got[k] = v.Get(k) } + if !reflect.DeepEqual(got, want) { //nolint:gosec // G304: reading a file the test created in t.TempDir b, _ := os.ReadFile(path) diff --git a/internal/config/print.go b/internal/config/print.go index bd374b3..8170b2f 100644 --- a/internal/config/print.go +++ b/internal/config/print.go @@ -15,13 +15,13 @@ func (c JobSearchConfig) PrintTerminal() { fmt.Print(c.String() + "\n") } -// PrintJson prints the configuration to stdout as JSON. -func (c JobSearchConfig) PrintJson() { +// PrintJSON prints the configuration to stdout as JSON. +func (c JobSearchConfig) PrintJSON() { fmt.Print(c.JSON() + "\n") } -// PrintYaml prints the configuration to stdout as YAML. -func (c JobSearchConfig) PrintYaml() { +// PrintYAML prints the configuration to stdout as YAML. +func (c JobSearchConfig) PrintYAML() { fmt.Print(c.YAML() + "\n") } @@ -29,7 +29,7 @@ func (c JobSearchConfig) PrintYaml() { // debug, warn, error, or info for any other level. On a terminal it logs one // line and prints the values under it as "key = value" lines; elsewhere each // value is a log entry of its own. -func (conf JobSearchConfig) PrintLog(atLevel slog.Level) { +func (c JobSearchConfig) PrintLog(atLevel slog.Level) { switch atLevel { case slog.LevelDebug, slog.LevelWarn, slog.LevelError: default: @@ -38,7 +38,7 @@ func (conf JobSearchConfig) PrintLog(atLevel slog.Level) { ctx := context.Background() logger := slog.Default() - flat := conf.Flatten() + flat := c.Flatten() keys := slices.Sorted(maps.Keys(flat)) if logging.WritesToTerminal(logger.Handler()) { @@ -55,4 +55,4 @@ func (conf JobSearchConfig) PrintLog(atLevel slog.Level) { for _, key := range keys { logger.Log(ctx, atLevel, "config value", "key", key, "value", flat[key]) } -} \ No newline at end of file +} diff --git a/internal/config/print_test.go b/internal/config/print_test.go index b8761bc..b283a55 100644 --- a/internal/config/print_test.go +++ b/internal/config/print_test.go @@ -20,17 +20,24 @@ func captureStdout(t *testing.T, fn func()) string { if err != nil { t.Fatal(err) } + old := os.Stdout os.Stdout = f + t.Cleanup(func() { + os.Stdout = old + }) fn() os.Stdout = old + if err := f.Close(); err != nil { t.Fatal(err) } + b, err := os.ReadFile(filepath.Clean(path)) if err != nil { t.Fatal(err) } + return string(b) } @@ -50,6 +57,7 @@ func TestStringSorted(t *testing.T) { key, _, _ := strings.Cut(line, " = ") keys = append(keys, key) } + if !slices.IsSorted(keys) || len(keys) != len(conf.Flatten()) { t.Errorf("String keys = %v; want each key once, sorted", keys) } @@ -58,27 +66,30 @@ func TestStringSorted(t *testing.T) { } } -func TestPrintJson(t *testing.T) { +// PrintJSON and PrintYAML print the configuration in their format. +func TestPrintFormats(t *testing.T) { conf := defaultConfig(t) - out := captureStdout(t, conf.PrintJson) - var m map[string]any - if err := json.Unmarshal([]byte(out), &m); err != nil { - t.Fatalf("PrintJson output is not JSON: %v\n%s", err, out) + tests := []struct { + name string + print func() + unmarshal func([]byte, any) error + }{ + {"PrintJSON", conf.PrintJSON, json.Unmarshal}, + {"PrintYAML", conf.PrintYAML, yaml.Unmarshal}, } - if got := m["logging"].(map[string]any)["level"]; got != "warn" { - t.Errorf("logging.level = %v; want warn", got) - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := captureStdout(t, tt.print) + var m map[string]any + if err := tt.unmarshal([]byte(out), &m); err != nil { + t.Fatalf("%s output doesn't parse: %v\n%s", tt.name, err, out) + } -func TestPrintYaml(t *testing.T) { - conf := defaultConfig(t) - out := captureStdout(t, conf.PrintYaml) - var m map[string]any - if err := yaml.Unmarshal([]byte(out), &m); err != nil { - t.Fatalf("PrintYaml output is not YAML: %v\n%s", err, out) - } - if got := m["logging"].(map[string]any)["level"]; got != "warn" { - t.Errorf("logging.level = %v; want warn", got) + level := m["logging"].(map[string]any)["level"] + if level != "warn" { + t.Errorf("logging.level = %v; want warn", level) + } + }) } } @@ -101,7 +112,12 @@ func TestPrintLog(t *testing.T) { lines := strings.Split(strings.TrimSpace(logs.String()), "\n") if len(lines) != len(conf.Flatten()) { - t.Errorf("PrintLog(%v) logged %d lines; want %d", tt.at, len(lines), len(conf.Flatten())) + t.Errorf( + "PrintLog(%v) logged %d lines; want %d", + tt.at, + len(lines), + len(conf.Flatten()), + ) } for _, l := range lines { if !strings.Contains(l, tt.want) { diff --git a/internal/config/section.go b/internal/config/section.go index 28429ee..2f77d61 100644 --- a/internal/config/section.go +++ b/internal/config/section.go @@ -43,6 +43,7 @@ import ( // // A section can check values before they are stored by implementing // validator on its pointer type. + // The struct tags section.go reads. const ( // keyTag holds a field's configuration key. @@ -54,7 +55,7 @@ const ( // The unknown key error and the field types section.go handles specially. var ( // errUnknownKey is returned for a key that names no field. - errUnknownKey = errors.New("unknown configuration key") + errUnknownKey = errors.New("config: unknown configuration key") // durationType is time.Duration, read and given out as a string. durationType = reflect.TypeOf(time.Duration(0)) // byteSizeType is util.ByteSize, read and given out as a string. @@ -67,9 +68,9 @@ var ( // Section is the method set of every configuration section, including // JobSearchConfig itself, implemented on the section's pointer type. Keys are -// dotted paths relative to the section, e.g. "sleeper.interval" in -// JSCSources. Durations and byte sizes are given out as strings such as -// "10m0s" and "32KiB", matching TypeOf's reflect.String for them. TestSections checks every +// dotted paths relative to the section, e.g. "sleeper.interval" in JSCSources. +// Durations and byte sizes are given out as strings such as "10m0s" and +// "32KiB", matching TypeOf's reflect.String for them. TestSections checks every // section, found by walking JobSearchConfig, implements it. type Section interface { // Map returns the section as a map of keys to values, with nested @@ -102,18 +103,19 @@ type validator interface { // field is a section struct field that is part of the configuration (see // fieldsOf). type field struct { - key string - index []int // path for reflect.Value.FieldByIndex - typ reflect.Type - section bool - def string - hasDefault bool + key string + index []int // path for reflect.Value.FieldByIndex + fieldType reflect.Type + section bool + defaultValue string + hasDefault bool } // isSectionList reports whether listType is a list of sections, []S for a // section struct S. func isSectionList(listType reflect.Type) bool { - return listType.Kind() == reflect.Slice && listType.Elem().Kind() == reflect.Struct && + return listType.Kind() == reflect.Slice && + listType.Elem().Kind() == reflect.Struct && listType.Elem() != timeType } @@ -123,6 +125,7 @@ func fieldsOf(structType reflect.Type) []field { var fields []field for i := 0; i < structType.NumField(); i++ { structField := structType.Field(i) + if structField.Anonymous && structField.Type.Kind() == reflect.Struct { for _, embeddedField := range fieldsOf(structField.Type) { embeddedField.index = append([]int{i}, embeddedField.index...) @@ -130,20 +133,24 @@ func fieldsOf(structType reflect.Type) []field { } continue } + key, ok := structField.Tag.Lookup(keyTag) if !ok || key == "" || key == "-" || !structField.IsExported() { continue } + defaultValue, hasDefault := structField.Tag.Lookup(defaultTag) fields = append(fields, field{ - key: key, - index: []int{i}, - typ: structField.Type, - section: structField.Type.Kind() == reflect.Struct && structField.Type != timeType, - def: defaultValue, - hasDefault: hasDefault, + key: key, + index: []int{i}, + fieldType: structField.Type, + section: structField.Type.Kind() == reflect.Struct && + structField.Type != timeType, + defaultValue: defaultValue, + hasDefault: hasDefault, }) } + return fields } @@ -249,6 +256,7 @@ func exportValue(fieldValue reflect.Value) any { case byteSizeType: return util.ByteSize(fieldValue.Int()).String() } + if isSectionList(fieldValue.Type()) { var items []map[string]any for i := range fieldValue.Len() { @@ -256,11 +264,13 @@ func exportValue(fieldValue reflect.Value) any { } return items } + value := fieldValue.Interface() if list, ok := value.([]string); ok { // A copy, so changing it doesn't change the configuration. return slices.Clone(list) } + return value } @@ -292,7 +302,7 @@ func typeOf(value any, key string) (reflect.Kind, error) { if err != nil { return reflect.Invalid, err } - return kindOf(configField.typ), nil + return kindOf(configField.fieldType), nil } // valueOf returns the value of the dotted key in the section value, or @@ -318,22 +328,25 @@ func setValue(ptr any, key string, value any) error { // validator, checks it, before setting the field. On error the field is left // unchanged. func setField(section reflect.Value, configField field, value any) error { - newValue, err := convert(value, configField.typ) + newValue, err := convert(value, configField.fieldType) if err != nil { return err } + if checker, ok := section.Addr().Interface().(validator); ok { - if err := checker.validate(configField.key, newValue.Interface()); err != nil { + err := checker.validate(configField.key, newValue.Interface()) + if err != nil { return err } } section.FieldByIndex(configField.index).Set(newValue) + return nil } -// convert turns value into a value of type targetType. Values already of the right -// type are used as-is, except that an empty []string becomes nil, as an empty -// comma-separated list does (see splitList); strings are parsed. Lists of +// convert turns value into a value of type targetType. Values already of the +// right type are used as-is, except that an empty []string becomes nil, as an +// empty comma-separated list does (see splitList); strings are parsed. Lists of // sections are converted by convertSectionList. func convert(value any, targetType reflect.Type) (reflect.Value, error) { if isSectionList(targetType) { @@ -351,36 +364,38 @@ func convert(value any, targetType reflect.Type) (reflect.Value, error) { return parseString(text, targetType) } } - return reflect.Value{}, fmt.Errorf("unsupported type %T", value) + return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } // convertSectionList turns value into a list of sections of type listType: a -// value of type listType, a list of maps of keys to values, or a JSON array of objects. -// nil and empty lists become a nil list. -func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) { +// value of type listType, a list of maps of keys to values, or a JSON array of +// objects. nil and empty lists become a nil list. +func convertSectionList( + value any, listType reflect.Type, +) (reflect.Value, error) { if text, ok := value.(string); ok { var items []map[string]any if strings.TrimSpace(text) != "" { if err := json.Unmarshal([]byte(text), &items); err != nil { return reflect.Value{}, fmt.Errorf( - "unable to convert to a list of %s: %w", - listType.Elem(), - err, + "unable to convert to a list of %s: %w", listType.Elem(), err, ) } } value = items } + if value == nil { return reflect.Zero(listType), nil } reflected := reflect.ValueOf(value) if reflected.Kind() != reflect.Slice { - return reflect.Value{}, fmt.Errorf("unsupported type %T", value) + return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } if reflected.Len() == 0 { return reflect.Zero(listType), nil } + if reflected.Type().AssignableTo(listType) { for i := range reflected.Len() { if err := validateSection(reflected.Index(i)); err != nil { @@ -389,6 +404,7 @@ func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) } return reflected, nil } + list := reflect.MakeSlice(listType, 0, reflected.Len()) for i := range reflected.Len() { item, err := sectionFromMap(reflected.Index(i).Interface(), listType.Elem()) @@ -397,6 +413,7 @@ func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) } list = reflect.Append(list, item) } + return list, nil } @@ -408,28 +425,32 @@ func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) func sectionFromMap(value any, structType reflect.Type) (reflect.Value, error) { item, ok := value.(map[string]any) if !ok { - return reflect.Value{}, fmt.Errorf("unsupported type %T", value) + return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } + section := reflect.New(structType).Elem() if err := applyDefaultsTo("", section); err != nil { return reflect.Value{}, err } + for itemKey, itemValue := range item { holder, configField, err := lookup(section, itemKey) if err != nil { return reflect.Value{}, fmt.Errorf("%w %q", err, itemKey) } if _, isString := itemValue.(string); itemValue != nil && !isString && - !reflect.TypeOf(itemValue).AssignableTo(configField.typ) { + !reflect.TypeOf(itemValue).AssignableTo(configField.fieldType) { itemValue = fmt.Sprint(itemValue) } if err := setField(holder, configField, itemValue); err != nil { return reflect.Value{}, fmt.Errorf("%s: %w", itemKey, err) } } + if err := validateSection(section); err != nil { return reflect.Value{}, err } + return section, nil } @@ -441,37 +462,41 @@ func validateSection(section reflect.Value) error { if !ok { return nil } + for _, configField := range fieldsOf(section.Type()) { if configField.section { continue } - if err := checker.validate( - configField.key, - section.FieldByIndex(configField.index).Interface(), - ); err != nil { + value := section.FieldByIndex(configField.index).Interface() + if err := checker.validate(configField.key, value); err != nil { return fmt.Errorf("%s: %w", configField.key, err) } } + return nil } -// parseString parses text as a value of type targetType. It supports strings, bools, -// signed and unsigned integers, floats, time.Duration, ByteSize and []string -// (from a comma-separated list, see splitList). +// parseString parses text as a value of type targetType. It supports strings, +// bools, signed and unsigned integers, floats, time.Duration, ByteSize and +// []string (from a comma-separated list, see splitList). func parseString(text string, targetType reflect.Type) (reflect.Value, error) { parsed := reflect.New(targetType).Elem() switch targetType { case durationType: duration, err := time.ParseDuration(text) if err != nil { - return reflect.Value{}, fmt.Errorf("unable to convert to duration: %w", err) + return reflect.Value{}, fmt.Errorf( + "unable to convert to duration: %w", err, + ) } parsed.SetInt(int64(duration)) return parsed, nil case byteSizeType: size, err := util.ParseByteSize(text) if err != nil { - return reflect.Value{}, fmt.Errorf("unable to convert to byte size: %w", err) + return reflect.Value{}, fmt.Errorf( + "unable to convert to byte size: %w", err, + ) } parsed.SetInt(int64(size)) return parsed, nil @@ -492,7 +517,11 @@ func parseString(text string, targetType reflect.Type) (reflect.Value, error) { var number int64 number, err = strconv.ParseInt(text, 10, targetType.Bits()) parsed.SetInt(number) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: var number uint64 number, err = strconv.ParseUint(text, 10, targetType.Bits()) parsed.SetUint(number) @@ -501,11 +530,16 @@ func parseString(text string, targetType reflect.Type) (reflect.Value, error) { number, err = strconv.ParseFloat(text, targetType.Bits()) parsed.SetFloat(number) default: - return reflect.Value{}, fmt.Errorf("unsupported type string for %s", targetType) + return reflect.Value{}, fmt.Errorf( + "config: unsupported type string for %s", targetType, + ) } if err != nil { - return reflect.Value{}, fmt.Errorf("unable to convert %q to %s: %w", text, targetType, err) + return reflect.Value{}, fmt.Errorf( + "unable to convert %q to %s: %w", text, targetType, err, + ) } + return parsed, nil } @@ -521,16 +555,21 @@ func applyDefaults(ptr any) error { func applyDefaultsTo(prefix string, section reflect.Value) error { for _, configField := range fieldsOf(section.Type()) { key := joinKey(prefix, configField.key) + if configField.section { - if err := applyDefaultsTo(key, section.FieldByIndex(configField.index)); err != nil { + nested := section.FieldByIndex(configField.index) + if err := applyDefaultsTo(key, nested); err != nil { return err } continue } + if !configField.hasDefault { continue } - if err := setField(section, configField, configField.def); err != nil { + + err := setField(section, configField, configField.defaultValue) + if err != nil { return fmt.Errorf("invalid default for %s: %w", key, err) } } diff --git a/internal/config/section_test.go b/internal/config/section_test.go index 3d90f72..c222d7a 100644 --- a/internal/config/section_test.go +++ b/internal/config/section_test.go @@ -222,7 +222,8 @@ func TestUnknownKeys(t *testing.T) { } var s testSection for _, key := range keys { - if k, err := typeOf(s, key); !errors.Is(err, errUnknownKey) || k != reflect.Invalid { + if k, err := typeOf(s, key); !errors.Is(err, errUnknownKey) || + k != reflect.Invalid { t.Errorf("typeOf(%q) = %v, %v; want Invalid, errUnknownKey", key, k, err) } if v, err := valueOf(s, key); !errors.Is(err, errUnknownKey) || v != nil { @@ -266,17 +267,24 @@ func TestSetValue(t *testing.T) { {"slice", []string{}, []string(nil)}, // empty lists are stored as nil // lists of sections are given out as lists of maps; items start // from their defaults and their values are parsed - {"items", `[{"n": "1", "wait": "2s"}, {"name": "b", "n": 2}]`, []map[string]any{ - {"name": "item", "n": 1, "wait": "2s"}, - {"name": "b", "n": 2, "wait": "0s"}, - }}, + { + "items", + `[{"n": "1", "wait": "2s"}, {"name": "b", "n": 2}]`, + []map[string]any{ + {"name": "item", "n": 1, "wait": "2s"}, + {"name": "b", "n": 2, "wait": "0s"}, + }, + }, {"items", []any{map[string]any{"name": "a", "n": 3}}, []map[string]any{ {"name": "a", "n": 3, "wait": "0s"}, }}, {"items", []map[string]any{{"name": "a"}}, []map[string]any{ {"name": "a", "n": 0, "wait": "0s"}, }}, - {"items", []testItem{{Name: "a", N: 4}}, []map[string]any{ + {"items", []testItem{{ + Name: "a", + N: 4, + }}, []map[string]any{ {"name": "a", "n": 4, "wait": "0s"}, }}, {"items", "", []map[string]any(nil)}, @@ -318,9 +326,9 @@ func TestSetValueErrors(t *testing.T) { {"on", "maybe", "unable to convert"}, {"dur", "soon", "unable to convert to duration"}, {"items", "not json", "unable to convert to a list of config.testItem"}, - {"items", `[{"bogus": 1}]`, `item 0: unknown configuration key "bogus"`}, + {"items", `[{"bogus": 1}]`, `item 0: config: unknown configuration key "bogus"`}, {"items", `[{"n": "x"}]`, "item 0: n: unable to convert"}, - {"items", []any{"a"}, "item 0: unsupported type string"}, + {"items", []any{"a"}, "item 0: config: unsupported type string"}, {"items", 5, "unsupported type int"}, {"size", "lots", "unable to convert to byte size"}, {"size", "-1KB", "unable to convert to byte size"}, @@ -344,7 +352,9 @@ func TestSetValueErrors(t *testing.T) { ) } if !reflect.DeepEqual(s, before) { - t.Errorf("setValue(%q, %#v) modified the section on error", tt.key, tt.value) + t.Errorf( + "setValue(%q, %#v) modified the section on error", tt.key, tt.value, + ) } } } diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index bf57171..9aa3084 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -33,7 +33,8 @@ func WithLogger(logger *slog.Logger) Option { } // logger returns the logger Run logs to: the given one, or slog.Default(). -// It isn't tagged here; each worker adds its source. +// It isn't tagged here, as it is also given to the clients the workers +// create, which add their own component; see tagged. func (opts options) logger() *slog.Logger { if opts.log == nil { return slog.Default() @@ -41,8 +42,15 @@ func (opts options) logger() *slog.Logger { return opts.log } -// runFunc runs one source's worker until ctx is cancelled, logging to -// logger. +// tagged returns logger tagged with the fetcher's component, for the lines +// Run and its workers log themselves. +func tagged(logger *slog.Logger) *slog.Logger { + return logger.With("component", "fetcher") +} + +// runFunc runs one source's worker until ctx is cancelled. logger is +// untagged: the worker tags its own lines with tagged, and passes logger as +// it is to any client it creates. type runFunc func( ctx context.Context, logger *slog.Logger, @@ -68,7 +76,7 @@ func Run( for _, opt := range opts { opt(&settings) } - logger := settings.logger().With("component", "fetcher") + logger := settings.logger() var ( wg sync.WaitGroup @@ -76,15 +84,16 @@ func Run( errs []error ) - start := func( - name string, - run runFunc, - ) { + start := func(name string, run runFunc) { wg.Add(1) go func() { defer wg.Done() if err := run(ctx, logger, conf, since); err != nil { - logger.Error("source stopped, shutting down", "source", name, "error", err) + tagged(logger).Error( + "source stopped, shutting down", + "source", name, + "error", err, + ) mu.Lock() errs = append(errs, fmt.Errorf("%s: %w", name, err)) mu.Unlock() @@ -104,5 +113,6 @@ func Run( } wg.Wait() + return errors.Join(errs...) } diff --git a/internal/fetcher/fetcher_test.go b/internal/fetcher/fetcher_test.go index 54ce736..f1c7735 100644 --- a/internal/fetcher/fetcher_test.go +++ b/internal/fetcher/fetcher_test.go @@ -20,7 +20,7 @@ import ( func TestMain(m *testing.M) { // Keep worker and fetch log lines out of the test output. - slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + slog.SetDefault(slog.New(slog.DiscardHandler)) os.Exit(m.Run()) } @@ -38,18 +38,22 @@ const onePackage = `{ const emptyPackage = `{"uri": "x", "releases": []}` // ftsServer serves body (with status) for every request and counts requests. -func ftsServer(t *testing.T, status int, body string) (*httptest.Server, *atomic.Int32) { +func ftsServer( + t *testing.T, status int, body string, +) (*httptest.Server, *atomic.Int32) { t.Helper() var hits atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - if r.URL.Path != "/ocdsReleasePackages" { - t.Errorf("unexpected path %q", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - io.WriteString(w, body) - })) + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + if r.URL.Path != "/ocdsReleasePackages" { + t.Errorf("unexpected path %q", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + io.WriteString(w, body) + }), + ) t.Cleanup(srv.Close) return srv, &hits } @@ -83,7 +87,8 @@ func TestFetchFunctions(t *testing.T) { "nil", }, {"no releases", http.StatusOK, emptyPackage, func(err error) bool { - return errors.Is(err, ErrNoNewReleases) && errors.Is(err, util.ErrWorkerNoWork) + return errors.Is(err, ErrNoNewReleases) && + errors.Is(err, util.ErrWorkerNoWork) }, "ErrNoNewReleases wrapping ErrWorkerNoWork"}, {"bad body", http.StatusInternalServerError, "oops", func(err error) bool { return err != nil && !errors.Is(err, util.ErrWorkerNoWork) @@ -119,8 +124,11 @@ func TestFetchUsesConfiguredAPIRoot(t *testing.T) { fetchFindAOpportunity(context.Background(), slog.Default(), conf, time.Time{}) if fatHits.Load() != 1 || mockHits.Load() != 0 { - t.Fatalf("FindAOpportunity hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + t.Fatalf( + "FindAOpportunity hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load(), + ) } + fetchFTSMock(context.Background(), slog.Default(), conf, time.Time{}) if fatHits.Load() != 1 || mockHits.Load() != 1 { t.Fatalf("FTSMock hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) @@ -144,9 +152,12 @@ func TestFetchCancelled(t *testing.T) { func TestSleepCancelled(t *testing.T) { for _, block := range []bool{false, true} { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + ctx, cancel := context.WithTimeout( + context.Background(), 10*time.Millisecond, + ) defer cancel() - if err := sleep(ctx, block, time.Hour); !errors.Is(err, context.DeadlineExceeded) { + err := sleep(ctx, block, time.Hour) + if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("block=%v: err = %v, want context.DeadlineExceeded", block, err) } } @@ -198,7 +209,9 @@ func TestRunWorkersRejectZeroInterval(t *testing.T) { "Sleeper": runSleeper, } for name, run := range runs { - if err := run(context.Background(), slog.Default(), conf, time.Time{}); err == nil { + if err := run( + context.Background(), slog.Default(), conf, time.Time{}, + ); err == nil { t.Errorf("%s: expected an error for a zero interval", name) } } @@ -228,7 +241,9 @@ func TestRunLogsSourceErrors(t *testing.T) { t.Errorf("returned error %v; want it to include %s", err, name) } } - if n := strings.Count(buf.String(), "interval must be greater than 0"); n != 3 { + if n := strings.Count( + buf.String(), "interval must be greater than 0", + ); n != 3 { t.Errorf("logged %d interval errors; want 3: %s", n, &buf) } } @@ -236,11 +251,14 @@ func TestRunLogsSourceErrors(t *testing.T) { func TestRunNoSourcesEnabled(t *testing.T) { done := make(chan struct{}) go func() { - if err := Run(context.Background(), &config.JobSearchConfig{}, time.Time{}); err != nil { + if err := Run( + context.Background(), &config.JobSearchConfig{}, time.Time{}, + ); err != nil { t.Errorf("Run = %v; want nil", err) } close(done) }() + select { case <-done: case <-time.After(time.Second): @@ -275,6 +293,7 @@ func TestRunShutsDownOnWorkerError(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("Run did not shut down after a worker failed") } + // FTSMock either polled once before the shutdown or never started a run. if n := mockHits.Load(); n > 1 { t.Errorf("FTSMock polled %d times; want at most 1", n) @@ -307,7 +326,9 @@ func TestRunEnabledSources(t *testing.T) { deadline := time.Now().Add(5 * time.Second) for fatHits.Load() == 0 || mockHits.Load() == 0 { if time.Now().After(deadline) { - t.Fatalf("sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + t.Fatalf( + "sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load(), + ) } time.Sleep(5 * time.Millisecond) } diff --git a/internal/fetcher/worker_findatender.go b/internal/fetcher/worker_findatender.go index eb2dfd1..519acdb 100644 --- a/internal/fetcher/worker_findatender.go +++ b/internal/fetcher/worker_findatender.go @@ -21,7 +21,7 @@ func runFindAOpportunity( since time.Time, ) error { worker := util.Worker{ - Logger: logger, + Logger: tagged(logger), Name: "FindAOpportunity", Interval: conf.Sources.FindATender.Interval, Task: func(ctx context.Context, since time.Time) error { @@ -40,15 +40,16 @@ func fetchFindAOpportunity( conf *config.JobSearchConfig, since time.Time, ) error { - opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) { - findATender := findatender.New( - conf, - conf.Sources.FindATender.APIRoot, - findatender.WithLogger(logger), - ) - defer findATender.Shutdown() - return findATender.With(ctx).FetchAll(since, true) - }) + opportunities, err := util.AwaitTask( + ctx, + func() ([]*types.Opportunity, error) { + findATender := findatender.New( + conf, conf.Sources.FindATender.APIRoot, findatender.WithLogger(logger), + ) + defer findATender.Shutdown() + return findATender.With(ctx).FetchAll(since, true) + }, + ) if errors.Is(err, findatender.ErrNoReleases) { return ErrNoNewReleases } @@ -56,7 +57,7 @@ func fetchFindAOpportunity( return err } - logger.Info("fetched opportunities", "count", len(opportunities)) + tagged(logger).Info("fetched opportunities", "count", len(opportunities)) return nil } diff --git a/internal/fetcher/worker_ftsmock.go b/internal/fetcher/worker_ftsmock.go index c08f840..faee530 100644 --- a/internal/fetcher/worker_ftsmock.go +++ b/internal/fetcher/worker_ftsmock.go @@ -21,7 +21,7 @@ func runFTSMock( since time.Time, ) error { worker := util.Worker{ - Logger: logger, + Logger: tagged(logger), Name: "FTSMock", Interval: conf.Sources.FTSMock.Interval, Task: func(ctx context.Context, since time.Time) error { @@ -40,15 +40,16 @@ func fetchFTSMock( conf *config.JobSearchConfig, since time.Time, ) error { - opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) { - findATender := findatender.New( - conf, - conf.Sources.FTSMock.APIRoot, - findatender.WithLogger(logger), - ) - defer findATender.Shutdown() - return findATender.With(ctx).FetchAll(since, true) - }) + opportunities, err := util.AwaitTask( + ctx, + func() ([]*types.Opportunity, error) { + findATender := findatender.New( + conf, conf.Sources.FTSMock.APIRoot, findatender.WithLogger(logger), + ) + defer findATender.Shutdown() + return findATender.With(ctx).FetchAll(since, true) + }, + ) if errors.Is(err, findatender.ErrNoReleases) { return ErrNoNewReleases } @@ -56,7 +57,7 @@ func fetchFTSMock( return err } - logger.Info("fetched opportunities", "count", len(opportunities)) + tagged(logger).Info("fetched opportunities", "count", len(opportunities)) return nil } diff --git a/internal/fetcher/worker_sleeper.go b/internal/fetcher/worker_sleeper.go index b72b272..10e40f2 100644 --- a/internal/fetcher/worker_sleeper.go +++ b/internal/fetcher/worker_sleeper.go @@ -22,15 +22,14 @@ func runSleeper( ) error { blockShutdown := conf.Sources.Sleeper.BlockShutdown worker := util.Worker{ - Logger: logger, + Logger: tagged(logger), Name: "Sleeper", Interval: conf.Sources.Sleeper.Interval, Task: func(taskCtx context.Context, since time.Time) error { if blockShutdown { return sleep(taskCtx, true, sleepDuration) - } else { - return sleep(ctx, false, sleepDuration) } + return sleep(ctx, false, sleepDuration) }, } return worker.Run(ctx, since) @@ -47,8 +46,10 @@ func sleep( time.Sleep(duration) return nil } + timer := time.NewTimer(duration) defer timer.Stop() + select { case <-timer.C: return nil diff --git a/internal/findatender/api.go b/internal/findatender/api.go index 1ffb559..40a194b 100644 --- a/internal/findatender/api.go +++ b/internal/findatender/api.go @@ -22,17 +22,17 @@ import ( // Errors returned when fetching from the API. var ( // ErrNoNext means a release package has no next page link. - ErrNoNext = errors.New("no next link") + ErrNoNext = errors.New("findatender: no next link") // ErrNoReleases means the API returned no releases. - ErrNoReleases = errors.New("no releases") + ErrNoReleases = errors.New("findatender: no releases") // ErrRateLimitExceeded means the API refused a request with 429 Too Many // Requests, and waiting for it was not allowed or did not help. - ErrRateLimitExceeded = errors.New("rate limit has been exceeded") + ErrRateLimitExceeded = errors.New("findatender: rate limit has been exceeded") // ErrFetchCancelled means the fetch's context was cancelled while waiting // out a rate limit. - ErrFetchCancelled = errors.New("fetch cancelled") + ErrFetchCancelled = errors.New("findatender: fetch cancelled") // ErrUnexpectedStatus wraps an API response status other than 200 OK. - ErrUnexpectedStatus = errors.New("unexpected response status") + ErrUnexpectedStatus = errors.New("findatender: unexpected response status") ) // RequestOptions selects the releases a fetch asks for. @@ -95,7 +95,7 @@ func repairDates(raw json.RawMessage) (json.RawMessage, error) { if _, err := decoder.Token(); !errors.Is(err, io.EOF) { return nil, errors.New( - "repairing dates: unexpected data after the JSON value", + "findatender: repairing dates: unexpected data after the JSON value", ) } @@ -106,8 +106,8 @@ func repairDates(raw json.RawMessage) (json.RawMessage, error) { return json.Marshal(doc) } -// recursiveRepairDates fixes date fields in value in place and reports whether it -// changed anything. +// recursiveRepairDates fixes date fields in value in place and reports whether +// it changed anything. func recursiveRepairDates(value any) bool { changed := false switch typed := value.(type) { @@ -155,10 +155,12 @@ func releaseIDs(raw json.RawMessage) (ocid, id string, err error) { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.UseNumber() // a numeric id keeps its exact digits - if token, err := decoder.Token(); err != nil { + token, err := decoder.Token() + if err != nil { return "", "", fmt.Errorf("reading release: %w", err) - } else if token != json.Delim('{') { - return "", "", errors.New("reading release: not a JSON object") + } + if token != json.Delim('{') { + return "", "", errors.New("findatender: reading release: not a JSON object") } var haveOCID, haveID bool @@ -205,7 +207,7 @@ func decodeReleaseID(decoder *json.Decoder) (string, error) { case json.Number: return typed.String(), nil default: - return "", fmt.Errorf("reading release id: unexpected %T", typed) + return "", fmt.Errorf("findatender: reading release id: unexpected %T", typed) } } @@ -213,11 +215,11 @@ func decodeReleaseID(decoder *json.Decoder) (string, error) { func missingIDsError(haveOCID, haveID bool) error { switch { case !haveOCID && !haveID: - return errors.New("reading release: no ocid or id") + return errors.New("findatender: reading release: no ocid or id") case !haveOCID: - return errors.New("reading release: no ocid") + return errors.New("findatender: reading release: no ocid") default: - return errors.New("reading release: no id") + return errors.New("findatender: reading release: no id") } } @@ -227,8 +229,7 @@ func missingIDsError(haveOCID, haveID bool) error { // // This method takes ownership of the response body and closes it when done. func safeUnpackReleases( - logger *slog.Logger, - response *resty.Response, + logger *slog.Logger, response *resty.Response, ) (*ocds.ReleasePackage, error) { var releasePackage safeReleasePackage @@ -251,7 +252,8 @@ func safeUnpackReleases( } // Correct PublishedDate of "" and "2006-01-02" - if releasePackage.PublishedDate != nil && len(*releasePackage.PublishedDate) > 0 { + if releasePackage.PublishedDate != nil && + *releasePackage.PublishedDate != "" { published, err := time.Parse(time.RFC3339, *releasePackage.PublishedDate) if err != nil { published, err = time.Parse(time.DateOnly, *releasePackage.PublishedDate) @@ -304,9 +306,9 @@ func closeBody(res *resty.Response) { } } -// fetchURI performs a single GET of uri, using findATender's context, and traces it at -// debug level. Unlike the method of the same name it does not check the -// response status. +// fetchURI performs a single GET of uri, using findATender's context, and +// traces it at debug level. Unlike the method of the same name it does not +// check the response status. func fetchURI(findATender *FindATender, uri string) (*resty.Response, error) { withTrace := findATender.config.Logging.LogLevel() <= slog.LevelDebug logger := findATender.logger() @@ -345,17 +347,21 @@ func logTrace(logger *slog.Logger, res *resty.Response) { if logging.IsJSONHandler(logger.Handler()) { trace = res.Request.TraceInfo().JSON() } - logger.Debug("fetched", "trace", trace, "status", res.Status(), "response", res.String()) + logger.Debug( + "fetched", + "trace", trace, + "status", res.Status(), + "response", res.String(), + ) } -// fetchURI performs a fetch against the API. If the API returns with a rate limit -// reached response and allowRateLimit is true, then wait and retry like a good -// client. If allowRateLimit is false, return to the caller informing them of -// the rate limit exceeded error. If, after waiting, the API still thinks we +// fetchURI performs a fetch against the API. If the API returns with a rate +// limit reached response and allowRateLimit is true, then wait and retry like a +// good client. If allowRateLimit is false, return to the caller informing them +// of the rate limit exceeded error. If, after waiting, the API still thinks we // have exceeded the rate limit, bail out anyway. func (ft *FindATender) fetchURI( - uri string, - allowRateLimit bool, + uri string, allowRateLimit bool, ) (*resty.Response, error) { res, err := fetchURI(ft, uri) if err != nil { @@ -381,7 +387,10 @@ func (ft *FindATender) fetchURI( } retryAfter := time.Duration(retryAfterSeconds) * time.Second - ft.logger().Warn("api rate limit hit, waiting as asked", "retry_after", retryAfter) + ft.logger().Warn( + "api rate limit hit, waiting as asked", + "retry_after", retryAfter, + ) if rateSleep(ctx, retryAfter) != nil { return nil, ErrFetchCancelled } @@ -430,10 +439,9 @@ func (ft *FindATender) fetchWithOpts( // ErrNoNext if there is no "Next" link; the caller can decide whether asking // for it was an error. func (ft *FindATender) fetchNext( - releasePackage *ocds.ReleasePackage, - allowRateLimit bool, + releasePackage *ocds.ReleasePackage, allowRateLimit bool, ) (*resty.Response, error) { - if releasePackage.Links == nil || len(releasePackage.Links.Next) == 0 { + if releasePackage.Links == nil || releasePackage.Links.Next == "" { return nil, ErrNoNext } return ft.fetchURI(releasePackage.Links.Next, allowRateLimit) @@ -478,16 +486,19 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) { // one cannot loop forever. seen := make(map[string]bool) for { - if releasePackage.Links != nil && len(releasePackage.Links.Next) > 0 { + if releasePackage.Links != nil && releasePackage.Links.Next != "" { if seen[releasePackage.Links.Next] { - ft.logger().Warn("stopping at repeated next link", "link", releasePackage.Links.Next) + ft.logger().Warn( + "stopping at repeated next link", + "link", releasePackage.Links.Next, + ) break } seen[releasePackage.Links.Next] = true } res, err := ft.fetchNext(releasePackage, opts.AllowRateLimit) - if err == ErrNoNext { + if errors.Is(err, ErrNoNext) { break } if err != nil { @@ -506,4 +517,4 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) { } return releases, nil -} \ No newline at end of file +} diff --git a/internal/findatender/api_test.go b/internal/findatender/api_test.go index f09f522..86fd3a6 100644 --- a/internal/findatender/api_test.go +++ b/internal/findatender/api_test.go @@ -34,7 +34,7 @@ func testConfig(level string) *config.JobSearchConfig { func quietLogs(t *testing.T, h slog.Handler) { t.Helper() if h == nil { - h = slog.NewTextHandler(io.Discard, nil) + h = slog.DiscardHandler } prev := slog.Default() slog.SetDefault(slog.New(h)) @@ -44,13 +44,18 @@ func quietLogs(t *testing.T, h slog.Handler) { } // newTestFT starts h on a test server and returns a FindATender pointed at it. -func newTestFT(t *testing.T, level string, h http.HandlerFunc) (*FindATender, *httptest.Server) { +func newTestFT( + t *testing.T, level string, h http.HandlerFunc, +) (*FindATender, *httptest.Server) { t.Helper() quietLogs(t, nil) + srv := httptest.NewServer(h) t.Cleanup(srv.Close) + ft := New(testConfig(level), srv.URL) t.Cleanup(ft.Shutdown) + return ft, srv } @@ -161,22 +166,40 @@ func TestFetchURIRateLimit(t *testing.T) { }{ {"not allowed", "0", false, ErrRateLimitExceeded, "", 1}, {"allowed, retries", "0", true, nil, "", 2}, - {"allowed, bad retry-after", "soon", true, nil, "unknown retry-after value", 1}, - {"allowed, missing retry-after", "", true, nil, "unknown retry-after value", 1}, + { + "allowed, bad retry-after", + "soon", + true, + nil, + "unknown retry-after value", + 1, + }, + { + "allowed, missing retry-after", + "", + true, + nil, + "unknown retry-after value", + 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var hits atomic.Int32 - ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) == 1 { - if tt.retryAfter != "" { - w.Header().Set("Retry-After", tt.retryAfter) + ft, srv := newTestFT( + t, + "warn", + func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + if tt.retryAfter != "" { + w.Header().Set("Retry-After", tt.retryAfter) + } + w.WriteHeader(http.StatusTooManyRequests) + return } - w.WriteHeader(http.StatusTooManyRequests) - return - } - io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) - }) + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }, + ) res, err := ft.fetchURI(srv.URL+"/x", tt.allowRateLimit) switch { @@ -217,7 +240,8 @@ func TestFetchURIRateLimitCancelled(t *testing.T) { }() start := time.Now() - if _, err := withCtx.fetchURI(srv.URL, true); !errors.Is(err, ErrFetchCancelled) { + _, err := withCtx.fetchURI(srv.URL, true) + if !errors.Is(err, ErrFetchCancelled) { t.Fatalf("err = %v, want ErrFetchCancelled", err) } if time.Since(start) > 5*time.Second { @@ -233,7 +257,8 @@ func TestFetchURIRateLimitTwice(t *testing.T) { w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) }) - if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrRateLimitExceeded) { + _, err := ft.fetchURI(srv.URL, true) + if !errors.Is(err, ErrRateLimitExceeded) { t.Fatalf("err = %v, want ErrRateLimitExceeded", err) } if n := hits.Load(); n != 2 { @@ -247,10 +272,14 @@ func TestFetchURIUnexpectedStatus(t *testing.T) { http.StatusNotFound, http.StatusInternalServerError, } { t.Run(http.StatusText(status), func(t *testing.T) { - ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(status) - io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) - }) + ft, srv := newTestFT( + t, + "warn", + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }, + ) _, err := ft.fetchURI(srv.URL, true) if !errors.Is(err, ErrUnexpectedStatus) { t.Fatalf("err = %v, want ErrUnexpectedStatus", err) @@ -273,7 +302,8 @@ func TestFetchURIRateLimitThenError(t *testing.T) { } w.WriteHeader(http.StatusServiceUnavailable) }) - if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrUnexpectedStatus) { + _, err := ft.fetchURI(srv.URL, true) + if !errors.Is(err, ErrUnexpectedStatus) { t.Fatalf("err = %v, want ErrUnexpectedStatus", err) } } @@ -314,9 +344,13 @@ func TestFetchURIDebugTrace(t *testing.T) { } for name, mk := range handlers { t.Run(name, func(t *testing.T) { - ft, _ := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) { - io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) - }) + ft, _ := newTestFT( + t, + "debug", + func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }, + ) var logs strings.Builder quietLogs(t, mk(&logs)) @@ -343,8 +377,12 @@ func TestFetchURIDebugTrace(t *testing.T) { } func TestFetchURIDebugTraceConnectionError(t *testing.T) { - ft, srv := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) { - }) + ft, srv := newTestFT( + t, + "debug", + func(w http.ResponseWriter, r *http.Request) { + }, + ) srv.Close() ft.httpClient.SetRetryCount(0) if _, err := fetchURI(ft, srv.URL); err == nil { @@ -419,13 +457,18 @@ func TestSafeUnpackReleases(t *testing.T) { {"ocid": "ocds-3", "id": "3", "date": "2026-10-08", "tag": [], "initiationType": "tender"} ] }` + pkg, err := safeUnpackReleases(slog.Default(), responseFor(t, body)) if err != nil { t.Fatal(err) } - if pkg.URI != "https://example/pkg" || pkg.Version != "1.1" || len(pkg.Extensions) != 1 || - pkg.Publisher == nil || pkg.Publisher.Name != "FTS" || pkg.License != "OGL" || - pkg.PublicationPolicy != "policy" || pkg.Links == nil || pkg.Links.Next != "https://example/next" { + + copied := pkg.URI == "https://example/pkg" && pkg.Version == "1.1" && + len(pkg.Extensions) == 1 && pkg.Publisher != nil && + pkg.Publisher.Name == "FTS" && pkg.License == "OGL" && + pkg.PublicationPolicy == "policy" && pkg.Links != nil && + pkg.Links.Next == "https://example/next" + if !copied { t.Errorf("package fields not copied: %+v", pkg) } if pkg.PublishedDate == nil || @@ -434,7 +477,9 @@ func TestSafeUnpackReleases(t *testing.T) { } if len(pkg.Releases) != 2 || pkg.Releases[0].OCID != "ocds-1" || pkg.Releases[1].OCID != "ocds-3" { - t.Errorf("releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases) + t.Errorf( + "releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases, + ) } } @@ -446,8 +491,16 @@ func TestSafeUnpackReleasesPublishedDate(t *testing.T) { }{ {"omitted", "", nil}, {"empty string", `""`, nil}, - {"RFC3339", `"2026-10-08T12:00:00Z"`, ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC))}, - {"date only", `"2026-10-08"`, ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC))}, + { + "RFC3339", + `"2026-10-08T12:00:00Z"`, + ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)), + }, + { + "date only", + `"2026-10-08"`, + ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC)), + }, {"unparseable", `"yesterday"`, nil}, } for _, tt := range tests { @@ -463,7 +516,8 @@ func TestSafeUnpackReleasesPublishedDate(t *testing.T) { switch { case tt.want == nil && pkg.PublishedDate != nil: t.Errorf("publishedDate = %v, want nil", pkg.PublishedDate) - case tt.want != nil && (pkg.PublishedDate == nil || !pkg.PublishedDate.Equal(*tt.want)): + case tt.want != nil && (pkg.PublishedDate == nil || + !pkg.PublishedDate.Equal(*tt.want)): t.Errorf("publishedDate = %v, want %v", pkg.PublishedDate, tt.want) } }) @@ -480,7 +534,9 @@ func TestSafeUnpackReleasesInvalidJSON(t *testing.T) { "publisher array": `{"publisher": []}`, } { t.Run(name, func(t *testing.T) { - if _, err := safeUnpackReleases(slog.Default(), responseFor(t, body)); err == nil { + if _, err := safeUnpackReleases( + slog.Default(), responseFor(t, body), + ); err == nil { t.Error("expected a decode error") } }) @@ -489,18 +545,23 @@ func TestSafeUnpackReleasesInvalidJSON(t *testing.T) { // pagedServer serves pages[i] for ?page=i (page 0 at the release package // path), linking each page to the next. -func pagedServer(t *testing.T, pages ...[]string) (*FindATender, *atomic.Int32) { +func pagedServer( + t *testing.T, pages ...[]string, +) (*FindATender, *atomic.Int32) { t.Helper() var hits atomic.Int32 var srvURL string ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { hits.Add(1) + i := 0 fmt.Sscan(r.URL.Query().Get("page"), &i) + next := "" if i+1 < len(pages) { next = fmt.Sprintf("%s/ocdsReleasePackages?page=%d", srvURL, i+1) } + io.WriteString(w, packageJSON(next, pages[i]...)) }) srvURL = srv.URL @@ -513,12 +574,14 @@ func TestFetchOnce(t *testing.T) { []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")}, []string{releaseJSON("ocds-3", "1")}, ) + releases, err := ft.fetchOnce(RequestOptions{ Since: time.Now(), }) if err != nil { t.Fatal(err) } + if len(releases) != 2 { t.Errorf("got %d releases, want 2 (first page only)", len(releases)) } @@ -551,10 +614,7 @@ func TestFetchOnceErrors(t *testing.T) { RequestOptions{ Since: time.Now(), }, - ); !errors.Is( - err, - ErrRateLimitExceeded, - ) { + ); !errors.Is(err, ErrRateLimitExceeded) { t.Errorf("rate limited: err = %v, want ErrRateLimitExceeded", err) } } @@ -566,12 +626,14 @@ func TestFetchAll(t *testing.T) { []string{}, []string{releaseJSON("ocds-1", "2")}, ) + releases, err := ft.fetchAll(RequestOptions{ Since: time.Now(), }) if err != nil { t.Fatal(err) } + var ids []string for _, r := range releases { ids = append(ids, r.OCID+"/"+r.ID) @@ -600,10 +662,7 @@ func TestFetchAllErrors(t *testing.T) { RequestOptions{ Since: time.Now(), }, - ); !errors.Is( - err, - ErrRateLimitExceeded, - ) { + ); !errors.Is(err, ErrRateLimitExceeded) { t.Errorf("first page rate limited: err = %v", err) } @@ -627,13 +686,19 @@ func TestFetchAllErrors(t *testing.T) { } { t.Run(name, func(t *testing.T) { var srvURL string - ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("page") == "" { - io.WriteString(w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1"))) - return - } - fail(w, r) - }) + ft, srv := newTestFT( + t, + "warn", + func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + io.WriteString( + w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1")), + ) + return + } + fail(w, r) + }, + ) srvURL = srv.URL if _, err := ft.fetchAll(RequestOptions{ Since: time.Now(), @@ -663,17 +728,22 @@ func TestFetchAllRepeatedNextLink(t *testing.T) { t.Run(name, func(t *testing.T) { var hits atomic.Int32 var srvURL string - ft, srv := newTestFT(t, "error", func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) > 10 { - t.Error("fetchAll did not stop following next links") - w.WriteHeader(http.StatusBadRequest) - return - } - page := r.URL.Query().Get("page") - io.WriteString(w, packageJSON( - nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"), - )) - }) + ft, srv := newTestFT( + t, + "error", + func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) > 10 { + t.Error("fetchAll did not stop following next links") + w.WriteHeader(http.StatusBadRequest) + return + } + page := r.URL.Query().Get("page") + body := packageJSON( + nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"), + ) + io.WriteString(w, body) + }, + ) srvURL = srv.URL releases, err := ft.fetchAll(RequestOptions{ @@ -683,7 +753,11 @@ func TestFetchAllRepeatedNextLink(t *testing.T) { t.Fatal(err) } if len(releases) != int(hits.Load()) { - t.Errorf("releases = %d, want one per request (%d)", len(releases), hits.Load()) + t.Errorf( + "releases = %d, want one per request (%d)", + len(releases), + hits.Load(), + ) } }) } @@ -730,23 +804,28 @@ func TestUnmarshalLenientDates(t *testing.T) { t.Errorf("date = %v", got.Date) } if got.PublishedDate != nil { - t.Errorf("empty publishedDate should decode as nil, got %v", got.PublishedDate) + t.Errorf( + "empty publishedDate should decode as nil, got %v", got.PublishedDate, + ) } + wantStart := time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600)) if start := got.Tender.TenderPeriod.StartDate; start == nil || - !start.Equal(time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600))) { + !start.Equal(wantStart) { t.Errorf("full timestamp changed: %v", start) } - if end := got.Tender.TenderPeriod.EndDate; end == nil || !end.Equal(midnight(2006, 11, 30)) { + if end := got.Tender.TenderPeriod.EndDate; end == nil || + !end.Equal(midnight(2006, 11, 30)) { t.Errorf("endDate = %v", end) } - if d := got.Tender.ExpressionOfInterestDeadline; d == nil || !d.Equal(midnight(2006, 10, 20)) { + if d := got.Tender.ExpressionOfInterestDeadline; d == nil || + !d.Equal(midnight(2006, 10, 20)) { t.Errorf("expressionOfInterestDeadline = %v", d) } if got.Tender.Description != "Bids close on 2006-11-30" { t.Errorf("free text changed: %q", got.Tender.Description) } - if a := got.Awards[0]; a.Date == nil || !a.Date.Equal(midnight(2006, 12, 1)) || - a.ID != "2006-12-01" { + if a := got.Awards[0]; a.Date == nil || + !a.Date.Equal(midnight(2006, 12, 1)) || a.ID != "2006-12-01" { t.Errorf("award = %+v (date should be repaired, id left alone)", a) } if got.Value != "1234567.891" { @@ -829,9 +908,27 @@ func TestReleaseIDs(t *testing.T) { {"missing ocid", `{"id":"r1"}`, "", "r1", "no ocid"}, {"missing id", `{"ocid":"ocds-1"}`, "ocds-1", "", "no id"}, {"ocid wrong type", `{"ocid":5,"id":"r1"}`, "", "", "reading release ocid"}, - {"id wrong type", `{"ocid":"ocds-1","id":true}`, "ocds-1", "", "unexpected bool"}, - {"id is an object", `{"ocid":"ocds-1","id":{}}`, "ocds-1", "", "unexpected map"}, - {"invalid id JSON", `{"ocid":"ocds-1","id":}`, "ocds-1", "", "reading release id"}, + { + "id wrong type", + `{"ocid":"ocds-1","id":true}`, + "ocds-1", + "", + "unexpected bool", + }, + { + "id is an object", + `{"ocid":"ocds-1","id":{}}`, + "ocds-1", + "", + "unexpected map", + }, + { + "invalid id JSON", + `{"ocid":"ocds-1","id":}`, + "ocds-1", + "", + "reading release id", + }, { "invalid field before ids", `{"date":nope,"ocid":"ocds-1","id":"r1"}`, @@ -839,7 +936,13 @@ func TestReleaseIDs(t *testing.T) { "", `reading release field "date"`, }, - {"truncated after ocid", `{"ocid":"ocds-1",`, "ocds-1", "", "reading release"}, + { + "truncated after ocid", + `{"ocid":"ocds-1",`, + "ocds-1", + "", + "reading release", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -852,7 +955,9 @@ func TestReleaseIDs(t *testing.T) { t.Fatalf("err = %v, want containing %q", err, tt.wantErr) } if ocid != tt.wantOCID || id != tt.wantID { - t.Errorf("got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID) + t.Errorf( + "got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID, + ) } }) } diff --git a/internal/findatender/findatender.go b/internal/findatender/findatender.go index c8c0638..fa82fa5 100644 --- a/internal/findatender/findatender.go +++ b/internal/findatender/findatender.go @@ -34,7 +34,9 @@ func WithLogger(logger *slog.Logger) Option { // New returns a client for the OCDS API at apiRoot. Requests time out after // 60 seconds and are retried once. Call Shutdown when finished with it. It // logs to slog.Default() unless given WithLogger. -func New(conf *config.JobSearchConfig, apiRoot string, opts ...Option) *FindATender { +func New( + conf *config.JobSearchConfig, apiRoot string, opts ...Option, +) *FindATender { findATender := &FindATender{ config: conf, apiRoot: apiRoot, @@ -42,9 +44,11 @@ func New(conf *config.JobSearchConfig, apiRoot string, opts ...Option) *FindATen for _, opt := range opts { opt(findATender) } + findATender.httpClient = resty.New(). - SetTimeout(60 * time.Duration(time.Second)). + SetTimeout(60 * time.Second). SetRetryCount(1) + return findATender } @@ -87,8 +91,7 @@ func (ft *FindATender) Shutdown() { // converts them to opportunities. If allowRateLimit is true, a 429 response // is waited out and retried; otherwise it fails with ErrRateLimitExceeded. func (ft *FindATender) FetchOnce( - since time.Time, - allowRateLimit bool, + since time.Time, allowRateLimit bool, ) ([]*types.Opportunity, error) { releases, err := ft.fetchOnce(RequestOptions{ Since: since, @@ -104,8 +107,7 @@ func (ft *FindATender) FetchOnce( // each page's next link, and converts them to opportunities. allowRateLimit // behaves as for FetchOnce. func (ft *FindATender) FetchAll( - since time.Time, - allowRateLimit bool, + since time.Time, allowRateLimit bool, ) ([]*types.Opportunity, error) { releases, err := ft.fetchAll(RequestOptions{ Since: since, diff --git a/internal/findatender/findatender_test.go b/internal/findatender/findatender_test.go index 3862135..2284df6 100644 --- a/internal/findatender/findatender_test.go +++ b/internal/findatender/findatender_test.go @@ -24,8 +24,13 @@ func TestNew(t *testing.T) { if ft.httpClient == nil { t.Fatal("no http client") } - if ft.httpClient.Timeout() != 60*time.Second || ft.httpClient.RetryCount() != 1 { - t.Errorf("client timeout %v, retries %d", ft.httpClient.Timeout(), ft.httpClient.RetryCount()) + if ft.httpClient.Timeout() != 60*time.Second || + ft.httpClient.RetryCount() != 1 { + t.Errorf( + "client timeout %v, retries %d", + ft.httpClient.Timeout(), + ft.httpClient.RetryCount(), + ) } if ft.ctx != nil { t.Error("New should not set a context") @@ -81,7 +86,11 @@ func opportunityIDs(ops []*types.Opportunity) []string { func TestFetchOncePublic(t *testing.T) { ft, _ := pagedServer( t, - []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-1", "2"), releaseJSON("ocds-2", "1")}, + []string{ + releaseJSON("ocds-1", "1"), + releaseJSON("ocds-1", "2"), + releaseJSON("ocds-2", "1"), + }, []string{releaseJSON("ocds-3", "1")}, ) ops, err := ft.FetchOnce(time.Now(), false) @@ -108,7 +117,9 @@ func TestFetchAllPublic(t *testing.T) { if err != nil { t.Fatal(err) } - if got := strings.Join(opportunityIDs(ops), ","); got != "ocds-1,ocds-2,ocds-3" { + if got := strings.Join( + opportunityIDs(ops), ",", + ); got != "ocds-1,ocds-2,ocds-3" { t.Errorf("opportunities = %s", got) } for _, op := range ops { diff --git a/internal/findatender/type_convert.go b/internal/findatender/type_convert.go index c2cc5c7..6952288 100644 --- a/internal/findatender/type_convert.go +++ b/internal/findatender/type_convert.go @@ -9,8 +9,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/types" ) -// ocdsGetBuyerId returns the buyer's id, or "" if the release has no buyer. -func ocdsGetBuyerId(release *ocds.Release) string { +// ocdsGetBuyerID returns the buyer's id, or "" if the release has no buyer. +func ocdsGetBuyerID(release *ocds.Release) string { if release.Buyer != nil { return string(release.Buyer.ID) } @@ -29,12 +29,12 @@ func ocdsGetBuyerName(release *ocds.Release) string { // ocdsConvertClassification converts an OCDS classification, noting whether // it is a CPV code. func ocdsConvertClassification( - classif *ocds.Classification, + classification *ocds.Classification, ) types.ClassificationCode { return types.ClassificationCode{ - IsCPV: strings.Compare(classif.Scheme, "CPV") == 0, - Code: string(classif.ID), - Classification: classif.Description, + IsCPV: strings.Compare(classification.Scheme, "CPV") == 0, + Code: string(classification.ID), + Classification: classification.Description, } } @@ -45,17 +45,19 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode { if release.Tender == nil { return classifications } - add := func(classif *ocds.Classification) { - newClassif := ocdsConvertClassification(classif) + + add := func(classification *ocds.Classification) { + converted := ocdsConvertClassification(classification) found := slices.ContainsFunc( classifications, func(code types.ClassificationCode) bool { - return code.Code == newClassif.Code + return code.Code == converted.Code }, ) if !found { - classifications = append(classifications, newClassif) + classifications = append(classifications, converted) } } + if release.Tender.Classification != nil { add(release.Tender.Classification) } @@ -63,10 +65,11 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode { if item.Classification != nil { add(item.Classification) } - for _, classif := range item.AdditionalClassifications { - add(&classif) + for _, classification := range item.AdditionalClassifications { + add(&classification) } } + return classifications } @@ -142,7 +145,7 @@ func convertOne(release *ocds.Release) *types.Opportunity { opportunity := &types.Opportunity{ UUID: release.OCID, UUIDType: "ocid", - BuyerId: ocdsGetBuyerId(release), + BuyerID: ocdsGetBuyerID(release), BuyerName: ocdsGetBuyerName(release), Source: "FindATender", } diff --git a/internal/findatender/type_convert_test.go b/internal/findatender/type_convert_test.go index e5006b5..87013a4 100644 --- a/internal/findatender/type_convert_test.go +++ b/internal/findatender/type_convert_test.go @@ -38,7 +38,7 @@ func TestOcdsGetBuyer(t *testing.T) { r := &ocds.Release{ Buyer: tt.buyer, } - if got := ocdsGetBuyerId(r); got != tt.wantID { + if got := ocdsGetBuyerID(r); got != tt.wantID { t.Errorf("buyer id = %q, want %q", got, tt.wantID) } if got := ocdsGetBuyerName(r); got != tt.wantName { @@ -101,6 +101,7 @@ func TestOcdsGetClassifications(t *testing.T) { Description: "d" + id, } } + codes := func(cs []types.ClassificationCode) []string { out := []string{} for _, c := range cs { @@ -108,6 +109,7 @@ func TestOcdsGetClassifications(t *testing.T) { } return out } + main := cpv("1") tests := []struct { name string @@ -122,13 +124,24 @@ func TestOcdsGetClassifications(t *testing.T) { {"main and item additionals, deduplicated", &ocds.Tender{ Classification: &main, Items: []ocds.Item{ - {ID: "a", AdditionalClassifications: []ocds.Classification{cpv("2"), cpv("1")}}, - {ID: "b"}, - {ID: "c", AdditionalClassifications: []ocds.Classification{cpv("3"), cpv("2")}}, + { + ID: "a", + AdditionalClassifications: []ocds.Classification{cpv("2"), cpv("1")}, + }, + { + ID: "b", + }, + { + ID: "c", + AdditionalClassifications: []ocds.Classification{cpv("3"), cpv("2")}, + }, }, }, []string{"1", "2", "3"}}, {"item additionals without main", &ocds.Tender{ - Items: []ocds.Item{{ID: "a", AdditionalClassifications: []ocds.Classification{cpv("5")}}}, + Items: []ocds.Item{{ + ID: "a", + AdditionalClassifications: []ocds.Classification{cpv("5")}, + }}, }, []string{"5"}}, } for _, tt := range tests { @@ -151,9 +164,15 @@ func TestOcdsGetClassificationsIncludesItemClassification(t *testing.T) { Scheme: "CPV", ID: "9", } - got := ocdsGetClassifications(&ocds.Release{Tender: &ocds.Tender{ - Items: []ocds.Item{{ID: "a", Classification: &item}}, - }}) + release := &ocds.Release{ + Tender: &ocds.Tender{ + Items: []ocds.Item{{ + ID: "a", + Classification: &item, + }}, + }, + } + got := ocdsGetClassifications(release) if len(got) != 1 || got[0].Code != "9" { t.Errorf("got %+v, want the item classification 9", got) } @@ -231,12 +250,21 @@ func TestOcdsRelease(t *testing.T) { } got := ocdsRelease(r) - if got.UUID != "ocds-x-1" || got.ReleaseID != "001" || !got.ReleaseDate.Equal(date) { + if got.UUID != "ocds-x-1" || got.ReleaseID != "001" || + !got.ReleaseDate.Equal(date) { t.Errorf("release = %+v", got) } want := []types.OpportunityReleaseTag{ - {UUID: "ocds-x-1", ReleaseID: "001", Tag: "planning"}, - {UUID: "ocds-x-1", ReleaseID: "001", Tag: "tender"}, + { + UUID: "ocds-x-1", + ReleaseID: "001", + Tag: "planning", + }, + { + UUID: "ocds-x-1", + ReleaseID: "001", + Tag: "tender", + }, } if !reflect.DeepEqual(got.Tags, want) { t.Errorf("tags = %+v, want %+v", got.Tags, want) @@ -250,7 +278,8 @@ func TestOcdsRelease(t *testing.T) { t.Errorf("round tripped release = %+v", back) } - if empty := ocdsRelease(&ocds.Release{}); empty.Tags == nil || len(empty.Tags) != 0 { + if empty := ocdsRelease(&ocds.Release{}); empty.Tags == nil || + len(empty.Tags) != 0 { t.Errorf("no tags should give an empty, non-nil slice, got %#v", empty.Tags) } } @@ -289,7 +318,9 @@ func TestConvertOne(t *testing.T) { Value: &ocds.Value{ Amount: &amount, }, - Documents: []ocds.Document{{ID: "doc"}}, + Documents: []ocds.Document{{ + ID: "doc", + }}, }, } @@ -300,7 +331,7 @@ func TestConvertOne(t *testing.T) { }{ {"UUID", got.UUID, "ocds-x-1"}, {"UUIDType", got.UUIDType, "ocid"}, - {"BuyerId", got.BuyerId, "GB-1"}, + {"BuyerID", got.BuyerID, "GB-1"}, {"BuyerName", got.BuyerName, "Council"}, {"Title", got.Title, "Website"}, {"Status", got.Status, "active"}, @@ -344,10 +375,11 @@ func TestConvertOneMinimal(t *testing.T) { ID: "1", Tender: tt.tender, }) - if got.UUID != "ocds-x-2" || got.BuyerId != "" || got.BuyerName != "" { + if got.UUID != "ocds-x-2" || got.BuyerID != "" || got.BuyerName != "" { t.Errorf("got %+v", got) } - if got.GrossValue != 0 || got.StartDate != nil || got.EndDate != nil || got.HasDocuments { + if got.GrossValue != 0 || got.StartDate != nil || got.EndDate != nil || + got.HasDocuments { t.Errorf("unexpected tender fields: %+v", got) } if len(got.Releases) != 1 { @@ -366,18 +398,46 @@ func TestConvert(t *testing.T) { return time.Date(2026, 1, d, 0, 0, 0, 0, time.UTC) } releases := []ocds.Release{ - {OCID: "ocds-b", ID: "1", Date: day(1), Tender: &ocds.Tender{ - Title: "B", - }}, - {OCID: "ocds-a", ID: "2", Date: day(3), Tender: &ocds.Tender{ - Title: "A latest", Lots: []ocds.Lot{{ID: "1"}, {ID: "2"}}, - }}, - {OCID: "ocds-a", ID: "1", Date: day(2), Tender: &ocds.Tender{ - Title: "A first", - }}, - {OCID: "ocds-a", ID: "3", Date: day(1), Tender: &ocds.Tender{ - Title: "A oldest", - }}, + { + OCID: "ocds-b", + ID: "1", + Date: day(1), + Tender: &ocds.Tender{ + Title: "B", + }, + }, + { + OCID: "ocds-a", + ID: "2", + Date: day(3), + Tender: &ocds.Tender{ + Title: "A latest", + Lots: []ocds.Lot{ + { + ID: "1", + }, + { + ID: "2", + }, + }, + }, + }, + { + OCID: "ocds-a", + ID: "1", + Date: day(2), + Tender: &ocds.Tender{ + Title: "A first", + }, + }, + { + OCID: "ocds-a", + ID: "3", + Date: day(1), + Tender: &ocds.Tender{ + Title: "A oldest", + }, + }, } got := Convert(releases) @@ -395,7 +455,9 @@ func TestConvert(t *testing.T) { t.Errorf("ocds-a title = %q, want %q", a.Title, "A latest") } if !a.LatestReleaseDate.Equal(day(3)) { - t.Errorf("ocds-a LatestReleaseDate = %v, want %v", a.LatestReleaseDate, day(3)) + t.Errorf( + "ocds-a LatestReleaseDate = %v, want %v", a.LatestReleaseDate, day(3), + ) } if !a.MultiLot { t.Error("ocds-a MultiLot = false, want true for two lots") @@ -408,7 +470,8 @@ func TestConvert(t *testing.T) { t.Errorf("ocds-a release ids = %v, want %v", ids, want) } - if b.Title != "B" || b.MultiLot || !b.LatestReleaseDate.Equal(day(1)) || len(b.Releases) != 1 { + if b.Title != "B" || b.MultiLot || !b.LatestReleaseDate.Equal(day(1)) || + len(b.Releases) != 1 { t.Errorf("ocds-b = %+v", b) } } @@ -417,12 +480,22 @@ func TestConvert(t *testing.T) { func TestConvertSameDate(t *testing.T) { date := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) got := Convert([]ocds.Release{ - {OCID: "ocds-a", ID: "1", Date: date, Tender: &ocds.Tender{ - Title: "first", - }}, - {OCID: "ocds-a", ID: "2", Date: date, Tender: &ocds.Tender{ - Title: "second", - }}, + { + OCID: "ocds-a", + ID: "1", + Date: date, + Tender: &ocds.Tender{ + Title: "first", + }, + }, + { + OCID: "ocds-a", + ID: "2", + Date: date, + Tender: &ocds.Tender{ + Title: "second", + }, + }, }) if len(got) != 1 || got[0].Title != "second" { t.Fatalf("got %+v, want one opportunity titled %q", got, "second") @@ -446,13 +519,24 @@ func TestConvertSetsSource(t *testing.T) { } for _, o := range Convert([]ocds.Release{ - {OCID: "ocds-a", ID: "1", Tender: &ocds.Tender{ - Title: "A", - }}, - {OCID: "ocds-a", ID: "2", Tender: &ocds.Tender{ - Title: "A amended", - }}, - {OCID: "ocds-b", ID: "1"}, + { + OCID: "ocds-a", + ID: "1", + Tender: &ocds.Tender{ + Title: "A", + }, + }, + { + OCID: "ocds-a", + ID: "2", + Tender: &ocds.Tender{ + Title: "A amended", + }, + }, + { + OCID: "ocds-b", + ID: "1", + }, }) { if o.Source != want { t.Errorf("Convert: %s Source = %q, want %q", o.UUID, o.Source, want) diff --git a/internal/httpx/httpx_test.go b/internal/httpx/httpx_test.go index bbda3ac..ae9e88f 100644 --- a/internal/httpx/httpx_test.go +++ b/internal/httpx/httpx_test.go @@ -20,7 +20,8 @@ func TestWriteProblem(t *testing.T) { if ct := rec.Header().Get("Content-Type"); ct != "application/problem+json" { t.Errorf("Content-Type = %q", ct) } - if nosniff := rec.Header().Get("X-Content-Type-Options"); nosniff != "nosniff" { + nosniff := rec.Header().Get("X-Content-Type-Options") + if nosniff != "nosniff" { t.Errorf("X-Content-Type-Options = %q", nosniff) } var p Problem @@ -42,7 +43,10 @@ func TestWriteProblem(t *testing.T) { // Without a request (or a request ID) the problem has no requestId, and an // empty detail is left out. func TestWriteProblemMinimal(t *testing.T) { - for _, req := range []*http.Request{nil, httptest.NewRequest("GET", "/", nil)} { + for _, req := range []*http.Request{ + nil, + httptest.NewRequest("GET", "/", nil), + } { rec := httptest.NewRecorder() WriteProblem(rec, req, http.StatusTooManyRequests, "") var body map[string]any diff --git a/internal/httpx/problem.go b/internal/httpx/problem.go index b5eb610..5bdaf8b 100644 --- a/internal/httpx/problem.go +++ b/internal/httpx/problem.go @@ -17,7 +17,9 @@ type Problem struct { } // WriteProblem writes a problem+json error response. -func WriteProblem(w http.ResponseWriter, r *http.Request, status int, detail string) { +func WriteProblem( + w http.ResponseWriter, r *http.Request, status int, detail string, +) { problem := Problem{ Type: "about:blank", Title: http.StatusText(status), @@ -27,6 +29,7 @@ func WriteProblem(w http.ResponseWriter, r *http.Request, status int, detail str if r != nil { problem.RequestID = RequestID(r.Context()) } + w.Header().Set("Content-Type", "application/problem+json") w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(status) diff --git a/internal/logging/accesslog.go b/internal/logging/accesslog.go index 204010d..2352118 100644 --- a/internal/logging/accesslog.go +++ b/internal/logging/accesslog.go @@ -19,19 +19,19 @@ type statusRecorder struct { bytes int } -func (r *statusRecorder) WriteHeader(code int) { - if r.status == 0 { - r.status = code +func (s *statusRecorder) WriteHeader(code int) { + if s.status == 0 { + s.status = code } - r.ResponseWriter.WriteHeader(code) + s.ResponseWriter.WriteHeader(code) } -func (r *statusRecorder) Write(data []byte) (int, error) { - if r.status == 0 { - r.status = http.StatusOK +func (s *statusRecorder) Write(data []byte) (int, error) { + if s.status == 0 { + s.status = http.StatusOK } - written, err := r.ResponseWriter.Write(data) - r.bytes += written + written, err := s.ResponseWriter.Write(data) + s.bytes += written return written, err } @@ -42,6 +42,7 @@ func logField(value string) string { if value == "" { return "-" } + var builder strings.Builder for i := 0; i < len(value); i++ { char := value[i] @@ -51,16 +52,17 @@ func logField(value string) string { builder.WriteByte(char) } } + return builder.String() } // clientKey attempts to produce a client host name as a key identifier to split // requests in the log between sources. Could be extended to also check // Forwarded-For or other proxy headers -func clientKey(req *http.Request) string { - host, _, err := net.SplitHostPort(req.RemoteAddr) +func clientKey(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return req.RemoteAddr + return r.RemoteAddr } return host } @@ -68,19 +70,21 @@ func clientKey(req *http.Request) string { // AccessLog wraps next, writing a line to logger for each request in the // Apache/nginx Common Log Format, or the Combined Log Format (which adds the // referer and user agent) if combined is true. -func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { +func AccessLog( + logger *log.Logger, combined bool, next http.Handler, +) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() recorder := &statusRecorder{ ResponseWriter: w, } - next.ServeHTTP(recorder, req) + next.ServeHTTP(recorder, r) if recorder.status == 0 { recorder.status = http.StatusOK } user := "-" - if username, _, ok := req.BasicAuth(); ok && username != "" { + if username, _, ok := r.BasicAuth(); ok && username != "" { user = logField(username) } size := "-" // CLF uses "-" for an empty body @@ -89,16 +93,16 @@ func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handle } line := fmt.Sprintf( `%s - %s [%s] "%s" %d %s`, - clientKey(req), + clientKey(r), user, start.Format("02/Jan/2006:15:04:05 -0700"), - logField(req.Method+" "+req.RequestURI+" "+req.Proto), + logField(r.Method+" "+r.RequestURI+" "+r.Proto), recorder.status, size, ) if combined { line += fmt.Sprintf( - ` "%s" "%s"`, logField(req.Referer()), logField(req.UserAgent()), + ` "%s" "%s"`, logField(r.Referer()), logField(r.UserAgent()), ) } logger.Println(line) diff --git a/internal/logging/accesslog_test.go b/internal/logging/accesslog_test.go index b70fb3a..3efeab0 100644 --- a/internal/logging/accesslog_test.go +++ b/internal/logging/accesslog_test.go @@ -65,6 +65,7 @@ func TestStatusRecorder(t *testing.T) { rec := &statusRecorder{ ResponseWriter: inner, } + rec.Write([]byte("hello")) rec.Write([]byte(" world")) if rec.status != http.StatusOK || rec.bytes != 11 { diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go index 44c88cf..0a2974f 100644 --- a/internal/logging/logging_test.go +++ b/internal/logging/logging_test.go @@ -29,7 +29,7 @@ func saveGlobals(t *testing.T) { func TestLevels(t *testing.T) { saveGlobals(t) - slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + slog.SetDefault(slog.New(slog.DiscardHandler)) SetLevel(slog.LevelInfo) if IsDebugEnabled() { @@ -57,6 +57,13 @@ func TestLevels(t *testing.T) { type wrapper struct{ slog.Handler } func TestIsJSONHandler(t *testing.T) { + // The handlers' types are what is tested, so slog.DiscardHandler won't do. + //nolint:sloglint // see above + jsonWithAttrs := slog.NewJSONHandler(io.Discard, nil).WithAttrs( + []slog.Attr{slog.String("k", "v")}, + ) + + //nolint:sloglint // as above tests := []struct { name string h slog.Handler @@ -66,9 +73,7 @@ func TestIsJSONHandler(t *testing.T) { {"text", slog.NewTextHandler(io.Discard, nil), false}, {"tint", tint.NewTextHandler(io.Discard, nil), false}, {"nil", nil, false}, - {"json with attrs", slog.NewJSONHandler(io.Discard, nil).WithAttrs( - []slog.Attr{slog.String("k", "v")}, - ), true}, + {"json with attrs", jsonWithAttrs, true}, // Only slog's own JSON handler is recognised, not one wrapped in another. {"wrapped json", wrapper{ Handler: slog.NewJSONHandler(io.Discard, nil), @@ -108,7 +113,8 @@ func TestLogToFileWritesJSON(t *testing.T) { t.Fatal(err) } s := string(data) - if !strings.Contains(s, `"msg":"hello file"`) || !strings.Contains(s, `"k":"v"`) { + if !strings.Contains(s, `"msg":"hello file"`) || + !strings.Contains(s, `"k":"v"`) { t.Fatalf("log file missing entry: %s", s) } if strings.Contains(s, "hidden") { @@ -118,7 +124,9 @@ func TestLogToFileWritesJSON(t *testing.T) { func TestLogToFileBadPath(t *testing.T) { saveGlobals(t) - if err := LogToFile(filepath.Join(t.TempDir(), "missing", "app.log")); err == nil { + if err := LogToFile( + filepath.Join(t.TempDir(), "missing", "app.log"), + ); err == nil { t.Fatal("expected an error for a path in a missing directory") } } @@ -131,6 +139,7 @@ func TestLogToFileTracksFile(t *testing.T) { if !IsFileLogger() { t.Fatal("IsFileLogger() = false after LogToFile") } + if err := Shutdown(); err != nil { t.Fatal(err) } @@ -155,6 +164,7 @@ func TestLogToTerminal(t *testing.T) { if IsJSONHandler(h) { t.Fatal("terminal handler reported as JSON") } + SetLevel(slog.LevelWarn) if h.Enabled(context.Background(), slog.LevelInfo) { t.Fatal("terminal handler ignores the package level") diff --git a/internal/middleware/ratelimit.go b/internal/middleware/ratelimit.go index 054c4d0..be3b500 100644 --- a/internal/middleware/ratelimit.go +++ b/internal/middleware/ratelimit.go @@ -53,7 +53,9 @@ func RateLimit(limiter Limiter, key KeyFunc) func(http.Handler) http.Handler { seconds = 1 } w.Header().Set("Retry-After", strconv.Itoa(seconds)) - httpx.WriteProblem(w, r, http.StatusTooManyRequests, "rate limit exceeded") + httpx.WriteProblem( + w, r, http.StatusTooManyRequests, "rate limit exceeded", + ) return } next.ServeHTTP(w, r) @@ -85,10 +87,7 @@ type limiterEntry struct { // requests per second with the given burst. A janitor goroutine evicts // entries idle for longer than ttl until ctx is cancelled. func NewKeyedLimiter( - ctx context.Context, - requestsPerSecond float64, - burst int, - ttl time.Duration, + ctx context.Context, requestsPerSecond float64, burst int, ttl time.Duration, ) *KeyedLimiter { limiter := &KeyedLimiter{ limit: rate.Limit(requestsPerSecond), @@ -99,9 +98,11 @@ func NewKeyedLimiter( for i := range limiter.shards { limiter.shards[i].entries = make(map[string]*limiterEntry) } + if ttl > 0 { go limiter.janitor(ctx) } + return limiter } @@ -125,14 +126,15 @@ func (k *KeyedLimiter) Allow(key string) (bool, time.Duration) { limiter := entry.limiter shard.mu.Unlock() - res := limiter.ReserveN(now, 1) - if !res.OK() { + reservation := limiter.ReserveN(now, 1) + if !reservation.OK() { return false, time.Second } - if delay := res.DelayFrom(now); delay > 0 { - res.CancelAt(now) + if delay := reservation.DelayFrom(now); delay > 0 { + reservation.CancelAt(now) return false, delay } + return true, 0 } diff --git a/internal/middleware/ratelimit_test.go b/internal/middleware/ratelimit_test.go index e74c6a0..e7d0c24 100644 --- a/internal/middleware/ratelimit_test.go +++ b/internal/middleware/ratelimit_test.go @@ -8,17 +8,21 @@ import ( func TestRateLimit(t *testing.T) { l := NewKeyedLimiter(t.Context(), 1, 2, 0) - h := RateLimit(l, nil)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - })) + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + }) + h := RateLimit(l, nil)(next) + var last int for i := 0; i < 4; i++ { rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest("GET", "/", nil)) last = rec.Code - if last == http.StatusTooManyRequests && rec.Header().Get("Retry-After") == "" { + if last == http.StatusTooManyRequests && + rec.Header().Get("Retry-After") == "" { t.Fatal("missing Retry-After") } } + if last != http.StatusTooManyRequests { t.Fatalf("want 429, got %d", last) } diff --git a/internal/middleware/real_ip.go b/internal/middleware/real_ip.go index a93e0ce..dfc425d 100644 --- a/internal/middleware/real_ip.go +++ b/internal/middleware/real_ip.go @@ -116,7 +116,8 @@ func RealIP(opts RealIPOptions) func(http.Handler) http.Handler { break } } - if proto := forwardedProto(r.Header); proto == "http" || proto == "https" { + proto := forwardedProto(r.Header) + if proto == "http" || proto == "https" { info.Scheme = proto } } else if opts.StripUntrusted { diff --git a/internal/middleware/real_ip_test.go b/internal/middleware/real_ip_test.go index 8254724..7ac94d5 100644 --- a/internal/middleware/real_ip_test.go +++ b/internal/middleware/real_ip_test.go @@ -9,18 +9,23 @@ import ( // serveRealIP runs a request from peer with the given headers through RealIP // and returns the ClientInfo the next handler sees. -func serveRealIP(t *testing.T, opts RealIPOptions, peer string, h map[string]string) ClientInfo { +func serveRealIP( + t *testing.T, opts RealIPOptions, peer string, h map[string]string, +) ClientInfo { t.Helper() var got ClientInfo next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = Client(r) }) + req := httptest.NewRequest("GET", "/", nil) req.RemoteAddr = peer for k, v := range h { req.Header.Set(k, v) } + RealIP(opts)(next).ServeHTTP(httptest.NewRecorder(), req) + return got } @@ -32,15 +37,19 @@ func TestRealIP(t *testing.T) { Headers: defaultHeaders, } // Spoofed left-most entry is ignored; right-most untrusted wins. - c := serveRealIP(t, opts, "127.0.0.1:1234", map[string]string{ - "X-Forwarded-For": "6.6.6.6, 203.0.113.9, 127.0.0.1", "X-Forwarded-Proto": "https", - }) + headers := map[string]string{ + "X-Forwarded-For": "6.6.6.6, 203.0.113.9, 127.0.0.1", + "X-Forwarded-Proto": "https", + } + c := serveRealIP(t, opts, "127.0.0.1:1234", headers) if c.IP.String() != "203.0.113.9" || c.Scheme != "https" || !c.ViaProxy { t.Fatalf("xff: %+v", c) } - c = serveRealIP(t, opts, "127.0.0.1:1234", map[string]string{ + + headers = map[string]string{ "Forwarded": `for="[2001:db8::7]:4711";proto=https`, - }) + } + c = serveRealIP(t, opts, "127.0.0.1:1234", headers) if c.IP.String() != "2001:db8::7" { t.Fatalf("forwarded: %+v", c) } @@ -51,7 +60,12 @@ func TestUntrustedPeerHeadersIgnored(t *testing.T) { Trusted: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, Headers: defaultHeaders, } - c := serveRealIP(t, opts, "127.0.0.1:1234", map[string]string{"X-Forwarded-For": "203.0.113.9"}) + c := serveRealIP( + t, + opts, + "127.0.0.1:1234", + map[string]string{"X-Forwarded-For": "203.0.113.9"}, + ) if c.IP.String() != "127.0.0.1" || c.ViaProxy { t.Fatalf("untrusted XFF honoured: %+v", c) } diff --git a/internal/mock/findatender/catalog.go b/internal/mock/findatender/catalog.go index 30215e2..6ff32b1 100644 --- a/internal/mock/findatender/catalog.go +++ b/internal/mock/findatender/catalog.go @@ -79,33 +79,19 @@ func parseCatalog(data []byte, name string) (*Catalog, error) { func (c *Catalog) validate() error { var errs []error - for _, pool := range []struct { - name string - n int + pools := []struct { + name string + count int }{ - { - name: "buyers", - n: len(c.Buyers), - }, - { - name: "suppliers", - n: len(c.Suppliers), - }, - { - name: "subjects", - n: len(c.Subjects), - }, - { - name: "methods", - n: len(c.Methods), - }, - { - name: "classifications", - n: len(c.Classifications), - }, - } { - if pool.n == 0 { - errs = append(errs, fmt.Errorf("no %s", pool.name)) + {name: "buyers", count: len(c.Buyers)}, + {name: "suppliers", count: len(c.Suppliers)}, + {name: "subjects", count: len(c.Subjects)}, + {name: "methods", count: len(c.Methods)}, + {name: "classifications", count: len(c.Classifications)}, + } + for _, pool := range pools { + if pool.count == 0 { + errs = append(errs, fmt.Errorf("mock: no %s", pool.name)) } } @@ -136,8 +122,9 @@ func (c *Catalog) validate() error { keys = append(keys[:5], "...") } errs = append(errs, fmt.Errorf( - "%d classification key(s) not found in classifications: %s", - len(missing), strings.Join(keys, ", "), + "mock: %d classification key(s) not found in classifications: %s", + len(missing), + strings.Join(keys, ", "), )) } @@ -164,13 +151,19 @@ func (c *Catalog) subject(rng *rand.Rand) ocds.Tender { subject := clone(c.Subjects[rng.Intn(len(c.Subjects))]) tender := subject.Tender tender.Classification = c.resolveOne(subject.Classification) - tender.AdditionalClassifications = c.resolve(subject.AdditionalClassifications) + tender.AdditionalClassifications = c.resolve( + subject.AdditionalClassifications, + ) + tender.Items = nil for _, subjectItem := range subject.Items { item := subjectItem.Item - item.AdditionalClassifications = c.resolve(subjectItem.AdditionalClassifications) + item.AdditionalClassifications = c.resolve( + subjectItem.AdditionalClassifications, + ) tender.Items = append(tender.Items, item) } + return tender } @@ -195,12 +188,13 @@ func (c *Catalog) resolve(keys []string) []ocds.Classification { return out } -// clone deep-copies a value through JSON, which is all the generated types need. +// clone deep-copies a value through JSON, which is all the generated types +// need. func clone[T any](value T) T { var out T data, err := json.Marshal(value) if err == nil { - json.Unmarshal(data, &out) //nolint:errcheck,gosec // b was just marshalled from a T + json.Unmarshal(data, &out) //nolint:errcheck,gosec // data was just marshalled from a T } return out } diff --git a/internal/mock/findatender/catalog_test.go b/internal/mock/findatender/catalog_test.go index fa0e717..0916931 100644 --- a/internal/mock/findatender/catalog_test.go +++ b/internal/mock/findatender/catalog_test.go @@ -31,8 +31,8 @@ func TestEmbeddedCatalogIsValid(t *testing.T) { if err != nil { t.Fatal(err) } - if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || len(cat.Subjects) == 0 || - len(cat.Methods) == 0 { + if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || + len(cat.Subjects) == 0 || len(cat.Methods) == 0 { t.Fatalf( "embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods", len(cat.Buyers), @@ -45,8 +45,8 @@ func TestEmbeddedCatalogIsValid(t *testing.T) { func TestParseCatalog(t *testing.T) { cat := loadTestCatalog(t) - if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || len(cat.Subjects) != 2 || - len(cat.Methods) != 2 { + if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || + len(cat.Subjects) != 2 || len(cat.Methods) != 2 { t.Fatalf( "got %d buyers, %d suppliers, %d subjects, %d methods", len(cat.Buyers), @@ -55,18 +55,22 @@ func TestParseCatalog(t *testing.T) { len(cat.Methods), ) } + if d := cat.Buyers[0].Details; d == nil || len(d.Classifications) != 1 || d.Classifications[0].Scheme != "UK_CA_TYPE" { t.Fatalf("buyer details.classifications not read: %+v", d) } + s := cat.Subjects[0] if s.Classification != "CPV-77314000" || len(s.Items) != 1 { t.Fatalf("subject classification key/items not read: %+v", s) } - if ac := s.Items[0].AdditionalClassifications; len(ac) != 1 || ac[0] != "CPV-77310000" { + if ac := s.Items[0].AdditionalClassifications; len(ac) != 1 || + ac[0] != "CPV-77310000" { t.Fatalf("items.additionalClassifications keys not read: %+v", ac) } - if v := s.Lots[0].Value; v == nil || v.AmountGross == nil || *v.AmountGross != 120000 { + if v := s.Lots[0].Value; v == nil || v.AmountGross == nil || + *v.AmountGross != 120000 { t.Fatalf("lot value not read: %+v", v) } } @@ -86,7 +90,8 @@ func TestSubjectResolvesClassifications(t *testing.T) { t.Fatalf("main classification = %+v", cl) } ac := tender.Items[0].AdditionalClassifications - if len(ac) != 1 || ac[0].Description != "Planting and maintenance services of green areas" { + if len(ac) != 1 || + ac[0].Description != "Planting and maintenance services of green areas" { t.Fatalf("item classifications = %+v", ac) } // The catalog itself still holds keys, untouched by picking. diff --git a/internal/mock/findatender/generator.go b/internal/mock/findatender/generator.go index 8c89d7c..74b040b 100644 --- a/internal/mock/findatender/generator.go +++ b/internal/mock/findatender/generator.go @@ -17,61 +17,63 @@ const ocidPrefix = "ocds-mock-" // moves an existing procedure on (planning -> tender -> award) so the same ocid // appears in several releases, as it does on the real service. type Generator struct { - cat *Catalog - store *Store - rnd *rand.Rand - opts options + catalog *Catalog + store *Store + rng *rand.Rand + opts options } // NewGenerator returns a generator drawing from catalog and storing releases in // store. The same seed and store contents give the same releases. It logs to // slog.Default() unless given WithLogger. -func NewGenerator(catalog *Catalog, store *Store, seed int64, opts ...Option) *Generator { +func NewGenerator( + catalog *Catalog, store *Store, seed int64, opts ...Option, +) *Generator { return &Generator{ - cat: catalog, - store: store, - rnd: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data - opts: applyOptions(opts), + catalog: catalog, + store: store, + rng: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data + opts: applyOptions(opts), } } -// Make returns a new release dated at (to the second). About a third of the -// time it moves on a procedure still open from the last 60 days; otherwise it -// starts a new one. -func (g *Generator) Make(at time.Time) ocds.Release { - at = at.UTC().Truncate(time.Second) - if g.rnd.Float64() < 0.35 { - if prev, ok := g.store.RandomOpen(g.rnd, at.AddDate(0, 0, -60)); ok { - return g.advance(prev, at) +// Make returns a new release with the given date (to the second). About a third +// of the time it moves on a procedure still open from the last 60 days; +// otherwise it starts a new one. +func (g *Generator) Make(date time.Time) ocds.Release { + date = date.UTC().Truncate(time.Second) + if g.rng.Float64() < 0.35 { + if previous, ok := g.store.RandomOpen(g.rng, date.AddDate(0, 0, -60)); ok { + return g.advance(previous, date) } } - return g.newProcedure(at) + return g.newProcedure(date) } // newProcedure returns the first release of a new procedure: a tender (or, // one time in five, a planning notice) from a random catalog buyer and // subject, using a random procurement method. Values are scaled by a random // factor, and a tender without lots gets a single lot covering it all. -func (g *Generator) newProcedure(at time.Time) ocds.Release { - buyer := g.cat.buyer(g.rnd) +func (g *Generator) newProcedure(date time.Time) ocds.Release { + buyer := g.catalog.buyer(g.rng) ensurePartyID(&buyer) buyer.Roles = []string{"buyer"} - method := g.cat.method(g.rnd) + method := g.catalog.method(g.rng) tag, status := "tender", "active" - if g.rnd.Float64() < 0.2 { + if g.rng.Float64() < 0.2 { tag, status = "planning", "planned" } ocid := g.newOCID() - tender := g.cat.subject(g.rnd) + tender := g.catalog.subject(g.rng) tender.ID = ocds.ID(ocid) tender.ProcurementMethod = method.Method tender.ProcurementMethodDetails = method.Details - factor := 0.5 + g.rnd.Float64()*1.5 + factor := 0.5 + g.rng.Float64()*1.5 if tender.Value == nil { - amount := float64((g.rnd.Intn(5000) + 10) * 1000) + amount := float64((g.rng.Intn(5000) + 10) * 1000) tender.Value = &ocds.Value{ Amount: &amount, Currency: "GBP", @@ -96,14 +98,14 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release { setStatus(&tender, status) if status == "active" { tender.TenderPeriod = &ocds.Period{ - EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))), + EndDate: ptr(date.AddDate(0, 0, 30+g.rng.Intn(30))), } } return ocds.Release{ OCID: ocid, - ID: g.store.NextNoticeID(at.Year()), - Date: at, + ID: g.store.NextNoticeID(date.Year()), + Date: date, Tag: []string{tag}, InitiationType: "tender", Parties: []ocds.Organization{buyer}, @@ -116,13 +118,15 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release { } } -// advance returns the next release in prev's procedure: planning moves to +// advance returns the next release in previous's procedure: planning moves to // tender, and a tender either has its deadline extended (tenderUpdate) or is // awarded to a supplier from the catalog. -func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { - release := clone(prev) - release.ID = g.store.NextNoticeID(at.Year()) - release.Date = at +func (g *Generator) advance( + previous ocds.Release, date time.Time, +) ocds.Release { + release := clone(previous) + release.ID = g.store.NextNoticeID(date.Year()) + release.Date = date release.Awards = nil if release.Tender == nil { release.Tender = &ocds.Tender{ @@ -131,11 +135,11 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { } next := "award" - switch prev.Tag[0] { + switch previous.Tag[0] { case "planning": next = "tender" case "tender": - if g.rnd.Float64() < 0.25 { + if g.rng.Float64() < 0.25 { next = "tenderUpdate" } } @@ -145,12 +149,13 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { case "tender": setStatus(release.Tender, "active") release.Tender.TenderPeriod = &ocds.Period{ - EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))), + EndDate: ptr(date.AddDate(0, 0, 30+g.rng.Intn(30))), } case "tenderUpdate": // A typical update: the deadline is extended by two weeks. - end := at.AddDate(0, 0, 30) - if period := release.Tender.TenderPeriod; period != nil && period.EndDate != nil { + end := date.AddDate(0, 0, 30) + if period := release.Tender.TenderPeriod; period != nil && + period.EndDate != nil { end = *period.EndDate } release.Tender.TenderPeriod = &ocds.Period{ @@ -158,7 +163,7 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { } case "award": setStatus(release.Tender, "complete") - supplier := g.cat.supplier(g.rnd) + supplier := g.catalog.supplier(g.rng) ensurePartyID(&supplier) supplier.Roles = []string{"supplier"} release.Parties = append(release.Parties, supplier) @@ -166,7 +171,7 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { var value *ocds.Value if release.Tender.Value != nil { scaled := clone(*release.Tender.Value) - scaleValue(&scaled, 0.8+g.rnd.Float64()*0.2) + scaleValue(&scaled, 0.8+g.rng.Float64()*0.2) value = &scaled } @@ -180,7 +185,7 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { release.Awards = []ocds.Award{{ ID: "1", Status: "active", - Date: ptr(at), + Date: ptr(date), Value: value, Suppliers: []ocds.OrganizationReference{ { @@ -198,26 +203,26 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { // newOCID returns a random ocid not already in the store. func (g *Generator) newOCID() string { for { - id := fmt.Sprintf("%s%06x", ocidPrefix, g.rnd.Intn(1<<24)) + id := fmt.Sprintf("%s%06x", ocidPrefix, g.rng.Intn(1<<24)) if !g.store.HasOCID(id) { return id } } } -// gap returns a random duration from min to max inclusive, or min if max is -// not larger. -func (g *Generator) gap(min, max time.Duration) time.Duration { - if max <= min { - return min +// gap returns a random duration from minGap to maxGap inclusive, or minGap if +// maxGap is not larger. +func (g *Generator) gap(minGap, maxGap time.Duration) time.Duration { + if maxGap <= minGap { + return minGap } - return min + time.Duration(g.rnd.Int63n(int64(max-min)+1)) + return minGap + time.Duration(g.rng.Int63n(int64(maxGap-minGap)+1)) } // CatchUp fills the period since the last stored release (or the last backfill // period, whichever is shorter) so restarts leave no gap in the timeline. func (g *Generator) CatchUp( - now time.Time, backfill, min, max time.Duration, + now time.Time, backfill, minGap, maxGap time.Duration, ) (int, error) { start := now.Add(-backfill) if last, ok := g.store.Last(); ok && last.After(start) { @@ -225,7 +230,8 @@ func (g *Generator) CatchUp( } count := 0 - for when := start.Add(g.gap(min, max)); !when.After(now); when = when.Add(g.gap(min, max)) { + when := start.Add(g.gap(minGap, maxGap)) + for ; !when.After(now); when = when.Add(g.gap(minGap, maxGap)) { if err := g.store.AppendNoSync(g.Make(when)); err != nil { return count, err } @@ -235,11 +241,12 @@ func (g *Generator) CatchUp( return count, g.store.Sync() } -// Run publishes a new release after a random gap between min and max until ctx ends. -func (g *Generator) Run(ctx context.Context, min, max time.Duration) { +// Run publishes a new release after a random gap between minGap and maxGap +// until ctx ends. +func (g *Generator) Run(ctx context.Context, minGap, maxGap time.Duration) { logger := g.opts.logger("generator") for { - gap := g.gap(min, max) + gap := g.gap(minGap, maxGap) logger.Info("next release", "in", gap.Round(time.Second)) timer := time.NewTimer(gap) diff --git a/internal/mock/findatender/generator_test.go b/internal/mock/findatender/generator_test.go index 92918af..a9eeca3 100644 --- a/internal/mock/findatender/generator_test.go +++ b/internal/mock/findatender/generator_test.go @@ -2,8 +2,7 @@ package mock import ( "context" - "io" - "log" + "log/slog" "math/rand" "path/filepath" "strings" @@ -13,19 +12,13 @@ import ( "git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds" ) -func quietLog(t *testing.T) { - t.Helper() - w := log.Writer() - log.SetOutput(io.Discard) - t.Cleanup(func() { - log.SetOutput(w) - }) -} +// quiet discards the logging of the Store and Generator it is given to. +var quiet = WithLogger(slog.New(slog.DiscardHandler)) func TestGap(t *testing.T) { //nolint:gosec // G404: seeded math/rand keeps the test deterministic g := &Generator{ - rnd: rand.New(rand.NewSource(1)), + rng: rand.New(rand.NewSource(1)), } if d := g.gap(time.Second, time.Second); d != time.Second { t.Errorf("min == max: %s", d) @@ -34,7 +27,8 @@ func TestGap(t *testing.T) { t.Errorf("max < min: %s, want min", d) } for i := 0; i < 1000; i++ { - if d := g.gap(time.Second, 2*time.Second); d < time.Second || d > 2*time.Second { + if d := g.gap(time.Second, 2*time.Second); d < time.Second || + d > 2*time.Second { t.Fatalf("gap out of range: %s", d) } } @@ -135,7 +129,14 @@ func TestAdvance(t *testing.T) { Amount: ptr(10000.0), Currency: "GBP", }, - Lots: []ocds.Lot{{ID: "1"}, {ID: ""}}, + Lots: []ocds.Lot{ + { + ID: "1", + }, + { + ID: "", + }, + }, TenderPeriod: &ocds.Period{ EndDate: &end, }, @@ -147,7 +148,8 @@ func TestAdvance(t *testing.T) { seen[r.Tag[0]] = true switch r.Tag[0] { case "tenderUpdate": - if got := *r.Tender.TenderPeriod.EndDate; !got.Equal(end.AddDate(0, 0, 14)) { + got := *r.Tender.TenderPeriod.EndDate + if !got.Equal(end.AddDate(0, 0, 14)) { t.Fatalf("updated deadline %s", got) } case "award": @@ -155,13 +157,15 @@ func TestAdvance(t *testing.T) { t.Fatalf("award: status %s, %d awards", r.Tender.Status, len(r.Awards)) } aw := r.Awards[0] - if aw.Value == nil || *aw.Value.Amount < 8000 || *aw.Value.Amount > 10000 { + if aw.Value == nil || *aw.Value.Amount < 8000 || + *aw.Value.Amount > 10000 { t.Fatalf("award value %+v", aw.Value) } if len(aw.RelatedLots) != 1 || aw.RelatedLots[0] != "1" { t.Fatalf("related lots %v", aw.RelatedLots) } - if len(aw.Suppliers) != 1 || len(r.Parties) != 1 || r.Parties[0].Roles[0] != "supplier" { + if len(aw.Suppliers) != 1 || len(r.Parties) != 1 || + r.Parties[0].Roles[0] != "supplier" { t.Fatalf("suppliers %v, parties %v", aw.Suppliers, r.Parties) } default: @@ -188,7 +192,8 @@ func TestAdvance(t *testing.T) { r := g.advance(prev, at) switch r.Tag[0] { case "tenderUpdate": - if got := *r.Tender.TenderPeriod.EndDate; !got.Equal(at.AddDate(0, 0, 44)) { + got := *r.Tender.TenderPeriod.EndDate + if !got.Equal(at.AddDate(0, 0, 44)) { t.Fatalf("default deadline %s", got) } case "award": @@ -200,9 +205,8 @@ func TestAdvance(t *testing.T) { } func TestRunPublishesUntilCancelled(t *testing.T) { - quietLog(t) cat := loadTestCatalog(t) - store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl"), quiet) if err != nil { t.Fatal(err) } @@ -210,8 +214,9 @@ func TestRunPublishesUntilCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) + generator := NewGenerator(cat, store, 1, quiet) go func() { - NewGenerator(cat, store, 1).Run(ctx, time.Millisecond, 2*time.Millisecond) + generator.Run(ctx, time.Millisecond, 2*time.Millisecond) close(done) }() @@ -234,7 +239,6 @@ func TestRunPublishesUntilCancelled(t *testing.T) { // A failing store is logged and Run carries on until cancelled. func TestRunSurvivesStoreErrors(t *testing.T) { - quietLog(t) cat := loadTestCatalog(t) store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) if err != nil { @@ -244,7 +248,8 @@ func TestRunSurvivesStoreErrors(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() - NewGenerator(cat, store, 1).Run(ctx, time.Millisecond, time.Millisecond) + generator := NewGenerator(cat, store, 1, quiet) + generator.Run(ctx, time.Millisecond, time.Millisecond) if store.Len() != 0 { t.Errorf("Len = %d, want 0", store.Len()) } diff --git a/internal/mock/findatender/handler.go b/internal/mock/findatender/handler.go index ea14610..ca812fd 100644 --- a/internal/mock/findatender/handler.go +++ b/internal/mock/findatender/handler.go @@ -31,8 +31,10 @@ const ( type server struct { store *Store baseURL string - loc *time.Location // zone for updatedFrom/updatedTo without an offset; nil means UTC - opts options + // location is the zone for updatedFrom/updatedTo without an offset; nil + // means UTC. + location *time.Location + opts options } // NewHandler serves release packages from store. baseURL is the public base @@ -42,16 +44,13 @@ type server struct { // Stored release dates are UTC; comparisons are on absolute instants. It // logs to slog.Default() unless given WithLogger. func NewHandler( - store *Store, - baseURL string, - location *time.Location, - opts ...Option, + store *Store, baseURL string, location *time.Location, opts ...Option, ) http.HandlerFunc { handler := &server{ - store: store, - baseURL: baseURL, - loc: location, - opts: applyOptions(opts), + store: store, + baseURL: baseURL, + location: location, + opts: applyOptions(opts), } return handler.handle } @@ -86,8 +85,8 @@ func badRequest(w http.ResponseWriter, msg string) { } // parseTime reads a query timestamp. Values without an offset (the documented -// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with an -// offset or Z (RFC 3339) mean exactly what they say. +// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with +// an offset or Z (RFC 3339) mean exactly what they say. func parseTime(value string, location *time.Location) (time.Time, error) { if location == nil { location = time.UTC @@ -98,7 +97,8 @@ func parseTime(value string, location *time.Location) (time.Time, error) { } for _, layout := range []string{timeLayout, "2006-01-02"} { - if parsed, err := time.ParseInLocation(layout, value, location); err == nil { + parsed, err := time.ParseInLocation(layout, value, location) + if err == nil { return parsed, nil } } @@ -115,22 +115,24 @@ type pageQuery struct { // handle answers a GET for a page of releases as an OCDS release package, // with a links.next when more releases match. -func (s *server) handle(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { +func (s *server) handle(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { w.WriteHeader(http.StatusMethodNotAllowed) return } - query := req.URL.Query() + query := r.URL.Query() page, err := s.parseQuery(query) if err != nil { badRequest(w, err.Error()) return } - releases, next, more := s.store.Page(page.from, page.upper, page.start, page.limit) + releases, next, more := s.store.Page( + page.from, page.upper, page.start, page.limit, + ) releasePackage := ocds.ReleasePackage{ - URI: s.baseURL + req.URL.RequestURI(), + URI: s.baseURL + r.URL.RequestURI(), Version: "1.1", Extensions: ocds.DeclaredExtensions, PublishedDate: ptr(time.Now().UTC().Truncate(time.Second)), @@ -167,13 +169,15 @@ func (s *server) parseQuery(query url.Values) (pageQuery, error) { if value := query.Get("limit"); value != "" { number, err := strconv.Atoi(value) if err != nil || number < 1 || number > maxLimit { - return page, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit) + return page, fmt.Errorf( + "limit must be an integer between 1 and %d", maxLimit, + ) } page.limit = number } if value := query.Get("updatedFrom"); value != "" { - from, err := parseTime(value, s.loc) + from, err := parseTime(value, s.location) if err != nil { return page, fmt.Errorf("updatedFrom %w", err) } @@ -181,15 +185,15 @@ func (s *server) parseQuery(query url.Values) (pageQuery, error) { } if value := query.Get("updatedTo"); value != "" { - to, err := parseTime(value, s.loc) + until, err := parseTime(value, s.location) if err != nil { return page, fmt.Errorf("updatedTo %w", err) } - if query.Get("updatedFrom") != "" && to.Before(page.from) { + if query.Get("updatedFrom") != "" && until.Before(page.from) { return page, errors.New("updatedTo must not be before updatedFrom") } - if to.Before(page.upper) { - page.upper = to + if until.Before(page.upper) { + page.upper = until } } diff --git a/internal/mock/findatender/handler_test.go b/internal/mock/findatender/handler_test.go index 7152191..83c4b45 100644 --- a/internal/mock/findatender/handler_test.go +++ b/internal/mock/findatender/handler_test.go @@ -120,7 +120,8 @@ func TestHandlerRejectsBadRequests(t *testing.T) { t.Errorf("%s: Content-Type %q", q, ct) } var body map[string]string - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil || body["error"] != want { + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil || + body["error"] != want { t.Errorf("%s: body %s, want error %q", q, rec.Body.String(), want) } } @@ -147,7 +148,12 @@ func TestParseTimeFormats(t *testing.T) { t.Errorf("parseTime(%q) = %s, %v; want %s", v, got, err, want) } } - for _, v := range []string{"", "08/10/2026", "2026-10-08 12:00:00", "2026-10-08T25:00:00"} { + for _, v := range []string{ + "", + "08/10/2026", + "2026-10-08 12:00:00", + "2026-10-08T25:00:00", + } { if _, err := parseTime(v, nil); err == nil { t.Errorf("parseTime(%q): expected error", v) } diff --git a/internal/mock/findatender/ratelimit.go b/internal/mock/findatender/ratelimit.go index 17354c0..119d90d 100644 --- a/internal/mock/findatender/ratelimit.go +++ b/internal/mock/findatender/ratelimit.go @@ -66,7 +66,8 @@ func (rl *RateLimiter) allow(key string) (bool, int) { // Move to whichever fixed window, counted from the anchor, now falls in. index := now.Sub(client.anchor) / rl.window - if windowStart := client.anchor.Add(index * rl.window); !windowStart.Equal(client.windowStart) { + windowStart := client.anchor.Add(index * rl.window) + if !windowStart.Equal(client.windowStart) { client.windowStart = windowStart client.count = 0 } @@ -82,10 +83,12 @@ func (rl *RateLimiter) allow(key string) (bool, int) { if nextHour := hour.Add(time.Hour); nextHour.Before(next) { next = nextHour } + seconds := int((next.Sub(now) + time.Second - 1) / time.Second) // round up if seconds < 1 { seconds = 1 } + return false, seconds } @@ -100,10 +103,10 @@ func (rl *RateLimiter) prune(hour time.Time) { // clientKey produces an identifier for the client based on remote address. // Could be updated to support proxy and forwarded-for headers. -func clientKey(req *http.Request) string { - host, _, err := net.SplitHostPort(req.RemoteAddr) +func clientKey(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return req.RemoteAddr + return r.RemoteAddr } return host } @@ -112,18 +115,20 @@ func clientKey(req *http.Request) string { // to make a request in their current window or returns a "TooManyRequests" // status and the time the client should wait before making another request. func (rl *RateLimiter) Middleware(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - ok, retry := rl.allow(clientKey(req)) + return func(w http.ResponseWriter, r *http.Request) { + ok, retry := rl.allow(clientKey(r)) if !ok { w.Header().Set("Retry-After", strconv.Itoa(retry)) w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusTooManyRequests) fmt.Fprintf( //nolint:errcheck // the client may have gone; nothing to do - w, "Rate limit of %d exceeded. Please retry after %d seconds.", - rl.limit, retry, + w, + "Rate limit of %d exceeded. Please retry after %d seconds.", + rl.limit, + retry, ) return } - next(w, req) + next(w, r) } } diff --git a/internal/mock/findatender/ratelimit_test.go b/internal/mock/findatender/ratelimit_test.go index bf2521d..754f99f 100644 --- a/internal/mock/findatender/ratelimit_test.go +++ b/internal/mock/findatender/ratelimit_test.go @@ -10,7 +10,8 @@ import ( ) func TestRateLimitWindows(t *testing.T) { - now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) // first request of the hour + // The first request of the hour. + now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) rl := NewRateLimiter(12, 5*time.Minute) rl.now = func() time.Time { return now diff --git a/internal/mock/findatender/store.go b/internal/mock/findatender/store.go index dee16e1..7cfa027 100644 --- a/internal/mock/findatender/store.go +++ b/internal/mock/findatender/store.go @@ -22,17 +22,17 @@ import ( // Store keeps every release in memory and appends each new one to a JSON Lines // file, so releases survive restarts. Releases are kept in date order. type Store struct { - mu sync.RWMutex - f *os.File - releases []ocds.Release - times []time.Time - latest map[string]int // ocid -> index of its most recent release - lastNum map[int]int // year -> highest notice number used + mu sync.RWMutex + file *os.File + releases []ocds.Release + times []time.Time + latest map[string]int // ocid -> index of its most recent release + lastNumber map[int]int // year -> highest notice number used // Procedures not yet awarded, as a slice plus positions so that adding, // removing and picking one at random are all constant time. - open []string - openPos map[string]int + open []string + openPosition map[string]int opts options } @@ -44,11 +44,12 @@ type Store struct { // It logs to slog.Default() unless given WithLogger. func OpenStore(path string, opts ...Option) (*Store, error) { store := &Store{ - latest: map[string]int{}, - lastNum: map[int]int{}, - openPos: map[string]int{}, - opts: applyOptions(opts), + latest: map[string]int{}, + lastNumber: map[int]int{}, + openPosition: map[string]int{}, + opts: applyOptions(opts), } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return nil, err } @@ -57,6 +58,7 @@ func OpenStore(path string, opts ...Option) (*Store, error) { if err != nil { return nil, err } + // The file is written in order, but sort anyway in case it was hand-edited. sort.SliceStable(releases, func(i, other int) bool { return releases[i].Date.Before(releases[other].Date) @@ -69,9 +71,10 @@ func OpenStore(path string, opts ...Option) (*Store, error) { store.index(i, release) } - if store.f, err = openForAppend(path); err != nil { + if store.file, err = openForAppend(path); err != nil { return nil, err } + return store, nil } @@ -106,6 +109,7 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) { if err := scanner.Err(); err != nil { return nil, fmt.Errorf("reading %s: %w", path, err) } + return releases, nil } @@ -113,18 +117,25 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) { // mid-write, it ends the partial record so the next one starts on a fresh // line. func openForAppend(path string) (*os.File, error) { - file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600) + file, err := os.OpenFile( + filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600, + ) if err != nil { return nil, err } + if info, err := file.Stat(); err == nil && info.Size() > 0 { last := make([]byte, 1) - if _, err := file.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' { + _, readErr := file.ReadAt(last, info.Size()-1) + if readErr == nil && last[0] != '\n' { if _, err := file.Write([]byte("\n")); err != nil { - return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), file.Close()) + return nil, errors.Join( + fmt.Errorf("repairing %s: %w", path, err), file.Close(), + ) } } } + return file, nil } @@ -133,51 +144,58 @@ func openForAppend(path string) (*os.File, error) { // highest notice number used each year. func (s *Store) index(i int, release ocds.Release) { s.latest[release.OCID] = i + if len(release.Tag) > 0 && release.Tag[0] == "award" { s.closeProcedure(release.OCID) - } else if _, ok := s.openPos[release.OCID]; !ok { - s.openPos[release.OCID] = len(s.open) + } else if _, ok := s.openPosition[release.OCID]; !ok { + s.openPosition[release.OCID] = len(s.open) s.open = append(s.open, release.OCID) } + if numberText, year, ok := strings.Cut(release.ID, "-"); ok { number, numberErr := strconv.Atoi(numberText) yearNumber, yearErr := strconv.Atoi(year) - if numberErr == nil && yearErr == nil && number > s.lastNum[yearNumber] { - s.lastNum[yearNumber] = number + if numberErr == nil && yearErr == nil && number > s.lastNumber[yearNumber] { + s.lastNumber[yearNumber] = number } } } // closeProcedure removes ocid from the open procedures, if it is there. func (s *Store) closeProcedure(ocid string) { - position, ok := s.openPos[ocid] + position, ok := s.openPosition[ocid] if !ok { return } + last := len(s.open) - 1 s.open[position] = s.open[last] - s.openPos[s.open[position]] = position + s.openPosition[s.open[position]] = position s.open = s.open[:last] - delete(s.openPos, ocid) + delete(s.openPosition, ocid) } // Append persists a release and flushes it to disk. -func (s *Store) Append(release ocds.Release) error { return s.append(release, true) } +func (s *Store) Append(release ocds.Release) error { + return s.append(release, true) +} // AppendNoSync persists a release without an fsync; call Sync afterwards. // Used for bulk catch-up, where a sync per record would be very slow. -func (s *Store) AppendNoSync(release ocds.Release) error { return s.append(release, false) } +func (s *Store) AppendNoSync(release ocds.Release) error { + return s.append(release, false) +} // Sync flushes appended releases to disk. Call it after AppendNoSync. func (s *Store) Sync() error { s.mu.Lock() defer s.mu.Unlock() - return s.f.Sync() + return s.file.Sync() } -// append stores release, syncing the file to disk if sync is true. A date earlier -// than the newest stored release (for example after a clock change) is moved -// up so the order stays valid. +// append stores release, syncing the file to disk if sync is true. A date +// earlier than the newest stored release (for example after a clock change) is +// moved up so the order stays valid. func (s *Store) append(release ocds.Release, sync bool) error { s.mu.Lock() defer s.mu.Unlock() @@ -186,27 +204,34 @@ func (s *Store) append(release ocds.Release, sync bool) error { if count := len(s.times); count > 0 && release.Date.Before(s.times[count-1]) { release.Date = s.times[count-1] } + data, err := json.Marshal(release) if err != nil { return err } - if _, err := s.f.Write(append(data, '\n')); err != nil { + if _, err := s.file.Write(append(data, '\n')); err != nil { return err } if sync { - if err := s.f.Sync(); err != nil { + if err := s.file.Sync(); err != nil { return err } } + s.releases = append(s.releases, release) s.times = append(s.times, release.Date) s.index(len(s.releases)-1, release) + return nil } // Page returns up to limit releases dated in [from, upper], starting no earlier -// than position start, plus the position to continue from and whether more exist. -func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, int, bool) { +// than position start, plus the position to continue from and whether more +// exist. +func (s *Store) Page( + from, upper time.Time, + start, limit int, +) ([]ocds.Release, int, bool) { s.mu.RLock() defer s.mu.RUnlock() @@ -216,12 +241,15 @@ func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, i if start > i { i = start } + out := make([]ocds.Release, 0, limit) for i < len(s.releases) && len(out) < limit && !s.times[i].After(upper) { out = append(out, s.releases[i]) i++ } + more := i < len(s.releases) && !s.times[i].After(upper) + return out, i, more } @@ -255,13 +283,15 @@ func (s *Store) HasOCID(ocid string) bool { func (s *Store) NextNoticeID(year int) string { s.mu.RLock() defer s.mu.RUnlock() - return fmt.Sprintf("%06d-%d", s.lastNum[year]+1, year) + return fmt.Sprintf("%06d-%d", s.lastNumber[year]+1, year) } // RandomOpen picks a procedure that has not been awarded and whose latest // release is after since. Procedures idle since before then are dropped from // the open set, as if they had been abandoned. -func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) { +func (s *Store) RandomOpen( + rng *rand.Rand, since time.Time, +) (ocds.Release, bool) { s.mu.Lock() defer s.mu.Unlock() for len(s.open) > 0 { @@ -279,5 +309,5 @@ func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) func (s *Store) Close() error { s.mu.Lock() defer s.mu.Unlock() - return s.f.Close() + return s.file.Close() } diff --git a/internal/mock/findatender/store_test.go b/internal/mock/findatender/store_test.go index ef15654..bf62d98 100644 --- a/internal/mock/findatender/store_test.go +++ b/internal/mock/findatender/store_test.go @@ -56,7 +56,9 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) { t.Fatalf("after reopen: %d releases, want %d", store2.Len(), len(all)) } gen2 := NewGenerator(cat, store2, 7) - m, _ := gen2.CatchUp(now.Add(time.Hour), 48*time.Hour, 3*time.Minute, 15*time.Minute) + m, _ := gen2.CatchUp( + now.Add(time.Hour), 48*time.Hour, 3*time.Minute, 15*time.Minute, + ) if m < 4 || m > 20 { t.Fatalf("one hour gap produced %d releases", m) } @@ -66,9 +68,9 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) { // The HTTP handler pages through the store without gaps or repeats. srv := &server{ - store: store2, - baseURL: "", - loc: time.UTC, + store: store2, + baseURL: "", + location: time.UTC, } seen := map[string]bool{} next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7" @@ -102,7 +104,9 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) { func writeLines(t *testing.T, path string, lines ...string) { t.Helper() - if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o600); err != nil { + if err := os.WriteFile( + path, []byte(strings.Join(lines, "\n")), 0o600, + ); err != nil { t.Fatal(err) } } @@ -122,7 +126,8 @@ func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string { } // Unreadable lines are skipped, blank lines ignored, a hand-edited file is put -// back in date order, and the index (open procedures, notice numbers) is rebuilt. +// back in date order, and the index (open procedures, notice numbers) is +// rebuilt. func TestOpenStoreRecoversFromMessyFile(t *testing.T) { path := filepath.Join(t.TempDir(), "r.jsonl") t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) @@ -148,7 +153,8 @@ func TestOpenStoreRecoversFromMessyFile(t *testing.T) { for _, r := range all { ids = append(ids, r.ID) } - if got, want := strings.Join(ids, ","), "000003-2026,000005-2026,000007-2026"; got != want { + want := "000003-2026,000005-2026,000007-2026" + if got := strings.Join(ids, ","); got != want { t.Fatalf("releases %s, want %s", got, want) } if got := store.NextNoticeID(2026); got != "000008-2026" { @@ -199,7 +205,11 @@ func TestOpenStoreAfterTruncatedWrite(t *testing.T) { } defer store2.Close() if store2.Len() != 2 || !store2.HasOCID("ocds-c") { - t.Fatalf("after reopen: Len = %d, has ocds-c = %v", store2.Len(), store2.HasOCID("ocds-c")) + t.Fatalf( + "after reopen: Len = %d, has ocds-c = %v", + store2.Len(), + store2.HasOCID("ocds-c"), + ) } } @@ -294,7 +304,8 @@ func TestRandomOpenDropsStaleProcedures(t *testing.T) { //nolint:gosec // G404: seeded math/rand keeps the test deterministic rnd := rand.New(rand.NewSource(1)) for i := 0; i < 10; i++ { - if r, ok := store.RandomOpen(rnd, t0.Add(time.Minute)); !ok || r.OCID != "new" { + if r, ok := store.RandomOpen(rnd, t0.Add(time.Minute)); !ok || + r.OCID != "new" { t.Fatalf("RandomOpen = %s, %v; want new", r.OCID, ok) } } diff --git a/internal/store/README.md b/internal/store/README.md index 003c8db..4d7e797 100644 --- a/internal/store/README.md +++ b/internal/store/README.md @@ -78,6 +78,18 @@ Reads select whole rows (`SELECT *`, `t.*`, `c.*`) which map to a Go struct. sql `ListOpportunities` is assembled by `listOpportunitiesQuery` from fragments in `queries.go`: a base `SELECT`, one `filterOpportunity*` clause per filter that is set, and the ordering and paging tail. Each filter fragment takes exactly one argument, except the `filterOpportunityMark*` fragments, which take none (`MarkUnset` puts `NOT` in front of them). Adding a filter means one fragment and one line in `listOpportunitiesQuery`. The classifications and the marks for a page of opportunities are then loaded with one `IN (...)` query each. +### SQL style + +A soft convention for the SQL in `queries.go`, followed so the queries look alike and read easily. A query that reads better another way can depart from it. + +- A statement starts on the line after the opening backtick. A fragment combined into another query (`filterOpportunity*`) starts straight after it. +- Keywords are uppercase (`SELECT`, `INSERT INTO`, `ON CONFLICT`); table and column names are the lowercase snake_case of the schema. +- A short statement may stay on one line (`SELECT t.* FROM opportunities t WHERE t.uuid = ?`). A longer one has one clause per line, each at the start of the line: `SELECT`, `FROM`, `JOIN`, `WHERE`, `ORDER BY`, `LIMIT`. +- A column list that doesn't fit on one line goes inside its brackets, indented one tab and filled to about 80 characters per line. The `VALUES` list wraps at the same columns, so each line of values matches the line of columns above it. +- `ON CONFLICT … DO UPDATE SET` puts one `column = excluded.column` per line, indented one tab. +- A subquery (`EXISTS (…)`, `IN (…)`) is indented one tab inside its brackets, with the closing bracket at the start of a line. +- Tables are aliased consistently: `t` for `opportunities`, `c` for `classification_codes`, `tc` for `opportunity_classifications`, `m` for `opportunity_marks`. + ## Store API ```go diff --git a/internal/store/migrate.go b/internal/store/migrate.go index 6239689..c5c88c0 100644 --- a/internal/store/migrate.go +++ b/internal/store/migrate.go @@ -17,21 +17,6 @@ import ( //go:embed migrations/*.sql var migrationsFS embed.FS -// ErrNeedsMigration is returned by Open when the database has pending -// migrations. Apply them with the migrate command, or pass -// WithAutoMigrate. -var ErrNeedsMigration = errors.New("store: database needs migrating") - -// ErrSchemaTooNew is returned by Open when the database has migrations -// this build doesn't know about, i.e. it was migrated by a newer build. -var ErrSchemaTooNew = errors.New("store: database schema is newer than this build") - -// ErrWouldEmptyDatabase is returned by StepDown when the migration would drop -// all of the tables and thus empty the database. -var ErrWouldEmptyDatabase = errors.New( - "migration cancelled; database would be emptied by undoing all migrations", -) - // NewMigrator returns a goose Provider over the store's embedded migrations, // for applying, rolling back or inspecting them. Each migration runs in its // own transaction, so a failed migration leaves the database at the previous @@ -96,13 +81,15 @@ func checkNotNewer(ctx context.Context, provider *goose.Provider) error { if err != nil { return err } + if latest := LatestVersion(provider); current > latest { return fmt.Errorf( - "database is at version %d but this build only knows up to %d; use a newer build", + "store: database is at version %d but this build only knows up to %d; use a newer build", current, latest, ) } + return nil } @@ -116,7 +103,7 @@ func prepareSchema(ctx context.Context, db *sql.DB, opts options) error { if opts.autoMigrate { if _, err := provider.Up(ctx); err != nil { - return fmt.Errorf("store: migrating database: %w", err) + return fmt.Errorf("migrating database: %w", err) } } @@ -184,6 +171,7 @@ func StepDown( applied++ } } + all := steps == 0 || steps >= applied if all && !confirmDrop { return results, ErrWouldEmptyDatabase diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index ec5eadf..c7cf115 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -29,10 +29,12 @@ func newMigrator(t *testing.T) (*goose.Provider, *sql.DB) { t.Cleanup(func() { db.Close() }) + m, err := store.NewMigrator(db) if err != nil { t.Fatal(err) } + return m, db } @@ -105,7 +107,8 @@ func TestOpenRejectsNewerSchema(t *testing.T) { "check": nil, "auto-migrate": {store.WithAutoMigrate()}, } { - if _, err := store.Open(ctx, path, opts...); !errors.Is(err, store.ErrSchemaTooNew) { + _, err := store.Open(ctx, path, opts...) + if !errors.Is(err, store.ErrSchemaTooNew) { t.Errorf("%s: err = %v, want ErrSchemaTooNew", name, err) } } @@ -135,10 +138,12 @@ func TestMigrateDownAllAndUpAgain(t *testing.T) { t.Fatalf("down to 0: %v", err) } var tables int - if err := db.QueryRow(` + query := ` SELECT count(*) FROM sqlite_master WHERE type IN ('table', 'index', 'trigger') - AND name NOT LIKE 'sqlite_%' AND name != 'goose_db_version'`).Scan(&tables); err != nil { + AND name NOT LIKE 'sqlite_%' + AND name != 'goose_db_version'` + if err := db.QueryRow(query).Scan(&tables); err != nil { t.Fatal(err) } if tables != 0 { @@ -229,7 +234,10 @@ func TestDownAllRequiresConfirmation(t *testing.T) { } hasErrored := false - for name, steps := range map[string]int{"steps 0": 0, "more steps than applied": 99} { + for name, steps := range map[string]int{ + "steps 0": 0, + "more steps than applied": 99, + } { _, err := store.StepDown(ctx, m, steps, false) if !errors.Is(err, store.ErrWouldEmptyDatabase) { t.Errorf("%s: %s", name, err) @@ -277,20 +285,11 @@ func TestNewerDatabaseIsRefused(t *testing.T) { t.Fatal(err) } - if _, err := store.StepUp( - ctx, - m, - 0, - ); err == nil || + if _, err := store.StepUp(ctx, m, 0); err == nil || !strings.Contains(err.Error(), "newer build") { t.Errorf("up: err = %v, want a newer-build error", err) } - if _, err := store.StepDown( - ctx, - m, - 1, - false, - ); err == nil || + if _, err := store.StepDown(ctx, m, 1, false); err == nil || !strings.Contains(err.Error(), "newer build") { t.Errorf("down: err = %v, want a newer-build error", err) } @@ -307,7 +306,9 @@ func TestFailedMigrationIsRolledBack(t *testing.T) { // Occupy migration 4's table name with a view. Its CREATE TABLE IF NOT // EXISTS then skips, and the CREATE INDEX that follows fails because // views can't be indexed, part-way through the migration. - if _, err := db.Exec(`CREATE VIEW classification_codes AS SELECT 1 AS x`); err != nil { + if _, err := db.Exec( + `CREATE VIEW classification_codes AS SELECT 1 AS x`, + ); err != nil { t.Fatal(err) } @@ -319,8 +320,9 @@ func TestFailedMigrationIsRolledBack(t *testing.T) { t.Errorf("version after failure: %d, want 3", v) } var indexes int - db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name LIKE 'classification_codes_%'`). - Scan(&indexes) + query := `SELECT count(*) FROM sqlite_master + WHERE name LIKE 'classification_codes_%'` + db.QueryRow(query).Scan(&indexes) if indexes != 0 { t.Errorf("failed migration left %d objects behind", indexes) } diff --git a/internal/store/queries.go b/internal/store/queries.go index 8a3d462..6c4c609 100644 --- a/internal/store/queries.go +++ b/internal/store/queries.go @@ -18,23 +18,27 @@ package store // ---------- opportunities ---------- -// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares against. +// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares +// against. const selectOpportunityLatestReleaseDate = ` SELECT latest_release_date FROM opportunities WHERE uuid = ?` -// upsertOpportunity inserts or updates a opportunity. Named parameters (types.Opportunity). +// upsertOpportunity inserts or updates a opportunity. Named parameters +// (types.Opportunity). // // An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first, // cascading to the opportunity's releases and changing its id. const upsertOpportunity = ` INSERT INTO opportunities ( uuid, uuid_type, title, status, description, buyer_name, buyer_id, - main_category, procurement_method, multi_lot, sme_suitable, vcse_suitable, - start_date, end_date, gross_value, has_documents, latest_release_date + main_category, procurement_method, multi_lot, sme_suitable, + vcse_suitable, start_date, end_date, gross_value, has_documents, + latest_release_date ) VALUES ( :uuid, :uuid_type, :title, :status, :description, :buyer_name, :buyer_id, - :main_category, :procurement_method, :multi_lot, :sme_suitable, :vcse_suitable, - :start_date, :end_date, :gross_value, :has_documents, :latest_release_date + :main_category, :procurement_method, :multi_lot, :sme_suitable, + :vcse_suitable, :start_date, :end_date, :gross_value, :has_documents, + :latest_release_date ) ON CONFLICT (uuid) DO UPDATE SET uuid_type = excluded.uuid_type, @@ -162,7 +166,8 @@ INSERT INTO opportunity_release_tags (uuid, release_id, tag) VALUES (:uuid, :release_id, :tag) ON CONFLICT DO NOTHING` -// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's releases. +// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's +// releases. const selectReleaseTagsByOpportunity = ` SELECT * FROM opportunity_release_tags @@ -180,13 +185,14 @@ ORDER BY release_id, tag` const upsertClassificationCodeReturningID = ` INSERT INTO classification_codes (is_cpv, code, classification) VALUES (:is_cpv, :code, :classification) -ON CONFLICT (is_cpv, code, classification) DO UPDATE SET is_cpv = excluded.is_cpv +ON CONFLICT (is_cpv, code, classification) + DO UPDATE SET is_cpv = excluded.is_cpv RETURNING id` // ---------- opportunity_classifications ---------- -// deleteOpportunityClassifications clears a opportunity's mappings before they are -// rewritten. +// deleteOpportunityClassifications clears a opportunity's mappings before they +// are rewritten. const deleteOpportunityClassifications = ` DELETE FROM opportunity_classifications WHERE opportunity_uuid = ?` @@ -197,8 +203,8 @@ VALUES (?, ?) ON CONFLICT DO NOTHING` // selectClassificationsForOpportunities reads the classifications of several -// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by Rebind. -// c.* must map entirely onto types.ClassificationCode. +// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by +// Rebind. c.* must map entirely onto types.ClassificationCode. const selectClassificationsForOpportunities = ` SELECT tc.opportunity_uuid, c.* FROM opportunity_classifications tc diff --git a/internal/store/queries_test.go b/internal/store/queries_test.go index 67c711d..e893fa4 100644 --- a/internal/store/queries_test.go +++ b/internal/store/queries_test.go @@ -30,26 +30,50 @@ type queryCheck struct { // missing here. var checks = map[string]queryCheck{ // opportunities - "selectOpportunityLatestReleaseDate": {sql: selectOpportunityLatestReleaseDate}, - "upsertOpportunity": {sql: upsertOpportunity, named: true}, - "selectOpportunityByUUID": {sql: selectOpportunityByUUID}, + "selectOpportunityLatestReleaseDate": { + sql: selectOpportunityLatestReleaseDate, + }, + "upsertOpportunity": {sql: upsertOpportunity, named: true}, + "selectOpportunityByUUID": {sql: selectOpportunityByUUID}, "listOpportunitiesSelect": { sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage, }, "listOpportunitiesOrderAndPage": { sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage, }, - "filterOpportunityStatus": {sql: withFilter(filterOpportunityStatus)}, - "filterOpportunityClassificationCode": {sql: withFilter(filterOpportunityClassificationCode)}, - "filterOpportunityTitleMatch": {sql: withFilter(filterOpportunityTitleMatch)}, - "filterOpportunityReleasedFrom": {sql: withFilter(filterOpportunityReleasedFrom)}, - "filterOpportunityReleasedBefore": {sql: withFilter(filterOpportunityReleasedBefore)}, - "filterOpportunityMarkHidden": {sql: withFilter(filterOpportunityMarkHidden)}, - "filterOpportunityMarkSkipped": {sql: withFilter(filterOpportunityMarkSkipped)}, - "filterOpportunityMarkInterested": {sql: withFilter(filterOpportunityMarkInterested)}, - "filterOpportunityMarkBidSubmitted": {sql: withFilter(filterOpportunityMarkBidSubmitted)}, - "filterOpportunityMarkWon": {sql: withFilter(filterOpportunityMarkWon)}, - "filterOpportunityMarkLost": {sql: withFilter(filterOpportunityMarkLost)}, + "filterOpportunityStatus": { + sql: withFilter(filterOpportunityStatus), + }, + "filterOpportunityClassificationCode": { + sql: withFilter(filterOpportunityClassificationCode), + }, + "filterOpportunityTitleMatch": { + sql: withFilter(filterOpportunityTitleMatch), + }, + "filterOpportunityReleasedFrom": { + sql: withFilter(filterOpportunityReleasedFrom), + }, + "filterOpportunityReleasedBefore": { + sql: withFilter(filterOpportunityReleasedBefore), + }, + "filterOpportunityMarkHidden": { + sql: withFilter(filterOpportunityMarkHidden), + }, + "filterOpportunityMarkSkipped": { + sql: withFilter(filterOpportunityMarkSkipped), + }, + "filterOpportunityMarkInterested": { + sql: withFilter(filterOpportunityMarkInterested), + }, + "filterOpportunityMarkBidSubmitted": { + sql: withFilter(filterOpportunityMarkBidSubmitted), + }, + "filterOpportunityMarkWon": { + sql: withFilter(filterOpportunityMarkWon), + }, + "filterOpportunityMarkLost": { + sql: withFilter(filterOpportunityMarkLost), + }, // opportunity_ocds_releases "insertRelease": {sql: insertRelease, named: true}, @@ -61,12 +85,22 @@ var checks = map[string]queryCheck{ "selectReleaseTagsByOpportunity": {sql: selectReleaseTagsByOpportunity}, // classification_codes - "upsertClassificationCodeReturningID": {sql: upsertClassificationCodeReturningID, named: true}, + "upsertClassificationCodeReturningID": { + sql: upsertClassificationCodeReturningID, + named: true, + }, // opportunity_classifications - "deleteOpportunityClassifications": {sql: deleteOpportunityClassifications}, - "insertOpportunityClassification": {sql: insertOpportunityClassification}, - "selectClassificationsForOpportunities": {sql: selectClassificationsForOpportunities, in: true}, + "deleteOpportunityClassifications": { + sql: deleteOpportunityClassifications, + }, + "insertOpportunityClassification": { + sql: insertOpportunityClassification, + }, + "selectClassificationsForOpportunities": { + sql: selectClassificationsForOpportunities, + in: true, + }, // opportunity_marks "insertOpportunityMarks": {sql: insertOpportunityMarks, named: true}, @@ -75,12 +109,17 @@ var checks = map[string]queryCheck{ } func withFilter(fragment string) string { - return listOpportunitiesSelect + "\nWHERE " + fragment + listOpportunitiesOrderAndPage + return listOpportunitiesSelect + "\nWHERE " + fragment + + listOpportunitiesOrderAndPage } func openTestDB(t *testing.T) *sqlx.DB { t.Helper() - s, err := Open(context.Background(), filepath.Join(t.TempDir(), "queries.db"), WithAutoMigrate()) + s, err := Open( + context.Background(), + filepath.Join(t.TempDir(), "queries.db"), + WithAutoMigrate(), + ) if err != nil { t.Fatalf("Open: %v", err) } @@ -144,11 +183,13 @@ func TestListOpportunitiesQueryAllFilters(t *testing.T) { Limit: 10, Offset: 5, }) + stmt, err := db.Preparex(query) if err != nil { t.Fatalf("prepare:\n%s\n%v", query, err) } defer stmt.Close() + var rows []opportunityRow if err := stmt.Select(&rows, args...); err != nil { t.Errorf("run with %d args:\n%s\n%v", len(args), query, err) @@ -162,6 +203,7 @@ func TestQueriesAllListed(t *testing.T) { if err != nil { t.Fatal(err) } + var declared []string for _, decl := range file.Decls { gen, ok := decl.(*ast.GenDecl) @@ -174,17 +216,21 @@ func TestQueriesAllListed(t *testing.T) { } } } + sort.Strings(declared) if len(declared) == 0 { t.Fatal("found no constants in queries.go") } + for _, name := range declared { if _, ok := checks[name]; !ok { t.Errorf("queries.go constant %s is not listed in checks", name) } } + for name := range checks { - if i := sort.SearchStrings(declared, name); i == len(declared) || declared[i] != name { + if i := sort.SearchStrings(declared, name); i == len(declared) || + declared[i] != name { t.Errorf("checks lists %s, which queries.go does not declare", name) } } @@ -194,20 +240,27 @@ func TestQueriesAllListed(t *testing.T) { // together, for writes that bypass OpportunityMarks.Validate. func TestMarksCheckConstraint(t *testing.T) { db := openTestDB(t) - if _, err := db.Exec(`INSERT INTO opportunities + insert := `INSERT INTO opportunities (uuid, uuid_type, title, status, latest_release_date) - VALUES ('ocds-1', 'ocid', 'T', 'active', '2026-01-01T00:00:00Z')`); err != nil { + VALUES ('ocds-1', 'ocid', 'T', 'active', '2026-01-01T00:00:00Z')` + if _, err := db.Exec(insert); err != nil { t.Fatal(err) } - if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won, lost) - VALUES ('ocds-1', 1, 1)`); err == nil { + + insertWonAndLost := `INSERT INTO opportunity_marks (uuid, won, lost) + VALUES ('ocds-1', 1, 1)` + if _, err := db.Exec(insertWonAndLost); err == nil { t.Error("inserting won and lost together succeeded") } - if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won) - VALUES ('ocds-1', 1)`); err != nil { + + insertWon := `INSERT INTO opportunity_marks (uuid, won) + VALUES ('ocds-1', 1)` + if _, err := db.Exec(insertWon); err != nil { t.Fatalf("inserting won: %v", err) } - if _, err := db.Exec(`UPDATE opportunity_marks SET lost = 1 WHERE uuid = 'ocds-1'`); err == nil { + + updateLost := `UPDATE opportunity_marks SET lost = 1 WHERE uuid = 'ocds-1'` + if _, err := db.Exec(updateLost); err == nil { t.Error("updating to won and lost together succeeded") } } diff --git a/internal/store/store.go b/internal/store/store.go index b0e0743..9571c5c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -18,8 +18,25 @@ import ( "git.avsdev.uk/Go/job-search/internal/types" ) -// ErrNotFound is returned when a requested opportunity does not exist. -var ErrNotFound = errors.New("store: not found") +// Errors returned by the store. +var ( + // ErrNotFound is returned when a requested opportunity does not exist. + ErrNotFound = errors.New("store: not found") + // ErrNeedsMigration is returned by Open when the database has pending + // migrations. Apply them with the migrate command, or pass + // WithAutoMigrate. + ErrNeedsMigration = errors.New("store: database needs migrating") + // ErrSchemaTooNew is returned by Open when the database has migrations + // this build doesn't know about, i.e. it was migrated by a newer build. + ErrSchemaTooNew = errors.New( + "store: database schema is newer than this build", + ) + // ErrWouldEmptyDatabase is returned by StepDown when the migration would + // drop all of the tables and thus empty the database. + ErrWouldEmptyDatabase = errors.New( + "store: migration cancelled; database would be emptied by undoing all migrations", + ) +) // DefaultListLimit is used when OpportunityFilter.Limit is zero or negative. const DefaultListLimit = 50 @@ -32,14 +49,15 @@ type Store struct { // SaveMultipleResult reports what SaveOpportunities changed. type SaveMultipleResult struct { - // NewOpportunities is the number of opportunities that were not already stored. + // NewOpportunities is the number of opportunities that were not already + // stored. NewOpportunities int // NewReleases is the number of releases that were not already stored. NewReleases int - // UpdatedOpportunities is incremented for each opportunity row and its classifications - // that are updated (when every release in the save was older than the - // one the stored opportunity was last updated from). It does not include new - // opportunities. + // UpdatedOpportunities is incremented for each opportunity row and its + // classifications that are updated (when every release in the save was older + // than the one the stored opportunity was last updated from). It does not + // include new opportunities. UpdatedOpportunities int // OpportunityResults is a slice of the results of each opportunity saved. OpportunityResults []SaveResult @@ -47,8 +65,8 @@ type SaveMultipleResult struct { // SaveResult reports what SaveOpportunity changed. type SaveResult struct { - // OpportunityUpdated is true if the opportunity row and its classifications were - // written. It is false when every release in the save was older than the + // OpportunityUpdated is true if the opportunity row and its classifications + // were written. It is false when every release in the save was older than the // one the stored opportunity was last updated from. OpportunityUpdated bool // NewReleases is the number of releases that were not already stored. @@ -58,6 +76,7 @@ type SaveResult struct { // MarkFilter filters ListOpportunities on one of an opportunity's marks. type MarkFilter int +// Values of MarkFilter. const ( // MarkAny does not filter. It is the zero value. MarkAny MarkFilter = iota @@ -68,8 +87,8 @@ const ( MarkUnset ) -// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued fields do not -// filter. +// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued +// fields do not filter. type OpportunityFilter struct { Status string // ClassificationCode matches opportunities with a classification of this code @@ -117,14 +136,18 @@ func prepareOpportunity(opportunity *types.Opportunity) error { opportunity.Marks.UUID = opportunity.UUID if err := opportunity.Marks.Validate(); err != nil { - return fmt.Errorf("store: opportunity %s: %w", opportunity.UUID, err) + return fmt.Errorf("opportunity %s: %w", opportunity.UUID, err) } latest := opportunity.LatestReleaseDate for i := range opportunity.Releases { release := &opportunity.Releases[i] if release.ReleaseID == "" { - return fmt.Errorf("store: opportunity %s: release %d has no release id", opportunity.UUID, i) + return fmt.Errorf( + "store: opportunity %s: release %d has no release id", + opportunity.UUID, + i, + ) } release.UUID = opportunity.UUID release.ReleaseDate = release.ReleaseDate.UTC() @@ -138,7 +161,9 @@ func prepareOpportunity(opportunity *types.Opportunity) error { } if latest.IsZero() { - return fmt.Errorf("store: opportunity %s has no release date", opportunity.UUID) + return fmt.Errorf( + "store: opportunity %s has no release date", opportunity.UUID, + ) } opportunity.LatestReleaseDate = latest.UTC() @@ -188,12 +213,15 @@ func Open(ctx context.Context, path string, opts ...Option) (*Store, error) { if err != nil { return nil, err } + if err := sqldb.PingContext(ctx); err != nil { return nil, errors.Join(err, sqldb.Close()) } + if err := prepareSchema(ctx, sqldb, settings); err != nil { return nil, errors.Join(err, sqldb.Close()) } + return &Store{ db: sqlx.NewDb(sqldb, "sqlite3"), }, nil @@ -207,10 +235,10 @@ func (s *Store) Close() error { return s.db.Close() } // reads fail loudly ("missing destination name ...") until it is mapped, // instead of being silently ignored. // -// opportunityRow gives the opportunities table's internal id column a home, so it stays -// out of types.Opportunity. sqlx maps the embedded Opportunity's fields as if declared -// here. The other tables have no internal columns and read straight into -// internal/types. +// opportunityRow gives the opportunities table's internal id column a home, so +// it stays out of types.Opportunity. sqlx maps the embedded Opportunity's +// fields as if declared here. The other tables have no internal columns and +// read straight into internal/types. type opportunityRow struct { ID int64 `db:"id"` types.Opportunity @@ -230,8 +258,7 @@ type opportunityRow struct { // opportunity is normalised in place (see prepareOpportunity), and when it is // written its classifications' IDs are filled in. func (s *Store) SaveOpportunity( - ctx context.Context, - opportunity *types.Opportunity, + ctx context.Context, opportunity *types.Opportunity, ) (SaveResult, error) { if err := prepareOpportunity(opportunity); err != nil { return SaveResult{}, err @@ -243,14 +270,16 @@ func (s *Store) SaveOpportunity( } defer tx.Rollback() //nolint:errcheck // no-op after Commit - res, _, err := saveOpportunity(ctx, tx, opportunity) + result, _, err := saveOpportunity(ctx, tx, opportunity) if err != nil { return SaveResult{}, err } + if err := tx.Commit(); err != nil { return SaveResult{}, err } - return res, nil + + return result, nil } // SaveOpportunities stores each of opportunities and its releases, all in one @@ -282,28 +311,29 @@ func (s *Store) SaveOpportunities( } defer tx.Rollback() //nolint:errcheck // no-op after Commit - allRes := SaveMultipleResult{ + total := SaveMultipleResult{ OpportunityResults: make([]SaveResult, len(opportunities)), } for i, opportunity := range opportunities { - res, isNew, err := saveOpportunity(ctx, tx, opportunity) + result, isNew, err := saveOpportunity(ctx, tx, opportunity) if err != nil { return SaveMultipleResult{}, err } - allRes.OpportunityResults[i] = res - allRes.NewReleases += res.NewReleases + total.OpportunityResults[i] = result + total.NewReleases += result.NewReleases switch { case isNew: - allRes.NewOpportunities++ - case res.OpportunityUpdated: - allRes.UpdatedOpportunities++ + total.NewOpportunities++ + case result.OpportunityUpdated: + total.UpdatedOpportunities++ } } if err := tx.Commit(); err != nil { return SaveMultipleResult{}, err } - return allRes, nil + + return total, nil } // saveOpportunity does the work of SaveOpportunity and SaveOpportunities for @@ -311,28 +341,31 @@ func (s *Store) SaveOpportunities( // not already stored. func saveOpportunity( ctx context.Context, tx *sqlx.Tx, opportunity *types.Opportunity, -) (res SaveResult, isNew bool, err error) { - var stored time.Time - err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, opportunity.UUID) +) (SaveResult, bool, error) { + var ( + result SaveResult + isNew bool + stored time.Time + ) + err := tx.GetContext( + ctx, &stored, selectOpportunityLatestReleaseDate, opportunity.UUID, + ) switch { case errors.Is(err, sql.ErrNoRows): isNew = true - res.OpportunityUpdated = true + result.OpportunityUpdated = true case err != nil: return SaveResult{}, false, fmt.Errorf( - "store: reading opportunity %s: %w", - opportunity.UUID, - err, + "reading opportunity %s: %w", opportunity.UUID, err, ) default: - res.OpportunityUpdated = !opportunity.LatestReleaseDate.Before(stored) + result.OpportunityUpdated = !opportunity.LatestReleaseDate.Before(stored) } - if res.OpportunityUpdated { + + if result.OpportunityUpdated { if err := writeOpportunity(ctx, tx, opportunity); err != nil { return SaveResult{}, false, fmt.Errorf( - "store: writing opportunity %s: %w", - opportunity.UUID, - err, + "writing opportunity %s: %w", opportunity.UUID, err, ) } } @@ -343,10 +376,11 @@ func saveOpportunity( return SaveResult{}, false, err } if inserted { - res.NewReleases++ + result.NewReleases++ } } - return res, isNew, nil + + return result, isNew, nil } // insertReleaseWithTags stores release and its tags unless it is already @@ -357,77 +391,96 @@ func insertReleaseWithTags( result, err := tx.NamedExecContext(ctx, insertRelease, release) if err != nil { return false, fmt.Errorf( - "store: inserting release %s/%s: %w", + "inserting release %s/%s: %w", release.UUID, release.ReleaseID, err, ) } + if affected, err := result.RowsAffected(); err != nil || affected == 0 { - return false, err // n == 0: already stored + return false, err // affected == 0: already stored } + for _, tag := range release.Tags { if _, err := tx.NamedExecContext(ctx, insertReleaseTag, tag); err != nil { return false, fmt.Errorf( - "store: inserting tag for %s/%s: %w", release.UUID, release.ReleaseID, err, + "inserting tag for %s/%s: %w", + release.UUID, + release.ReleaseID, + err, ) } } + return true, nil } -// writeOpportunity upserts opportunity's row, gives it a marks row if it has none -// (existing marks are kept), and replaces its classifications, filling in +// writeOpportunity upserts opportunity's row, gives it a marks row if it has +// none (existing marks are kept), and replaces its classifications, filling in // their IDs. func writeOpportunity( - ctx context.Context, - tx *sqlx.Tx, - opportunity *types.Opportunity, + ctx context.Context, tx *sqlx.Tx, opportunity *types.Opportunity, ) (err error) { - if _, err := tx.NamedExecContext(ctx, upsertOpportunity, opportunity); err != nil { - return err - } - if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, opportunity.Marks); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, opportunity.UUID); err != nil { + _, err = tx.NamedExecContext(ctx, upsertOpportunity, opportunity) + if err != nil { return err } - statement, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID) + _, err = tx.NamedExecContext(ctx, insertOpportunityMarks, opportunity.Marks) + if err != nil { + return err + } + + _, err = tx.ExecContext( + ctx, deleteOpportunityClassifications, opportunity.UUID, + ) + if err != nil { + return err + } + + statement, err := tx.PrepareNamedContext( + ctx, upsertClassificationCodeReturningID, + ) if err != nil { return err } defer func() { err = errors.Join(err, statement.Close()) }() + for i := range opportunity.Classifications { classification := &opportunity.Classifications[i] - if err := statement.GetContext(ctx, &classification.ID, classification); err != nil { + + err = statement.GetContext(ctx, &classification.ID, classification) + if err != nil { return err } - if _, err := tx.ExecContext( - ctx, - insertOpportunityClassification, - opportunity.UUID, - classification.ID, - ); err != nil { + + _, err = tx.ExecContext( + ctx, insertOpportunityClassification, opportunity.UUID, classification.ID, + ) + if err != nil { return err } } + return nil } // SetMarks replaces the marks of the opportunity marks.UUID, or returns // ErrNotFound. Marks that fail marks.Validate (e.g. types.ErrWonAndLost) are // not stored. -func (s *Store) SetMarks(ctx context.Context, marks types.OpportunityMarks) error { +func (s *Store) SetMarks( + ctx context.Context, marks types.OpportunityMarks, +) error { if marks.UUID == "" { return errors.New("store: marks have no UUID") } if err := marks.Validate(); err != nil { - return fmt.Errorf("store: opportunity %s: %w", marks.UUID, err) + return fmt.Errorf("opportunity %s: %w", marks.UUID, err) } + _, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, marks) if isForeignKeyError(err) { return ErrNotFound @@ -439,12 +492,15 @@ func (s *Store) SetMarks(ctx context.Context, marks types.OpportunityMarks) erro // breaks a foreign key, e.g. one referring to a missing opportunity. func isForeignKeyError(err error) bool { var sqliteErr sqlite3.Error - return errors.As(err, &sqliteErr) && sqliteErr.ExtendedCode == sqlite3.ErrConstraintForeignKey + return errors.As(err, &sqliteErr) && + sqliteErr.ExtendedCode == sqlite3.ErrConstraintForeignKey } // GetOpportunity returns the opportunity with its classifications, marks and // all of its releases (with tags), or ErrNotFound. -func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) { +func (s *Store) GetOpportunity( + ctx context.Context, uuid string, +) (*types.Opportunity, error) { var row opportunityRow err := s.db.GetContext(ctx, &row, selectOpportunityByUUID, uuid) if errors.Is(err, sql.ErrNoRows) { @@ -453,6 +509,7 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu if err != nil { return nil, err } + opportunities := []types.Opportunity{row.Opportunity} if err := s.loadClassifications(ctx, opportunities); err != nil { return nil, err @@ -460,9 +517,11 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu if err := s.loadMarks(ctx, opportunities); err != nil { return nil, err } + if opportunities[0].Releases, err = s.ListReleases(ctx, uuid); err != nil { return nil, err } + return &opportunities[0], nil } @@ -470,29 +529,32 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu // classifications and marks but without releases (use ListReleases or // GetOpportunity for those). func (s *Store) ListOpportunities( - ctx context.Context, - filter OpportunityFilter, + ctx context.Context, filter OpportunityFilter, ) ([]types.Opportunity, error) { query, args := listOpportunitiesQuery(filter) var rows []opportunityRow if err := s.db.SelectContext(ctx, &rows, query, args...); err != nil { return nil, err } + opportunities := make([]types.Opportunity, len(rows)) for i, row := range rows { opportunities[i] = row.Opportunity } + if err := s.loadClassifications(ctx, opportunities); err != nil { return nil, err } if err := s.loadMarks(ctx, opportunities); err != nil { return nil, err } + return opportunities, nil } -// listOpportunitiesQuery assembles the ListOpportunities query from the fragments in -// queries.go, with one argument per filter that is set (none for mark filters). +// listOpportunitiesQuery assembles the ListOpportunities query from the +// fragments in queries.go, with one argument per filter that is set (none for +// mark filters). func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { var where []string var args []any @@ -500,6 +562,7 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { where = append(where, clause) args = append(args, argument) } + if filter.Status != "" { add(filterOpportunityStatus, filter.Status) } @@ -515,35 +578,19 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { if !filter.ReleasedBefore.IsZero() { add(filterOpportunityReleasedBefore, filter.ReleasedBefore.UTC()) } - for _, mark := range []struct { + + marks := []struct { filter MarkFilter clause string }{ - { - filter: filter.Hidden, - clause: filterOpportunityMarkHidden, - }, - { - filter: filter.Skipped, - clause: filterOpportunityMarkSkipped, - }, - { - filter: filter.Interested, - clause: filterOpportunityMarkInterested, - }, - { - filter: filter.BidSubmitted, - clause: filterOpportunityMarkBidSubmitted, - }, - { - filter: filter.Won, - clause: filterOpportunityMarkWon, - }, - { - filter: filter.Lost, - clause: filterOpportunityMarkLost, - }, - } { + {filter: filter.Hidden, clause: filterOpportunityMarkHidden}, + {filter: filter.Skipped, clause: filterOpportunityMarkSkipped}, + {filter: filter.Interested, clause: filterOpportunityMarkInterested}, + {filter: filter.BidSubmitted, clause: filterOpportunityMarkBidSubmitted}, + {filter: filter.Won, clause: filterOpportunityMarkWon}, + {filter: filter.Lost, clause: filterOpportunityMarkLost}, + } + for _, mark := range marks { switch mark.filter { case MarkSet: where = append(where, mark.clause) @@ -557,14 +604,19 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { query += "\nWHERE " + strings.Join(where, "\n\tAND ") } query += listOpportunitiesOrderAndPage + return query, append(args, filter.limit(), filter.Offset) } -// loadClassifications fills in Classifications for all opportunities with one query. -func (s *Store) loadClassifications(ctx context.Context, opportunities []types.Opportunity) error { +// loadClassifications fills in Classifications for all opportunities with one +// query. +func (s *Store) loadClassifications( + ctx context.Context, opportunities []types.Opportunity, +) error { if len(opportunities) == 0 { return nil } + uuids := make([]string, len(opportunities)) byUUID := make(map[string]*types.Opportunity, len(opportunities)) for i := range opportunities { @@ -576,27 +628,36 @@ func (s *Store) loadClassifications(ctx context.Context, opportunities []types.O if err != nil { return err } + // opportunity_uuid says which opportunity each row belongs to. var rows []struct { OpportunityUUID string `db:"opportunity_uuid"` types.ClassificationCode } - if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil { + err = s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...) + if err != nil { return err } - for _, releaseRow := range rows { - opportunity := byUUID[releaseRow.OpportunityUUID] - opportunity.Classifications = append(opportunity.Classifications, releaseRow.ClassificationCode) + + for _, classificationRow := range rows { + opportunity := byUUID[classificationRow.OpportunityUUID] + opportunity.Classifications = append( + opportunity.Classifications, classificationRow.ClassificationCode, + ) } + return nil } // loadMarks fills in Marks for all opportunities with one query. An // opportunity with no marks row gets zero marks carrying its UUID. -func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity) error { +func (s *Store) loadMarks( + ctx context.Context, opportunities []types.Opportunity, +) error { if len(opportunities) == 0 { return nil } + uuids := make([]string, len(opportunities)) byUUID := make(map[string]*types.Opportunity, len(opportunities)) for i := range opportunities { @@ -611,42 +672,55 @@ func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity if err != nil { return err } + var rows []types.OpportunityMarks - if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil { + err = s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...) + if err != nil { return err } + for _, row := range rows { byUUID[row.UUID].Marks = row } + return nil } // ListReleases returns a opportunity's releases with their tags, oldest first. -func (s *Store) ListReleases(ctx context.Context, uuid string) ([]types.OpportunityRelease, error) { +func (s *Store) ListReleases( + ctx context.Context, uuid string, +) ([]types.OpportunityRelease, error) { releases := []types.OpportunityRelease{} - if err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid); err != nil { + err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid) + if err != nil { return nil, err } + byID := make(map[string]*types.OpportunityRelease, len(releases)) for i := range releases { byID[releases[i].ReleaseID] = &releases[i] } var tags []types.OpportunityReleaseTag - if err := s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid); err != nil { + err = s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid) + if err != nil { return nil, err } + for _, tag := range tags { if release := byID[tag.ReleaseID]; release != nil { release.Tags = append(release.Tags, tag) } } + return releases, nil } // ReleaseExists reports whether a release is already stored, so a fetcher // can skip converting it. -func (s *Store) ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) { +func (s *Store) ReleaseExists( + ctx context.Context, uuid, releaseID string, +) (bool, error) { var exists bool err := s.db.GetContext(ctx, &exists, selectReleaseExists, uuid, releaseID) return exists, err diff --git a/internal/store/store_test.go b/internal/store/store_test.go index c383787..59bca09 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -30,7 +30,11 @@ func day(d int) time.Time { return time.Date(2026, time.March, d, 9, 30, 0, 0, time.UTC) } -func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *types.Opportunity { +func opportunity( + uuid, title string, + releaseDay int, + releaseIDs ...string, +) *types.Opportunity { start, end := day(20), day(28) t := &types.Opportunity{ UUID: uuid, @@ -39,7 +43,7 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type Status: "active", Description: "A description", BuyerName: "Example Council", - BuyerId: "GB-EXAMPLE", + BuyerID: "GB-EXAMPLE", MainCategory: "services", ProcurementMethod: "open", MultiLot: true, @@ -49,8 +53,16 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type GrossValue: 125000.50, HasDocuments: true, Classifications: []types.ClassificationCode{ - {IsCPV: true, Code: "72000000", Classification: "IT services"}, - {IsCPV: true, Code: "03000000", Classification: "Agricultural products"}, + { + IsCPV: true, + Code: "72000000", + Classification: "IT services", + }, + { + IsCPV: true, + Code: "03000000", + Classification: "Agricultural products", + }, }, // No UUID: SaveOpportunity must fill it in. Marks: types.OpportunityMarks{ @@ -58,18 +70,29 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type BidSubmitted: true, }, } + for i, id := range releaseIDs { t.Releases = append(t.Releases, types.OpportunityRelease{ ReleaseID: id, ReleaseDate: day(releaseDay).Add(time.Duration(i) * time.Minute), JSON: `{"id":"` + id + `"}`, - Tags: []types.OpportunityReleaseTag{{Tag: "opportunity"}, {Tag: "planning"}}, + Tags: []types.OpportunityReleaseTag{ + { + Tag: "opportunity", + }, + { + Tag: "planning", + }, + }, }) } + return t } -func mustSave(t *testing.T, s *store.Store, td *types.Opportunity) store.SaveResult { +func mustSave( + t *testing.T, s *store.Store, td *types.Opportunity, +) store.SaveResult { t.Helper() res, err := s.SaveOpportunity(context.Background(), td) if err != nil { @@ -116,24 +139,30 @@ func TestSaveAndGetRoundTrip(t *testing.T) { } got := mustGet(t, s, "ocds-1") - if got.Title != in.Title || got.BuyerName != in.BuyerName || got.BuyerId != in.BuyerId || - got.MainCategory != in.MainCategory || got.ProcurementMethod != in.ProcurementMethod || - got.UUIDType != "ocid" || got.Status != "active" || got.Description != in.Description || - !got.MultiLot || !got.SMESuitable || got.VCSESuitable || !got.HasDocuments || - got.GrossValue != in.GrossValue { + if got.Title != in.Title || got.BuyerName != in.BuyerName || + got.BuyerID != in.BuyerID || got.MainCategory != in.MainCategory || + got.ProcurementMethod != in.ProcurementMethod || got.UUIDType != "ocid" || + got.Status != "active" || got.Description != in.Description || + !got.MultiLot || !got.SMESuitable || got.VCSESuitable || + !got.HasDocuments || got.GrossValue != in.GrossValue { t.Errorf("opportunity fields differ:\n got %+v\nwant %+v", got, in) } - if got.StartDate == nil || !got.StartDate.Equal(day(20)) || got.EndDate == nil || - !got.EndDate.Equal(day(28)) { + if got.StartDate == nil || !got.StartDate.Equal(day(20)) || + got.EndDate == nil || !got.EndDate.Equal(day(28)) { t.Errorf("dates: got %v / %v", got.StartDate, got.EndDate) } if want := day(1).Add(time.Minute); !got.LatestReleaseDate.Equal(want) { t.Errorf("LatestReleaseDate = %v, want %v", got.LatestReleaseDate, want) } - if len(got.Classifications) != 2 || got.Classifications[0].Code != "03000000" || - got.Classifications[1].Code != "72000000" || got.Classifications[0].ID == 0 { - t.Errorf("classifications: got %+v (leading zero must survive)", got.Classifications) + if len(got.Classifications) != 2 || + got.Classifications[0].Code != "03000000" || + got.Classifications[1].Code != "72000000" || + got.Classifications[0].ID == 0 { + t.Errorf( + "classifications: got %+v (leading zero must survive)", + got.Classifications, + ) } wantMarks := types.OpportunityMarks{ @@ -153,9 +182,8 @@ func TestSaveAndGetRoundTrip(t *testing.T) { !r.ReleaseDate.Equal(day(1)) { t.Errorf("release: got %+v", r) } - if len(r.Tags) != 2 || - r.Tags[0].Tag != "opportunity" || r.Tags[1].Tag != "planning" || - r.Tags[0].ReleaseID != "r1" || + if len(r.Tags) != 2 || r.Tags[0].Tag != "opportunity" || + r.Tags[1].Tag != "planning" || r.Tags[0].ReleaseID != "r1" || r.Tags[0].UUID != "ocds-1" { t.Errorf("tags: got %+v", r.Tags) } @@ -178,7 +206,9 @@ func TestReleasesAreWriteOnce(t *testing.T) { again := opportunity("ocds-1", "Title", 1, "r1") again.Releases[0].JSON = `{"changed":true}` - again.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}} + again.Releases[0].Tags = []types.OpportunityReleaseTag{{ + Tag: "award", + }} if res := mustSave(t, s, again); res.NewReleases != 0 { t.Errorf("re-save: NewReleases = %d, want 0", res.NewReleases) } @@ -206,7 +236,9 @@ func TestOlderReleaseDoesNotOverwriteOpportunity(t *testing.T) { t.Errorf("opportunity was overwritten by an older release: %+v", got) } if len(got.Releases) != 2 || got.Releases[0].ReleaseID != "r1" { - t.Errorf("older release should still be stored, oldest first: %+v", got.Releases) + t.Errorf( + "older release should still be stored, oldest first: %+v", got.Releases, + ) } } @@ -220,7 +252,11 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) { newer := opportunity("ocds-1", "Website redesign", 2, "r2") newer.Status = "complete" newer.Classifications = []types.ClassificationCode{ - {IsCPV: true, Code: "72000000", Classification: "IT services (amended)"}, + { + IsCPV: true, + Code: "72000000", + Classification: "IT services (amended)", + }, } if res := mustSave(t, s, newer); !res.OpportunityUpdated { t.Fatalf("newer save not applied: %+v", res) @@ -246,7 +282,12 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) { t.Fatal(err) } if len(ts) != want { - t.Errorf("search %q after title change: %d results, want %d", title, len(ts), want) + t.Errorf( + "search %q after title change: %d results, want %d", + title, + len(ts), + want, + ) } } } @@ -257,7 +298,8 @@ func TestClassificationRowsAreShared(t *testing.T) { b := opportunity("ocds-b", "B", 1, "r1") mustSave(t, s, a) mustSave(t, s, b) - if a.Classifications[0].ID == 0 || a.Classifications[0].ID != b.Classifications[0].ID { + if a.Classifications[0].ID == 0 || + a.Classifications[0].ID != b.Classifications[0].ID { t.Errorf( "identical classifications should share an ID: %d vs %d", a.Classifications[0].ID, @@ -267,7 +309,11 @@ func TestClassificationRowsAreShared(t *testing.T) { c := opportunity("ocds-c", "C", 1, "r1") c.Classifications = []types.ClassificationCode{ - {IsCPV: false, Code: "72000000", Classification: "IT services"}, + { + IsCPV: false, + Code: "72000000", + Classification: "IT services", + }, } mustSave(t, s, c) if c.Classifications[0].ID == a.Classifications[0].ID { @@ -279,13 +325,17 @@ func TestSameReleaseIDInDifferentOpportunities(t *testing.T) { s := openStore(t) a := opportunity("ocds-a", "A", 1, "001") b := opportunity("ocds-b", "B", 1, "001") - b.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}} + b.Releases[0].Tags = []types.OpportunityReleaseTag{{ + Tag: "award", + }} + if res := mustSave(t, s, a); res.NewReleases != 1 { t.Fatal(res) } if res := mustSave(t, s, b); res.NewReleases != 1 { t.Fatalf("release id reused under another OCID must be stored: %+v", res) } + got := mustGet(t, s, "ocds-b").Releases if len(got) != 1 || len(got[0].Tags) != 1 || got[0].Tags[0].Tag != "award" { t.Errorf("tags leaked between opportunities: %+v", got) @@ -300,7 +350,11 @@ func TestListOpportunitiesFilters(t *testing.T) { roads := opportunity("ocds-2", "Road resurfacing works", 2, "r1") roads.Status = "complete" roads.Classifications = []types.ClassificationCode{ - {IsCPV: true, Code: "45233142", Classification: "Road-repair works"}, + { + IsCPV: true, + Code: "45233142", + Classification: "Road-repair works", + }, } hosting := opportunity("ocds-3", "Managed cloud hosting", 3, "r1") for _, td := range []*types.Opportunity{cloud, roads, hosting} { @@ -312,7 +366,11 @@ func TestListOpportunitiesFilters(t *testing.T) { f store.OpportunityFilter want []string }{ - {"all, newest first", store.OpportunityFilter{}, []string{"ocds-3", "ocds-2", "ocds-1"}}, + { + "all, newest first", + store.OpportunityFilter{}, + []string{"ocds-3", "ocds-2", "ocds-1"}, + }, {"status", store.OpportunityFilter{ Status: "complete", }, []string{"ocds-2"}}, @@ -385,9 +443,13 @@ func TestListOpportunitiesFilters(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got) != 1 || len(got[0].Classifications) != 1 || got[0].Releases != nil || - got[0].Marks.UUID != "ocds-2" || !got[0].Marks.Interested { - t.Errorf("list should include classifications and marks and omit releases: %+v", got) + if len(got) != 1 || len(got[0].Classifications) != 1 || + got[0].Releases != nil || got[0].Marks.UUID != "ocds-2" || + !got[0].Marks.Interested { + t.Errorf( + "list should include classifications and marks and omit releases: %+v", + got, + ) } } @@ -402,26 +464,28 @@ func TestReleaseExistsAndListReleases(t *testing.T) { }{{"ocds-1", "r1", true}, {"ocds-1", "r3", false}, {"ocds-2", "r1", false}} { got, err := s.ReleaseExists(ctx, tc.uuid, tc.id) if err != nil || got != tc.want { - t.Errorf("ReleaseExists(%s, %s) = %v, %v; want %v", tc.uuid, tc.id, got, err, tc.want) + t.Errorf( + "ReleaseExists(%s, %s) = %v, %v; want %v", + tc.uuid, + tc.id, + got, + err, + tc.want, + ) } } rs, err := s.ListReleases(ctx, "ocds-1") - if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" || rs[1].ReleaseID != "r2" || - len(rs[1].Tags) != 2 { + if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" || + rs[1].ReleaseID != "r2" || len(rs[1].Tags) != 2 { t.Errorf("ListReleases: %+v, %v", rs, err) } } func TestGetOpportunityNotFound(t *testing.T) { s := openStore(t) - if _, err := s.GetOpportunity( - context.Background(), - "missing", - ); !errors.Is( - err, - store.ErrNotFound, - ) { + _, err := s.GetOpportunity(context.Background(), "missing") + if !errors.Is(err, store.ErrNotFound) { t.Errorf("err = %v, want ErrNotFound", err) } } @@ -473,8 +537,12 @@ func TestResaveLoadedOpportunity(t *testing.T) { ids := []int64{loaded.Classifications[0].ID, loaded.Classifications[1].ID} loaded.Title = "Amended" loaded.Releases = append(loaded.Releases, types.OpportunityRelease{ - ReleaseID: "r2", ReleaseDate: day(2), JSON: `{}`, - Tags: []types.OpportunityReleaseTag{{Tag: "opportunityAmendment"}}, + ReleaseID: "r2", + ReleaseDate: day(2), + JSON: `{}`, + Tags: []types.OpportunityReleaseTag{{ + Tag: "opportunityAmendment", + }}, }) res := mustSave(t, s, loaded) if !res.OpportunityUpdated || res.NewReleases != 1 { @@ -482,10 +550,12 @@ func TestResaveLoadedOpportunity(t *testing.T) { } got := mustGet(t, s, "ocds-1") - if got.Title != "Amended" || len(got.Releases) != 2 || !got.LatestReleaseDate.Equal(day(2)) { + if got.Title != "Amended" || len(got.Releases) != 2 || + !got.LatestReleaseDate.Equal(day(2)) { t.Errorf("resave not applied: %+v", got) } - if got.Classifications[0].ID != ids[0] || got.Classifications[1].ID != ids[1] { + if got.Classifications[0].ID != ids[0] || + got.Classifications[1].ID != ids[1] { t.Errorf( "classification IDs changed on resave: %v -> %d, %d", ids, @@ -527,10 +597,7 @@ func TestSetMarksErrors(t *testing.T) { UUID: "missing", Hidden: true, }, - ); !errors.Is( - err, - store.ErrNotFound, - ) { + ); !errors.Is(err, store.ErrNotFound) { t.Errorf("unknown opportunity: err = %v, want ErrNotFound", err) } if err := s.SetMarks(ctx, types.OpportunityMarks{ @@ -565,7 +632,9 @@ func TestSaveKeepsExistingMarks(t *testing.T) { newest := opportunity("ocds-1", "Amended again", 3, "r3") newest.Marks = types.OpportunityMarks{} - if _, err := s.SaveOpportunities(ctx, []*types.Opportunity{newest}); err != nil { + if _, err := s.SaveOpportunities( + ctx, []*types.Opportunity{newest}, + ); err != nil { t.Fatal(err) } if got := mustGet(t, s, "ocds-1").Marks; got != want { @@ -580,9 +649,13 @@ func TestSaveOpportunitiesStoresMarks(t *testing.T) { b.Marks = types.OpportunityMarks{ Skipped: true, } - if _, err := s.SaveOpportunities(context.Background(), []*types.Opportunity{a, b}); err != nil { + + if _, err := s.SaveOpportunities( + context.Background(), []*types.Opportunity{a, b}, + ); err != nil { t.Fatal(err) } + for uuid, want := range map[string]types.OpportunityMarks{ "ocds-a": {UUID: "ocds-a", Interested: true, BidSubmitted: true}, "ocds-b": {UUID: "ocds-b", Skipped: true}, @@ -610,7 +683,10 @@ func TestListOpportunitiesMarkFilters(t *testing.T) { Lost: true, }) mustSetMarks(t, s, types.OpportunityMarks{ - UUID: "ocds-4", Interested: true, BidSubmitted: true, Won: true, + UUID: "ocds-4", + Interested: true, + BidSubmitted: true, + Won: true, }) set, unset := store.MarkSet, store.MarkUnset @@ -619,7 +695,11 @@ func TestListOpportunitiesMarkFilters(t *testing.T) { f store.OpportunityFilter want []string }{ - {"any", store.OpportunityFilter{}, []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}}, + { + "any", + store.OpportunityFilter{}, + []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}, + }, {"hidden", store.OpportunityFilter{ Hidden: set, }, []string{"ocds-2"}}, @@ -698,8 +778,8 @@ func TestListOpportunitiesMarkFilters(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost || !got[2].Marks.Hidden || - got[3].Marks.Hidden || !got[3].Marks.Interested { + if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost || + !got[2].Marks.Hidden || got[3].Marks.Hidden || !got[3].Marks.Interested { t.Errorf("list marks: %+v", got) } } @@ -727,19 +807,20 @@ func TestWonAndLostAreExclusive(t *testing.T) { Won: true, Lost: true, } - if _, err := s.SaveOpportunity(ctx, in); !errors.Is(err, types.ErrWonAndLost) { + _, err := s.SaveOpportunity(ctx, in) + if !errors.Is(err, types.ErrWonAndLost) { t.Errorf("SaveOpportunity(won and lost): err = %v, want ErrWonAndLost", err) } - if _, err := s.SaveOpportunities( - ctx, - []*types.Opportunity{in}, - ); !errors.Is( - err, - types.ErrWonAndLost, - ) { - t.Errorf("SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err) + + _, err = s.SaveOpportunities(ctx, []*types.Opportunity{in}) + if !errors.Is(err, types.ErrWonAndLost) { + t.Errorf( + "SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err, + ) } - if _, err := s.GetOpportunity(ctx, "ocds-2"); !errors.Is(err, store.ErrNotFound) { + + _, err = s.GetOpportunity(ctx, "ocds-2") + if !errors.Is(err, store.ErrNotFound) { t.Errorf("rejected opportunity was stored: err = %v", err) } diff --git a/internal/types/opportunities.go b/internal/types/opportunities.go index f2585b9..c233890 100644 --- a/internal/types/opportunities.go +++ b/internal/types/opportunities.go @@ -17,7 +17,7 @@ type Opportunity struct { Description string `db:"description" json:"description"` BuyerName string `db:"buyer_name" json:"buyer_name"` Classifications []ClassificationCode `db:"-" json:"classifications"` - BuyerId string `db:"buyer_id" json:"buyer_id"` + BuyerID string `db:"buyer_id" json:"buyer_id"` MainCategory string `db:"main_category" json:"main_category"` ProcurementMethod string `db:"procurement_method" json:"procurement_method"` MultiLot bool `db:"multi_lot" json:"multi_lot"` @@ -53,7 +53,7 @@ type OpportunityReleaseTag struct { // ErrWonAndLost is returned by OpportunityMarks.Validate when both Won and // Lost are set. -var ErrWonAndLost = errors.New("an opportunity cannot be both won and lost") +var ErrWonAndLost = errors.New("types: an opportunity cannot be both won and lost") // OpportunityMarks are the user's own marks on an opportunity, kept apart // from the fetched data so saving a newer release never changes them. The diff --git a/internal/util/bytesize.go b/internal/util/bytesize.go index c42557c..9868245 100644 --- a/internal/util/bytesize.go +++ b/internal/util/bytesize.go @@ -83,15 +83,19 @@ func ParseByteSize(text string) (ByteSize, error) { return ByteSizeNoLimit, nil } if strings.HasPrefix(trimmed, "-") { - return 0, fmt.Errorf("byte size %q is negative; only -1 (no limit) is allowed", text) + return 0, fmt.Errorf( + "util: byte size %q is negative; only -1 (no limit) is allowed", text, + ) } + parsed, err := humanize.ParseBytes(text) if err != nil { return 0, err } if parsed > math.MaxInt64 { - return 0, fmt.Errorf("byte size %q is too large", text) + return 0, fmt.Errorf("util: byte size %q is too large", text) } + return ByteSize(parsed), nil } diff --git a/internal/util/bytesize_test.go b/internal/util/bytesize_test.go index 0861f7e..3557f32 100644 --- a/internal/util/bytesize_test.go +++ b/internal/util/bytesize_test.go @@ -35,7 +35,9 @@ func TestByteSizeNoLimit(t *testing.T) { for _, text := range []string{"-1", "-1B", "-1b", " -1 "} { size, err := ParseByteSize(text) if err != nil || size != ByteSizeNoLimit { - t.Errorf("ParseByteSize(%q) = %d, %v; want ByteSizeNoLimit", text, size, err) + t.Errorf( + "ParseByteSize(%q) = %d, %v; want ByteSizeNoLimit", text, size, err, + ) } } if got := ByteSizeNoLimit.String(); got != "-1" { @@ -44,7 +46,12 @@ func TestByteSizeNoLimit(t *testing.T) { for _, text := range []string{"-2", "-1KB", "-0.5", "-1KiB", "-100"} { size, err := ParseByteSize(text) if err == nil || !strings.Contains(err.Error(), "only -1 (no limit)") { - t.Errorf("ParseByteSize(%q) = %d, %v; want a negative size error", text, size, err) + t.Errorf( + "ParseByteSize(%q) = %d, %v; want a negative size error", + text, + size, + err, + ) } } } diff --git a/internal/util/context.go b/internal/util/context.go new file mode 100644 index 0000000..b966efc --- /dev/null +++ b/internal/util/context.go @@ -0,0 +1,19 @@ +package util + +import ( + "context" + "time" +) + +// WithTimeout returns ctx with the timeout applied, or ctx itself if the +// timeout is 0 or less (no limit). Either way the returned cancel function +// must be called. +func WithTimeout( + ctx context.Context, timeout time.Duration, +) (context.Context, context.CancelFunc) { + if timeout <= 0 { + return ctx, func() { + } + } + return context.WithTimeout(ctx, timeout) +} diff --git a/internal/util/context_test.go b/internal/util/context_test.go new file mode 100644 index 0000000..575935d --- /dev/null +++ b/internal/util/context_test.go @@ -0,0 +1,20 @@ +package util + +import ( + "context" + "testing" + "time" +) + +func TestWithTimeout(t *testing.T) { + ctx, cancel := WithTimeout(context.Background(), 0) + defer cancel() + if _, ok := ctx.Deadline(); ok { + t.Error("timeout 0 set a deadline") + } + ctx, cancel = WithTimeout(context.Background(), time.Minute) + defer cancel() + if _, ok := ctx.Deadline(); !ok { + t.Error("timeout 1m set no deadline") + } +} diff --git a/internal/util/worker.go b/internal/util/worker.go index b7029d9..d0e5608 100644 --- a/internal/util/worker.go +++ b/internal/util/worker.go @@ -11,18 +11,18 @@ import ( // DefaultWorkerShutdownTimeout is used when Worker.ShutdownTimeout is zero. const DefaultWorkerShutdownTimeout = 10 * time.Second -// ErrWorkerNoWork tells the Worker that a run succeeded but found nothing to do. -// Tasks return it, or an error wrapping it, to have the run treated as a +// ErrWorkerNoWork tells the Worker that a run succeeded but found nothing to +// do. Tasks return it, or an error wrapping it, to have the run treated as a // success (checkpoint advances) while being logged distinctly. For example: // // var ErrNoNewReleases = fmt.Errorf("%w: no new releases", ErrWorkerNoWork) -var ErrWorkerNoWork = errors.New("no work") +var ErrWorkerNoWork = errors.New("util: no work") // Task processes everything that has changed since the given checkpoint. // -// Returning nil or an error matching ErrNoWork advances the checkpoint to the -// time this run started. Any other error leaves the checkpoint unchanged, so -// the next run retries from the same point. +// Returning nil or an error matching ErrWorkerNoWork advances the checkpoint to +// the time this run started. Any other error leaves the checkpoint unchanged, +// so the next run retries from the same point. type Task func(ctx context.Context, since time.Time) error // Worker runs a Task periodically until its context is cancelled. @@ -32,7 +32,8 @@ type Worker struct { // Interval is how often Task runs. Must be greater than zero. Interval time.Duration // ShutdownTimeout is how long an in-flight Task may keep running after - // the parent context is cancelled. Zero means DefaultShutdownTimeout. + // the parent context is cancelled. 0 or less means + // DefaultWorkerShutdownTimeout. ShutdownTimeout time.Duration // Task is the work to perform on each tick. Task Task @@ -45,14 +46,16 @@ type Worker struct { // cancelled. It blocks until shutdown has finished. func (w Worker) Run(ctx context.Context, since time.Time) error { if w.Task == nil { - return fmt.Errorf("worker %q: no task configured", w.Name) + return fmt.Errorf("util: worker %q: no task configured", w.Name) } if w.Interval <= 0 { return fmt.Errorf( - "worker %q: interval must be greater than 0, got %s", - w.Name, w.Interval, + "util: worker %q: interval must be greater than 0, got %s", + w.Name, + w.Interval, ) } + timeout := w.ShutdownTimeout if timeout <= 0 { timeout = DefaultWorkerShutdownTimeout @@ -124,7 +127,11 @@ func (w Worker) execute( logger.Info("task completed", "elapsed", elapsed) return start case errors.Is(err, ErrWorkerNoWork): - logger.Info("task completed with nothing to do", "reason", err, "elapsed", elapsed) + logger.Info( + "task completed with nothing to do", + "reason", err, + "elapsed", elapsed, + ) return start case errors.Is(err, context.Canceled): logger.Warn("task cancelled", "elapsed", elapsed) @@ -142,22 +149,22 @@ func (w Worker) execute( // The result channel is buffered, so if ctx wins the race the goroutine can // still deliver its result and exit instead of blocking forever. func AwaitTask[T any](ctx context.Context, task func() (T, error)) (T, error) { - type result struct { - val T - err error + type outcome struct { + value T + err error } - results := make(chan result, 1) + results := make(chan outcome, 1) go func() { value, err := task() - results <- result{ - val: value, - err: err, + results <- outcome{ + value: value, + err: err, } }() select { case result := <-results: - return result.val, result.err + return result.value, result.err case <-ctx.Done(): var zero T return zero, ctx.Err() diff --git a/internal/util/worker_test.go b/internal/util/worker_test.go index 7d8b81d..4029e48 100644 --- a/internal/util/worker_test.go +++ b/internal/util/worker_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "log/slog" "os" "sync" @@ -14,7 +13,7 @@ import ( func TestMain(m *testing.M) { // Keep worker log lines out of the test output. - slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + slog.SetDefault(slog.New(slog.DiscardHandler)) os.Exit(m.Run()) } @@ -53,10 +52,15 @@ func TestWorkerRunCancelledBeforeStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() called := false - w := Worker{Name: "w", Interval: time.Millisecond, Task: func(context.Context, time.Time) error { - called = true - return nil - }} + w := Worker{ + Name: "w", + Interval: time.Millisecond, + Task: func(context.Context, time.Time) error { + called = true + return nil + }, + } + if err := w.Run(ctx, time.Time{}); err != nil { t.Fatal(err) } @@ -91,12 +95,14 @@ func TestWorkerRunCheckpoints(t *testing.T) { Task: func(_ context.Context, since time.Time) error { mu.Lock() defer mu.Unlock() + i := len(seen) seen = append(seen, since) starts = append(starts, time.Now()) if i == len(results)-1 { cancel() } + return results[i] }, } @@ -129,11 +135,20 @@ func TestWorkerRunCheckpoints(t *testing.T) { } } if !seen[1].After(since0) || seen[1].After(starts[1]) { - t.Fatalf("checkpoint after success = %v, want between %v and %v", seen[1], since0, starts[1]) + t.Fatalf( + "checkpoint after success = %v, want between %v and %v", + seen[1], + since0, + starts[1], + ) } // Run 3 returned ErrWorkerNoWork, so run 4 sees a newer checkpoint. if !seen[4].After(seen[3]) { - t.Fatalf("checkpoint did not advance after ErrWorkerNoWork: %v -> %v", seen[3], seen[4]) + t.Fatalf( + "checkpoint did not advance after ErrWorkerNoWork: %v -> %v", + seen[3], + seen[4], + ) } } @@ -143,7 +158,9 @@ func TestWorkerRunGracefulShutdown(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) var workErr error w := Worker{ - Name: "w", Interval: time.Hour, ShutdownTimeout: time.Second, + Name: "w", + Interval: time.Hour, + ShutdownTimeout: time.Second, Task: func(taskCtx context.Context, _ time.Time) error { cancel() time.Sleep(10 * time.Millisecond) @@ -151,6 +168,7 @@ func TestWorkerRunGracefulShutdown(t *testing.T) { return nil }, } + if err := w.Run(ctx, time.Time{}); err != nil { t.Fatal(err) } @@ -163,7 +181,9 @@ func TestWorkerRunGracefulShutdown(t *testing.T) { func TestWorkerRunShutdownTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) w := Worker{ - Name: "w", Interval: time.Hour, ShutdownTimeout: 20 * time.Millisecond, + Name: "w", + Interval: time.Hour, + ShutdownTimeout: 20 * time.Millisecond, Task: func(taskCtx context.Context, _ time.Time) error { cancel() select { @@ -174,6 +194,7 @@ func TestWorkerRunShutdownTimeout(t *testing.T) { } }, } + start := time.Now() if err := w.Run(ctx, time.Time{}); err != nil { t.Fatal(err) @@ -209,7 +230,9 @@ func TestWorkerExecute(t *testing.T) { got := w.execute(context.Background(), logger, since) if tc.advance { if got.Before(before) { - t.Fatalf("checkpoint %v not advanced to run start (>= %v)", got, before) + t.Fatalf( + "checkpoint %v not advanced to run start (>= %v)", got, before, + ) } } else if !got.Equal(since) { t.Fatalf("checkpoint = %v, want unchanged %v", got, since) @@ -261,10 +284,7 @@ func TestAwaitTaskE(t *testing.T) { func() error { return errFail }, - ); !errors.Is( - err, - errFail, - ) { + ); !errors.Is(err, errFail) { t.Fatalf("got %v, want %v", err, errFail) } @@ -278,10 +298,7 @@ func TestAwaitTaskE(t *testing.T) { <-release return nil }, - ); !errors.Is( - err, - context.DeadlineExceeded, - ) { + ); !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("got %v, want context.DeadlineExceeded", err) } } diff --git a/internal/websocket/client.go b/internal/websocket/client.go index f9a475e..5504f3e 100644 --- a/internal/websocket/client.go +++ b/internal/websocket/client.go @@ -12,6 +12,8 @@ import ( "github.com/coder/websocket" "golang.org/x/time/rate" + + "git.avsdev.uk/Go/job-search/internal/util" ) // clientSendBuffer is how many messages are queued for a client before it @@ -26,7 +28,9 @@ const logMessageLimit = 1 << 10 // *websocket.Conn, or a tcpConn for plain TCP clients (see tcp.go). type msgConn interface { Read(ctx context.Context) (websocket.MessageType, []byte, error) - Write(ctx context.Context, messageType websocket.MessageType, payload []byte) error + Write( + ctx context.Context, messageType websocket.MessageType, payload []byte, + ) error Ping(ctx context.Context) error Close(code websocket.StatusCode, reason string) error CloseNow() error @@ -49,7 +53,7 @@ type client struct { // errUnsupportedData is returned by readLoop (and a tcpConn's Write) for a // binary message. -var errUnsupportedData = errors.New("binary frames are not supported") +var errUnsupportedData = errors.New("websocket: binary frames are not supported") // newClient creates a client for conn. cancel cancels the context the client // runs with; the client calls it when it finishes or is closed. The client's @@ -59,10 +63,7 @@ func newClient( ) *client { limiter := rate.NewLimiter(rate.Inf, 0) if rule := hub.conf.MessageRateLimit; rule.Enabled && rule.PerSecond > 0 { - limiter = rate.NewLimiter( - rate.Limit(rule.PerSecond), - max(rule.Burst, 1), - ) + limiter = rate.NewLimiter(rate.Limit(rule.PerSecond), max(rule.Burst, 1)) } client := &client{ @@ -95,7 +96,7 @@ func (c *client) run(ctx context.Context) { c.sendEnvelope(Envelope{ Type: TypeWelcome, - Payload: mustRaw(map[string]any{ + Payload: rawJSON(map[string]any{ "clientId": c.id, }), }) @@ -129,16 +130,20 @@ func (c *client) readLoop(ctx context.Context) error { if err != nil { return err } + c.logMessage(ctx, "message received", data) c.lastSeen.Store(time.Now().UnixNano()) + if messageType != websocket.MessageText { return errUnsupportedData } + // Check if the client is rate limited for early discard if !c.limiter.Allow() { c.sendError("", "rate limited") continue } + // Unpack the envelope and pass to handle to process if valid var envelope Envelope if err := json.Unmarshal(data, &envelope); err != nil { @@ -157,7 +162,7 @@ func (c *client) handle(envelope Envelope) { c.sendEnvelope(Envelope{ Type: TypePong, ID: envelope.ID, - Payload: mustRaw(map[string]any{"time": time.Now().UTC()}), + Payload: rawJSON(map[string]any{"time": time.Now().UTC()}), }) case TypeEcho: @@ -180,7 +185,7 @@ func (c *client) writeLoop(ctx context.Context) { case <-ctx.Done(): return case msg := <-c.send: - writeCtx, cancel := withTimeout(ctx, c.hub.conf.WriteTimeout) + writeCtx, cancel := util.WithTimeout(ctx, c.hub.conf.WriteTimeout) err := c.conn.Write(writeCtx, websocket.MessageText, msg) cancel() if err != nil { @@ -217,7 +222,7 @@ func (c *client) pingLoop(ctx context.Context) { if time.Now().Before(lastSeen.Add(pingInterval)) { continue } - pingCtx, cancel := withTimeout(ctx, pongTimeout) + pingCtx, cancel := util.WithTimeout(ctx, pongTimeout) err := c.conn.Ping(pingCtx) cancel() if err != nil { @@ -283,29 +288,19 @@ func (c *client) logMessage(ctx context.Context, event string, data []byte) { if !logger.Enabled(ctx, slog.LevelDebug) { return } + logged, truncated := data, false if len(logged) > logMessageLimit { logged, truncated = logged[:logMessageLimit], true } + logger.LogAttrs( ctx, slog.LevelDebug, - event, + event, //nolint:sloglint // a constant message from the caller slog.String("client", c.id), slog.Int("bytes", len(data)), slog.String("message", string(logged)), slog.Bool("truncated", truncated), ) } - -// withTimeout returns ctx with the timeout applied, or ctx itself if the -// timeout is 0 or less (no limit). -func withTimeout( - ctx context.Context, timeout time.Duration, -) (context.Context, context.CancelFunc) { - if timeout <= 0 { - return ctx, func() { - } - } - return context.WithTimeout(ctx, timeout) -} diff --git a/internal/websocket/client_test.go b/internal/websocket/client_test.go index 0312708..646d872 100644 --- a/internal/websocket/client_test.go +++ b/internal/websocket/client_test.go @@ -34,7 +34,9 @@ func (f *fakeConn) Read(ctx context.Context) (ws.MessageType, []byte, error) { return 0, nil, ctx.Err() } -func (f *fakeConn) Write(context.Context, ws.MessageType, []byte) error { return nil } +func (f *fakeConn) Write(context.Context, ws.MessageType, []byte) error { + return nil +} func (f *fakeConn) Ping(context.Context) error { return nil } @@ -102,8 +104,10 @@ func TestSendEnvelopeSlowConsumer(t *testing.T) { func TestClientCloseOnce(t *testing.T) { f := newFakeConn() c, _ := newTestClient(f) + c.close(ws.StatusGoingAway, "bye") c.close(ws.StatusNormalClosure, "again") + if code := <-f.closed; code != ws.StatusGoingAway { t.Errorf("closed with %v; want going away", code) } @@ -147,12 +151,17 @@ func TestClientMessageRateLimit(t *testing.T) { }) c.limiter.Allow() if limited := !c.limiter.Allow(); limited != tt.limited { - t.Errorf("rule %+v: second message limited = %v; want %v", tt.rule, limited, tt.limited) + t.Errorf( + "rule %+v: second message limited = %v; want %v", + tt.rule, + limited, + tt.limited, + ) } } } -func TestMustRaw(t *testing.T) { +func TestRawJSON(t *testing.T) { raw := json.RawMessage(`{"a":1}`) tests := []struct { name string @@ -166,25 +175,12 @@ func TestMustRaw(t *testing.T) { }, ""}, } for _, tt := range tests { - if got := string(mustRaw(tt.v)); got != tt.want { - t.Errorf("%s: mustRaw = %q; want %q", tt.name, got, tt.want) + if got := string(rawJSON(tt.v)); got != tt.want { + t.Errorf("%s: rawJSON = %q; want %q", tt.name, got, tt.want) } } } -func TestWithTimeout(t *testing.T) { - ctx, cancel := withTimeout(context.Background(), 0) - defer cancel() - if _, ok := ctx.Deadline(); ok { - t.Error("timeout 0 set a deadline") - } - ctx, cancel = withTimeout(context.Background(), time.Minute) - defer cancel() - if _, ok := ctx.Deadline(); !ok { - t.Error("timeout 1m set no deadline") - } -} - // syncBuffer is a bytes.Buffer safe for the client's goroutines to log to. type syncBuffer struct { mu sync.Mutex @@ -202,6 +198,7 @@ func (b *syncBuffer) entries(t *testing.T) []map[string]any { t.Helper() b.mu.Lock() defer b.mu.Unlock() + var entries []map[string]any for line := range strings.Lines(b.buf.String()) { var e map[string]any @@ -210,6 +207,7 @@ func (b *syncBuffer) entries(t *testing.T) []map[string]any { } entries = append(entries, e) } + return entries } @@ -283,8 +281,9 @@ func TestLogMessageLimit(t *testing.T) { t.Fatalf("logged %d entries; want 1", len(entries)) } e := entries[0] - if e["message"] != long[:logMessageLimit] || e["bytes"] != float64(len(long)) || - e["truncated"] != true || e["client"] != c.id { + if e["message"] != long[:logMessageLimit] || + e["bytes"] != float64(len(long)) || e["truncated"] != true || + e["client"] != c.id { t.Errorf("entry = %v", e) } diff --git a/internal/websocket/http_test.go b/internal/websocket/http_test.go index 14a7f84..57ef039 100644 --- a/internal/websocket/http_test.go +++ b/internal/websocket/http_test.go @@ -38,10 +38,12 @@ func dialWS(t *testing.T, ctx context.Context, url string) *ws.Conn { t.Cleanup(func() { conn.CloseNow() }) + var e Envelope if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypeWelcome { t.Fatalf("want welcome, got %+v, %v", e, err) } + return conn } @@ -66,7 +68,8 @@ func TestHTTPMessages(t *testing.T) { t.Fatal(err) } var e Envelope - if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || e.ID != "1" { + if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || + e.ID != "1" { t.Fatalf("want pong 1, got %+v, %v", e, err) } } @@ -75,6 +78,7 @@ func TestHTTPMessages(t *testing.T) { func TestPushReachesBothTransports(t *testing.T) { h, addr := tcpHub(t, nil) url := wsHub(t, h) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() conn := dialWS(t, ctx, url) @@ -155,7 +159,8 @@ func TestHTTPMessageRateLimit(t *testing.T) { } } var e Envelope - if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || e.ID != "1" { + if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || + e.ID != "1" { t.Fatalf("first: want pong 1, got %+v, %v", e, err) } err := wsjson.Read(ctx, conn, &e) @@ -173,11 +178,13 @@ func TestHTTPNotAnUpgrade(t *testing.T) { h := NewHub(config.JSCServerWebSocket{}, WithLogger(discardLog)) srv := httptest.NewServer(h) defer srv.Close() + res, err := http.Get(srv.URL) if err != nil { t.Fatal(err) } res.Body.Close() + if res.StatusCode < 400 { t.Errorf("status = %d; want an error", res.StatusCode) } @@ -194,6 +201,7 @@ func TestHTTPRefusedAfterShutdown(t *testing.T) { if err := h.Shutdown(context.Background()); err != nil { t.Fatal(err) } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() conn, _, err := ws.Dial(ctx, url, nil) @@ -201,6 +209,7 @@ func TestHTTPRefusedAfterShutdown(t *testing.T) { t.Fatalf("dial: %v", err) } defer conn.CloseNow() + if _, _, err := conn.Read(ctx); ws.CloseStatus(err) != ws.StatusGoingAway { t.Fatalf("read = %v; want close 1001", err) } diff --git a/internal/websocket/hub.go b/internal/websocket/hub.go index 12935a9..085c30f 100644 --- a/internal/websocket/hub.go +++ b/internal/websocket/hub.go @@ -91,7 +91,11 @@ func (h *Hub) Push(msgType string, payload any) { h.mu.RUnlock() for _, client := range slow { - h.logger().Warn("dropping slow client", "client", client.id, "ip", client.ip) + h.logger().Warn( + "dropping slow client", + "client", client.id, + "ip", client.ip, + ) go client.close(ws.StatusPolicyViolation, "slow consumer") } } @@ -166,11 +170,14 @@ func (h *Hub) maxMessageSize() int { func (h *Hub) register(client *client) bool { h.mu.Lock() defer h.mu.Unlock() + if h.closed { return false } + h.clients[client] = struct{}{} h.wg.Add(1) + return true } diff --git a/internal/websocket/hub_test.go b/internal/websocket/hub_test.go index 6366853..e8b5832 100644 --- a/internal/websocket/hub_test.go +++ b/internal/websocket/hub_test.go @@ -21,16 +21,19 @@ func pipeClient(t *testing.T, h *Hub) (net.Conn, chan struct{}) { server.Close() peer.Close() }) + done := make(chan struct{}) go func() { h.ServeTCP(context.Background(), server) close(done) }() + peer.SetReadDeadline(time.Now().Add(time.Second)) if _, err := bufio.NewReader(peer).ReadString('\n'); err != nil { t.Fatalf("welcome: %v", err) } waitCount(t, h, 1) + return peer, done } @@ -100,7 +103,12 @@ func TestMaxMessageSize(t *testing.T) { for _, tt := range tests { h := NewHub(tt.conf, WithLogger(discardLog)) if got := h.maxMessageSize(); got != tt.want { - t.Errorf("maxMessageSize(%d) = %d; want %d", tt.conf.MaxMessageSize, got, tt.want) + t.Errorf( + "maxMessageSize(%d) = %d; want %d", + tt.conf.MaxMessageSize, + got, + tt.want, + ) } } } diff --git a/internal/websocket/messages.go b/internal/websocket/messages.go index d804a26..72d67f1 100644 --- a/internal/websocket/messages.go +++ b/internal/websocket/messages.go @@ -55,15 +55,16 @@ const ( TypeError = "error" ) -// mustRaw returns value as raw JSON: value itself if it is already +// rawJSON returns value as raw JSON: value itself if it is already // json.RawMessage, nil if value is nil or can't be marshalled. -func mustRaw(value any) json.RawMessage { +func rawJSON(value any) json.RawMessage { if value == nil { return nil } if raw, ok := value.(json.RawMessage); ok { return raw } + encoded, err := json.Marshal(value) if err != nil { return nil diff --git a/internal/websocket/tcp.go b/internal/websocket/tcp.go index 47d9d9d..52f6724 100644 --- a/internal/websocket/tcp.go +++ b/internal/websocket/tcp.go @@ -13,6 +13,8 @@ import ( "time" ws "github.com/coder/websocket" + + "git.avsdev.uk/Go/job-search/internal/util" ) // Plain TCP clients join the hub as if they were WebSocket clients. Instead @@ -55,7 +57,8 @@ func (h *Hub) ServeTCP(ctx context.Context, newConn net.Conn) { } ip := "" - if addrPort, err := netip.ParseAddrPort(newConn.RemoteAddr().String()); err == nil { + addrPort, err := netip.ParseAddrPort(newConn.RemoteAddr().String()) + if err == nil { ip = addrPort.Addr().Unmap().String() } @@ -71,10 +74,20 @@ func (h *Hub) ServeTCP(ctx context.Context, newConn net.Conn) { } defer h.unregister(client) - h.logger().Info("client connected", "client", client.id, "ip", ip, "transport", "tcp") + h.logger().Info( + "client connected", + "client", client.id, + "ip", ip, + "transport", "tcp", + ) client.run(ctx) conn.CloseNow() //nolint:errcheck,gosec // finished with; it may already be closed - h.logger().Info("client disconnected", "client", client.id, "ip", ip, "transport", "tcp") + h.logger().Info( + "client disconnected", + "client", client.id, + "ip", ip, + "transport", "tcp", + ) } // ---------- tcpConn ---------- @@ -88,7 +101,7 @@ type tcpConn struct { scan *bufio.Scanner closeTimeout time.Duration // caps Close's message; 0 or less for none - wm sync.Mutex // serialises writes + writeMu sync.Mutex // serialises writes closeOnce sync.Once // records the first close closed chan struct{} // closed by the first Close or CloseNow @@ -153,6 +166,7 @@ func (t *tcpConn) Read(ctx context.Context) (ws.MessageType, []byte, error) { t.Close(ws.StatusMessageTooBig, "message too long") //nolint:errcheck,gosec // dropped anyway return 0, nil, t.closeErr } + return 0, nil, err } @@ -185,12 +199,9 @@ func (t *tcpConn) Close(code ws.StatusCode, reason string) error { return nil } - ctx := context.Background() - if t.closeTimeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, t.closeTimeout) - defer cancel() - } + // No deadline if closeTimeout is 0 or less. + ctx, cancel := util.WithTimeout(context.Background(), t.closeTimeout) + defer cancel() message, err := json.Marshal(Envelope{ Type: TypeError, @@ -203,6 +214,7 @@ func (t *tcpConn) Close(code ws.StatusCode, reason string) error { if err := t.conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { return err } + return nil } @@ -244,8 +256,8 @@ func (t *tcpConn) isClosed() bool { // write writes data whole, giving up at ctx's deadline or when ctx is // cancelled. func (t *tcpConn) write(ctx context.Context, data []byte) error { - t.wm.Lock() - defer t.wm.Unlock() + t.writeMu.Lock() + defer t.writeMu.Unlock() deadline, _ := ctx.Deadline() // zero means none if err := t.conn.SetWriteDeadline(deadline); err != nil { diff --git a/internal/websocket/tcp_test.go b/internal/websocket/tcp_test.go index 46ae1d5..d37fd09 100644 --- a/internal/websocket/tcp_test.go +++ b/internal/websocket/tcp_test.go @@ -18,11 +18,13 @@ import ( "git.avsdev.uk/Go/job-search/internal/config" ) -var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil)) +var discardLog = slog.New(slog.DiscardHandler) // tcpHub starts a hub serving plain TCP clients, via ServeTCP, on a local // port and returns it with the address. It is shut down when the test ends. -func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string) { +func tcpHub( + t *testing.T, mutate func(*config.JSCServerWebSocket), +) (*Hub, string) { t.Helper() conf := config.JSCServerWebSocket{ MaxMessageSize: 1 << 10, @@ -31,11 +33,13 @@ func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string if mutate != nil { mutate(&conf) } + h := NewHub(conf, WithLogger(discardLog)) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } + ctx, cancel := context.WithCancel(context.Background()) go func() { for { @@ -46,6 +50,7 @@ func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string go h.ServeTCP(ctx, conn) } }() + t.Cleanup(func() { ln.Close() sctx, scancel := context.WithTimeout(context.Background(), time.Second) @@ -53,6 +58,7 @@ func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string h.Shutdown(sctx) cancel() }) + return h, ln.Addr().String() } @@ -94,10 +100,12 @@ func (c *tcpClient) read() Envelope { if err != nil { c.t.Fatalf("read: %v (got %q)", err, line) } + var env Envelope if err := json.Unmarshal([]byte(line), &env); err != nil { c.t.Fatalf("bad envelope %q: %v", line, err) } + return env } @@ -107,7 +115,8 @@ func (c *tcpClient) read() Envelope { func (c *tcpClient) waitClosed() { c.t.Helper() c.conn.SetReadDeadline(time.Now().Add(time.Second)) - if _, err := io.ReadAll(c.r); err != nil && !errors.Is(err, syscall.ECONNRESET) { + if _, err := io.ReadAll(c.r); err != nil && + !errors.Is(err, syscall.ECONNRESET) { c.t.Fatalf("connection not closed: %v", err) } } @@ -139,7 +148,8 @@ func TestTCPMessages(t *testing.T) { t.Fatalf("want pong 1, got %+v", e) } c.send(`{"type":"echo","id":"2","payload":{"a":1}}` + "\n") - if e := c.read(); e.Type != TypeEcho || e.ID != "2" || string(e.Payload) != `{"a":1}` { + if e := c.read(); e.Type != TypeEcho || e.ID != "2" || + string(e.Payload) != `{"a":1}` { t.Fatalf("want echo 2, got %+v", e) } c.send("not json\n") @@ -153,7 +163,8 @@ func TestTCPMessages(t *testing.T) { // Pushed messages reach TCP clients. h.Push("item.created", map[string]string{"id": "x"}) - if e := c.read(); e.Type != "item.created" || string(e.Payload) != `{"id":"x"}` { + if e := c.read(); e.Type != "item.created" || + string(e.Payload) != `{"id":"x"}` { t.Fatalf("want item.created, got %+v", e) } } @@ -164,10 +175,12 @@ func TestTCPNoWriteTimeout(t *testing.T) { _, addr := tcpHub(t, func(c *config.JSCServerWebSocket) { c.WriteTimeout = 0 }) + c := dialTCP(t, addr) if e := c.read(); e.Type != TypeWelcome { t.Fatalf("want welcome, got %+v", e) } + c.send(`{"type":"ping","id":"1"}` + "\n") if e := c.read(); e.Type != TypePong { t.Fatalf("want pong, got %+v", e) @@ -179,6 +192,7 @@ func TestTCPClientDisconnect(t *testing.T) { c := dialTCP(t, addr) c.read() // welcome waitCount(t, h, 1) + c.conn.Close() waitCount(t, h, 0) } @@ -190,6 +204,7 @@ func TestTCPLineTooLong(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome + c.send(`{"type":"echo","payload":"` + strings.Repeat("x", 100) + `"}` + "\n") c.waitClosed() waitCount(t, h, 0) @@ -214,7 +229,8 @@ func TestTCPShutdown(t *testing.T) { c.waitClosed() late := dialTCP(t, addr) - if e := late.read(); e.Type != TypeError || e.Error != "server shutting down" { + if e := late.read(); e.Type != TypeError || + e.Error != "server shutting down" { t.Fatalf("late client: want shutdown error, got %+v", e) } late.waitClosed() @@ -229,11 +245,13 @@ func TestTCPPing(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome + c.conn.SetReadDeadline(time.Now().Add(time.Second)) line, err := c.r.ReadString('\n') if err != nil || line != "\n" { t.Fatalf("want a blank ping line, got %q, %v", line, err) } + for i := range 5 { c.send(`{"type":"ping"}` + "\n") for { @@ -291,12 +309,14 @@ func TestTCPNoPongTimeout(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome + for range 3 { c.conn.SetReadDeadline(time.Now().Add(time.Second)) if line, err := c.r.ReadString('\n'); err != nil || line != "\n" { t.Fatalf("want a blank ping line, got %q, %v", line, err) } } + waitCount(t, h, 1) } @@ -307,7 +327,9 @@ func TestTCPDefaultMessageSize(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome - c.send(`{"type":"echo","id":"1","payload":"` + strings.Repeat("x", 4096) + `"}` + "\n") + + payload := strings.Repeat("x", 4096) + c.send(`{"type":"echo","id":"1","payload":"` + payload + `"}` + "\n") if e := c.read(); e.Type != TypeEcho || e.ID != "1" { t.Fatalf("want echo, got %+v", e) } @@ -333,10 +355,12 @@ func TestTCPConnRead(t *testing.T) { io.WriteString(peer, "\r\n{\"a\":1}\r\n") peer.Close() }() + typ, data, err := tc.Read(context.Background()) if err != nil || typ != ws.MessageText || string(data) != `{"a":1}` { t.Fatalf("Read = %v, %q, %v", typ, data, err) } + // The peer closing is a normal closure. _, _, err = tc.Read(context.Background()) if ws.CloseStatus(err) != ws.StatusNormalClosure { @@ -370,17 +394,21 @@ func TestTCPConnClose(t *testing.T) { line, _ := bufio.NewReader(peer).ReadString('\n') got <- line }() + if err := tc.Close(ws.StatusGoingAway, "bye"); err != nil { t.Fatalf("Close: %v", err) } if line := <-got; line != `{"type":"error","error":"bye"}`+"\n" { t.Errorf("peer got %q", line) } + _, _, err := tc.Read(context.Background()) var ce ws.CloseError - if !errors.As(err, &ce) || ce.Code != ws.StatusGoingAway || ce.Reason != "bye" { + if !errors.As(err, &ce) || ce.Code != ws.StatusGoingAway || + ce.Reason != "bye" { t.Errorf("Read after Close = %v; want going away, bye", err) } + if err := tc.Close(ws.StatusNormalClosure, "again"); err != nil { t.Errorf("second Close: %v", err) } diff --git a/internal/websocket/tcpserver/server.go b/internal/websocket/tcpserver/server.go index 4be4965..36ca994 100644 --- a/internal/websocket/tcpserver/server.go +++ b/internal/websocket/tcpserver/server.go @@ -17,6 +17,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/middleware" + "git.avsdev.uk/Go/job-search/internal/util" "git.avsdev.uk/Go/job-search/internal/websocket" ) @@ -64,7 +65,7 @@ func (s *Server) logger() *slog.Logger { if log == nil { log = slog.Default() } - return log.With("component", "tcp") + return log.With("component", "tcp_server") } // Run listens on the configured host and port and serves plain TCP clients @@ -102,19 +103,17 @@ func (s *Server) serve(ctx context.Context, listen net.Listener) error { listen.Close() //nolint:errcheck,gosec // closed already when ctx was cancelled s.logger().Info("shutting down") - shutdownCtx := context.Background() - if s.conf.Timeouts.ShutdownTimeout > 0 { - var cancel context.CancelFunc - shutdownCtx, cancel = context.WithTimeout( - shutdownCtx, s.conf.Timeouts.ShutdownTimeout, - ) - defer cancel() - } + // No deadline if the shutdown timeout is 0 or less. + shutdownCtx, cancel := util.WithTimeout( + context.Background(), s.conf.Timeouts.ShutdownTimeout, + ) + defer cancel() errs := []error{acceptErr} if err := s.hub.Shutdown(shutdownCtx); err != nil { errs = append(errs, fmt.Errorf("websocket shutdown: %w", err)) } + return errors.Join(errs...) } diff --git a/internal/websocket/tcpserver/server_test.go b/internal/websocket/tcpserver/server_test.go index 0c9d076..5adbf5b 100644 --- a/internal/websocket/tcpserver/server_test.go +++ b/internal/websocket/tcpserver/server_test.go @@ -17,7 +17,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/websocket" ) -var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil)) +var discardLog = slog.New(slog.DiscardHandler) // testConf is a server configuration for tests, changed by mutate if it is // not nil. @@ -55,12 +55,14 @@ func startServer( if err != nil { t.Fatal(err) } + ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) done := make(chan error, 1) go func() { done <- s.serve(ctx, ln) }() + return h, ln.Addr().String(), cancel, done } @@ -95,10 +97,12 @@ func (c *tcpClient) read() websocket.Envelope { if err != nil { c.t.Fatalf("read: %v (got %q)", err, line) } + var env websocket.Envelope if err := json.Unmarshal([]byte(line), &env); err != nil { c.t.Fatalf("bad envelope %q: %v", line, err) } + return env } @@ -107,7 +111,8 @@ func (c *tcpClient) read() websocket.Envelope { func (c *tcpClient) waitClosed() { c.t.Helper() c.conn.SetReadDeadline(time.Now().Add(time.Second)) - if _, err := io.ReadAll(c.r); err != nil && !errors.Is(err, syscall.ECONNRESET) { + if _, err := io.ReadAll(c.r); err != nil && + !errors.Is(err, syscall.ECONNRESET) { c.t.Fatalf("connection not closed: %v", err) } } @@ -129,7 +134,8 @@ func TestServerShutdown(t *testing.T) { } cancel() - if e := c.read(); e.Type != websocket.TypeError || e.Error != "server shutting down" { + if e := c.read(); e.Type != websocket.TypeError || + e.Error != "server shutting down" { t.Fatalf("want shutdown error, got %+v", e) } c.waitClosed() @@ -152,7 +158,9 @@ func TestServerShutdown(t *testing.T) { func TestServerConnectionLimit(t *testing.T) { conf := testConf(func(c *config.JSCServer) { c.WebSocket.ConnectionLimit = config.JSCRateLimitRule{ - Enabled: true, PerSecond: 1, Burst: 1, + Enabled: true, + PerSecond: 1, + Burst: 1, } c.RateLimits.IdleTTL = time.Minute }) @@ -164,7 +172,8 @@ func TestServerConnectionLimit(t *testing.T) { t.Fatalf("first: want welcome, got %+v", e) } second := dial(t, addr) - if e := second.read(); e.Type != websocket.TypeError || e.Error != "rate limited" { + if e := second.read(); e.Type != websocket.TypeError || + e.Error != "rate limited" { t.Fatalf("second: want rate limited, got %+v", e) } second.waitClosed() @@ -203,7 +212,8 @@ func TestServerRunListenError(t *testing.T) { s, _ := newServer(testConf(func(c *config.JSCServer) { c.Port = -1 })) - if err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "listen") { + if err := s.Run(context.Background()); err == nil || + !strings.Contains(err.Error(), "listen") { t.Fatalf("Run = %v; want a listen error", err) } } @@ -231,13 +241,17 @@ func TestServerAcceptRetries(t *testing.T) { t.Fatal(err) } defer ln.Close() + fl := &flakyListener{ Listener: ln, fails: 3, } + err = s.accept(context.Background(), context.Background(), fl) if !errors.Is(err, net.ErrClosed) || fl.fails != 0 { - t.Fatalf("accept = %v with %d failures left; want net.ErrClosed", err, fl.fails) + t.Fatalf( + "accept = %v with %d failures left; want net.ErrClosed", err, fl.fails, + ) } } @@ -249,11 +263,13 @@ func TestServerAcceptCancelledInBackOff(t *testing.T) { t.Fatal(err) } defer ln.Close() + ctx, cancel := context.WithCancel(context.Background()) fl := &flakyListener{ Listener: ln, fails: 1 << 30, } + time.AfterFunc(20*time.Millisecond, cancel) if err := s.accept(ctx, ctx, fl); !errors.Is(err, context.Canceled) { t.Fatalf("accept = %v; want context.Canceled", err)