diff --git a/.golangci.json b/.golangci.json index 1100652..b5a9593 100644 --- a/.golangci.json +++ b/.golangci.json @@ -9,19 +9,111 @@ "cyclop", "dogsled", "errcheck", + "errorlint", + "gocritic", "godox", - "govet", "gosec", + "govet", "ineffassign", + "misspell", "nilerr", "nilnil", "nolintlint", + "revive", + "sloglint", + "thelper", "unused", - "whitespace" + "varnamelen", + "whitespace", + "wsl_v5" ], "settings": { "cyclop": { "max-complexity": 15 + }, + "misspell": { + "locale": "UK", + "ignore-rules": [ + "organization", + "humanize" + ] + }, + "sloglint": { + "no-mixed-args": true, + "key-naming-case": "snake", + "static-msg": true + }, + "revive": { + "rules": [ + { + "name": "context-as-argument", + "arguments": [ + { + "allowTypesBefore": "*testing.T" + } + ] + }, + { + "name": "error-naming" + }, + { + "name": "error-strings" + }, + { + "name": "errorf" + }, + { + "name": "exported" + }, + { + "name": "indent-error-flow" + }, + { + "name": "receiver-naming" + }, + { + "name": "superfluous-else" + }, + { + "name": "var-naming" + } + ] + }, + "varnamelen": { + "ignore-names": [ + "err", + "ctx", + "i", + "idx", + "msg", + "w", + "r", + "req", + "res", + "db", + "tx", + "wg", + "mu", + "t", + "ok", + "id", + "ip" + ], + "ignore-decls": [] + }, + "gocritic": { + "disable-all": true, + "enabled-checks": [ + "builtinShadow", + "emptyStringTest", + "paramTypeCombine" + ] + }, + "wsl_v5": { + "default": "none", + "enable": [ + "err" + ] } }, "exclusions": { @@ -39,6 +131,12 @@ "gosec" ], "text": "G104" + }, + { + "path": "_test\\.go", + "linters": [ + "varnamelen" + ] } ] } diff --git a/CODE_CONVENTIONS.md b/CODE_CONVENTIONS.md new file mode 100644 index 0000000..9858de8 --- /dev/null +++ b/CODE_CONVENTIONS.md @@ -0,0 +1,786 @@ +# Code conventions + +These are the conventions for Go code in this repository. Most formatting +is enforced by `golangci-lint` (see [Tooling](#tooling)); the rest are +applied by hand and checked in review. + +Generated code is never edited to follow these conventions. It is +recognised by Go's standard marker, a comment before the package clause: + +```go +// Code generated by ocdsgen from extensions.json. DO NOT EDIT. +``` + + +## Tooling + +```bash +golangci-lint run --build-tags sqlite_fts5 ./... # lint +golangci-lint fmt ./... # format (gofmt, goimports, gci, golines) +``` + +- Anything that touches the store needs the `sqlite_fts5` build tag (the + title search uses SQLite FTS5); `scripts/build.sh` passes it. +- `golines` limits lines to 100 characters, counting a tab as 2. See + [Line length](#line-length) for the length to aim for. +- In `_test.go` files, `.golangci.json` turns off `errcheck` and + `cyclop`, and gosec's G104 only (it duplicates errcheck). Every other + linter still runs on tests. + +### Linters that check these conventions + +Several of the conventions below are checked by linters enabled in +`.golangci.json`, so `golangci-lint run` reports them rather than review: + +| Linter | Checks | Settings | +|---|---|---| +| `revive` | doc comments on exported names ([Doc comments](#doc-comments)), consistent receiver names, `ctx` first ([Functions and types](#functions-and-types)), [initialisms](#initialisms), lowercase error strings and `Err…` names ([Error messages](#error-messages)), [no `else` after a `return`](#no-else-after-a-return) | Only these rules are enabled. `context-as-argument` allows a `*testing.T` before `ctx`, for test helpers. | +| `errorlint` | errors are compared with `errors.Is` and `errors.As`, not `==` or a type assertion, so wrapped errors still match | | +| `misspell` | British spelling ([Spelling](#spelling)) in comments, strings and names | `locale: UK`. `organization` and `humanize` are ignored: they are external names (OCDS's `Organization`, the `go-humanize` package). | +| `sloglint` | snake_case keys, no mixing of key/value pairs with `slog.Attr`, constant messages ([Logging](#logging)); `slog.DiscardHandler` for discarded output | | +| `thelper` | test helpers call `t.Helper()` first ([Tests](#tests)) | | +| `gocritic` | neighbouring parameters and results of the same type share one type name, and strings are tested against `""` ([Functions and types](#functions-and-types)); no names shadowing built-ins ([Naming](#naming)) | Only `paramTypeCombine`, `emptyStringTest` and `builtinShadow` are enabled. | +| `wsl_v5` | no blank line between a call and the `if err != nil` that checks it ([Blank lines](#blank-lines)) | Only the `err` check is enabled; the rest of wsl's rules conflict with the Blank lines convention. | +| `varnamelen` | short names used over a wide scope ([Naming](#naming)) | Off in `_test.go` files. The short names in the Naming table are allowed. | + +A finding that is a false positive is silenced with a `//nolint` +directive and a reason, as for any linter (see +[Other `//nolint` directives](#other-nolint-directives)). + + +## Files and packages + +- File names are lowercase. Compound names run together (`findatender`, + `ftsmock`, `ratelimits`); underscores separate parts + (`conf_source_ftsmock.go`, `confirm_action.go`). +- Never end a file name in `_` or `_` (e.g. `_js`, `_arm`): + Go treats it as a build constraint. +- Package directories are lowercase, with no underscores. + + +## Imports + +Three groups, each separated by a blank line: the standard library, then +third-party packages, then this module's own packages. + +```go +import ( + "context" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.avsdev.uk/Go/job-search/internal/config" +) +``` + +`golangci-lint fmt` applies this (gci sections `standard`, `default`, +`localmodule`; goimports' local prefix is the module path). + + +## Naming + +Variables and parameters in application code have descriptive full-word +names, not single letters or abbreviations. + +| Use | Not | +|---|---| +| `listen`, `newConn`, `client`, `limiter` | `ln`, `nc`, `c`, `lim` | +| `shutdownCtx`, `writeCtx` | `shCtx`, `wctx` | +| `release`, `opportunity`, `entry`, `value` | `r`, `t`, `e`, `v` | + +These short names are conventional enough to keep: + +| Name | For | +|---|---| +| a short method receiver | e.g. `h *Hub`, `t *tcpConn`, `s *Store` | +| `err`, `ctx` | errors and contexts | +| `i`, `idx` | a plain index loop | +| `msg` | a message | +| `w`, `r` | `http.ResponseWriter` and `*http.Request` in HTTP handlers and middleware, as in the standard library | +| `req`, `res` | a request and a response elsewhere, e.g. building an `*http.Request` or receiving an `*http.Response` in client code | +| `db`, `tx` | `*sql.DB` and a database transaction (`*sql.Tx`, `*sqlx.Tx`) | +| `wg`, `mu` | `sync.WaitGroup` and `sync.Mutex` | +| `t` | `*testing.T`, including in test helper packages such as `internal/cli/cmdtest` | + +`w` and `r` are used for nothing else: a release is `release`, an +`io.Writer` is `out`, a `*rand.Rand` is `rng`. + +Names never shadow Go's predeclared identifiers: not `min`, `max`, +`len`, `cap`, `new`, `copy`, `clear`, `close`, `error`, `string` and so +on. A gap's bounds are `minGap` and `maxGap`, not `min` and `max`, so the +built-in `min` and `max` still work inside the function. + +`tx` means only a database transaction. It is not used for transmit (or +`rx` for receive), as in tx/rx: name those `transmit` and `receive`, or +more specifically (e.g. `sent`, `received`). + +Test files are exempt from this rule: short names (`h`, `c`, `e`, `ln`, +`tc`) are fine there, and existing ones shouldn't be renamed. + +### Initialisms + +Initialisms and acronyms keep one case throughout a name, as in the +standard library: `ID`, `URL`, `URI`, `JSON`, `HTTP`, `API`, `UUID`, +`SQL`. So `ReleaseID`, `APIRoot`, `UUIDType` and `baseURL`, not +`ReleaseId`, `ApiRoot` or `baseUrl`. At the start of an unexported name +the whole initialism is lowercase: `uuid`, `apiRoot`, `jsonHandler`. + +`revive` checks this (`var-naming`). + +### `must` functions + +A name starting with `must` (or `Must`) promises that the function never +returns an error to its caller, as in the standard library's +`regexp.MustCompile` and `template.Must`: + +- In tests, a `must…` helper takes `t` first, calls `t.Helper()`, and + stops the test with `t.Fatal` on error: `mustSave(t, s, opportunity)`. +- In application code, a `must…` function panics on error, and is only + used where the input is known to be valid, such as constants or + package-level setup. +- A function that hides an error some other way, such as returning a zero + value or `nil`, isn't named `must…`: `rawJSON` returns `nil` for a value + it can't marshal. + + +## Functions and types + +- A function that takes a context takes it first, named `ctx`: + `func (s *Store) Get(ctx context.Context, id string)`. +- Constructors are `New` when a package has one main type + (`tcpserver.New`), otherwise `NewX` (`NewHub`, `NewKeyedLimiter`). +- All of a type's methods use the same receiver name, usually its first + letter (`h *Hub`, `t *tcpConn`), for value and pointer receivers alike. +- A string is tested for emptiness against `""`, not with `len`: + `if file == ""`, not `if len(file) == 0`. It says what is meant, and + reads the same as any other string comparison. +- Neighbouring parameters or results of the same type share one type + name, as in the standard library: `func f(ocid, id string)`, not + `func f(ocid string, id string)`. Parameters are never reordered just + to share a type. + +### No `else` after a `return` + +When an `if` block ends in a `return` (or `break`, `continue`, `panic`), +there is no `else`: the code that would be in it follows the `if`, +un-indented. The usual path then reads straight down the left edge of +the function, with the early exits indented, which is how Go code is +normally read (see Effective Go and the Go Code Review Comments, "Indent +Error Flow"). + +```go +if blockShutdown { + return sleep(taskCtx, true, sleepDuration) +} +return sleep(ctx, false, sleepDuration) +``` + +Not: + +```go +if blockShutdown { + return sleep(taskCtx, true, sleepDuration) +} else { + return sleep(ctx, false, sleepDuration) +} +``` + +Where there are several outcomes, none of which is the usual path, a +`switch` with no condition reads better than nested `if`/`else`: + +```go +switch { +case errors.As(err, ¬Found): + slog.Warn("no config file found; using defaults") +case err != nil: + return err +default: + slog.Info("using config file", "file", viper.ConfigFileUsed()) +} +``` + +`revive` checks this (`indent-error-flow`, `superfluous-else`). + + +## Layout + +These rules are applied by hand and checked in review; `golines` only +enforces the 100-character limit. They apply to tests as well as +application code, except where a rule says otherwise. + +### Line length + +Lines should be at most 80 characters (counting a tab as 2), so code +reads well side by side and in diffs. 100 is the absolute maximum, +enforced by `golines`. + +Go past 80 only when breaking the line would make it harder to read: + +- a line ending in a `//nolint` directive, which must stay on the line it + applies to; +- struct tags; +- string literals that would lose meaning or searchability if split: + URLs, error messages, regular expressions, JSON fixtures, help text; +- rows of table-driven tests. + +Don't run `golines` at 80 to do this: it splits method chains, separates +slog keys from their values and moves `//nolint` directives off their +lines. + +### Assignments in `if` and `for` + +When an `if` statement's assignment and check don't fit within the line +length together, the assignment is taken out of the `if`, rather than the +call being wrapped inside it. That keeps the condition readable, +particularly an `if err != nil`, which would otherwise be left on the +call's closing line, after its arguments: + +```go +_, err = tx.ExecContext( + ctx, + deleteOpportunityClassifications, + opportunity.UUID, +) +if err != nil { + return err +} +``` + +Not: + +```go +if _, err := tx.ExecContext( + ctx, + deleteOpportunityClassifications, + opportunity.UUID, +); err != nil { + return err +} +``` + +The same applies to a `for` statement's init statement, and to a range +expression that doesn't fit on the `for` line: it is assigned to a +variable first, so the loop's header stays on one line, next to its body. + +```go +marks := []struct { + filter MarkFilter + clause string +}{ + {filter: filter.Hidden, clause: filterOpportunityMarkHidden}, + {filter: filter.Skipped, clause: filterOpportunityMarkSkipped}, +} +for _, mark := range marks { +``` + +Tests are exempt from this rule. + +### Wrapped calls and declarations + +When a call doesn't fit on one line, it breaks straight after the opening +bracket and the closing bracket goes on a line of its own. If the +arguments all fit on one line between them, they may stay together on +that line, followed by a comma: + +```go +ctx, stop := signal.NotifyContext( + context.Background(), os.Interrupt, syscall.SIGTERM, +) +``` + +Otherwise each argument goes on its own line, followed by a comma. The +arguments are either all on one line or one per line, never grouped two +or three to a line: + +```go +server := tcpserver.New( + &conf.Server, + websocket.NewHub(conf.Server.WebSocket, websocket.WithLogger(log)), + tcpserver.WithLogger(log), +) +``` + +Not: + +```go +server := tcpserver.New(&conf.Server, + websocket.NewHub(conf.Server.WebSocket, websocket.WithLogger(log)), + tcpserver.WithLogger(log)) +``` + +A function declaration whose parameters don't fit on one line wraps the +same way: + +```go +func insertReleaseWithTags( + ctx context.Context, tx *sqlx.Tx, release types.OpportunityRelease, +) (bool, error) { +``` + +A `log/slog` call that fits on one line stays on one line, like any other +call. When it doesn't, its attributes are the exception to the rule above: +the message goes on its own line, then each field name stays on the same +line as its value, one field per line, even when the fields would all fit +together on one line. + +```go +slog.Info( + "fetched opportunities", + "source", "find_a_tender", + "count", len(opportunities), + "since", since, +) +``` + +A `//nolint` directive on a wrapped call goes on its first line, after the +opening bracket (see [Ignoring an error](#ignoring-an-error)). + +### Method chains + +A chain of calls that fits on one line stays on one line. One that +doesn't breaks after each `.`, one call per line, indented once, with +the dot at the end of the line: + +```go +findATender.httpClient = resty.New(). + SetTimeout(60 * time.Second). + SetRetryCount(1) +``` + +A short accessor at the start of a chain, such as `h.logger()` or +`cmd.Flags()`, stays with the call after it; if that call doesn't fit, +its arguments are wrapped instead (see +[Wrapped calls and declarations](#wrapped-calls-and-declarations)): + +```go +h.logger().Info( + "client connected", + "client", client.id, + "ip", ip, +) +``` + +Not: + +```go +h.logger(). + Info("client connected", "client", client.id, "ip", ip) +``` + +### Long conditions + +A condition that doesn't fit on one line breaks after a `||` or `&&`, +which ends the line, and continues indented once. Each line is filled up +to the line length, keeping related comparisons together, rather than +one comparison per line: + +```go +if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || + len(cat.Subjects) == 0 || len(cat.Methods) == 0 { +``` + +A condition is never broken inside a call's brackets; if a call in it +doesn't fit, its result is assigned to a variable first (see +[Assignments in `if` and `for`](#assignments-in-if-and-for)). A +condition that runs to more than about three lines is better given a +name, as a boolean variable or a small function. + +### Blank lines + +A function body is split into steps, with one blank line between them. A +step is a statement, or a few closely related ones, together with its +error handling. Go itself has no rule for this (`gofmt` only collapses +several blank lines into one, and the `whitespace` linter only rejects +blank lines at the start or end of a block), so it is applied by hand. + +- A call and the `if err != nil` that checks it are never separated by a + blank line. `wsl_v5` checks this; the rest of the rule is applied + by hand. +- A `defer` that releases something goes straight after the check of the + call that acquired it, with no blank line, so acquiring and releasing + read as one step. +- A function or block of only one or two steps needs no blank lines. +- A final `return` after several (three or more) steps is set off by a + blank line. The `return` itself isn't counted as a step. + +```go +statement, err := tx.PrepareNamedContext( + ctx, upsertClassificationCodeReturningID, +) +if err != nil { + return err +} +defer func() { + err = errors.Join(err, statement.Close()) +}() + +for i := range opportunity.Classifications { + ... +} + +return nil +``` + +### Function literals + +A function literal (`func(...) { ... }` written inline) is never written on +one line, however short its body: + +```go +t.Cleanup(func() { + server.Close() +}) +``` + +Not `t.Cleanup(func() { server.Close() })`. + +This is about function literals only. Named one-line functions and methods, +such as a config section's `Map() map[string]any { return mapOf(c) }`, are +unaffected. + +### Struct literals + +A struct literal is always wrapped, one field per line, and names its +fields: + +```go +envelope := websocket.Envelope{ + Type: websocket.TypePing, + ID: "1", +} +``` + +Not `websocket.Envelope{Type: websocket.TypePing, ID: "1"}`, and never +positional (`websocket.Envelope{"ping", "1", nil, ""}`). + +- Empty struct literals (`netip.Addr{}`, `time.Time{}`, `struct{}{}`, + `&Server{}`) have nothing to wrap and stay on one line. +- Rows of a table-driven test (`{"size", "4096", "4KiB"},`) stay one row per + line, so the table stays easy to scan. A struct literal used as a value + inside a row is still wrapped. +- Rows of a short lookup table in application code (a slice or map + literal of small structs, used as data) may each stay on one line if + they fit within the line length, so the table is easy to scan. They + still name their fields: + `{filter: filter.Hidden, clause: filterOpportunityMarkHidden},`. +- This is about struct literals only; slice and map literals + (`[]string{"a", "b"}`) may stay on one line. + + +## Comments + +### Doc comments + +Every exported identifier has a doc comment, and it starts with the name +it documents. + +```go +// NewHub creates a hub. +func NewHub(conf config.JSCServerWebSocket, opts ...Option) *Hub { +``` + +### Grouped declarations + +Related package-level constants and variables go in one block, with a +comment for the group and one for each member. + +```go +// Errors returned by CSRF.Validate for a token that is rejected. +var ( + // ErrCSRFMissing is returned when no token is presented. + ErrCSRFMissing = errors.New("csrf token missing") + // ErrCSRFExpired is returned for a token older than the token TTL. + ErrCSRFExpired = errors.New("csrf token expired") +) +``` + +### Section banners + +A file is divided into sections with a banner, followed by a blank line +so godoc doesn't attach the banner to the next declaration: + +```go +// ---------- tcpConn ---------- + +// tcpConn adapts a net.Conn ... +type tcpConn struct { +``` + +In a file with more than one type (such as a config section and its +nested section), each type gets a `==========` sub-section, which then +has its own `----------` banners: + +```go +// ========== HSTS Headers ========== + +// JSCServerHeadersHSTS configures the Strict-Transport-Security header. +type JSCServerHeadersHSTS struct { + ... +} + +// ---------- Section methods (see Section) ---------- +``` + +`internal/store/queries.go` uses `----------` banners for its per-table +groups of queries. + +### Trailing comments + +A comment after code on the same line is for a short note, and only when +the line still fits within the [line length](#line-length). Otherwise +the comment goes on its own line above the code it describes: + +```go + // location is the zone for updatedFrom/updatedTo without an offset; + // nil means UTC. + location *time.Location +``` + +Not a trailing comment that runs the line past 80, and never a comment +after the closing bracket of a wrapped call. + +A comment on a row of a table-driven test stays on the row, like the row +itself (see [Line length](#line-length)). + +`//nolint` directives are exempt: a directive must be on the line the +linter reports, so it stays a trailing comment even when that takes the +line past 80 (see [Ignoring an error](#ignoring-an-error)). + +### Spelling + +Comments and documentation use British English ("normalised", "honoured", +"recognise", "behaviour"). Identifiers, and anything that must match an +external name, keep their own spelling (`Authorization`, `Color` in a +third-party API). + + +## Errors + +### Error messages + +- Error strings are lowercase and don't end with punctuation, as they are + usually wrapped into a longer message (`revive`'s `error-strings` + reports capitalised ones): `errors.New("store: not found")`. +- An error from another call is wrapped with `%w`, adding what was being + done, so the chain reads as a sentence: + `fmt.Errorf("listen %s: %w", addr, err)`. + +### Package prefixes + +An error names the package it came from once, where it is created, as +the standard library does (`sql: no rows in result set`, `http: server +closed`) and Effective Go recommends: + +- An error a package creates itself, a sentinel, `errors.New`, or + `fmt.Errorf` without `%w`, starts with the package name: + `errors.New("store: not found")`, + `fmt.Errorf("util: byte size %q is too large", text)`. +- Wrapping an error adds what was being done, never a package name: + `fmt.Errorf("inserting release %s/%s: %w", uuid, id, err)`, not + `"store: inserting release …"`. The wrapped error already says where it + came from, so a chain reads `writing opportunity ocds-1: store: …` + rather than repeating `store:` at every level. + +Two kinds of error are exempt and have no prefix: + +- Errors from `package main` and the command packages in `internal/cli`. + Their text is the final message shown to the user, who has no use for + an internal package name. +- Error text that is part of a protocol sent to a client, such as the + mock Find a Tender API's error responses, which match the real API's. +- Errors that callers test for are sentinel values named `Err…` + (exported) or `err…` (unexported), checked with `errors.Is`. A + package's sentinels go in one commented `var` block (see + [Grouped declarations](#grouped-declarations)). + +### Ignoring an error + +An error is never ignored with `_ = f()`. Instead, the call says why +with a `//nolint` directive naming only the linters that actually report +the line (`nolintlint` reports any that don't): + +```go +json.NewEncoder(w).Encode(problem) //nolint:errcheck,gosec // the client may have gone +``` + +- Most calls need `errcheck,gosec`. `hash.Write` needs only `gosec` + (errcheck excludes it); `fmt.Fprint*` needs only `errcheck` (gosec + excludes it). +- A deferred call needs only `errcheck`, since gosec doesn't check + deferred calls: + + ```go + defer rows.Close() //nolint:errcheck // read only + ``` + +- For a call spanning several lines, the directive goes on the first + line, after the opening parenthesis: + + ```go + tcpSocket.SetKeepAliveConfig(net.KeepAliveConfig{ //nolint:errcheck,gosec // best effort + ``` + + A linter that reports one argument rather than the call (such as + `sloglint` on a message that isn't a constant) needs the directive on + that argument's line instead. One that reports several lines of a + statement, such as a table of test cases, takes the directive on a line + of its own directly above the statement. + +- A call whose error both linters already exclude (such as + `crypto/rand.Read`, which never returns one) gets a plain comment + rather than a directive, which would be reported as unused: + + ```go + rand.Read(key) // crypto/rand.Read never returns an error + ``` + +- A multi-value assignment such as `value, _ := f()` can't take a + directive (errcheck doesn't check blank assignments, so the directive + would be unused) and stays as it is. These are mostly cobra flag + getters, which only fail for an undefined flag. Where ignoring the + error is ambiguous or potentially dangerous in future, an explanation + comment can be added. + +### Errors that matter + +When closing something can lose data (a file being written, a database +handle), its error is joined into the function's returned error with +`errors.Join`, through a named return. + +### Named results + +A function names its results only when it needs to: + +- **A deferred function changes the error.** This is the only reason to + name `err` for its own sake (see [Errors that matter](#errors-that-matter)). +- **The names say which result is which.** When several results share a + type, naming them documents the order, as the Go Code Review Comments + suggest: `func releaseIDs(raw json.RawMessage) (ocid, id string, err error)`. + Go names all of a function's results or none, so `err` is named here too. + +Otherwise results are unnamed. A function with named results: + +- never uses a bare `return`: each `return` lists what it returns, so a + reader doesn't have to track the named variables to know; +- where a deferred function changes `err`, assigns to it with `=`, never + `:=`, in nested blocks. `err :=` there declares a new variable that + shadows the result, and the error assigned to it never reaches the + deferred `errors.Join`: + + ```go + for i := range opportunity.Classifications { + err = statement.GetContext(ctx, &classification.ID, classification) + if err != nil { + return err + } + } + ``` + +### Other `//nolint` directives + +Otherwise `//nolint` is only for false positives that the code can't +express, always with a reason. For example, a seeded `math/rand` in the +mock, or a gosec finding explained in the comment above it: + +```go +// Secure is false only on plain HTTP (local development), where +// browsers refuse Secure cookies; HttpOnly and SameSite are always set. +http.SetCookie(w, &http.Cookie{ //nolint:gosec // G124: see above +``` + + +## Logging + +- Logging uses `log/slog`. A component (a type such as `Hub`, or a + package-level entry point such as `fetcher.Run`) logs to `slog.Default()` + unless it is given a logger with a `WithLogger` option, so callers that + are happy with the default pass nothing, and tests can still capture or + discard its output. +- The logger is given to the constructor (or entry point) as an option, + never set afterwards: a setter called once goroutines are running would + be a data race. Options follow the functional options pattern used by + `store.Open`. +- With no logger given, the component looks up `slog.Default()` when it + logs, not when it is created, so it follows `logging.Setup` and a test's + `slog.SetDefault` even if it was created first. + + ```go + // WithLogger makes the hub log to logger instead of slog.Default(). + func WithLogger(logger *slog.Logger) Option { + return func(h *Hub) { + h.log = logger + } + } + + // logger returns the logger the hub logs to, tagged with its component. + func (h *Hub) logger() *slog.Logger { + log := h.log + if log == nil { + log = slog.Default() + } + return log.With("component", "ws") + } + ``` + + ```go + hub := websocket.NewHub(conf.Server.WebSocket) // slog.Default() + hub := websocket.NewHub(conf.Server.WebSocket, websocket.WithLogger(log)) // given logger + ``` + +- Field keys are snake_case and name one thing each: `client`, `ip`, + `request_id`, `retry_in`. Durations, sizes and counts are logged with + the matching `slog` type, not as pre-formatted strings. +- A component tags its logger with `"component"` in one place (its + `logger()` method, as above), whether the logger was given or is the + default, rather than on every line. +- Messages are short lowercase phrases ("client connected", "tcp accept + failed"); the details go in fields. + + +## Configuration + +### Keys + +Configuration keys (the `cfg` struct tags) are snake_case: +`max_message_size`, `connection_limit`. Viper lowercases every key it +reads, so a camelCase key such as `minVersion` is never loaded from a +config file or environment variable. + +### Zero values + +Every setting's doc comment (and its row in `internal/config/README.md`) +says what 0 or empty means, and the code treats it consistently: + +- A timeout of 0 or less means **no limit**, and an interval of 0 or less + means **off** (e.g. `ping_interval`), never "already expired". Where off + makes no sense the setting rejects 0 instead (a source's `interval`: a + source is turned off with `enabled`). Code applies a timeout with a helper + rather than `context.WithTimeout(ctx, 0)`: + + ```go + writeCtx, cancel := util.WithTimeout(ctx, conf.WriteTimeout) // no deadline if 0 or less + ``` + +- A size of 0 means **the default** (`max_message_size` of `0B` reads + 32KiB), and -1 means **no limit** (`util.ByteSizeNoLimit`); no other size + may be negative. A setting for which no limit is unsafe rejects -1 in its + validation and says why (`max_message_size` does: a client could exhaust + the server's memory). +- An empty list means **none** (no trusted origins, no exempt paths), and + is stored as `nil`. + + +## Tests + +- Tests are table-driven where several cases share a check, with a name + or input field identifying each case in failure messages. +- Helpers call `t.Helper()` first, and register their own teardown with + `t.Cleanup` rather than returning a cleanup function. +- Tests that touch global state (Viper, `slog.Default`, the working + directory, environment variables) don't call `t.Parallel()`, and reset + what they change: `resetViper(t)`, `t.Chdir`, `t.Setenv`. +- Network tests listen on `127.0.0.1:0` and use the address they were + given, never a fixed port. +- Short names are fine in tests (see [Naming](#naming)), and setup calls + may ignore errors with a bare call (errcheck is off for tests). diff --git a/cmd/jobsearch/main_test.go b/cmd/jobsearch/main_test.go index b7d83d1..b0c29a2 100644 --- a/cmd/jobsearch/main_test.go +++ b/cmd/jobsearch/main_test.go @@ -12,15 +12,19 @@ import ( func TestCommandTree(t *testing.T) { root := newRootCmd() + var names []string for _, c := range root.Commands() { names = append(names, c.Name()) } - for _, want := range []string{"config", "fetch", "migrate", "version", "wsserve"} { + + commands := []string{"config", "fetch", "migrate", "version", "wsserve"} + for _, want := range commands { if !slices.Contains(names, want) { t.Errorf("job-search has no %q command; has %v", want, names) } } + for _, flag := range []string{"config", "debug", "quiet"} { if root.PersistentFlags().Lookup(flag) == nil { t.Errorf("job-search has no --%s flag", flag) diff --git a/cmd/mock-fts/main.go b/cmd/mock-fts/main.go index 8521fd1..ef5408e 100644 --- a/cmd/mock-fts/main.go +++ b/cmd/mock-fts/main.go @@ -52,7 +52,7 @@ type options struct { rateWindow time.Duration accessLogPath string logFormat string - tz string + timeZone string } const longHelp = `Mock of the Find a Tender (FTS) OCDS release package endpoint: @@ -170,10 +170,19 @@ func newRootCmd() *cobra.Command { "maximum time between generated releases", ) - flags.IntVar(&opts.rateLimit, "rate-limit", 12, "requests allowed per window per client") - flags.DurationVar(&opts.rateWindow, "rate-window", 5*time.Minute, "rate limit window length") + flags.IntVar( + &opts.rateLimit, "rate-limit", 12, "requests allowed per window per client", + ) + flags.DurationVar( + &opts.rateWindow, "rate-window", 5*time.Minute, "rate limit window length", + ) - flags.StringVar(&opts.accessLogPath, "access-log", "-", `access log file, or "-" for stdout`) + flags.StringVar( + &opts.accessLogPath, + "access-log", + "-", + `access log file, or "-" for stdout`, + ) flags.StringVar( &opts.logFormat, "log-format", @@ -181,7 +190,7 @@ func newRootCmd() *cobra.Command { `access log format: "combined" or "common"`, ) flags.StringVar( - &opts.tz, + &opts.timeZone, "tz", "Europe/London", `time zone for updatedFrom/updatedTo without an offset ("Local" for this machine's)`, @@ -193,34 +202,40 @@ func newRootCmd() *cobra.Command { // validate checks flag values before anything is started. func (o *options) validate() error { if o.logFormat != "combined" && o.logFormat != "common" { - return fmt.Errorf(`--log-format must be "combined" or "common", got %q`, o.logFormat) + return fmt.Errorf( + `--log-format must be "combined" or "common", got %q`, o.logFormat, + ) } if o.minGap <= 0 || o.maxGap < o.minGap { return errors.New("--min-gap must be positive and no larger than --max-gap") } if o.rateLimit < 1 || o.rateWindow <= 0 { - return errors.New("--rate-limit must be at least 1 and --rate-window positive") + return errors.New( + "--rate-limit must be at least 1 and --rate-window positive", + ) } - if _, err := time.LoadLocation(o.tz); err != nil { + if _, err := time.LoadLocation(o.timeZone); err != nil { return fmt.Errorf("--tz: %w", err) } + return nil } // run starts the mock and blocks until ctx is cancelled or the server fails. func run(ctx context.Context, opts *options) (err error) { - location, err := time.LoadLocation(opts.tz) // already validated + location, err := time.LoadLocation(opts.timeZone) // already validated if err != nil { return fmt.Errorf("--tz: %w", err) } - if err := os.MkdirAll(opts.dataDir, 0o750); err != nil { - return err + + if err = os.MkdirAll(opts.dataDir, 0o750); err != nil { + return fmt.Errorf("creating data directory: %w", err) } - // ---------- catalog ---------- + // Load the catalog. catalog, err := mock.LoadCatalog() if err != nil { - return err + return fmt.Errorf("loading catalog: %w", err) } slog.Info( "catalog loaded", @@ -231,27 +246,30 @@ func run(ctx context.Context, opts *options) (err error) { "classifications", len(catalog.Classifications), ) - // ---------- store ---------- + // Open (or reset) the store. storePath := filepath.Join(opts.dataDir, mock.ReleasesFile) if opts.reset { - if err := os.Remove(storePath); err != nil && !errors.Is(err, os.ErrNotExist) { - return err + if err = os.Remove(storePath); err != nil && + !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resetting store: %w", err) } slog.Info("store reset", "path", storePath) } store, err := mock.OpenStore(storePath) if err != nil { - return err + return fmt.Errorf("opening store: %w", err) } defer func() { err = errors.Join(err, store.Close()) }() - // ---------- generator ---------- + // Backfill releases, then keep generating them. generator := mock.NewGenerator(catalog, store, time.Now().UnixNano()) - generated, err := generator.CatchUp(time.Now(), opts.backfill, opts.minGap, opts.maxGap) + generated, err := generator.CatchUp( + time.Now(), opts.backfill, opts.minGap, opts.maxGap, + ) if err != nil { - return err + return fmt.Errorf("backfilling releases: %w", err) } slog.Info("store ready", "releases", store.Len(), "generated", generated) @@ -267,10 +285,11 @@ func run(ctx context.Context, opts *options) (err error) { generator.Run(ctx, opts.minGap, opts.maxGap) }() - // ---------- HTTP ---------- + // Serve the API. out := os.Stdout if opts.accessLogPath != "-" { - logFile, err := os.OpenFile( + var logFile *os.File + logFile, err = os.OpenFile( opts.accessLogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600, ) if err != nil { @@ -288,12 +307,15 @@ func run(ctx context.Context, opts *options) (err error) { rateLimiter := mock.NewRateLimiter(opts.rateLimit, opts.rateWindow) mux := http.NewServeMux() mux.HandleFunc( - mock.BasePath, rateLimiter.Middleware(mock.NewHandler(store, opts.baseURL, location)), + mock.BasePath, + rateLimiter.Middleware(mock.NewHandler(store, opts.baseURL, location)), ) - httpSrv := &http.Server{ - Addr: opts.addr, - Handler: logging.AccessLog(accessLogger, opts.logFormat == "combined", mux), + httpServer := &http.Server{ + Addr: opts.addr, + Handler: logging.AccessLog( + accessLogger, opts.logFormat == "combined", mux, + ), ReadHeaderTimeout: 10 * time.Second, } go func() { @@ -303,7 +325,7 @@ func run(ctx context.Context, opts *options) (err error) { context.WithoutCancel(ctx), 5*time.Second, ) defer cancelShutdown() - if err := httpSrv.Shutdown(shutdownCtx); err != nil { + if err := httpServer.Shutdown(shutdownCtx); err != nil { slog.Error("shutdown failed", "error", err) } }() @@ -314,15 +336,15 @@ func run(ctx context.Context, opts *options) (err error) { "path", mock.BasePath, "time_zone", location.String(), ) - serveErr := httpSrv.ListenAndServe() + serveErr := httpServer.ListenAndServe() cancel() // stop the worker (and the shutdown goroutine) if the server failed <-workerDone if serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) { - return serveErr + return fmt.Errorf("serving: %w", serveErr) } - log.Print("stopped") + slog.Info("stopped") return nil } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 66a18eb..931f065 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -52,9 +52,11 @@ func newCommand(t *testing.T, opts *Options, args ...string) *cobra.Command { Use: "test", } opts.Bind(cmd) + if err := cmd.ParseFlags(args); err != nil { t.Fatal(err) } + return cmd } @@ -144,12 +146,16 @@ func TestRunSetupWithoutLoadingConfig(t *testing.T) { "-d -q": "debug", // --debug wins } { opts := &Options{} - conf, err := opts.RunSetup(newCommand(t, opts, strings.Fields(args)...), false, true) + conf, err := opts.RunSetup( + newCommand(t, opts, strings.Fields(args)...), false, true, + ) if err != nil { t.Fatal(err) } if conf.Logging.Level != want { - t.Errorf("%q: Logging.Level = %q, want %q", args, conf.Logging.Level, want) + t.Errorf( + "%q: Logging.Level = %q, want %q", args, conf.Logging.Level, want, + ) } } } @@ -192,22 +198,18 @@ func confirmWithInput(t *testing.T, input string) bool { dir := t.TempDir() in := filepath.Join(dir, "stdin") writeFile(t, in, input) + stdin, err := os.Open(filepath.Clean(in)) if err != nil { t.Fatal(err) } defer stdin.Close() + stdout, err := os.Create(filepath.Clean(filepath.Join(dir, "stdout"))) if err != nil { t.Fatal(err) } defer stdout.Close() - oldIn, oldOut := os.Stdin, os.Stdout - os.Stdin, os.Stdout = stdin, stdout - defer func() { - os.Stdin, os.Stdout = oldIn, oldOut - }() - - return Confirm("Proceed?") + return Confirm(stdout, stdin, "Proceed?") } diff --git a/internal/cli/cmdtest/cmdtest.go b/internal/cli/cmdtest/cmdtest.go index a3814e9..03ba651 100644 --- a/internal/cli/cmdtest/cmdtest.go +++ b/internal/cli/cmdtest/cmdtest.go @@ -28,10 +28,12 @@ func Setup(t *testing.T, config string) string { t.Helper() dir := t.TempDir() t.Chdir(dir) + WriteFile(t, ".env", "") if config != "" { WriteFile(t, "config.yaml", config) } + return dir } @@ -78,6 +80,7 @@ func NewRoot(opts *cli.Options, cmds ...*cobra.Command) *cobra.Command { cobra.EnableTraverseRunHooks = true opts.Bind(root) root.AddCommand(cmds...) + return root } @@ -95,18 +98,19 @@ func Run( dir := t.TempDir() inPath, outPath := filepath.Join(dir, "stdin"), filepath.Join(dir, "stdout") WriteFile(t, inPath, stdin) - in, err := os.Open(filepath.Clean(inPath)) + input, err := os.Open(filepath.Clean(inPath)) if err != nil { t.Fatal(err) } - defer in.Close() //nolint:errcheck // read only + defer input.Close() //nolint:errcheck // read only + out, err := os.Create(filepath.Clean(outPath)) if err != nil { t.Fatal(err) } oldIn, oldOut, oldLog := os.Stdin, os.Stdout, slog.Default() - os.Stdin, os.Stdout = in, out + os.Stdin, os.Stdout = input, out defer func() { os.Stdin, os.Stdout = oldIn, oldOut slog.SetDefault(oldLog) @@ -120,11 +124,14 @@ func Run( if err := out.Close(); err != nil { t.Fatal(err) } + return ReadFile(t, outPath), runErr } // MustRun is Run with no input, for commands that must succeed. -func MustRun(t *testing.T, newRoot func() *cobra.Command, args ...string) string { +func MustRun( + t *testing.T, newRoot func() *cobra.Command, args ...string, +) string { t.Helper() out, err := Run(t, newRoot, "", args...) if err != nil { @@ -136,9 +143,9 @@ func MustRun(t *testing.T, newRoot func() *cobra.Command, args ...string) string // Contains fails the test unless out contains every string in want. func Contains(t *testing.T, out string, want ...string) { t.Helper() - for _, w := range want { - if !strings.Contains(out, w) { - t.Errorf("output does not contain %q:\n%s", w, out) + for _, wanted := range want { + if !strings.Contains(out, wanted) { + t.Errorf("output does not contain %q:\n%s", wanted, out) } } } diff --git a/internal/cli/config/command.go b/internal/cli/config/command.go index c898930..4b7944c 100644 --- a/internal/cli/config/command.go +++ b/internal/cli/config/command.go @@ -8,7 +8,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "config" command, which shows and edits the configuration. +// NewCommand returns the "config" command, which shows and edits the +// configuration. func NewCommand(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "config", @@ -24,5 +25,6 @@ func NewCommand(opts *cli.Options) *cobra.Command { cmd.AddCommand(commandGet(opts)) cmd.AddCommand(commandSet(opts)) cmd.AddCommand(commandInit()) + return cmd } diff --git a/internal/cli/config/config_test.go b/internal/cli/config/config_test.go index ec11cee..de28a19 100644 --- a/internal/cli/config/config_test.go +++ b/internal/cli/config/config_test.go @@ -41,9 +41,12 @@ func mustRun(t *testing.T, args ...string) string { func fileSettings(t *testing.T, path string) map[string]any { t.Helper() var nested map[string]any - if err := yaml.Unmarshal([]byte(cmdtest.ReadFile(t, path)), &nested); err != nil { + if err := yaml.Unmarshal( + []byte(cmdtest.ReadFile(t, path)), &nested, + ); err != nil { t.Fatal(err) } + flat := map[string]any{} var walk func(prefix string, m map[string]any) walk = func(prefix string, m map[string]any) { @@ -56,6 +59,7 @@ func fileSettings(t *testing.T, path string) map[string]any { } } walk("", nested) + return flat } @@ -69,11 +73,17 @@ func TestConfigShow(t *testing.T) { "logging.level = info ", "sources.sleeper.interval = 2m0s ", ) + // Keys are listed in order, the same every time. if again := mustRun(t, "config", "show"); again != out { - t.Errorf("config show output changed between runs:\n%s\n---\n%s", out, again) + t.Errorf( + "config show output changed between runs:\n%s\n---\n%s", out, again, + ) } - if i, j := strings.Index(out, "database.file"), strings.Index(out, "logging.level"); i > j { + + database := strings.Index(out, "database.file") + logging := strings.Index(out, "logging.level") + if database > logging { t.Errorf("keys not sorted:\n%s", out) } @@ -87,9 +97,17 @@ func TestConfigShow(t *testing.T) { func TestConfigGet(t *testing.T) { cmdtest.Setup(t, appConfig) - cmdtest.Contains(t, mustRun(t, "config", "get", "logging.level"), "info ") - cmdtest.Contains(t, mustRun(t, "config", "get", "sources.sleeper.interval"), "2m0s ") - cmdtest.Contains(t, mustRun(t, "config", "get", "sources.sleeper.enabled"), "false ") + for _, tt := range []struct { + key, want string + }{ + {"logging.level", "info "}, + {"sources.sleeper.interval", "2m0s "}, + {"sources.sleeper.enabled", "false "}, + } { + t.Run(tt.key, func(t *testing.T) { + cmdtest.Contains(t, mustRun(t, "config", "get", tt.key), tt.want) + }) + } if _, err := run(t, "config", "get", "no.such.key"); err == nil { t.Error("get of an unknown key succeeded") @@ -104,7 +122,9 @@ func TestConfigSet(t *testing.T) { out := mustRun(t, "config", "set", "logging.level", "debug") cmdtest.Contains(t, out, "logging.level = debug ") - cmdtest.Contains(t, mustRun(t, "config", "get", "logging.level"), "debug ") + cmdtest.Contains( + t, mustRun(t, "config", "get", "logging.level"), "debug ", + ) out = mustRun(t, "config", "set", "sources.sleeper.enabled", "true") cmdtest.Contains(t, out, "sources.sleeper.enabled = true ") @@ -156,11 +176,13 @@ func TestConfigSetUnset(t *testing.T) { cmdtest.Setup(t, appConfig) mustRun(t, "config", "set", "--unset", "sources.sleeper.interval") + // Only the unset key is gone: no defaults are written in its place. want := map[string]any{"logging.level": "info"} if got := fileSettings(t, "config.yaml"); !equalSettings(got, want) { t.Errorf("config.yaml settings = %v; want %v", got, want) } + // Back to the default. cmdtest.Contains( t, @@ -180,7 +202,9 @@ func TestConfigInit(t *testing.T) { out := mustRun(t, "config", "init", "./new.yaml") cmdtest.Contains(t, out, "writing default configuration to file: ./new.yaml") - cmdtest.Contains(t, cmdtest.ReadFile(t, "new.yaml"), "level: warn", "interval: 10m0s") + cmdtest.Contains( + t, cmdtest.ReadFile(t, "new.yaml"), "level: warn", "interval: 10m0s", + ) _, err := run(t, "config", "init", "./new.yaml") cmdtest.ErrContains(t, err, "already exists") @@ -199,8 +223,10 @@ func TestConfigInitDefaultLocations(t *testing.T) { {"config", "init"}, {"config", "init", " "}, } { - _, err := run(t, args...) - cmdtest.ErrContains(t, err, "already exists") + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := run(t, args...) + cmdtest.ErrContains(t, err, "already exists") + }) } } diff --git a/internal/cli/config/get.go b/internal/cli/config/get.go index a636c14..86aff14 100644 --- a/internal/cli/config/get.go +++ b/internal/cli/config/get.go @@ -29,7 +29,7 @@ func commandGet(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } value, err := conf.ValueOf(args[0]) diff --git a/internal/cli/config/init.go b/internal/cli/config/init.go index 5b89e69..67164cf 100644 --- a/internal/cli/config/init.go +++ b/internal/cli/config/init.go @@ -39,13 +39,13 @@ func commandInit() *cobra.Command { cmd.SilenceUsage = true // With no path, the default locations are used. - cfgFile := "" + configFile := "" if len(args) > 0 { - cfgFile = strings.TrimSpace(args[0]) + configFile = strings.TrimSpace(args[0]) } - if cfgFile != "" { + if configFile != "" { // Use the config file given. - viper.SetConfigFile(cfgFile) + viper.SetConfigFile(configFile) } else { // Search config in /etc or local directory. viper.AddConfigPath(config.EtcConfigPath) @@ -58,8 +58,8 @@ func commandInit() *cobra.Command { return errors.New("configuration file already exists") } - _, ok := err.(viper.ConfigFileNotFoundError) - if !errors.Is(err, os.ErrNotExist) && !ok { + var notFound viper.ConfigFileNotFoundError + if !errors.Is(err, os.ErrNotExist) && !errors.As(err, ¬Found) { // Config file was found but another error was produced return err } @@ -80,13 +80,15 @@ func commandInit() *cobra.Command { if err := viper.WriteConfig(); err != nil { return err } + return nil }, } // Config flag should be hidden as the config path should be provided - cmd.SetHelpFunc(func(command *cobra.Command, strings []string) { + cmd.SetHelpFunc(func(command *cobra.Command, args []string) { command.Flags().MarkHidden("config") //nolint:errcheck,gosec // a root flag, so it always exists - command.Parent().HelpFunc()(command, strings) + command.Parent().HelpFunc()(command, args) }) + return cmd } diff --git a/internal/cli/config/set.go b/internal/cli/config/set.go index 7a69012..35078a8 100644 --- a/internal/cli/config/set.go +++ b/internal/cli/config/set.go @@ -43,7 +43,7 @@ func commandSet(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } if viper.ConfigFileUsed() == "" { @@ -65,5 +65,6 @@ func commandSet(opts *cli.Options) *cobra.Command { }, } cmd.Flags().BoolP("unset", "x", false, "clear configuration setting") + return cmd } diff --git a/internal/cli/config/show.go b/internal/cli/config/show.go index 1a89822..9946a61 100644 --- a/internal/cli/config/show.go +++ b/internal/cli/config/show.go @@ -24,13 +24,13 @@ func commandShow(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } if yaml, _ := cmd.Flags().GetBool("yaml"); yaml { - conf.PrintYaml() + conf.PrintYAML() } else if json, _ := cmd.Flags().GetBool("json"); json { - conf.PrintJson() + conf.PrintJSON() } else { conf.PrintTerminal() } @@ -40,5 +40,6 @@ func commandShow(opts *cli.Options) *cobra.Command { } cmd.Flags().Bool("json", false, "format configuration as json") cmd.Flags().Bool("yaml", false, "format configuration as yaml") + return cmd } diff --git a/internal/cli/confirm_action.go b/internal/cli/confirm_action.go index a282625..a0fcc14 100644 --- a/internal/cli/confirm_action.go +++ b/internal/cli/confirm_action.go @@ -3,7 +3,7 @@ package cli import ( "bufio" "fmt" - "log/slog" + "io" "os" "strings" ) @@ -11,15 +11,19 @@ import ( // Confirm asks the user the yes/no question prompt on stdin and reports whether // they answered yes. "y" and "yes" confirm, "n" and "no" refuse, in any case; // anything else asks again. It exits the program if stdin can't be read. -func Confirm(prompt string) bool { - reader := bufio.NewReader(os.Stdin) +func Confirm(out io.Writer, in io.Reader, prompt string) bool { + reader := bufio.NewReader(in) for { - fmt.Printf("%s [y/n]: ", prompt) + _, err := fmt.Fprintf(out, "%s [y/n]: ", prompt) + if err != nil { + fmt.Fprintf(os.Stderr, "error asking question: %s\n", err) + os.Exit(1) + } response, err := reader.ReadString('\n') if err != nil { - slog.Error("reading the answer", "error", err) + fmt.Fprintf(os.Stderr, "error reading answer: %s\n", err) os.Exit(1) } diff --git a/internal/cli/fetch/command.go b/internal/cli/fetch/command.go index cdc1709..0e0cccf 100644 --- a/internal/cli/fetch/command.go +++ b/internal/cli/fetch/command.go @@ -8,7 +8,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "fetch" command, which fetches opportunities once or as a service. +// NewCommand returns the "fetch" command, which fetches opportunities once or +// as a service. func NewCommand(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "fetch", @@ -22,5 +23,6 @@ func NewCommand(opts *cli.Options) *cobra.Command { } cmd.AddCommand(commandOnce(opts)) cmd.AddCommand(commandService(opts)) + return cmd } diff --git a/internal/cli/fetch/once.go b/internal/cli/fetch/once.go index b111887..d21e14f 100644 --- a/internal/cli/fetch/once.go +++ b/internal/cli/fetch/once.go @@ -29,7 +29,9 @@ var stageTags = map[string][]string{ // filter keeps the opportunities whose latest release is at stage. Releases // are oldest first, so the last one is the procedure's current stage. -func filter(opportunities []*types.Opportunity, stage string) []*types.Opportunity { +func filter( + opportunities []*types.Opportunity, stage string, +) []*types.Opportunity { var matching []*types.Opportunity for _, opportunity := range opportunities { @@ -63,7 +65,7 @@ func commandOnce(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } since, sinceErr := cmd.Flags().GetTime("since") @@ -96,15 +98,15 @@ func commandOnce(opts *cli.Options) *cobra.Command { filterTag = "award" } - if len(filterTag) > 0 { + if filterTag != "" { results = filter(results, filterTag) } - jbytes, err := json.MarshalIndent(results, "", " ") + encoded, err := json.MarshalIndent(results, "", " ") if err != nil { return err } - fmt.Println(string(jbytes)) + fmt.Println(string(encoded)) return nil }, @@ -118,17 +120,24 @@ func commandOnce(opts *cli.Options) *cobra.Command { "contract updates published since YYYY-MM-DD[ hh:mm:ss]", ) cmd.Flags().Bool( - "planning", false, "only return planning (i.e. exclude opportunities and awards)", + "planning", + false, + "only return planning (i.e. exclude opportunities and awards)", ) cmd.Flags().Bool( - "opportunities", false, "only return opportunities (i.e. exclude awards and planning)", + "opportunities", + false, + "only return opportunities (i.e. exclude awards and planning)", ) cmd.Flags().Bool( - "awards", false, "only return awards (i.e. exclude planning and opportunities)", + "awards", + false, + "only return awards (i.e. exclude planning and opportunities)", ) cmd.Flags().Bool( "ignore-rate-limit", false, "ignore the rate limit and fail if exceeded", ) cmd.MarkFlagsMutuallyExclusive("planning", "opportunities", "awards") + return cmd } diff --git a/internal/cli/fetch/once_test.go b/internal/cli/fetch/once_test.go index d078bd7..ad0c1b8 100644 --- a/internal/cli/fetch/once_test.go +++ b/internal/cli/fetch/once_test.go @@ -17,7 +17,9 @@ func opp(uuid string, tags ...string) *types.Opportunity { } for _, tag := range tags { o.Releases = append(o.Releases, types.OpportunityRelease{ - Tags: []types.OpportunityReleaseTag{{Tag: tag}}, + Tags: []types.OpportunityReleaseTag{{ + Tag: tag, + }}, }) } return o @@ -56,6 +58,7 @@ func TestOnceFiltersExclusive(t *testing.T) { cmd.SetArgs([]string{"--planning", "--awards"}) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) + err := cmd.Execute() if err == nil || !strings.Contains(err.Error(), "none of the others can be") { t.Fatalf("err = %v; want a mutually exclusive flags error", err) diff --git a/internal/cli/fetch/service.go b/internal/cli/fetch/service.go index 9fbd54d..40c7d2e 100644 --- a/internal/cli/fetch/service.go +++ b/internal/cli/fetch/service.go @@ -32,7 +32,7 @@ func commandService(opts *cli.Options) *cobra.Command { conf, err := opts.RunSetup(cmd, true, foreground) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } since, sinceErr := cmd.Flags().GetTime("since") @@ -65,5 +65,6 @@ func commandService(opts *cli.Options) *cobra.Command { "contract updates published since YYYY-MM-DD[ hh:mm:ss]", ) cmd.Flags().Bool("foreground", false, "run in the foreground") + return cmd } diff --git a/internal/cli/flags_test.go b/internal/cli/flags_test.go index 3a69eac..22c8b2d 100644 --- a/internal/cli/flags_test.go +++ b/internal/cli/flags_test.go @@ -3,6 +3,7 @@ package cli_test import ( "fmt" "os" + "strings" "testing" "github.com/spf13/cobra" @@ -49,8 +50,10 @@ func TestLogLevelFlags(t *testing.T) { {[]string{"-q"}, "level: error"}, {[]string{"-q", "-d"}, "level: debug"}, } { - out := cmdtest.MustRun(t, newRoot, append([]string{"level"}, tt.args...)...) - cmdtest.Contains(t, out, tt.want) + args := append([]string{"level"}, tt.args...) + t.Run(strings.Join(args, " "), func(t *testing.T) { + cmdtest.Contains(t, cmdtest.MustRun(t, newRoot, args...), tt.want) + }) } } diff --git a/internal/cli/migrate/common.go b/internal/cli/migrate/common.go index e2b4c84..25488d7 100644 --- a/internal/cli/migrate/common.go +++ b/internal/cli/migrate/common.go @@ -24,15 +24,15 @@ import ( // If the database file doesn't exist, create decides between asking whether // to create it (up) and failing (down, status: nothing to work on). func openMigrator( - cmd *cobra.Command, opts *cli.Options, create bool, createConfirmed bool, + cmd *cobra.Command, opts *cli.Options, create, createConfirmed bool, ) (*goose.Provider, *sql.DB, error) { conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return nil, nil, fmt.Errorf("no configuration loaded (%w)", err) + return nil, nil, fmt.Errorf("load configuration: %w", err) } file := conf.Database.File - if len(file) == 0 { + if file == "" { return nil, nil, errors.New( "no database file specified. Update your configuration", ) @@ -46,8 +46,8 @@ func openMigrator( question := fmt.Sprintf( "Database file '%s' does not exist. Would you like to create it?", file, ) - if !cli.Confirm(question) { - return nil, nil, errors.New("migrate: no database file") + if !cli.Confirm(cmd.OutOrStdout(), cmd.InOrStdin(), question) { + return nil, nil, errors.New("no database file") } } } else if err != nil { @@ -75,14 +75,14 @@ func closeDB(db *sql.DB, err *error) { // errWriter writes until a write fails, then keeps the error, so a run of // prints needs checking only once. type errWriter struct { - w io.Writer + out io.Writer err error } // printf writes like fmt.Fprintf, unless an earlier write failed. func (e *errWriter) printf(format string, args ...any) { if e.err == nil { - _, e.err = fmt.Fprintf(e.w, format, args...) + _, e.err = fmt.Fprintf(e.out, format, args...) } } @@ -91,7 +91,7 @@ func (e *errWriter) printf(format string, args ...any) { // applied 00003_create_release_tags_table.sql (2ms) func printResults(out io.Writer, results []*goose.MigrationResult) error { output := &errWriter{ - w: out, + out: out, } for _, result := range results { verb := "applied" @@ -136,7 +136,9 @@ func failure(out io.Writer, err error) error { } // printVersion reports where the database ended up. -func printVersion(ctx context.Context, out io.Writer, migrator *goose.Provider) error { +func printVersion( + ctx context.Context, out io.Writer, migrator *goose.Provider, +) error { dbVersion, err := migrator.GetDBVersion(ctx) if err != nil { return err diff --git a/internal/cli/migrate/common_test.go b/internal/cli/migrate/common_test.go index ac53364..5c6d28d 100644 --- a/internal/cli/migrate/common_test.go +++ b/internal/cli/migrate/common_test.go @@ -39,7 +39,11 @@ func TestFailurePartial(t *testing.T) { if !errors.Is(err, cause) { t.Errorf("err = %v, want it to wrap the migration error", err) } - for _, want := range []string{"00002_second.sql", "rolled back", "last version that succeeded"} { + for _, want := range []string{ + "00002_second.sql", + "rolled back", + "last version that succeeded", + } { if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("err = %v, want it to mention %q", err, want) } @@ -58,7 +62,10 @@ func TestFailureWithoutFailedSource(t *testing.T) { func TestFailureOtherError(t *testing.T) { cause := errors.New("database is locked") var out bytes.Buffer - if err := failure(&out, cause); err != cause { + if err := failure( + &out, + cause, + ); err != cause { //nolint:errorlint // checks the error is returned unchanged t.Errorf("err = %v, want the error unchanged", err) } if out.Len() != 0 { @@ -75,7 +82,8 @@ func TestPrintResults(t *testing.T) { if err != nil { t.Fatal(err) } - want := " applied 00001_first.sql (2ms)\n reverted 00002_second.sql (2ms)\n" + want := " applied 00001_first.sql (2ms)\n" + + " reverted 00002_second.sql (2ms)\n" if out.String() != want { t.Errorf("output = %q, want %q", out.String(), want) } @@ -97,7 +105,9 @@ func TestWriteErrors(t *testing.T) { cause := errors.New("boom") err := failure(failingWriter{}, &goose.PartialError{ - Applied: results, Failed: result("00002_second.sql", "up"), Err: cause, + Applied: results, + Failed: result("00002_second.sql", "up"), + Err: cause, }) if !errors.Is(err, cause) || !errors.Is(err, errWrite) { t.Errorf("failure: err = %v, want both the migration and write errors", err) diff --git a/internal/cli/migrate/down.go b/internal/cli/migrate/down.go index 11387ad..81081e0 100644 --- a/internal/cli/migrate/down.go +++ b/internal/cli/migrate/down.go @@ -39,12 +39,11 @@ confirmed by providing -yes.`, defer closeDB(db, &err) results, err := store.StepDown(cmd.Context(), migrator, int(steps), yes) - if len(results) == 0 && err == nil { fmt.Println("No changes made to database (empty)") return nil } - if err == store.ErrWouldEmptyDatabase { + if errors.Is(err, store.ErrWouldEmptyDatabase) { return fmt.Errorf( "%w\n\nTo confirm you wish to proceed, pass the --yes flag", err, ) @@ -62,17 +61,14 @@ confirmed by providing -yes.`, }, } cmd.Flags().Uint( - "steps", - 0, - "number of migrations to roll back. 0 = all (deletes all data)", + "steps", 0, "number of migrations to roll back. 0 = all (deletes all data)", ) if err := cmd.MarkFlagRequired("steps"); err != nil { panic(err) // only fails if the flag isn't defined } cmd.Flags().BoolP( - "yes", "y", - false, - "confirm you are intending to roll back every migration", + "yes", "y", false, "confirm you are intending to roll back every migration", ) + return cmd } diff --git a/internal/cli/migrate/migrate.go b/internal/cli/migrate/migrate.go index d04a9bd..8ac457a 100644 --- a/internal/cli/migrate/migrate.go +++ b/internal/cli/migrate/migrate.go @@ -14,8 +14,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/cli" ) -// NewCommand returns the "migrate" command, which migrates the database schema (see -// internal/store). +// NewCommand returns the "migrate" command, which migrates the database schema +// (see internal/store). func NewCommand(opts *cli.Options) *cobra.Command { cmd := &cobra.Command{ Use: "migrate", @@ -39,5 +39,6 @@ migration is applied.`, cmd.AddCommand(commandUp(opts)) cmd.AddCommand(commandDown(opts)) cmd.AddCommand(commandStatus(opts)) + return cmd } diff --git a/internal/cli/migrate/migrate_test.go b/internal/cli/migrate/migrate_test.go index d936864..60cbe4d 100644 --- a/internal/cli/migrate/migrate_test.go +++ b/internal/cli/migrate/migrate_test.go @@ -3,6 +3,7 @@ package migrate import ( "fmt" "path/filepath" + "strings" "testing" "github.com/spf13/cobra" @@ -43,10 +44,12 @@ func latestVersion(t *testing.T) int64 { t.Fatal(err) } defer db.Close() + m, err := store.NewMigrator(db) if err != nil { t.Fatal(err) } + return store.LatestVersion(m) } @@ -83,7 +86,9 @@ func TestMigrateWithoutSubcommand(t *testing.T) { cmdtest.WriteFile(t, "test.db", "") // exists, so no question is asked out := mustRun(t, "migrate") - cmdtest.Contains(t, out, fmt.Sprintf("Database migrated to version %d", latestVersion(t))) + cmdtest.Contains( + t, out, fmt.Sprintf("Database migrated to version %d", latestVersion(t)), + ) } func TestMigrateUpAsksBeforeCreating(t *testing.T) { @@ -130,16 +135,24 @@ func TestMigrateDown(t *testing.T) { mustRun(t, "migrate", "up", "--create") out := mustRun(t, "migrate", "down", "--steps", "1") - cmdtest.Contains(t, out, "reverted", fmt.Sprintf("Database migrated to version %d", latest-1)) + cmdtest.Contains( + t, + out, + "reverted", + fmt.Sprintf("Database migrated to version %d", latest-1), + ) // Rolling back everything deletes all data, so it must be confirmed. for _, args := range [][]string{ {"migrate", "down", "--steps", "0"}, {"migrate", "down", "--steps", "99"}, } { - _, err := run(t, "", args...) - cmdtest.ErrContains(t, err, "pass the --yes flag") + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := run(t, "", args...) + cmdtest.ErrContains(t, err, "pass the --yes flag") + }) } + cmdtest.Contains( t, mustRun(t, "migrate", "status"), @@ -166,9 +179,12 @@ func TestMigrateMissingDatabase(t *testing.T) { {"migrate", "status"}, {"migrate", "down", "--steps", "1"}, } { - _, err := run(t, "", args...) - cmdtest.ErrContains(t, err, "does not exist") + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := run(t, "", args...) + cmdtest.ErrContains(t, err, "does not exist") + }) } + if cmdtest.Exists("test.db") { t.Error("status or down created the database") } diff --git a/internal/cli/migrate/status.go b/internal/cli/migrate/status.go index e155424..2855829 100644 --- a/internal/cli/migrate/status.go +++ b/internal/cli/migrate/status.go @@ -22,7 +22,9 @@ import ( // 1 applied 2026-10-07 10:12:01 00001_create_opportunities_table.sql // ... // 5 pending 00005_create_opportunity_classifications_table.sql -func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) error { +func printStatus( + ctx context.Context, out io.Writer, migrator *goose.Provider, +) error { statuses, err := migrator.Status(ctx) if err != nil { return err @@ -43,7 +45,7 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e } output := &errWriter{ - w: out, + out: out, } switch { case current > latest: @@ -55,7 +57,9 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e case pending == 0: output.printf("Database version %d of %d (up-to-date)\n", current, latest) default: - output.printf("Database version %d of %d (%d pending)\n", current, latest, pending) + output.printf( + "Database version %d of %d (%d pending)\n", current, latest, pending, + ) } if output.err != nil { return output.err @@ -63,7 +67,7 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e table := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) output = &errWriter{ - w: table, + out: table, } for _, status := range statuses { applied := "" @@ -78,6 +82,7 @@ func printStatus(ctx context.Context, out io.Writer, migrator *goose.Provider) e filepath.Base(status.Source.Path), ) } + return errors.Join(output.err, table.Flush()) } diff --git a/internal/cli/migrate/up.go b/internal/cli/migrate/up.go index 71961b9..82bf663 100644 --- a/internal/cli/migrate/up.go +++ b/internal/cli/migrate/up.go @@ -1,6 +1,7 @@ package migrate import ( + "errors" "fmt" "log/slog" @@ -23,14 +24,15 @@ func runStepUp( defer closeDB(db, &err) results, err := store.StepUp(cmd.Context(), migrator, int(steps)) - if len(results) == 0 && err == nil { - _, err := fmt.Fprintln(cmd.OutOrStdout(), "No changes made to database (up-to-date)") + _, err = fmt.Fprintln( + cmd.OutOrStdout(), "No changes made to database (up-to-date)", + ) return err } - if err := printResults(cmd.OutOrStdout(), results); err != nil { - return err + if printErr := printResults(cmd.OutOrStdout(), results); printErr != nil { + return errors.Join(err, printErr) } if err != nil { @@ -60,15 +62,10 @@ func commandUp(opts *cli.Options) *cobra.Command { return runStepUp(cmd, opts, int(steps), create) }, } - cmd.Flags().Uint( - "steps", - 0, - "number of migrations to apply. 0 = all pending", - ) + cmd.Flags().Uint("steps", 0, "number of migrations to apply. 0 = all pending") cmd.Flags().Bool( - "create", - false, - "create the database file if it does not already exist", + "create", false, "create the database file if it does not already exist", ) + return cmd } diff --git a/internal/cli/options.go b/internal/cli/options.go index 1c76243..1214b86 100644 --- a/internal/cli/options.go +++ b/internal/cli/options.go @@ -34,13 +34,13 @@ func (o *Options) Bind(cmd *cobra.Command) { // terminalLog is true, switches logging to the configured log file (staying // on the terminal, with a warning, if none is configured). func (o *Options) RunSetup( - cmd *cobra.Command, loadConfig bool, terminalLog bool, + cmd *cobra.Command, loadConfig, terminalLog bool, ) (*config.JobSearchConfig, error) { // Flags and arguments are valid from here on, so errors are runtime // failures: report them without cobra's usage text. cmd.SilenceUsage = true - // Check for debug level or quiet supression via args + // Check for debug level or quiet suppression via args conf := &config.JobSearchConfig{ Logging: config.JSCLogging{}, } @@ -64,7 +64,7 @@ func (o *Options) RunSetup( // Check if the logging should be switched to file if !terminalLog { - if len(conf.Logging.File) == 0 { + if conf.Logging.File == "" { slog.Warn("no log file configured; logging to the terminal") } else if err := logging.LogToFile(conf.Logging.File); err != nil { return nil, fmt.Errorf("opening log file: %w", err) diff --git a/internal/cli/wsserve/command.go b/internal/cli/wsserve/command.go index 237c30f..f5c446f 100644 --- a/internal/cli/wsserve/command.go +++ b/internal/cli/wsserve/command.go @@ -2,6 +2,7 @@ package wsserve import ( "context" + "errors" "fmt" "log/slog" "os" @@ -39,21 +40,21 @@ func NewCommand(opts *cli.Options) *cobra.Command { if addrOverride != "" { listenParts := strings.Split(addrOverride, ":") if len(listenParts) != 2 { - return fmt.Errorf( + return errors.New( "--listen should be in the format of 127.0.0.1:9393", ) } - i, err := strconv.Atoi(listenParts[1]) + port, err := strconv.Atoi(listenParts[1]) if err != nil { return fmt.Errorf("failed to parse --listen port: %w", err) } hostOverride = listenParts[0] - portOverride = i + portOverride = port } conf, err := opts.RunSetup(cmd, true, true) if err != nil { - return fmt.Errorf("no configuration loaded (%w)", err) + return fmt.Errorf("load configuration: %w", err) } if portOverride != 0 { @@ -82,11 +83,16 @@ func NewCommand(opts *cli.Options) *cobra.Command { return server.Run(ctx) }, } - cmd.Flags().String("listen", "", "listen address (default from config, 127.0.0.1:9393)") - cmd.Flags().String("host", "", "listen address (default from config, 127.0.0.1)") + cmd.Flags().String( + "listen", "", "listen address (default from config, 127.0.0.1:9393)", + ) + cmd.Flags().String( + "host", "", "listen address (default from config, 127.0.0.1)", + ) cmd.Flags().Int("port", 0, "listen address (default from config, 9393)") cmd.Flags().Bool("foreground", false, "run in the foreground") cmd.MarkFlagsMutuallyExclusive("listen", "host") cmd.MarkFlagsMutuallyExclusive("listen", "port") + return cmd } diff --git a/internal/config/README.md b/internal/config/README.md index 3cc47ac..c55185f 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -48,6 +48,9 @@ is kept. | `server.websocket.max_message_size` | byte size | `32KiB` | Largest message read from a client; `0B` for 32KiB. `-1` (no limit) is rejected, as a client could then exhaust the server's memory | | `server.websocket.connection_limit.*` | | | A rate limit rule for new connections from each client IP, WebSocket and plain TCP alike; limiters are kept for `server.rate_limits.idle_ttl` | | `server.websocket.message_rate_limit.*` | | | A rate limit rule for messages from each connection | +| `server.websocket.*.enabled` | bool | `false` | In a rate limit rule: whether it applies | +| `server.websocket.*.per_second` | int | `50` | In a rate limit rule: sustained rate. `0` or less: the rule is off | +| `server.websocket.*.burst` | int | `100` | In a rate limit rule: largest burst. Below `1`: 1 | | `server.websocket.write_timeout` | duration | `10s` | `0s` for none | | `server.websocket.ping_interval` | duration | `30s` | How long a connection may be idle before it is pinged; `0s`: no pings | | `server.websocket.pong_timeout` | duration | `10s` | `0s` for none | @@ -91,7 +94,7 @@ per key: passed to `SetValue` (e.g. `"10m"`, `"true"`). - A tagged field whose type is a struct is a nested section, addressed as `section.key`. An embedded struct has its fields promoted, as Go does; - every source embeds `JobSearchConfigSource` for `enabled` and `interval`. + every source embeds `JSCSourcesSource` for `enabled` and `interval`. - `time.Duration` fields report `reflect.String` from `TypeOf`, accept either a `time.Duration` or a duration string in `SetValue`, and are given out as strings (`"10m0s"`) by `Map`, `Flatten` and `ValueOf`, and so by `JSON`, @@ -125,7 +128,7 @@ To add a new section: its key, and an implementation of `Section` (copy an existing section's methods). 2. Add the struct as a tagged field of its parent: `JobSearchConfig` in - `config.go`, or `JobSearchConfigSources` in `conf_sources.go` for a source, + `config.go`, or `JSCSources` in `conf_sources.go` for a source, etc. The field's `cfg` tag must match the `JSCK*` constant. Nothing in `section.go` needs changing. diff --git a/internal/config/conf_database.go b/internal/config/conf_database.go index a8a3fe7..c644797 100644 --- a/internal/config/conf_database.go +++ b/internal/config/conf_database.go @@ -17,20 +17,25 @@ type JSCDatabase struct { // Keys returns the keys directly in the section. func (c *JSCDatabase) Keys() []string { return keysOf(c) } +// Map implements Section. func (c JSCDatabase) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCDatabase) Flatten() map[string]any { return flattenOf(JSCKDatabase, c) } +// TypeOf implements Section. func (c JSCDatabase) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCDatabase) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCDatabase) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_logging.go b/internal/config/conf_logging.go index 2cb4f80..97f3de3 100644 --- a/internal/config/conf_logging.go +++ b/internal/config/conf_logging.go @@ -19,20 +19,25 @@ type JSCLogging struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCLogging) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCLogging) Flatten() map[string]any { return flattenOf(JSCKLogging, c) } +// TypeOf implements Section. func (c JSCLogging) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCLogging) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCLogging) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -50,16 +55,16 @@ func (c *JSCLogging) validate(key string, value any) error { case "debug", "info", "warn", "error": default: return errors.New( - "invalid log level. Must be one of: debug, info, warn, error", + "config: invalid log level: must be one of debug, info, warn, error", ) } } return nil } -// ConfigLevelToLogLevel converts a configured level name to an slog level. An +// LevelToLogLevel converts a configured level name to an slog level. An // unknown name gives 99, above every real level, so nothing is logged. -func ConfigLevelToLogLevel(level string) slog.Level { +func LevelToLogLevel(level string) slog.Level { switch level { case "debug": return slog.LevelDebug @@ -77,5 +82,5 @@ func ConfigLevelToLogLevel(level string) slog.Level { // LogLevel returns the configured level as an slog level. An unknown level // gives 99, above every real level, so nothing is logged. func (c *JSCLogging) LogLevel() slog.Level { - return ConfigLevelToLogLevel(c.Level) + return LevelToLogLevel(c.Level) } diff --git a/internal/config/conf_ratelimitrule.go b/internal/config/conf_ratelimitrule.go index 3e7bf0c..42aaebd 100644 --- a/internal/config/conf_ratelimitrule.go +++ b/internal/config/conf_ratelimitrule.go @@ -14,18 +14,23 @@ type JSCRateLimitRule struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCRateLimitRule) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCRateLimitRule) Flatten() map[string]any { return flattenOf("", c) } +// TypeOf implements Section. func (c JSCRateLimitRule) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCRateLimitRule) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCRateLimitRule) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_sections_test.go b/internal/config/conf_sections_test.go index 0424e4d..0fe2d12 100644 --- a/internal/config/conf_sections_test.go +++ b/internal/config/conf_sections_test.go @@ -25,9 +25,10 @@ func captureLogs(t *testing.T, level slog.Level) *bytes.Buffer { t.Helper() var buf bytes.Buffer def := slog.Default() - slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{ + handler := slog.NewTextHandler(&buf, &slog.HandlerOptions{ Level: level, - }))) + }) + slog.SetDefault(slog.New(handler)) t.Cleanup(func() { slog.SetDefault(def) }) @@ -48,7 +49,8 @@ func TestDefaults(t *testing.T) { if conf.Logging.Level != "warn" { t.Errorf("Logging.Level = %q; want warn", conf.Logging.Level) } - if conf.Sources.FindATender.APIRoot != "https://www.find-tender.service.gov.uk/api/1.0" { + const apiRoot = "https://www.find-tender.service.gov.uk/api/1.0" + if conf.Sources.FindATender.APIRoot != apiRoot { t.Errorf("FindATender.APIRoot = %q", conf.Sources.FindATender.APIRoot) } if conf.Sources.FTSMock.APIRoot != "" { @@ -71,25 +73,22 @@ func TestLoggingSection(t *testing.T) { File: "/tmp/x.log", } - if got, want := c.Keys(), []string{"level", "file"}; !reflect.DeepEqual(got, want) { + if got, want := c.Keys(), []string{ + "level", + "file", + }; !reflect.DeepEqual(got, want) { t.Errorf("Keys = %v; want %v", got, want) } if got, want := c.Map(), map[string]any{ "level": "info", "file": "/tmp/x.log", - }; !reflect.DeepEqual( - got, - want, - ) { + }; !reflect.DeepEqual(got, want) { t.Errorf("Map = %v; want %v", got, want) } if got, want := c.Flatten(), map[string]any{ "logging.level": "info", "logging.file": "/tmp/x.log", - }; !reflect.DeepEqual( - got, - want, - ) { + }; !reflect.DeepEqual(got, want) { t.Errorf("Flatten = %v; want %v", got, want) } if k, err := c.TypeOf("file"); err != nil || k != reflect.String { @@ -101,7 +100,8 @@ func TestLoggingSection(t *testing.T) { if _, err := c.TypeOf("nope"); err == nil { t.Error("TypeOf(nope) succeeded") } - if err := c.SetValue("file", "/var/log/js.log"); err != nil || c.File != "/var/log/js.log" { + if err := c.SetValue("file", "/var/log/js.log"); err != nil || + c.File != "/var/log/js.log" { t.Errorf("SetValue(file) = %v, File = %q", err, c.File) } } @@ -114,10 +114,7 @@ func TestLoggingLevelValidation(t *testing.T) { } } for _, lvl := range []string{"", "trace", "WARN", "fatal"} { - if err := c.SetValue( - "level", - lvl, - ); err == nil || + if err := c.SetValue("level", lvl); err == nil || !strings.Contains(err.Error(), "invalid log level") { t.Errorf("SetValue(level, %q) = %v; want invalid log level", lvl, err) } @@ -136,7 +133,7 @@ func TestLoggingLevelValidation(t *testing.T) { } } -func TestConfigLevelToLogLevel(t *testing.T) { +func TestLevelToLogLevel(t *testing.T) { tests := map[string]slog.Level{ "debug": slog.LevelDebug, "info": slog.LevelInfo, @@ -146,8 +143,8 @@ func TestConfigLevelToLogLevel(t *testing.T) { "bogus": 99, } for in, want := range tests { - if got := ConfigLevelToLogLevel(in); got != want { - t.Errorf("ConfigLevelToLogLevel(%q) = %v; want %v", in, got, want) + if got := LevelToLogLevel(in); got != want { + t.Errorf("LevelToLogLevel(%q) = %v; want %v", in, got, want) } c := JSCLogging{ Level: in, @@ -164,7 +161,10 @@ func TestSourceSection(t *testing.T) { Interval: time.Minute, } - if got, want := c.Keys(), []string{"enabled", "interval"}; !reflect.DeepEqual(got, want) { + if got, want := c.Keys(), []string{ + "enabled", + "interval", + }; !reflect.DeepEqual(got, want) { t.Errorf("Keys = %v; want %v", got, want) } want := map[string]any{"enabled": true, "interval": "1m0s"} @@ -184,7 +184,8 @@ func TestSourceSection(t *testing.T) { if v, err := c.ValueOf("interval"); err != nil || v != "1m0s" { t.Errorf("ValueOf(interval) = %v, %v", v, err) } - if err := c.SetValue("interval", "5s"); err != nil || c.Interval != 5*time.Second { + if err := c.SetValue("interval", "5s"); err != nil || + c.Interval != 5*time.Second { t.Errorf("SetValue(interval, 5s) = %v, Interval = %v", err, c.Interval) } if err := c.SetValue("interval", "never"); err == nil { @@ -229,7 +230,8 @@ func TestConcreteSourceSections(t *testing.T) { if err := s.SetValue("api_root", "http://localhost:8080"); err != nil { t.Fatalf("SetValue(api_root): %v", err) } - if v, err := s.ValueOf("api_root"); err != nil || v != "http://localhost:8080" { + if v, err := s.ValueOf("api_root"); err != nil || + v != "http://localhost:8080" { t.Errorf("ValueOf(api_root) = %v, %v", v, err) } wantMap["api_root"] = "http://localhost:8080" @@ -241,7 +243,8 @@ func TestConcreteSourceSections(t *testing.T) { if err := s.SetValue("block_shutdown", "true"); err != nil { t.Fatalf("SetValue(block_shutdown): %v", err) } - if k, err := s.TypeOf("block_shutdown"); err != nil || k != reflect.Bool { + if k, err := s.TypeOf("block_shutdown"); err != nil || + k != reflect.Bool { t.Errorf("TypeOf(block_shutdown) = %v, %v", k, err) } wantMap["block_shutdown"] = true @@ -271,7 +274,8 @@ func TestSourcesSection(t *testing.T) { if c.FTSMock.APIRoot != "http://mock" { t.Errorf("FTSMock.APIRoot = %q", c.FTSMock.APIRoot) } - if v, err := c.ValueOf("fts_mock.api_root"); err != nil || v != "http://mock" { + if v, err := c.ValueOf("fts_mock.api_root"); err != nil || + v != "http://mock" { t.Errorf("ValueOf = %v, %v", v, err) } if k, err := c.TypeOf("sleeper.enabled"); err != nil || k != reflect.Bool { @@ -299,6 +303,7 @@ func TestSourcesSection(t *testing.T) { func TestJobSearchConfigMap(t *testing.T) { conf := defaultConfig(t) m := conf.Map() + logging, ok := m["logging"].(map[string]any) if !ok || logging["level"] != "warn" { t.Errorf("Map()[logging] = %#v", m["logging"]) @@ -330,31 +335,33 @@ func TestString(t *testing.T) { } } -func TestJSON(t *testing.T) { +// JSON and YAML give the configuration as nested sections. +func TestSerialised(t *testing.T) { conf := defaultConfig(t) - var m map[string]map[string]any - if err := json.Unmarshal([]byte(conf.JSON()), &m); err != nil { - t.Fatalf("JSON() is not valid JSON: %v\n%s", err, conf.JSON()) + tests := []struct { + name string + output string + unmarshal func([]byte, any) error + }{ + {"JSON", conf.JSON(), json.Unmarshal}, + {"YAML", conf.YAML(), yaml.Unmarshal}, } - if m["logging"]["level"] != "warn" { - t.Errorf("JSON logging.level = %#v", m["logging"]["level"]) - } - if _, ok := m["sources"]["fts_mock"]; !ok { - t.Errorf("JSON missing sources.fts_mock: %v", m["sources"]) - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m map[string]map[string]any + if err := tt.unmarshal([]byte(tt.output), &m); err != nil { + t.Fatalf("%s() doesn't parse: %v\n%s", tt.name, err, tt.output) + } -func TestYAML(t *testing.T) { - conf := defaultConfig(t) - var m map[string]map[string]any - if err := yaml.Unmarshal([]byte(conf.YAML()), &m); err != nil { - t.Fatalf("YAML() is not valid YAML: %v\n%s", err, conf.YAML()) - } - if m["logging"]["level"] != "warn" { - t.Errorf("YAML logging.level = %#v", m["logging"]["level"]) - } - if _, ok := m["sources"]["sleeper"]; !ok { - t.Errorf("YAML missing sources.sleeper: %v", m["sources"]) + if m["logging"]["level"] != "warn" { + t.Errorf("logging.level = %#v", m["logging"]["level"]) + } + for _, source := range []string{"fts_mock", "sleeper"} { + if _, ok := m["sources"][source]; !ok { + t.Errorf("missing sources.%s: %v", source, m["sources"]) + } + } + }) } } @@ -365,6 +372,7 @@ func TestSerialisedRoundTrip(t *testing.T) { orig.Sources.FTSMock.Enabled = true orig.Sources.FTSMock.APIRoot = "http://mock" orig.Server.WebSocket.MaxMessageSize = 64 << 10 + for _, format := range []string{"json", "yaml"} { t.Run(format, func(t *testing.T) { resetViper(t) @@ -372,10 +380,12 @@ func TestSerialisedRoundTrip(t *testing.T) { if format == "yaml" { out = orig.YAML() } + viper.SetConfigType(format) if err := viper.ReadConfig(strings.NewReader(out)); err != nil { t.Fatalf("ReadConfig: %v", err) } + var got JobSearchConfig LoadFromViper(&got) if !reflect.DeepEqual(got, orig) { @@ -402,7 +412,9 @@ func TestLoadDefaultConfig(t *testing.T) { if got := viper.GetString("logging.level"); got != "warn" { t.Errorf("logging.level = %q; want warn", got) } - if got := viper.GetDuration("sources.sleeper.interval"); got != 10*time.Minute { + if got := viper.GetDuration( + "sources.sleeper.interval", + ); got != 10*time.Minute { t.Errorf("sources.sleeper.interval = %v; want 10m", got) } } @@ -460,7 +472,9 @@ func TestLoadFromViperInvalidValues(t *testing.T) { t.Errorf("Logging.Level = %q; want warn kept", conf.Logging.Level) } if conf.Sources.Sleeper.Interval != 10*time.Minute { - t.Errorf("Sleeper.Interval = %v; want 10m kept", conf.Sources.Sleeper.Interval) + t.Errorf( + "Sleeper.Interval = %v; want 10m kept", conf.Sources.Sleeper.Interval, + ) } for _, key := range []string{"logging.level", "sources.sleeper.interval"} { if !strings.Contains(logs.String(), "key="+key) { @@ -477,7 +491,9 @@ func TestDatabaseSection(t *testing.T) { if got, want := c.Keys(), []string{"file"}; !reflect.DeepEqual(got, want) { t.Errorf("Keys = %v; want %v", got, want) } - if got, want := c.Map(), map[string]any{"file": "./jobsearch.db"}; !reflect.DeepEqual(got, want) { + if got, want := c.Map(), map[string]any{ + "file": "./jobsearch.db", + }; !reflect.DeepEqual(got, want) { t.Errorf("Map = %v; want %v", got, want) } want := map[string]any{"database.file": "./jobsearch.db"} @@ -496,7 +512,8 @@ func TestDatabaseSection(t *testing.T) { if _, err := c.ValueOf("nope"); err == nil { t.Error("ValueOf(nope) succeeded") } - if err := c.SetValue("file", "/var/lib/js.db"); err != nil || c.File != "/var/lib/js.db" { + if err := c.SetValue("file", "/var/lib/js.db"); err != nil || + c.File != "/var/lib/js.db" { t.Errorf("SetValue(file) = %v, File = %q", err, c.File) } if err := c.SetValue("file", 42); err == nil { @@ -532,10 +549,18 @@ func TestViperValue(t *testing.T) { } { got, ok := viperValue(tt.key, tt.kind) if !ok || !reflect.DeepEqual(got, tt.want) { - t.Errorf("viperValue(%q, %v) = %#v, %v; want %#v", tt.key, tt.kind, got, ok, tt.want) + t.Errorf( + "viperValue(%q, %v) = %#v, %v; want %#v", + tt.key, + tt.kind, + got, + ok, + tt.want, + ) } } - for _, kind := range []reflect.Kind{reflect.Float64, reflect.Int8, reflect.Map} { + unhandled := []reflect.Kind{reflect.Float64, reflect.Int8, reflect.Map} + for _, kind := range unhandled { if got, ok := viperValue("i", kind); ok { t.Errorf("viperValue(i, %v) = %#v, true; want unhandled", kind, got) } @@ -553,14 +578,20 @@ func TestWebSocketMaxMessageSizeNoLimit(t *testing.T) { if c.MaxMessageSize != 0 { t.Errorf("rejected size was stored: %v", c.MaxMessageSize) } + for _, text := range []string{"0B", "64KiB"} { if err := c.SetValue("max_message_size", text); err != nil { t.Errorf("SetValue(max_message_size, %s): %v", text, err) } } + conf := defaultConfig(t) - if err := conf.SetValue("server.websocket.max_message_size", "-1"); err == nil { - t.Error("JobSearchConfig.SetValue(server.websocket.max_message_size, -1) succeeded") + if err := conf.SetValue( + "server.websocket.max_message_size", "-1", + ); err == nil { + t.Error( + "JobSearchConfig.SetValue(server.websocket.max_message_size, -1) succeeded", + ) } } @@ -584,6 +615,8 @@ func TestSourceIntervalMustBePositive(t *testing.T) { } } if conf.Sources.FindATender.Interval.Minutes() != 1 { - t.Errorf("FindATender.Interval = %v; want 1m", conf.Sources.FindATender.Interval) + t.Errorf( + "FindATender.Interval = %v; want 1m", conf.Sources.FindATender.Interval, + ) } } diff --git a/internal/config/conf_server.go b/internal/config/conf_server.go index ea08ce4..dfeeb13 100644 --- a/internal/config/conf_server.go +++ b/internal/config/conf_server.go @@ -20,20 +20,25 @@ type JSCServer struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServer) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServer) Flatten() map[string]any { return flattenOf(JSCKServer, c) } +// TypeOf implements Section. func (c JSCServer) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServer) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServer) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_server_ratelimits.go b/internal/config/conf_server_ratelimits.go index ebc0e46..8341468 100644 --- a/internal/config/conf_server_ratelimits.go +++ b/internal/config/conf_server_ratelimits.go @@ -17,20 +17,25 @@ type JSCServerRateLimits struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServerRateLimits) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServerRateLimits) Flatten() map[string]any { return flattenOf(JSCKServerRateLimits, c) } +// TypeOf implements Section. func (c JSCServerRateLimits) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServerRateLimits) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServerRateLimits) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_server_timeouts.go b/internal/config/conf_server_timeouts.go index ccf60e4..ee70f9f 100644 --- a/internal/config/conf_server_timeouts.go +++ b/internal/config/conf_server_timeouts.go @@ -17,20 +17,25 @@ type JSCServerTimeouts struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServerTimeouts) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServerTimeouts) Flatten() map[string]any { return flattenOf(JSCKServerTimeouts, c) } +// TypeOf implements Section. func (c JSCServerTimeouts) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServerTimeouts) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServerTimeouts) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_server_websocket.go b/internal/config/conf_server_websocket.go index cef9239..92e3b02 100644 --- a/internal/config/conf_server_websocket.go +++ b/internal/config/conf_server_websocket.go @@ -28,20 +28,25 @@ type JSCServerWebSocket struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCServerWebSocket) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCServerWebSocket) Flatten() map[string]any { return flattenOf(JSCKServerWebSocket, c) } +// TypeOf implements Section. func (c JSCServerWebSocket) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCServerWebSocket) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCServerWebSocket) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -52,9 +57,10 @@ func (c *JSCServerWebSocket) SetValue(key string, value any) error { // max_message_size can't be -1 (no limit): a client could then send a // message of any size, and the server holds each message in memory. func (c *JSCServerWebSocket) validate(key string, value any) error { - if key == "max_message_size" && value.(util.ByteSize) == util.ByteSizeNoLimit { + if key == "max_message_size" && + value.(util.ByteSize) == util.ByteSizeNoLimit { return errors.New( - "max_message_size can't be -1 (no limit): set an explicit size, or 0 for the default", + "config: max_message_size can't be -1 (no limit): set an explicit size, or 0 for the default", ) } return nil diff --git a/internal/config/conf_source_findatender.go b/internal/config/conf_source_findatender.go index 50bb9c5..a871f98 100644 --- a/internal/config/conf_source_findatender.go +++ b/internal/config/conf_source_findatender.go @@ -14,20 +14,25 @@ type JSCSourceFindATender struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourceFindATender) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourceFindATender) Flatten() map[string]any { return flattenOf(JSCKSourceFindATender, c) } +// TypeOf implements Section. func (c JSCSourceFindATender) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourceFindATender) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourceFindATender) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_source_ftsmock.go b/internal/config/conf_source_ftsmock.go index 45a618e..98b8a30 100644 --- a/internal/config/conf_source_ftsmock.go +++ b/internal/config/conf_source_ftsmock.go @@ -15,20 +15,25 @@ type JSCSourceFTSMock struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourceFTSMock) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourceFTSMock) Flatten() map[string]any { return flattenOf(JSCKSourceFTSMock, c) } +// TypeOf implements Section. func (c JSCSourceFTSMock) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourceFTSMock) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourceFTSMock) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_source_sleeper.go b/internal/config/conf_source_sleeper.go index 2f9b308..1e09fb2 100644 --- a/internal/config/conf_source_sleeper.go +++ b/internal/config/conf_source_sleeper.go @@ -15,20 +15,25 @@ type JSCSourceSleeper struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourceSleeper) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourceSleeper) Flatten() map[string]any { return flattenOf(JSCKSourceSleeper, c) } +// TypeOf implements Section. func (c JSCSourceSleeper) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourceSleeper) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourceSleeper) SetValue(key string, value any) error { return setValue(c, key, value) } diff --git a/internal/config/conf_sources.go b/internal/config/conf_sources.go index d1132f9..da542d4 100644 --- a/internal/config/conf_sources.go +++ b/internal/config/conf_sources.go @@ -6,6 +6,8 @@ import ( "time" ) +// ========== Sources ========== + // JSCKSources is the sources section's configuration key. const JSCKSources string = "sources" @@ -19,20 +21,25 @@ type JSCSources struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSources) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSources) Flatten() map[string]any { return flattenOf(JSCKSources, c) } +// TypeOf implements Section. func (c JSCSources) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSources) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSources) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -50,20 +57,25 @@ type JSCSourcesSource struct { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JSCSourcesSource) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JSCSourcesSource) Flatten() map[string]any { return flattenOf("", c) } +// TypeOf implements Section. func (c JSCSourcesSource) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JSCSourcesSource) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JSCSourcesSource) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -76,7 +88,7 @@ func (c *JSCSourcesSource) Keys() []string { return keysOf(c) } func (c *JSCSourcesSource) validate(key string, value any) error { if key == "interval" && value.(time.Duration) <= 0 { return errors.New( - "interval must be greater than 0; set enabled to false to turn the source off", + "config: interval must be greater than 0; set enabled to false to turn the source off", ) } return nil diff --git a/internal/config/config.go b/internal/config/config.go index 731b92b..a6c84fe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -42,18 +42,23 @@ func LoadDefaultConfig() { // ---------- Section methods (see Section) ---------- +// Map implements Section. func (c JobSearchConfig) Map() map[string]any { return mapOf(c) } +// Flatten implements Section. func (c JobSearchConfig) Flatten() map[string]any { return flattenOf("", c) } +// TypeOf implements Section. func (c JobSearchConfig) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } +// ValueOf implements Section. func (c JobSearchConfig) ValueOf(key string) (any, error) { return valueOf(c, key) } +// SetValue implements Section. func (c *JobSearchConfig) SetValue(key string, value any) error { return setValue(c, key, value) } @@ -70,7 +75,11 @@ func LoadFromViper(conf *JobSearchConfig) { } value, ok := viperValue(key, keyType) if !ok { - slog.Error("unknown config key type", "key", key, "kind", keyType.String()) + slog.Error( + "unknown config key type", + "key", key, + "kind", keyType.String(), + ) continue } if err := conf.SetValue(key, value); err != nil { @@ -79,9 +88,9 @@ func LoadFromViper(conf *JobSearchConfig) { } } -// viperValue reads key from Viper as a value of kind. ok is false for a kind -// LoadFromViper doesn't handle. -func viperValue(key string, kind reflect.Kind) (value any, ok bool) { +// viperValue reads key from Viper as a value of kind. Its bool is false for a +// kind LoadFromViper doesn't handle. +func viperValue(key string, kind reflect.Kind) (any, bool) { switch kind { case reflect.String: return viper.GetString(key), true @@ -134,13 +143,13 @@ func (c JobSearchConfig) String() string { // JSON formats the configuration as indented JSON. func (c JobSearchConfig) JSON() string { confMap := c.Map() - jbytes, _ := json.MarshalIndent(confMap, "", " ") - return string(jbytes) + encoded, _ := json.MarshalIndent(confMap, "", " ") + return string(encoded) } // YAML formats the configuration as YAML. func (c JobSearchConfig) YAML() string { confMap := c.Map() - ybytes, _ := yaml.Marshal(confMap) - return string(ybytes) + encoded, _ := yaml.Marshal(confMap) + return string(encoded) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 53b3316..8c1f785 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -10,7 +10,10 @@ import ( // Kinds that LoadFromViper knows how to read. var viperKinds = map[reflect.Kind]bool{ - reflect.String: true, reflect.Int: true, reflect.Bool: true, reflect.Slice: true, + reflect.String: true, + reflect.Int: true, + reflect.Bool: true, + reflect.Slice: true, } // Every key round-trips through TypeOf/ValueOf/SetValue and has a type that @@ -25,7 +28,9 @@ func TestKeysRoundTrip(t *testing.T) { if err != nil { t.Errorf("TypeOf(%q): %v", key, err) } else if !viperKinds[kind] { - t.Errorf("TypeOf(%q) = %v, which LoadFromViper does not handle", key, kind) + t.Errorf( + "TypeOf(%q) = %v, which LoadFromViper does not handle", key, kind, + ) } got, err := conf.ValueOf(key) if err != nil || !reflect.DeepEqual(got, want) { @@ -53,6 +58,7 @@ func TestSections(t *testing.T) { if !reflect.PointerTo(reflect.TypeOf(conf)).Implements(sectionType) { t.Error("*JobSearchConfig does not implement Section") } + all := conf.Flatten() uses := map[reflect.Type]int{} countSections(uses, reflect.TypeOf(conf)) @@ -64,8 +70,8 @@ func TestSections(t *testing.T) { func countSections(uses map[reflect.Type]int, section reflect.Type) { for _, f := range fieldsOf(section) { if f.section { - uses[f.typ]++ - countSections(uses, f.typ) + uses[f.fieldType]++ + countSections(uses, f.fieldType) } } } @@ -78,14 +84,16 @@ func checkSections( prefix string, section reflect.Value, ) { + t.Helper() for _, f := range fieldsOf(section.Type()) { key := joinKey(prefix, f.key) - if isSectionList(f.typ) { + if isSectionList(f.fieldType) { checkSectionList(t, all, key, section.FieldByIndex(f.index)) } if !f.section { continue } + fv := section.FieldByIndex(f.index) name := fv.Type().Name() if !fv.Addr().Type().Implements(sectionType) { @@ -93,18 +101,16 @@ func checkSections( continue } checkEmbedded(t, fv.Type()) + keys := fv.Addr().Interface().(Section).Flatten() keyPrefix := prefix - if uses[f.typ] > 1 { + if uses[f.fieldType] > 1 { keyPrefix = key } for k := range keys { if _, ok := all[joinKey(keyPrefix, k)]; !ok { t.Errorf( - "%s.Flatten() key %q does not match its cfg tag %q", - name, - k, - key, + "%s.Flatten() key %q does not match its cfg tag %q", name, k, key, ) } } @@ -126,29 +132,36 @@ func checkSections( // checkSectionList checks the items of the list of sections at key implement // Section and that their methods agree with the list's value in all. -func checkSectionList(t *testing.T, all map[string]any, key string, list reflect.Value) { +func checkSectionList( + t *testing.T, all map[string]any, key string, list reflect.Value, +) { t.Helper() name := list.Type().Elem().Name() if !reflect.PointerTo(list.Type().Elem()).Implements(sectionType) { t.Errorf("*%s (items of %s) does not implement Section", name, key) return } + items, _ := all[key].([]map[string]any) if len(items) != list.Len() { t.Errorf("%s has %d items; Flatten gives %d", key, list.Len(), len(items)) return } + for i := range list.Len() { item, itemName := list.Index(i), fmt.Sprintf("%s[%d]", key, i) want := flattenMap("", items[i]) + // Items are not prefixed, and their kinds are their fields' kinds. - if got := item.Addr().Interface().(Section).Flatten(); !reflect.DeepEqual(got, want) { + got := item.Addr().Interface().(Section).Flatten() + if !reflect.DeepEqual(got, want) { t.Errorf("%s.Flatten() =\n%v\nwant\n%v", itemName, got, want) } + kinds := map[string]reflect.Kind{} for k := range want { _, f, _ := lookup(item, k) - kinds[k] = kindOf(f.typ) + kinds[k] = kindOf(f.fieldType) } checkSectionMethods(t, itemName, item, want, kinds) } @@ -164,10 +177,12 @@ func checkSectionMethods( want map[string]any, kinds map[string]reflect.Kind, ) { + t.Helper() s := section.Addr().Interface().(Section) if got := flattenMap("", s.Map()); !reflect.DeepEqual(got, want) { t.Errorf("%s.Map() =\n%v\nwant\n%v", name, got, want) } + before := section.Interface() for k, v := range want { if kind, err := s.TypeOf(k); err != nil || kind != kinds[k] { @@ -180,6 +195,7 @@ func checkSectionMethods( t.Errorf("%s.SetValue(%q, %#v): %v", name, k, v, err) } } + if !reflect.DeepEqual(section.Interface(), before) { t.Errorf("%s changed by setting its keys to their own values", name) } diff --git a/internal/config/load.go b/internal/config/load.go index 62afb56..782e4d9 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "log/slog" "os" @@ -41,13 +42,13 @@ func loadEnv() error { return nil } -// loadConfig sets up Viper with the defaults, cfgFile (or config.* found in +// loadConfig sets up Viper with the defaults, configFile (or config.* found in // EtcConfigPath or the working directory) and JOBSEARCH_* environment // variables. A missing config file is only a warning. -func loadConfig(cfgFile string) error { - if cfgFile != "" { +func loadConfig(configFile string) error { + if configFile != "" { // Use config file from the flag. - viper.SetConfigFile(cfgFile) + viper.SetConfigFile(configFile) } else { // Search config in /etc or local directory. viper.AddConfigPath(EtcConfigPath) @@ -64,26 +65,25 @@ func loadConfig(cfgFile string) error { viper.AutomaticEnv() // read in environment variables that match err := viper.ReadInConfig() - if err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - // Indicates an explicitly set config file is not found or - // that no config file was found in any search path - slog.Warn("no config file found; using defaults") - } else { - // Config file was found but another error was produced - return err - } - } else { - // Config file found and successfully parsed + var notFound viper.ConfigFileNotFoundError + switch { + case errors.As(err, ¬Found): + // No config file was found in any search path. + slog.Warn("no config file found; using defaults") + case err != nil: + // A config file was found but couldn't be read or parsed. + return err + default: slog.Info("using config file", "file", viper.ConfigFileUsed()) } + return nil } // Init loads the .env file and the configuration (see loadConfig) into conf. // A debug or error log level already set in conf, from --debug or --quiet, // is kept over the loaded one. -func Init(cfgFile string, conf *JobSearchConfig) error { +func Init(configFile string, conf *JobSearchConfig) error { if err := loadEnv(); err != nil { return err } @@ -91,7 +91,7 @@ func Init(cfgFile string, conf *JobSearchConfig) error { printEnv() } - if err := loadConfig(cfgFile); err != nil { + if err := loadConfig(configFile); err != nil { return err } diff --git a/internal/config/load_test.go b/internal/config/load_test.go index 757efd9..cf938ca 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -18,6 +18,7 @@ func initDir(t *testing.T, env []string, config string) string { resetViper(t) dir := t.TempDir() t.Chdir(dir) + if env != nil { // godotenv sets variables without cleaning up after the test. for _, kv := range env { @@ -28,9 +29,11 @@ func initDir(t *testing.T, env []string, config string) string { } writeFile(t, filepath.Join(dir, ".env"), strings.Join(env, "\n")+"\n") } + if config != "" { writeFile(t, filepath.Join(dir, "config.yaml"), config) } + return dir } @@ -63,12 +66,16 @@ func TestInitLoadsEnvAndConfigFile(t *testing.T) { if conf.Logging.Level != "info" { t.Errorf("Logging.Level = %q; want info", conf.Logging.Level) } - if !conf.Sources.FTSMock.Enabled || conf.Sources.FTSMock.Interval != 30*time.Second { + if !conf.Sources.FTSMock.Enabled || + conf.Sources.FTSMock.Interval != 30*time.Second { t.Errorf("FTSMock = %+v; want enabled every 30s", conf.Sources.FTSMock) } // Keys missing from the file keep their defaults. if conf.Sources.FindATender.Interval != 10*time.Minute { - t.Errorf("FindATender.Interval = %v; want default 10m", conf.Sources.FindATender.Interval) + t.Errorf( + "FindATender.Interval = %v; want default 10m", + conf.Sources.FindATender.Interval, + ) } } @@ -83,7 +90,9 @@ func TestInitExplicitConfigFile(t *testing.T) { t.Fatal(err) } if conf.Logging.Level != "error" { - t.Errorf("Logging.Level = %q; want error from %s", conf.Logging.Level, other) + t.Errorf( + "Logging.Level = %q; want error from %s", conf.Logging.Level, other, + ) } } @@ -124,7 +133,9 @@ func TestInitKeepsCommandLineLevel(t *testing.T) { // At debug, the environment and loaded config are logged. gotDebug := strings.Contains(logs.String(), "loaded config value") if gotDebug != (level == "debug") { - t.Errorf("debug output logged = %v at level %s: %s", gotDebug, level, logs) + t.Errorf( + "debug output logged = %v at level %s: %s", gotDebug, level, logs, + ) } }) } @@ -134,7 +145,8 @@ func TestInitErrors(t *testing.T) { t.Run("no .env file", func(t *testing.T) { initDir(t, nil, testConfigYAML) var conf JobSearchConfig - if err := Init("", &conf); err == nil || !strings.Contains(err.Error(), ".env") { + if err := Init("", &conf); err == nil || + !strings.Contains(err.Error(), ".env") { t.Fatalf("err = %v; want an invalid .env file error", err) } }) @@ -169,10 +181,15 @@ func TestInitEnvOverride(t *testing.T) { t.Fatal(err) } if conf.Logging.Level != "error" { - t.Errorf("Logging.Level = %q; want error from the environment", conf.Logging.Level) + t.Errorf( + "Logging.Level = %q; want error from the environment", conf.Logging.Level, + ) } if conf.Sources.FTSMock.Interval != 45*time.Second { - t.Errorf("FTSMock.Interval = %v; want 45s from the environment", conf.Sources.FTSMock.Interval) + t.Errorf( + "FTSMock.Interval = %v; want 45s from the environment", + conf.Sources.FTSMock.Interval, + ) } } @@ -204,11 +221,15 @@ func loadExplicit(t *testing.T) { t.Helper() resetViper(t) LoadDefaultConfig() + viper.SetEnvPrefix(EnvPrefix) viper.SetEnvKeyReplacer(envKeyReplacer) viper.AutomaticEnv() + viper.SetConfigType("yaml") - if err := viper.ReadConfig(strings.NewReader(explicitConfigYAML)); err != nil { + if err := viper.ReadConfig( + strings.NewReader(explicitConfigYAML), + ); err != nil { t.Fatal(err) } } @@ -255,7 +276,11 @@ func TestPrintLoadedConfig(t *testing.T) { t.Errorf("missing %q in:\n%s", want, out) } } - for _, unwanted := range []string{"sources.", "logging.file", "connection_limit.burst"} { + for _, unwanted := range []string{ + "sources.", + "logging.file", + "connection_limit.burst", + } { if strings.Contains(out, unwanted) { t.Errorf("logged default key %q:\n%s", unwanted, out) } diff --git a/internal/config/modify.go b/internal/config/modify.go index 7ae3212..7638b2b 100644 --- a/internal/config/modify.go +++ b/internal/config/modify.go @@ -14,14 +14,16 @@ import ( // file's own settings are written back, not defaults or environment // variables. It returns the stored value (as ValueOf gives it) and its kind. func StoreValue( - conf *JobSearchConfig, key string, value string, + conf *JobSearchConfig, key, value string, ) (any, reflect.Kind, error) { return storeValue(conf, key, value) } // storeValue is StoreValue for any section, so key types that // JobSearchConfig doesn't have yet can be tested. -func storeValue(section Section, key string, value string) (any, reflect.Kind, error) { +func storeValue( + section Section, key, value string, +) (any, reflect.Kind, error) { confType, err := section.TypeOf(key) if err != nil { return nil, confType, err @@ -31,7 +33,7 @@ func storeValue(section Section, key string, value string) (any, reflect.Kind, e // SetValue gives. if _, err := parseValue(confType, value); err != nil { return nil, confType, fmt.Errorf( - "config value type not accepted, expected <%s>", confType, + "config: value type not accepted, expected <%s>", confType, ) } @@ -69,7 +71,11 @@ func parseValue(kind reflect.Kind, value string) (any, error) { return value, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return strconv.ParseInt(value, 10, 64) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: return strconv.ParseUint(value, 10, 64) case reflect.Float32, reflect.Float64: return strconv.ParseFloat(value, 64) @@ -91,6 +97,7 @@ func PurgeValue(conf *JobSearchConfig, key string) error { if err != nil { return err } + out := viper.New() out.SetConfigFile(file.ConfigFileUsed()) for _, fileKey := range file.AllKeys() { @@ -98,6 +105,7 @@ func PurgeValue(conf *JobSearchConfig, key string) error { out.Set(fileKey, file.Get(fileKey)) } } + return out.WriteConfig() } @@ -106,12 +114,14 @@ func PurgeValue(conf *JobSearchConfig, key string) error { func readUsedConfigFile() (*viper.Viper, error) { path := viper.ConfigFileUsed() if path == "" { - return nil, errors.New("no configuration file in use") + return nil, errors.New("config: no configuration file in use") } + file := viper.New() file.SetConfigFile(path) if err := file.ReadInConfig(); err != nil { return nil, err } + return file, nil } diff --git a/internal/config/modify_test.go b/internal/config/modify_test.go index 1c264f1..c7591e2 100644 --- a/internal/config/modify_test.go +++ b/internal/config/modify_test.go @@ -20,10 +20,12 @@ func useConfigFile(t *testing.T, contents string) string { if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { t.Fatal(err) } + viper.SetConfigFile(path) if err := viper.ReadInConfig(); err != nil { t.Fatal(err) } + return path } @@ -74,6 +76,7 @@ func TestStoreValueKeepsOtherKeys(t *testing.T) { if _, _, err := StoreValue(&conf, "logging.level", "error"); err != nil { t.Fatal(err) } + v := readConfigFile(t, path) if got := v.GetString("logging.file"); got != "/tmp/js.log" { t.Errorf("logging.file = %q; want it kept", got) @@ -159,7 +162,9 @@ func TestPurgeValueErrors(t *testing.T) { func TestStoreValueUpdatesConf(t *testing.T) { useConfigFile(t, "logging:\n level: warn\n") conf := defaultConfig(t) - if _, _, err := StoreValue(&conf, "sources.fts_mock.interval", "30s"); err != nil { + if _, _, err := StoreValue( + &conf, "sources.fts_mock.interval", "30s", + ); err != nil { t.Fatal(err) } if conf.Sources.FTSMock.Interval != 30*time.Second { @@ -185,7 +190,14 @@ func TestParseValue(t *testing.T) { } { got, err := parseValue(tt.kind, tt.value) if err != nil || !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseValue(%v, %q) = %#v, %v; want %#v", tt.kind, tt.value, got, err, tt.want) + t.Errorf( + "parseValue(%v, %q) = %#v, %v; want %#v", + tt.kind, + tt.value, + got, + err, + tt.want, + ) } } @@ -199,11 +211,14 @@ func TestParseValue(t *testing.T) { {reflect.Float64, "x"}, } { if got, err := parseValue(tt.kind, tt.value); err == nil { - t.Errorf("parseValue(%v, %q) = %#v; want an error", tt.kind, tt.value, got) + t.Errorf( + "parseValue(%v, %q) = %#v; want an error", tt.kind, tt.value, got, + ) } } - if _, err := parseValue(reflect.Map, "a"); !errors.Is(err, errors.ErrUnsupported) { + _, err := parseValue(reflect.Map, "a") + if !errors.Is(err, errors.ErrUnsupported) { t.Errorf("parseValue(map) err = %v; want ErrUnsupported", err) } } @@ -217,11 +232,21 @@ type numberSection struct { Ratio float32 `cfg:"ratio"` } -func (c numberSection) Map() map[string]any { return mapOf(c) } -func (c numberSection) Flatten() map[string]any { return flattenOf("", c) } -func (c numberSection) TypeOf(key string) (reflect.Kind, error) { return typeOf(c, key) } -func (c numberSection) ValueOf(key string) (any, error) { return valueOf(c, key) } -func (c *numberSection) SetValue(key string, value any) error { return setValue(c, key, value) } +func (c numberSection) Map() map[string]any { return mapOf(c) } + +func (c numberSection) Flatten() map[string]any { return flattenOf("", c) } + +func (c numberSection) TypeOf(key string) (reflect.Kind, error) { + return typeOf(c, key) +} + +func (c numberSection) ValueOf(key string) (any, error) { + return valueOf(c, key) +} + +func (c *numberSection) SetValue(key string, value any) error { + return setValue(c, key, value) +} // Numbers are stored as the field's own type, not as the int64, uint64 or // float64 that parsing produces. @@ -276,7 +301,9 @@ func TestStoreAndPurgeWriteOnlyFileSettings(t *testing.T) { LoadDefaultConfig() conf := defaultConfig(t) - if _, _, err := StoreValue(&conf, "sources.sleeper.enabled", "true"); err != nil { + if _, _, err := StoreValue( + &conf, "sources.sleeper.enabled", "true", + ); err != nil { t.Fatal(err) } want := map[string]any{ @@ -301,6 +328,7 @@ func checkFileSettings(t *testing.T, path string, want map[string]any) { for _, k := range v.AllKeys() { got[k] = v.Get(k) } + if !reflect.DeepEqual(got, want) { //nolint:gosec // G304: reading a file the test created in t.TempDir b, _ := os.ReadFile(path) diff --git a/internal/config/print.go b/internal/config/print.go index bd374b3..8170b2f 100644 --- a/internal/config/print.go +++ b/internal/config/print.go @@ -15,13 +15,13 @@ func (c JobSearchConfig) PrintTerminal() { fmt.Print(c.String() + "\n") } -// PrintJson prints the configuration to stdout as JSON. -func (c JobSearchConfig) PrintJson() { +// PrintJSON prints the configuration to stdout as JSON. +func (c JobSearchConfig) PrintJSON() { fmt.Print(c.JSON() + "\n") } -// PrintYaml prints the configuration to stdout as YAML. -func (c JobSearchConfig) PrintYaml() { +// PrintYAML prints the configuration to stdout as YAML. +func (c JobSearchConfig) PrintYAML() { fmt.Print(c.YAML() + "\n") } @@ -29,7 +29,7 @@ func (c JobSearchConfig) PrintYaml() { // debug, warn, error, or info for any other level. On a terminal it logs one // line and prints the values under it as "key = value" lines; elsewhere each // value is a log entry of its own. -func (conf JobSearchConfig) PrintLog(atLevel slog.Level) { +func (c JobSearchConfig) PrintLog(atLevel slog.Level) { switch atLevel { case slog.LevelDebug, slog.LevelWarn, slog.LevelError: default: @@ -38,7 +38,7 @@ func (conf JobSearchConfig) PrintLog(atLevel slog.Level) { ctx := context.Background() logger := slog.Default() - flat := conf.Flatten() + flat := c.Flatten() keys := slices.Sorted(maps.Keys(flat)) if logging.WritesToTerminal(logger.Handler()) { @@ -55,4 +55,4 @@ func (conf JobSearchConfig) PrintLog(atLevel slog.Level) { for _, key := range keys { logger.Log(ctx, atLevel, "config value", "key", key, "value", flat[key]) } -} \ No newline at end of file +} diff --git a/internal/config/print_test.go b/internal/config/print_test.go index b8761bc..b283a55 100644 --- a/internal/config/print_test.go +++ b/internal/config/print_test.go @@ -20,17 +20,24 @@ func captureStdout(t *testing.T, fn func()) string { if err != nil { t.Fatal(err) } + old := os.Stdout os.Stdout = f + t.Cleanup(func() { + os.Stdout = old + }) fn() os.Stdout = old + if err := f.Close(); err != nil { t.Fatal(err) } + b, err := os.ReadFile(filepath.Clean(path)) if err != nil { t.Fatal(err) } + return string(b) } @@ -50,6 +57,7 @@ func TestStringSorted(t *testing.T) { key, _, _ := strings.Cut(line, " = ") keys = append(keys, key) } + if !slices.IsSorted(keys) || len(keys) != len(conf.Flatten()) { t.Errorf("String keys = %v; want each key once, sorted", keys) } @@ -58,27 +66,30 @@ func TestStringSorted(t *testing.T) { } } -func TestPrintJson(t *testing.T) { +// PrintJSON and PrintYAML print the configuration in their format. +func TestPrintFormats(t *testing.T) { conf := defaultConfig(t) - out := captureStdout(t, conf.PrintJson) - var m map[string]any - if err := json.Unmarshal([]byte(out), &m); err != nil { - t.Fatalf("PrintJson output is not JSON: %v\n%s", err, out) + tests := []struct { + name string + print func() + unmarshal func([]byte, any) error + }{ + {"PrintJSON", conf.PrintJSON, json.Unmarshal}, + {"PrintYAML", conf.PrintYAML, yaml.Unmarshal}, } - if got := m["logging"].(map[string]any)["level"]; got != "warn" { - t.Errorf("logging.level = %v; want warn", got) - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := captureStdout(t, tt.print) + var m map[string]any + if err := tt.unmarshal([]byte(out), &m); err != nil { + t.Fatalf("%s output doesn't parse: %v\n%s", tt.name, err, out) + } -func TestPrintYaml(t *testing.T) { - conf := defaultConfig(t) - out := captureStdout(t, conf.PrintYaml) - var m map[string]any - if err := yaml.Unmarshal([]byte(out), &m); err != nil { - t.Fatalf("PrintYaml output is not YAML: %v\n%s", err, out) - } - if got := m["logging"].(map[string]any)["level"]; got != "warn" { - t.Errorf("logging.level = %v; want warn", got) + level := m["logging"].(map[string]any)["level"] + if level != "warn" { + t.Errorf("logging.level = %v; want warn", level) + } + }) } } @@ -101,7 +112,12 @@ func TestPrintLog(t *testing.T) { lines := strings.Split(strings.TrimSpace(logs.String()), "\n") if len(lines) != len(conf.Flatten()) { - t.Errorf("PrintLog(%v) logged %d lines; want %d", tt.at, len(lines), len(conf.Flatten())) + t.Errorf( + "PrintLog(%v) logged %d lines; want %d", + tt.at, + len(lines), + len(conf.Flatten()), + ) } for _, l := range lines { if !strings.Contains(l, tt.want) { diff --git a/internal/config/section.go b/internal/config/section.go index 28429ee..2f77d61 100644 --- a/internal/config/section.go +++ b/internal/config/section.go @@ -43,6 +43,7 @@ import ( // // A section can check values before they are stored by implementing // validator on its pointer type. + // The struct tags section.go reads. const ( // keyTag holds a field's configuration key. @@ -54,7 +55,7 @@ const ( // The unknown key error and the field types section.go handles specially. var ( // errUnknownKey is returned for a key that names no field. - errUnknownKey = errors.New("unknown configuration key") + errUnknownKey = errors.New("config: unknown configuration key") // durationType is time.Duration, read and given out as a string. durationType = reflect.TypeOf(time.Duration(0)) // byteSizeType is util.ByteSize, read and given out as a string. @@ -67,9 +68,9 @@ var ( // Section is the method set of every configuration section, including // JobSearchConfig itself, implemented on the section's pointer type. Keys are -// dotted paths relative to the section, e.g. "sleeper.interval" in -// JSCSources. Durations and byte sizes are given out as strings such as -// "10m0s" and "32KiB", matching TypeOf's reflect.String for them. TestSections checks every +// dotted paths relative to the section, e.g. "sleeper.interval" in JSCSources. +// Durations and byte sizes are given out as strings such as "10m0s" and +// "32KiB", matching TypeOf's reflect.String for them. TestSections checks every // section, found by walking JobSearchConfig, implements it. type Section interface { // Map returns the section as a map of keys to values, with nested @@ -102,18 +103,19 @@ type validator interface { // field is a section struct field that is part of the configuration (see // fieldsOf). type field struct { - key string - index []int // path for reflect.Value.FieldByIndex - typ reflect.Type - section bool - def string - hasDefault bool + key string + index []int // path for reflect.Value.FieldByIndex + fieldType reflect.Type + section bool + defaultValue string + hasDefault bool } // isSectionList reports whether listType is a list of sections, []S for a // section struct S. func isSectionList(listType reflect.Type) bool { - return listType.Kind() == reflect.Slice && listType.Elem().Kind() == reflect.Struct && + return listType.Kind() == reflect.Slice && + listType.Elem().Kind() == reflect.Struct && listType.Elem() != timeType } @@ -123,6 +125,7 @@ func fieldsOf(structType reflect.Type) []field { var fields []field for i := 0; i < structType.NumField(); i++ { structField := structType.Field(i) + if structField.Anonymous && structField.Type.Kind() == reflect.Struct { for _, embeddedField := range fieldsOf(structField.Type) { embeddedField.index = append([]int{i}, embeddedField.index...) @@ -130,20 +133,24 @@ func fieldsOf(structType reflect.Type) []field { } continue } + key, ok := structField.Tag.Lookup(keyTag) if !ok || key == "" || key == "-" || !structField.IsExported() { continue } + defaultValue, hasDefault := structField.Tag.Lookup(defaultTag) fields = append(fields, field{ - key: key, - index: []int{i}, - typ: structField.Type, - section: structField.Type.Kind() == reflect.Struct && structField.Type != timeType, - def: defaultValue, - hasDefault: hasDefault, + key: key, + index: []int{i}, + fieldType: structField.Type, + section: structField.Type.Kind() == reflect.Struct && + structField.Type != timeType, + defaultValue: defaultValue, + hasDefault: hasDefault, }) } + return fields } @@ -249,6 +256,7 @@ func exportValue(fieldValue reflect.Value) any { case byteSizeType: return util.ByteSize(fieldValue.Int()).String() } + if isSectionList(fieldValue.Type()) { var items []map[string]any for i := range fieldValue.Len() { @@ -256,11 +264,13 @@ func exportValue(fieldValue reflect.Value) any { } return items } + value := fieldValue.Interface() if list, ok := value.([]string); ok { // A copy, so changing it doesn't change the configuration. return slices.Clone(list) } + return value } @@ -292,7 +302,7 @@ func typeOf(value any, key string) (reflect.Kind, error) { if err != nil { return reflect.Invalid, err } - return kindOf(configField.typ), nil + return kindOf(configField.fieldType), nil } // valueOf returns the value of the dotted key in the section value, or @@ -318,22 +328,25 @@ func setValue(ptr any, key string, value any) error { // validator, checks it, before setting the field. On error the field is left // unchanged. func setField(section reflect.Value, configField field, value any) error { - newValue, err := convert(value, configField.typ) + newValue, err := convert(value, configField.fieldType) if err != nil { return err } + if checker, ok := section.Addr().Interface().(validator); ok { - if err := checker.validate(configField.key, newValue.Interface()); err != nil { + err := checker.validate(configField.key, newValue.Interface()) + if err != nil { return err } } section.FieldByIndex(configField.index).Set(newValue) + return nil } -// convert turns value into a value of type targetType. Values already of the right -// type are used as-is, except that an empty []string becomes nil, as an empty -// comma-separated list does (see splitList); strings are parsed. Lists of +// convert turns value into a value of type targetType. Values already of the +// right type are used as-is, except that an empty []string becomes nil, as an +// empty comma-separated list does (see splitList); strings are parsed. Lists of // sections are converted by convertSectionList. func convert(value any, targetType reflect.Type) (reflect.Value, error) { if isSectionList(targetType) { @@ -351,36 +364,38 @@ func convert(value any, targetType reflect.Type) (reflect.Value, error) { return parseString(text, targetType) } } - return reflect.Value{}, fmt.Errorf("unsupported type %T", value) + return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } // convertSectionList turns value into a list of sections of type listType: a -// value of type listType, a list of maps of keys to values, or a JSON array of objects. -// nil and empty lists become a nil list. -func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) { +// value of type listType, a list of maps of keys to values, or a JSON array of +// objects. nil and empty lists become a nil list. +func convertSectionList( + value any, listType reflect.Type, +) (reflect.Value, error) { if text, ok := value.(string); ok { var items []map[string]any if strings.TrimSpace(text) != "" { if err := json.Unmarshal([]byte(text), &items); err != nil { return reflect.Value{}, fmt.Errorf( - "unable to convert to a list of %s: %w", - listType.Elem(), - err, + "unable to convert to a list of %s: %w", listType.Elem(), err, ) } } value = items } + if value == nil { return reflect.Zero(listType), nil } reflected := reflect.ValueOf(value) if reflected.Kind() != reflect.Slice { - return reflect.Value{}, fmt.Errorf("unsupported type %T", value) + return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } if reflected.Len() == 0 { return reflect.Zero(listType), nil } + if reflected.Type().AssignableTo(listType) { for i := range reflected.Len() { if err := validateSection(reflected.Index(i)); err != nil { @@ -389,6 +404,7 @@ func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) } return reflected, nil } + list := reflect.MakeSlice(listType, 0, reflected.Len()) for i := range reflected.Len() { item, err := sectionFromMap(reflected.Index(i).Interface(), listType.Elem()) @@ -397,6 +413,7 @@ func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) } list = reflect.Append(list, item) } + return list, nil } @@ -408,28 +425,32 @@ func convertSectionList(value any, listType reflect.Type) (reflect.Value, error) func sectionFromMap(value any, structType reflect.Type) (reflect.Value, error) { item, ok := value.(map[string]any) if !ok { - return reflect.Value{}, fmt.Errorf("unsupported type %T", value) + return reflect.Value{}, fmt.Errorf("config: unsupported type %T", value) } + section := reflect.New(structType).Elem() if err := applyDefaultsTo("", section); err != nil { return reflect.Value{}, err } + for itemKey, itemValue := range item { holder, configField, err := lookup(section, itemKey) if err != nil { return reflect.Value{}, fmt.Errorf("%w %q", err, itemKey) } if _, isString := itemValue.(string); itemValue != nil && !isString && - !reflect.TypeOf(itemValue).AssignableTo(configField.typ) { + !reflect.TypeOf(itemValue).AssignableTo(configField.fieldType) { itemValue = fmt.Sprint(itemValue) } if err := setField(holder, configField, itemValue); err != nil { return reflect.Value{}, fmt.Errorf("%s: %w", itemKey, err) } } + if err := validateSection(section); err != nil { return reflect.Value{}, err } + return section, nil } @@ -441,37 +462,41 @@ func validateSection(section reflect.Value) error { if !ok { return nil } + for _, configField := range fieldsOf(section.Type()) { if configField.section { continue } - if err := checker.validate( - configField.key, - section.FieldByIndex(configField.index).Interface(), - ); err != nil { + value := section.FieldByIndex(configField.index).Interface() + if err := checker.validate(configField.key, value); err != nil { return fmt.Errorf("%s: %w", configField.key, err) } } + return nil } -// parseString parses text as a value of type targetType. It supports strings, bools, -// signed and unsigned integers, floats, time.Duration, ByteSize and []string -// (from a comma-separated list, see splitList). +// parseString parses text as a value of type targetType. It supports strings, +// bools, signed and unsigned integers, floats, time.Duration, ByteSize and +// []string (from a comma-separated list, see splitList). func parseString(text string, targetType reflect.Type) (reflect.Value, error) { parsed := reflect.New(targetType).Elem() switch targetType { case durationType: duration, err := time.ParseDuration(text) if err != nil { - return reflect.Value{}, fmt.Errorf("unable to convert to duration: %w", err) + return reflect.Value{}, fmt.Errorf( + "unable to convert to duration: %w", err, + ) } parsed.SetInt(int64(duration)) return parsed, nil case byteSizeType: size, err := util.ParseByteSize(text) if err != nil { - return reflect.Value{}, fmt.Errorf("unable to convert to byte size: %w", err) + return reflect.Value{}, fmt.Errorf( + "unable to convert to byte size: %w", err, + ) } parsed.SetInt(int64(size)) return parsed, nil @@ -492,7 +517,11 @@ func parseString(text string, targetType reflect.Type) (reflect.Value, error) { var number int64 number, err = strconv.ParseInt(text, 10, targetType.Bits()) parsed.SetInt(number) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: var number uint64 number, err = strconv.ParseUint(text, 10, targetType.Bits()) parsed.SetUint(number) @@ -501,11 +530,16 @@ func parseString(text string, targetType reflect.Type) (reflect.Value, error) { number, err = strconv.ParseFloat(text, targetType.Bits()) parsed.SetFloat(number) default: - return reflect.Value{}, fmt.Errorf("unsupported type string for %s", targetType) + return reflect.Value{}, fmt.Errorf( + "config: unsupported type string for %s", targetType, + ) } if err != nil { - return reflect.Value{}, fmt.Errorf("unable to convert %q to %s: %w", text, targetType, err) + return reflect.Value{}, fmt.Errorf( + "unable to convert %q to %s: %w", text, targetType, err, + ) } + return parsed, nil } @@ -521,16 +555,21 @@ func applyDefaults(ptr any) error { func applyDefaultsTo(prefix string, section reflect.Value) error { for _, configField := range fieldsOf(section.Type()) { key := joinKey(prefix, configField.key) + if configField.section { - if err := applyDefaultsTo(key, section.FieldByIndex(configField.index)); err != nil { + nested := section.FieldByIndex(configField.index) + if err := applyDefaultsTo(key, nested); err != nil { return err } continue } + if !configField.hasDefault { continue } - if err := setField(section, configField, configField.def); err != nil { + + err := setField(section, configField, configField.defaultValue) + if err != nil { return fmt.Errorf("invalid default for %s: %w", key, err) } } diff --git a/internal/config/section_test.go b/internal/config/section_test.go index 3d90f72..c222d7a 100644 --- a/internal/config/section_test.go +++ b/internal/config/section_test.go @@ -222,7 +222,8 @@ func TestUnknownKeys(t *testing.T) { } var s testSection for _, key := range keys { - if k, err := typeOf(s, key); !errors.Is(err, errUnknownKey) || k != reflect.Invalid { + if k, err := typeOf(s, key); !errors.Is(err, errUnknownKey) || + k != reflect.Invalid { t.Errorf("typeOf(%q) = %v, %v; want Invalid, errUnknownKey", key, k, err) } if v, err := valueOf(s, key); !errors.Is(err, errUnknownKey) || v != nil { @@ -266,17 +267,24 @@ func TestSetValue(t *testing.T) { {"slice", []string{}, []string(nil)}, // empty lists are stored as nil // lists of sections are given out as lists of maps; items start // from their defaults and their values are parsed - {"items", `[{"n": "1", "wait": "2s"}, {"name": "b", "n": 2}]`, []map[string]any{ - {"name": "item", "n": 1, "wait": "2s"}, - {"name": "b", "n": 2, "wait": "0s"}, - }}, + { + "items", + `[{"n": "1", "wait": "2s"}, {"name": "b", "n": 2}]`, + []map[string]any{ + {"name": "item", "n": 1, "wait": "2s"}, + {"name": "b", "n": 2, "wait": "0s"}, + }, + }, {"items", []any{map[string]any{"name": "a", "n": 3}}, []map[string]any{ {"name": "a", "n": 3, "wait": "0s"}, }}, {"items", []map[string]any{{"name": "a"}}, []map[string]any{ {"name": "a", "n": 0, "wait": "0s"}, }}, - {"items", []testItem{{Name: "a", N: 4}}, []map[string]any{ + {"items", []testItem{{ + Name: "a", + N: 4, + }}, []map[string]any{ {"name": "a", "n": 4, "wait": "0s"}, }}, {"items", "", []map[string]any(nil)}, @@ -318,9 +326,9 @@ func TestSetValueErrors(t *testing.T) { {"on", "maybe", "unable to convert"}, {"dur", "soon", "unable to convert to duration"}, {"items", "not json", "unable to convert to a list of config.testItem"}, - {"items", `[{"bogus": 1}]`, `item 0: unknown configuration key "bogus"`}, + {"items", `[{"bogus": 1}]`, `item 0: config: unknown configuration key "bogus"`}, {"items", `[{"n": "x"}]`, "item 0: n: unable to convert"}, - {"items", []any{"a"}, "item 0: unsupported type string"}, + {"items", []any{"a"}, "item 0: config: unsupported type string"}, {"items", 5, "unsupported type int"}, {"size", "lots", "unable to convert to byte size"}, {"size", "-1KB", "unable to convert to byte size"}, @@ -344,7 +352,9 @@ func TestSetValueErrors(t *testing.T) { ) } if !reflect.DeepEqual(s, before) { - t.Errorf("setValue(%q, %#v) modified the section on error", tt.key, tt.value) + t.Errorf( + "setValue(%q, %#v) modified the section on error", tt.key, tt.value, + ) } } } diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index bf57171..9aa3084 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -33,7 +33,8 @@ func WithLogger(logger *slog.Logger) Option { } // logger returns the logger Run logs to: the given one, or slog.Default(). -// It isn't tagged here; each worker adds its source. +// It isn't tagged here, as it is also given to the clients the workers +// create, which add their own component; see tagged. func (opts options) logger() *slog.Logger { if opts.log == nil { return slog.Default() @@ -41,8 +42,15 @@ func (opts options) logger() *slog.Logger { return opts.log } -// runFunc runs one source's worker until ctx is cancelled, logging to -// logger. +// tagged returns logger tagged with the fetcher's component, for the lines +// Run and its workers log themselves. +func tagged(logger *slog.Logger) *slog.Logger { + return logger.With("component", "fetcher") +} + +// runFunc runs one source's worker until ctx is cancelled. logger is +// untagged: the worker tags its own lines with tagged, and passes logger as +// it is to any client it creates. type runFunc func( ctx context.Context, logger *slog.Logger, @@ -68,7 +76,7 @@ func Run( for _, opt := range opts { opt(&settings) } - logger := settings.logger().With("component", "fetcher") + logger := settings.logger() var ( wg sync.WaitGroup @@ -76,15 +84,16 @@ func Run( errs []error ) - start := func( - name string, - run runFunc, - ) { + start := func(name string, run runFunc) { wg.Add(1) go func() { defer wg.Done() if err := run(ctx, logger, conf, since); err != nil { - logger.Error("source stopped, shutting down", "source", name, "error", err) + tagged(logger).Error( + "source stopped, shutting down", + "source", name, + "error", err, + ) mu.Lock() errs = append(errs, fmt.Errorf("%s: %w", name, err)) mu.Unlock() @@ -104,5 +113,6 @@ func Run( } wg.Wait() + return errors.Join(errs...) } diff --git a/internal/fetcher/fetcher_test.go b/internal/fetcher/fetcher_test.go index 54ce736..f1c7735 100644 --- a/internal/fetcher/fetcher_test.go +++ b/internal/fetcher/fetcher_test.go @@ -20,7 +20,7 @@ import ( func TestMain(m *testing.M) { // Keep worker and fetch log lines out of the test output. - slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + slog.SetDefault(slog.New(slog.DiscardHandler)) os.Exit(m.Run()) } @@ -38,18 +38,22 @@ const onePackage = `{ const emptyPackage = `{"uri": "x", "releases": []}` // ftsServer serves body (with status) for every request and counts requests. -func ftsServer(t *testing.T, status int, body string) (*httptest.Server, *atomic.Int32) { +func ftsServer( + t *testing.T, status int, body string, +) (*httptest.Server, *atomic.Int32) { t.Helper() var hits atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - if r.URL.Path != "/ocdsReleasePackages" { - t.Errorf("unexpected path %q", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - io.WriteString(w, body) - })) + srv := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + if r.URL.Path != "/ocdsReleasePackages" { + t.Errorf("unexpected path %q", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + io.WriteString(w, body) + }), + ) t.Cleanup(srv.Close) return srv, &hits } @@ -83,7 +87,8 @@ func TestFetchFunctions(t *testing.T) { "nil", }, {"no releases", http.StatusOK, emptyPackage, func(err error) bool { - return errors.Is(err, ErrNoNewReleases) && errors.Is(err, util.ErrWorkerNoWork) + return errors.Is(err, ErrNoNewReleases) && + errors.Is(err, util.ErrWorkerNoWork) }, "ErrNoNewReleases wrapping ErrWorkerNoWork"}, {"bad body", http.StatusInternalServerError, "oops", func(err error) bool { return err != nil && !errors.Is(err, util.ErrWorkerNoWork) @@ -119,8 +124,11 @@ func TestFetchUsesConfiguredAPIRoot(t *testing.T) { fetchFindAOpportunity(context.Background(), slog.Default(), conf, time.Time{}) if fatHits.Load() != 1 || mockHits.Load() != 0 { - t.Fatalf("FindAOpportunity hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + t.Fatalf( + "FindAOpportunity hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load(), + ) } + fetchFTSMock(context.Background(), slog.Default(), conf, time.Time{}) if fatHits.Load() != 1 || mockHits.Load() != 1 { t.Fatalf("FTSMock hits: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) @@ -144,9 +152,12 @@ func TestFetchCancelled(t *testing.T) { func TestSleepCancelled(t *testing.T) { for _, block := range []bool{false, true} { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + ctx, cancel := context.WithTimeout( + context.Background(), 10*time.Millisecond, + ) defer cancel() - if err := sleep(ctx, block, time.Hour); !errors.Is(err, context.DeadlineExceeded) { + err := sleep(ctx, block, time.Hour) + if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("block=%v: err = %v, want context.DeadlineExceeded", block, err) } } @@ -198,7 +209,9 @@ func TestRunWorkersRejectZeroInterval(t *testing.T) { "Sleeper": runSleeper, } for name, run := range runs { - if err := run(context.Background(), slog.Default(), conf, time.Time{}); err == nil { + if err := run( + context.Background(), slog.Default(), conf, time.Time{}, + ); err == nil { t.Errorf("%s: expected an error for a zero interval", name) } } @@ -228,7 +241,9 @@ func TestRunLogsSourceErrors(t *testing.T) { t.Errorf("returned error %v; want it to include %s", err, name) } } - if n := strings.Count(buf.String(), "interval must be greater than 0"); n != 3 { + if n := strings.Count( + buf.String(), "interval must be greater than 0", + ); n != 3 { t.Errorf("logged %d interval errors; want 3: %s", n, &buf) } } @@ -236,11 +251,14 @@ func TestRunLogsSourceErrors(t *testing.T) { func TestRunNoSourcesEnabled(t *testing.T) { done := make(chan struct{}) go func() { - if err := Run(context.Background(), &config.JobSearchConfig{}, time.Time{}); err != nil { + if err := Run( + context.Background(), &config.JobSearchConfig{}, time.Time{}, + ); err != nil { t.Errorf("Run = %v; want nil", err) } close(done) }() + select { case <-done: case <-time.After(time.Second): @@ -275,6 +293,7 @@ func TestRunShutsDownOnWorkerError(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("Run did not shut down after a worker failed") } + // FTSMock either polled once before the shutdown or never started a run. if n := mockHits.Load(); n > 1 { t.Errorf("FTSMock polled %d times; want at most 1", n) @@ -307,7 +326,9 @@ func TestRunEnabledSources(t *testing.T) { deadline := time.Now().Add(5 * time.Second) for fatHits.Load() == 0 || mockHits.Load() == 0 { if time.Now().After(deadline) { - t.Fatalf("sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load()) + t.Fatalf( + "sources not polled: fat=%d mock=%d", fatHits.Load(), mockHits.Load(), + ) } time.Sleep(5 * time.Millisecond) } diff --git a/internal/fetcher/worker_findatender.go b/internal/fetcher/worker_findatender.go index eb2dfd1..519acdb 100644 --- a/internal/fetcher/worker_findatender.go +++ b/internal/fetcher/worker_findatender.go @@ -21,7 +21,7 @@ func runFindAOpportunity( since time.Time, ) error { worker := util.Worker{ - Logger: logger, + Logger: tagged(logger), Name: "FindAOpportunity", Interval: conf.Sources.FindATender.Interval, Task: func(ctx context.Context, since time.Time) error { @@ -40,15 +40,16 @@ func fetchFindAOpportunity( conf *config.JobSearchConfig, since time.Time, ) error { - opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) { - findATender := findatender.New( - conf, - conf.Sources.FindATender.APIRoot, - findatender.WithLogger(logger), - ) - defer findATender.Shutdown() - return findATender.With(ctx).FetchAll(since, true) - }) + opportunities, err := util.AwaitTask( + ctx, + func() ([]*types.Opportunity, error) { + findATender := findatender.New( + conf, conf.Sources.FindATender.APIRoot, findatender.WithLogger(logger), + ) + defer findATender.Shutdown() + return findATender.With(ctx).FetchAll(since, true) + }, + ) if errors.Is(err, findatender.ErrNoReleases) { return ErrNoNewReleases } @@ -56,7 +57,7 @@ func fetchFindAOpportunity( return err } - logger.Info("fetched opportunities", "count", len(opportunities)) + tagged(logger).Info("fetched opportunities", "count", len(opportunities)) return nil } diff --git a/internal/fetcher/worker_ftsmock.go b/internal/fetcher/worker_ftsmock.go index c08f840..faee530 100644 --- a/internal/fetcher/worker_ftsmock.go +++ b/internal/fetcher/worker_ftsmock.go @@ -21,7 +21,7 @@ func runFTSMock( since time.Time, ) error { worker := util.Worker{ - Logger: logger, + Logger: tagged(logger), Name: "FTSMock", Interval: conf.Sources.FTSMock.Interval, Task: func(ctx context.Context, since time.Time) error { @@ -40,15 +40,16 @@ func fetchFTSMock( conf *config.JobSearchConfig, since time.Time, ) error { - opportunities, err := util.AwaitTask(ctx, func() ([]*types.Opportunity, error) { - findATender := findatender.New( - conf, - conf.Sources.FTSMock.APIRoot, - findatender.WithLogger(logger), - ) - defer findATender.Shutdown() - return findATender.With(ctx).FetchAll(since, true) - }) + opportunities, err := util.AwaitTask( + ctx, + func() ([]*types.Opportunity, error) { + findATender := findatender.New( + conf, conf.Sources.FTSMock.APIRoot, findatender.WithLogger(logger), + ) + defer findATender.Shutdown() + return findATender.With(ctx).FetchAll(since, true) + }, + ) if errors.Is(err, findatender.ErrNoReleases) { return ErrNoNewReleases } @@ -56,7 +57,7 @@ func fetchFTSMock( return err } - logger.Info("fetched opportunities", "count", len(opportunities)) + tagged(logger).Info("fetched opportunities", "count", len(opportunities)) return nil } diff --git a/internal/fetcher/worker_sleeper.go b/internal/fetcher/worker_sleeper.go index b72b272..10e40f2 100644 --- a/internal/fetcher/worker_sleeper.go +++ b/internal/fetcher/worker_sleeper.go @@ -22,15 +22,14 @@ func runSleeper( ) error { blockShutdown := conf.Sources.Sleeper.BlockShutdown worker := util.Worker{ - Logger: logger, + Logger: tagged(logger), Name: "Sleeper", Interval: conf.Sources.Sleeper.Interval, Task: func(taskCtx context.Context, since time.Time) error { if blockShutdown { return sleep(taskCtx, true, sleepDuration) - } else { - return sleep(ctx, false, sleepDuration) } + return sleep(ctx, false, sleepDuration) }, } return worker.Run(ctx, since) @@ -47,8 +46,10 @@ func sleep( time.Sleep(duration) return nil } + timer := time.NewTimer(duration) defer timer.Stop() + select { case <-timer.C: return nil diff --git a/internal/findatender/api.go b/internal/findatender/api.go index 1ffb559..40a194b 100644 --- a/internal/findatender/api.go +++ b/internal/findatender/api.go @@ -22,17 +22,17 @@ import ( // Errors returned when fetching from the API. var ( // ErrNoNext means a release package has no next page link. - ErrNoNext = errors.New("no next link") + ErrNoNext = errors.New("findatender: no next link") // ErrNoReleases means the API returned no releases. - ErrNoReleases = errors.New("no releases") + ErrNoReleases = errors.New("findatender: no releases") // ErrRateLimitExceeded means the API refused a request with 429 Too Many // Requests, and waiting for it was not allowed or did not help. - ErrRateLimitExceeded = errors.New("rate limit has been exceeded") + ErrRateLimitExceeded = errors.New("findatender: rate limit has been exceeded") // ErrFetchCancelled means the fetch's context was cancelled while waiting // out a rate limit. - ErrFetchCancelled = errors.New("fetch cancelled") + ErrFetchCancelled = errors.New("findatender: fetch cancelled") // ErrUnexpectedStatus wraps an API response status other than 200 OK. - ErrUnexpectedStatus = errors.New("unexpected response status") + ErrUnexpectedStatus = errors.New("findatender: unexpected response status") ) // RequestOptions selects the releases a fetch asks for. @@ -95,7 +95,7 @@ func repairDates(raw json.RawMessage) (json.RawMessage, error) { if _, err := decoder.Token(); !errors.Is(err, io.EOF) { return nil, errors.New( - "repairing dates: unexpected data after the JSON value", + "findatender: repairing dates: unexpected data after the JSON value", ) } @@ -106,8 +106,8 @@ func repairDates(raw json.RawMessage) (json.RawMessage, error) { return json.Marshal(doc) } -// recursiveRepairDates fixes date fields in value in place and reports whether it -// changed anything. +// recursiveRepairDates fixes date fields in value in place and reports whether +// it changed anything. func recursiveRepairDates(value any) bool { changed := false switch typed := value.(type) { @@ -155,10 +155,12 @@ func releaseIDs(raw json.RawMessage) (ocid, id string, err error) { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.UseNumber() // a numeric id keeps its exact digits - if token, err := decoder.Token(); err != nil { + token, err := decoder.Token() + if err != nil { return "", "", fmt.Errorf("reading release: %w", err) - } else if token != json.Delim('{') { - return "", "", errors.New("reading release: not a JSON object") + } + if token != json.Delim('{') { + return "", "", errors.New("findatender: reading release: not a JSON object") } var haveOCID, haveID bool @@ -205,7 +207,7 @@ func decodeReleaseID(decoder *json.Decoder) (string, error) { case json.Number: return typed.String(), nil default: - return "", fmt.Errorf("reading release id: unexpected %T", typed) + return "", fmt.Errorf("findatender: reading release id: unexpected %T", typed) } } @@ -213,11 +215,11 @@ func decodeReleaseID(decoder *json.Decoder) (string, error) { func missingIDsError(haveOCID, haveID bool) error { switch { case !haveOCID && !haveID: - return errors.New("reading release: no ocid or id") + return errors.New("findatender: reading release: no ocid or id") case !haveOCID: - return errors.New("reading release: no ocid") + return errors.New("findatender: reading release: no ocid") default: - return errors.New("reading release: no id") + return errors.New("findatender: reading release: no id") } } @@ -227,8 +229,7 @@ func missingIDsError(haveOCID, haveID bool) error { // // This method takes ownership of the response body and closes it when done. func safeUnpackReleases( - logger *slog.Logger, - response *resty.Response, + logger *slog.Logger, response *resty.Response, ) (*ocds.ReleasePackage, error) { var releasePackage safeReleasePackage @@ -251,7 +252,8 @@ func safeUnpackReleases( } // Correct PublishedDate of "" and "2006-01-02" - if releasePackage.PublishedDate != nil && len(*releasePackage.PublishedDate) > 0 { + if releasePackage.PublishedDate != nil && + *releasePackage.PublishedDate != "" { published, err := time.Parse(time.RFC3339, *releasePackage.PublishedDate) if err != nil { published, err = time.Parse(time.DateOnly, *releasePackage.PublishedDate) @@ -304,9 +306,9 @@ func closeBody(res *resty.Response) { } } -// fetchURI performs a single GET of uri, using findATender's context, and traces it at -// debug level. Unlike the method of the same name it does not check the -// response status. +// fetchURI performs a single GET of uri, using findATender's context, and +// traces it at debug level. Unlike the method of the same name it does not +// check the response status. func fetchURI(findATender *FindATender, uri string) (*resty.Response, error) { withTrace := findATender.config.Logging.LogLevel() <= slog.LevelDebug logger := findATender.logger() @@ -345,17 +347,21 @@ func logTrace(logger *slog.Logger, res *resty.Response) { if logging.IsJSONHandler(logger.Handler()) { trace = res.Request.TraceInfo().JSON() } - logger.Debug("fetched", "trace", trace, "status", res.Status(), "response", res.String()) + logger.Debug( + "fetched", + "trace", trace, + "status", res.Status(), + "response", res.String(), + ) } -// fetchURI performs a fetch against the API. If the API returns with a rate limit -// reached response and allowRateLimit is true, then wait and retry like a good -// client. If allowRateLimit is false, return to the caller informing them of -// the rate limit exceeded error. If, after waiting, the API still thinks we +// fetchURI performs a fetch against the API. If the API returns with a rate +// limit reached response and allowRateLimit is true, then wait and retry like a +// good client. If allowRateLimit is false, return to the caller informing them +// of the rate limit exceeded error. If, after waiting, the API still thinks we // have exceeded the rate limit, bail out anyway. func (ft *FindATender) fetchURI( - uri string, - allowRateLimit bool, + uri string, allowRateLimit bool, ) (*resty.Response, error) { res, err := fetchURI(ft, uri) if err != nil { @@ -381,7 +387,10 @@ func (ft *FindATender) fetchURI( } retryAfter := time.Duration(retryAfterSeconds) * time.Second - ft.logger().Warn("api rate limit hit, waiting as asked", "retry_after", retryAfter) + ft.logger().Warn( + "api rate limit hit, waiting as asked", + "retry_after", retryAfter, + ) if rateSleep(ctx, retryAfter) != nil { return nil, ErrFetchCancelled } @@ -430,10 +439,9 @@ func (ft *FindATender) fetchWithOpts( // ErrNoNext if there is no "Next" link; the caller can decide whether asking // for it was an error. func (ft *FindATender) fetchNext( - releasePackage *ocds.ReleasePackage, - allowRateLimit bool, + releasePackage *ocds.ReleasePackage, allowRateLimit bool, ) (*resty.Response, error) { - if releasePackage.Links == nil || len(releasePackage.Links.Next) == 0 { + if releasePackage.Links == nil || releasePackage.Links.Next == "" { return nil, ErrNoNext } return ft.fetchURI(releasePackage.Links.Next, allowRateLimit) @@ -478,16 +486,19 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) { // one cannot loop forever. seen := make(map[string]bool) for { - if releasePackage.Links != nil && len(releasePackage.Links.Next) > 0 { + if releasePackage.Links != nil && releasePackage.Links.Next != "" { if seen[releasePackage.Links.Next] { - ft.logger().Warn("stopping at repeated next link", "link", releasePackage.Links.Next) + ft.logger().Warn( + "stopping at repeated next link", + "link", releasePackage.Links.Next, + ) break } seen[releasePackage.Links.Next] = true } res, err := ft.fetchNext(releasePackage, opts.AllowRateLimit) - if err == ErrNoNext { + if errors.Is(err, ErrNoNext) { break } if err != nil { @@ -506,4 +517,4 @@ func (ft *FindATender) fetchAll(opts RequestOptions) ([]ocds.Release, error) { } return releases, nil -} \ No newline at end of file +} diff --git a/internal/findatender/api_test.go b/internal/findatender/api_test.go index f09f522..86fd3a6 100644 --- a/internal/findatender/api_test.go +++ b/internal/findatender/api_test.go @@ -34,7 +34,7 @@ func testConfig(level string) *config.JobSearchConfig { func quietLogs(t *testing.T, h slog.Handler) { t.Helper() if h == nil { - h = slog.NewTextHandler(io.Discard, nil) + h = slog.DiscardHandler } prev := slog.Default() slog.SetDefault(slog.New(h)) @@ -44,13 +44,18 @@ func quietLogs(t *testing.T, h slog.Handler) { } // newTestFT starts h on a test server and returns a FindATender pointed at it. -func newTestFT(t *testing.T, level string, h http.HandlerFunc) (*FindATender, *httptest.Server) { +func newTestFT( + t *testing.T, level string, h http.HandlerFunc, +) (*FindATender, *httptest.Server) { t.Helper() quietLogs(t, nil) + srv := httptest.NewServer(h) t.Cleanup(srv.Close) + ft := New(testConfig(level), srv.URL) t.Cleanup(ft.Shutdown) + return ft, srv } @@ -161,22 +166,40 @@ func TestFetchURIRateLimit(t *testing.T) { }{ {"not allowed", "0", false, ErrRateLimitExceeded, "", 1}, {"allowed, retries", "0", true, nil, "", 2}, - {"allowed, bad retry-after", "soon", true, nil, "unknown retry-after value", 1}, - {"allowed, missing retry-after", "", true, nil, "unknown retry-after value", 1}, + { + "allowed, bad retry-after", + "soon", + true, + nil, + "unknown retry-after value", + 1, + }, + { + "allowed, missing retry-after", + "", + true, + nil, + "unknown retry-after value", + 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var hits atomic.Int32 - ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) == 1 { - if tt.retryAfter != "" { - w.Header().Set("Retry-After", tt.retryAfter) + ft, srv := newTestFT( + t, + "warn", + func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + if tt.retryAfter != "" { + w.Header().Set("Retry-After", tt.retryAfter) + } + w.WriteHeader(http.StatusTooManyRequests) + return } - w.WriteHeader(http.StatusTooManyRequests) - return - } - io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) - }) + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }, + ) res, err := ft.fetchURI(srv.URL+"/x", tt.allowRateLimit) switch { @@ -217,7 +240,8 @@ func TestFetchURIRateLimitCancelled(t *testing.T) { }() start := time.Now() - if _, err := withCtx.fetchURI(srv.URL, true); !errors.Is(err, ErrFetchCancelled) { + _, err := withCtx.fetchURI(srv.URL, true) + if !errors.Is(err, ErrFetchCancelled) { t.Fatalf("err = %v, want ErrFetchCancelled", err) } if time.Since(start) > 5*time.Second { @@ -233,7 +257,8 @@ func TestFetchURIRateLimitTwice(t *testing.T) { w.Header().Set("Retry-After", "0") w.WriteHeader(http.StatusTooManyRequests) }) - if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrRateLimitExceeded) { + _, err := ft.fetchURI(srv.URL, true) + if !errors.Is(err, ErrRateLimitExceeded) { t.Fatalf("err = %v, want ErrRateLimitExceeded", err) } if n := hits.Load(); n != 2 { @@ -247,10 +272,14 @@ func TestFetchURIUnexpectedStatus(t *testing.T) { http.StatusNotFound, http.StatusInternalServerError, } { t.Run(http.StatusText(status), func(t *testing.T) { - ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(status) - io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) - }) + ft, srv := newTestFT( + t, + "warn", + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }, + ) _, err := ft.fetchURI(srv.URL, true) if !errors.Is(err, ErrUnexpectedStatus) { t.Fatalf("err = %v, want ErrUnexpectedStatus", err) @@ -273,7 +302,8 @@ func TestFetchURIRateLimitThenError(t *testing.T) { } w.WriteHeader(http.StatusServiceUnavailable) }) - if _, err := ft.fetchURI(srv.URL, true); !errors.Is(err, ErrUnexpectedStatus) { + _, err := ft.fetchURI(srv.URL, true) + if !errors.Is(err, ErrUnexpectedStatus) { t.Fatalf("err = %v, want ErrUnexpectedStatus", err) } } @@ -314,9 +344,13 @@ func TestFetchURIDebugTrace(t *testing.T) { } for name, mk := range handlers { t.Run(name, func(t *testing.T) { - ft, _ := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) { - io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) - }) + ft, _ := newTestFT( + t, + "debug", + func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, packageJSON("", releaseJSON("ocds-1", "1"))) + }, + ) var logs strings.Builder quietLogs(t, mk(&logs)) @@ -343,8 +377,12 @@ func TestFetchURIDebugTrace(t *testing.T) { } func TestFetchURIDebugTraceConnectionError(t *testing.T) { - ft, srv := newTestFT(t, "debug", func(w http.ResponseWriter, r *http.Request) { - }) + ft, srv := newTestFT( + t, + "debug", + func(w http.ResponseWriter, r *http.Request) { + }, + ) srv.Close() ft.httpClient.SetRetryCount(0) if _, err := fetchURI(ft, srv.URL); err == nil { @@ -419,13 +457,18 @@ func TestSafeUnpackReleases(t *testing.T) { {"ocid": "ocds-3", "id": "3", "date": "2026-10-08", "tag": [], "initiationType": "tender"} ] }` + pkg, err := safeUnpackReleases(slog.Default(), responseFor(t, body)) if err != nil { t.Fatal(err) } - if pkg.URI != "https://example/pkg" || pkg.Version != "1.1" || len(pkg.Extensions) != 1 || - pkg.Publisher == nil || pkg.Publisher.Name != "FTS" || pkg.License != "OGL" || - pkg.PublicationPolicy != "policy" || pkg.Links == nil || pkg.Links.Next != "https://example/next" { + + copied := pkg.URI == "https://example/pkg" && pkg.Version == "1.1" && + len(pkg.Extensions) == 1 && pkg.Publisher != nil && + pkg.Publisher.Name == "FTS" && pkg.License == "OGL" && + pkg.PublicationPolicy == "policy" && pkg.Links != nil && + pkg.Links.Next == "https://example/next" + if !copied { t.Errorf("package fields not copied: %+v", pkg) } if pkg.PublishedDate == nil || @@ -434,7 +477,9 @@ func TestSafeUnpackReleases(t *testing.T) { } if len(pkg.Releases) != 2 || pkg.Releases[0].OCID != "ocds-1" || pkg.Releases[1].OCID != "ocds-3" { - t.Errorf("releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases) + t.Errorf( + "releases = %+v, want ocds-1 and ocds-3 (bad one skipped)", pkg.Releases, + ) } } @@ -446,8 +491,16 @@ func TestSafeUnpackReleasesPublishedDate(t *testing.T) { }{ {"omitted", "", nil}, {"empty string", `""`, nil}, - {"RFC3339", `"2026-10-08T12:00:00Z"`, ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC))}, - {"date only", `"2026-10-08"`, ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC))}, + { + "RFC3339", + `"2026-10-08T12:00:00Z"`, + ptrTime(time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)), + }, + { + "date only", + `"2026-10-08"`, + ptrTime(time.Date(2026, 10, 8, 0, 0, 0, 0, time.UTC)), + }, {"unparseable", `"yesterday"`, nil}, } for _, tt := range tests { @@ -463,7 +516,8 @@ func TestSafeUnpackReleasesPublishedDate(t *testing.T) { switch { case tt.want == nil && pkg.PublishedDate != nil: t.Errorf("publishedDate = %v, want nil", pkg.PublishedDate) - case tt.want != nil && (pkg.PublishedDate == nil || !pkg.PublishedDate.Equal(*tt.want)): + case tt.want != nil && (pkg.PublishedDate == nil || + !pkg.PublishedDate.Equal(*tt.want)): t.Errorf("publishedDate = %v, want %v", pkg.PublishedDate, tt.want) } }) @@ -480,7 +534,9 @@ func TestSafeUnpackReleasesInvalidJSON(t *testing.T) { "publisher array": `{"publisher": []}`, } { t.Run(name, func(t *testing.T) { - if _, err := safeUnpackReleases(slog.Default(), responseFor(t, body)); err == nil { + if _, err := safeUnpackReleases( + slog.Default(), responseFor(t, body), + ); err == nil { t.Error("expected a decode error") } }) @@ -489,18 +545,23 @@ func TestSafeUnpackReleasesInvalidJSON(t *testing.T) { // pagedServer serves pages[i] for ?page=i (page 0 at the release package // path), linking each page to the next. -func pagedServer(t *testing.T, pages ...[]string) (*FindATender, *atomic.Int32) { +func pagedServer( + t *testing.T, pages ...[]string, +) (*FindATender, *atomic.Int32) { t.Helper() var hits atomic.Int32 var srvURL string ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { hits.Add(1) + i := 0 fmt.Sscan(r.URL.Query().Get("page"), &i) + next := "" if i+1 < len(pages) { next = fmt.Sprintf("%s/ocdsReleasePackages?page=%d", srvURL, i+1) } + io.WriteString(w, packageJSON(next, pages[i]...)) }) srvURL = srv.URL @@ -513,12 +574,14 @@ func TestFetchOnce(t *testing.T) { []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-2", "1")}, []string{releaseJSON("ocds-3", "1")}, ) + releases, err := ft.fetchOnce(RequestOptions{ Since: time.Now(), }) if err != nil { t.Fatal(err) } + if len(releases) != 2 { t.Errorf("got %d releases, want 2 (first page only)", len(releases)) } @@ -551,10 +614,7 @@ func TestFetchOnceErrors(t *testing.T) { RequestOptions{ Since: time.Now(), }, - ); !errors.Is( - err, - ErrRateLimitExceeded, - ) { + ); !errors.Is(err, ErrRateLimitExceeded) { t.Errorf("rate limited: err = %v, want ErrRateLimitExceeded", err) } } @@ -566,12 +626,14 @@ func TestFetchAll(t *testing.T) { []string{}, []string{releaseJSON("ocds-1", "2")}, ) + releases, err := ft.fetchAll(RequestOptions{ Since: time.Now(), }) if err != nil { t.Fatal(err) } + var ids []string for _, r := range releases { ids = append(ids, r.OCID+"/"+r.ID) @@ -600,10 +662,7 @@ func TestFetchAllErrors(t *testing.T) { RequestOptions{ Since: time.Now(), }, - ); !errors.Is( - err, - ErrRateLimitExceeded, - ) { + ); !errors.Is(err, ErrRateLimitExceeded) { t.Errorf("first page rate limited: err = %v", err) } @@ -627,13 +686,19 @@ func TestFetchAllErrors(t *testing.T) { } { t.Run(name, func(t *testing.T) { var srvURL string - ft, srv := newTestFT(t, "warn", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("page") == "" { - io.WriteString(w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1"))) - return - } - fail(w, r) - }) + ft, srv := newTestFT( + t, + "warn", + func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + io.WriteString( + w, packageJSON(srvURL+"/next?page=1", releaseJSON("ocds-1", "1")), + ) + return + } + fail(w, r) + }, + ) srvURL = srv.URL if _, err := ft.fetchAll(RequestOptions{ Since: time.Now(), @@ -663,17 +728,22 @@ func TestFetchAllRepeatedNextLink(t *testing.T) { t.Run(name, func(t *testing.T) { var hits atomic.Int32 var srvURL string - ft, srv := newTestFT(t, "error", func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) > 10 { - t.Error("fetchAll did not stop following next links") - w.WriteHeader(http.StatusBadRequest) - return - } - page := r.URL.Query().Get("page") - io.WriteString(w, packageJSON( - nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"), - )) - }) + ft, srv := newTestFT( + t, + "error", + func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) > 10 { + t.Error("fetchAll did not stop following next links") + w.WriteHeader(http.StatusBadRequest) + return + } + page := r.URL.Query().Get("page") + body := packageJSON( + nextFor(srvURL, page), releaseJSON("ocds-"+page, "1"), + ) + io.WriteString(w, body) + }, + ) srvURL = srv.URL releases, err := ft.fetchAll(RequestOptions{ @@ -683,7 +753,11 @@ func TestFetchAllRepeatedNextLink(t *testing.T) { t.Fatal(err) } if len(releases) != int(hits.Load()) { - t.Errorf("releases = %d, want one per request (%d)", len(releases), hits.Load()) + t.Errorf( + "releases = %d, want one per request (%d)", + len(releases), + hits.Load(), + ) } }) } @@ -730,23 +804,28 @@ func TestUnmarshalLenientDates(t *testing.T) { t.Errorf("date = %v", got.Date) } if got.PublishedDate != nil { - t.Errorf("empty publishedDate should decode as nil, got %v", got.PublishedDate) + t.Errorf( + "empty publishedDate should decode as nil, got %v", got.PublishedDate, + ) } + wantStart := time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600)) if start := got.Tender.TenderPeriod.StartDate; start == nil || - !start.Equal(time.Date(2006, 10, 2, 15, 33, 2, 0, time.FixedZone("", 7*3600))) { + !start.Equal(wantStart) { t.Errorf("full timestamp changed: %v", start) } - if end := got.Tender.TenderPeriod.EndDate; end == nil || !end.Equal(midnight(2006, 11, 30)) { + if end := got.Tender.TenderPeriod.EndDate; end == nil || + !end.Equal(midnight(2006, 11, 30)) { t.Errorf("endDate = %v", end) } - if d := got.Tender.ExpressionOfInterestDeadline; d == nil || !d.Equal(midnight(2006, 10, 20)) { + if d := got.Tender.ExpressionOfInterestDeadline; d == nil || + !d.Equal(midnight(2006, 10, 20)) { t.Errorf("expressionOfInterestDeadline = %v", d) } if got.Tender.Description != "Bids close on 2006-11-30" { t.Errorf("free text changed: %q", got.Tender.Description) } - if a := got.Awards[0]; a.Date == nil || !a.Date.Equal(midnight(2006, 12, 1)) || - a.ID != "2006-12-01" { + if a := got.Awards[0]; a.Date == nil || + !a.Date.Equal(midnight(2006, 12, 1)) || a.ID != "2006-12-01" { t.Errorf("award = %+v (date should be repaired, id left alone)", a) } if got.Value != "1234567.891" { @@ -829,9 +908,27 @@ func TestReleaseIDs(t *testing.T) { {"missing ocid", `{"id":"r1"}`, "", "r1", "no ocid"}, {"missing id", `{"ocid":"ocds-1"}`, "ocds-1", "", "no id"}, {"ocid wrong type", `{"ocid":5,"id":"r1"}`, "", "", "reading release ocid"}, - {"id wrong type", `{"ocid":"ocds-1","id":true}`, "ocds-1", "", "unexpected bool"}, - {"id is an object", `{"ocid":"ocds-1","id":{}}`, "ocds-1", "", "unexpected map"}, - {"invalid id JSON", `{"ocid":"ocds-1","id":}`, "ocds-1", "", "reading release id"}, + { + "id wrong type", + `{"ocid":"ocds-1","id":true}`, + "ocds-1", + "", + "unexpected bool", + }, + { + "id is an object", + `{"ocid":"ocds-1","id":{}}`, + "ocds-1", + "", + "unexpected map", + }, + { + "invalid id JSON", + `{"ocid":"ocds-1","id":}`, + "ocds-1", + "", + "reading release id", + }, { "invalid field before ids", `{"date":nope,"ocid":"ocds-1","id":"r1"}`, @@ -839,7 +936,13 @@ func TestReleaseIDs(t *testing.T) { "", `reading release field "date"`, }, - {"truncated after ocid", `{"ocid":"ocds-1",`, "ocds-1", "", "reading release"}, + { + "truncated after ocid", + `{"ocid":"ocds-1",`, + "ocds-1", + "", + "reading release", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -852,7 +955,9 @@ func TestReleaseIDs(t *testing.T) { t.Fatalf("err = %v, want containing %q", err, tt.wantErr) } if ocid != tt.wantOCID || id != tt.wantID { - t.Errorf("got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID) + t.Errorf( + "got (%q, %q), want (%q, %q)", ocid, id, tt.wantOCID, tt.wantID, + ) } }) } diff --git a/internal/findatender/findatender.go b/internal/findatender/findatender.go index c8c0638..fa82fa5 100644 --- a/internal/findatender/findatender.go +++ b/internal/findatender/findatender.go @@ -34,7 +34,9 @@ func WithLogger(logger *slog.Logger) Option { // New returns a client for the OCDS API at apiRoot. Requests time out after // 60 seconds and are retried once. Call Shutdown when finished with it. It // logs to slog.Default() unless given WithLogger. -func New(conf *config.JobSearchConfig, apiRoot string, opts ...Option) *FindATender { +func New( + conf *config.JobSearchConfig, apiRoot string, opts ...Option, +) *FindATender { findATender := &FindATender{ config: conf, apiRoot: apiRoot, @@ -42,9 +44,11 @@ func New(conf *config.JobSearchConfig, apiRoot string, opts ...Option) *FindATen for _, opt := range opts { opt(findATender) } + findATender.httpClient = resty.New(). - SetTimeout(60 * time.Duration(time.Second)). + SetTimeout(60 * time.Second). SetRetryCount(1) + return findATender } @@ -87,8 +91,7 @@ func (ft *FindATender) Shutdown() { // converts them to opportunities. If allowRateLimit is true, a 429 response // is waited out and retried; otherwise it fails with ErrRateLimitExceeded. func (ft *FindATender) FetchOnce( - since time.Time, - allowRateLimit bool, + since time.Time, allowRateLimit bool, ) ([]*types.Opportunity, error) { releases, err := ft.fetchOnce(RequestOptions{ Since: since, @@ -104,8 +107,7 @@ func (ft *FindATender) FetchOnce( // each page's next link, and converts them to opportunities. allowRateLimit // behaves as for FetchOnce. func (ft *FindATender) FetchAll( - since time.Time, - allowRateLimit bool, + since time.Time, allowRateLimit bool, ) ([]*types.Opportunity, error) { releases, err := ft.fetchAll(RequestOptions{ Since: since, diff --git a/internal/findatender/findatender_test.go b/internal/findatender/findatender_test.go index 3862135..2284df6 100644 --- a/internal/findatender/findatender_test.go +++ b/internal/findatender/findatender_test.go @@ -24,8 +24,13 @@ func TestNew(t *testing.T) { if ft.httpClient == nil { t.Fatal("no http client") } - if ft.httpClient.Timeout() != 60*time.Second || ft.httpClient.RetryCount() != 1 { - t.Errorf("client timeout %v, retries %d", ft.httpClient.Timeout(), ft.httpClient.RetryCount()) + if ft.httpClient.Timeout() != 60*time.Second || + ft.httpClient.RetryCount() != 1 { + t.Errorf( + "client timeout %v, retries %d", + ft.httpClient.Timeout(), + ft.httpClient.RetryCount(), + ) } if ft.ctx != nil { t.Error("New should not set a context") @@ -81,7 +86,11 @@ func opportunityIDs(ops []*types.Opportunity) []string { func TestFetchOncePublic(t *testing.T) { ft, _ := pagedServer( t, - []string{releaseJSON("ocds-1", "1"), releaseJSON("ocds-1", "2"), releaseJSON("ocds-2", "1")}, + []string{ + releaseJSON("ocds-1", "1"), + releaseJSON("ocds-1", "2"), + releaseJSON("ocds-2", "1"), + }, []string{releaseJSON("ocds-3", "1")}, ) ops, err := ft.FetchOnce(time.Now(), false) @@ -108,7 +117,9 @@ func TestFetchAllPublic(t *testing.T) { if err != nil { t.Fatal(err) } - if got := strings.Join(opportunityIDs(ops), ","); got != "ocds-1,ocds-2,ocds-3" { + if got := strings.Join( + opportunityIDs(ops), ",", + ); got != "ocds-1,ocds-2,ocds-3" { t.Errorf("opportunities = %s", got) } for _, op := range ops { diff --git a/internal/findatender/type_convert.go b/internal/findatender/type_convert.go index c2cc5c7..6952288 100644 --- a/internal/findatender/type_convert.go +++ b/internal/findatender/type_convert.go @@ -9,8 +9,8 @@ import ( "git.avsdev.uk/Go/job-search/internal/types" ) -// ocdsGetBuyerId returns the buyer's id, or "" if the release has no buyer. -func ocdsGetBuyerId(release *ocds.Release) string { +// ocdsGetBuyerID returns the buyer's id, or "" if the release has no buyer. +func ocdsGetBuyerID(release *ocds.Release) string { if release.Buyer != nil { return string(release.Buyer.ID) } @@ -29,12 +29,12 @@ func ocdsGetBuyerName(release *ocds.Release) string { // ocdsConvertClassification converts an OCDS classification, noting whether // it is a CPV code. func ocdsConvertClassification( - classif *ocds.Classification, + classification *ocds.Classification, ) types.ClassificationCode { return types.ClassificationCode{ - IsCPV: strings.Compare(classif.Scheme, "CPV") == 0, - Code: string(classif.ID), - Classification: classif.Description, + IsCPV: strings.Compare(classification.Scheme, "CPV") == 0, + Code: string(classification.ID), + Classification: classification.Description, } } @@ -45,17 +45,19 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode { if release.Tender == nil { return classifications } - add := func(classif *ocds.Classification) { - newClassif := ocdsConvertClassification(classif) + + add := func(classification *ocds.Classification) { + converted := ocdsConvertClassification(classification) found := slices.ContainsFunc( classifications, func(code types.ClassificationCode) bool { - return code.Code == newClassif.Code + return code.Code == converted.Code }, ) if !found { - classifications = append(classifications, newClassif) + classifications = append(classifications, converted) } } + if release.Tender.Classification != nil { add(release.Tender.Classification) } @@ -63,10 +65,11 @@ func ocdsGetClassifications(release *ocds.Release) []types.ClassificationCode { if item.Classification != nil { add(item.Classification) } - for _, classif := range item.AdditionalClassifications { - add(&classif) + for _, classification := range item.AdditionalClassifications { + add(&classification) } } + return classifications } @@ -142,7 +145,7 @@ func convertOne(release *ocds.Release) *types.Opportunity { opportunity := &types.Opportunity{ UUID: release.OCID, UUIDType: "ocid", - BuyerId: ocdsGetBuyerId(release), + BuyerID: ocdsGetBuyerID(release), BuyerName: ocdsGetBuyerName(release), Source: "FindATender", } diff --git a/internal/findatender/type_convert_test.go b/internal/findatender/type_convert_test.go index e5006b5..87013a4 100644 --- a/internal/findatender/type_convert_test.go +++ b/internal/findatender/type_convert_test.go @@ -38,7 +38,7 @@ func TestOcdsGetBuyer(t *testing.T) { r := &ocds.Release{ Buyer: tt.buyer, } - if got := ocdsGetBuyerId(r); got != tt.wantID { + if got := ocdsGetBuyerID(r); got != tt.wantID { t.Errorf("buyer id = %q, want %q", got, tt.wantID) } if got := ocdsGetBuyerName(r); got != tt.wantName { @@ -101,6 +101,7 @@ func TestOcdsGetClassifications(t *testing.T) { Description: "d" + id, } } + codes := func(cs []types.ClassificationCode) []string { out := []string{} for _, c := range cs { @@ -108,6 +109,7 @@ func TestOcdsGetClassifications(t *testing.T) { } return out } + main := cpv("1") tests := []struct { name string @@ -122,13 +124,24 @@ func TestOcdsGetClassifications(t *testing.T) { {"main and item additionals, deduplicated", &ocds.Tender{ Classification: &main, Items: []ocds.Item{ - {ID: "a", AdditionalClassifications: []ocds.Classification{cpv("2"), cpv("1")}}, - {ID: "b"}, - {ID: "c", AdditionalClassifications: []ocds.Classification{cpv("3"), cpv("2")}}, + { + ID: "a", + AdditionalClassifications: []ocds.Classification{cpv("2"), cpv("1")}, + }, + { + ID: "b", + }, + { + ID: "c", + AdditionalClassifications: []ocds.Classification{cpv("3"), cpv("2")}, + }, }, }, []string{"1", "2", "3"}}, {"item additionals without main", &ocds.Tender{ - Items: []ocds.Item{{ID: "a", AdditionalClassifications: []ocds.Classification{cpv("5")}}}, + Items: []ocds.Item{{ + ID: "a", + AdditionalClassifications: []ocds.Classification{cpv("5")}, + }}, }, []string{"5"}}, } for _, tt := range tests { @@ -151,9 +164,15 @@ func TestOcdsGetClassificationsIncludesItemClassification(t *testing.T) { Scheme: "CPV", ID: "9", } - got := ocdsGetClassifications(&ocds.Release{Tender: &ocds.Tender{ - Items: []ocds.Item{{ID: "a", Classification: &item}}, - }}) + release := &ocds.Release{ + Tender: &ocds.Tender{ + Items: []ocds.Item{{ + ID: "a", + Classification: &item, + }}, + }, + } + got := ocdsGetClassifications(release) if len(got) != 1 || got[0].Code != "9" { t.Errorf("got %+v, want the item classification 9", got) } @@ -231,12 +250,21 @@ func TestOcdsRelease(t *testing.T) { } got := ocdsRelease(r) - if got.UUID != "ocds-x-1" || got.ReleaseID != "001" || !got.ReleaseDate.Equal(date) { + if got.UUID != "ocds-x-1" || got.ReleaseID != "001" || + !got.ReleaseDate.Equal(date) { t.Errorf("release = %+v", got) } want := []types.OpportunityReleaseTag{ - {UUID: "ocds-x-1", ReleaseID: "001", Tag: "planning"}, - {UUID: "ocds-x-1", ReleaseID: "001", Tag: "tender"}, + { + UUID: "ocds-x-1", + ReleaseID: "001", + Tag: "planning", + }, + { + UUID: "ocds-x-1", + ReleaseID: "001", + Tag: "tender", + }, } if !reflect.DeepEqual(got.Tags, want) { t.Errorf("tags = %+v, want %+v", got.Tags, want) @@ -250,7 +278,8 @@ func TestOcdsRelease(t *testing.T) { t.Errorf("round tripped release = %+v", back) } - if empty := ocdsRelease(&ocds.Release{}); empty.Tags == nil || len(empty.Tags) != 0 { + if empty := ocdsRelease(&ocds.Release{}); empty.Tags == nil || + len(empty.Tags) != 0 { t.Errorf("no tags should give an empty, non-nil slice, got %#v", empty.Tags) } } @@ -289,7 +318,9 @@ func TestConvertOne(t *testing.T) { Value: &ocds.Value{ Amount: &amount, }, - Documents: []ocds.Document{{ID: "doc"}}, + Documents: []ocds.Document{{ + ID: "doc", + }}, }, } @@ -300,7 +331,7 @@ func TestConvertOne(t *testing.T) { }{ {"UUID", got.UUID, "ocds-x-1"}, {"UUIDType", got.UUIDType, "ocid"}, - {"BuyerId", got.BuyerId, "GB-1"}, + {"BuyerID", got.BuyerID, "GB-1"}, {"BuyerName", got.BuyerName, "Council"}, {"Title", got.Title, "Website"}, {"Status", got.Status, "active"}, @@ -344,10 +375,11 @@ func TestConvertOneMinimal(t *testing.T) { ID: "1", Tender: tt.tender, }) - if got.UUID != "ocds-x-2" || got.BuyerId != "" || got.BuyerName != "" { + if got.UUID != "ocds-x-2" || got.BuyerID != "" || got.BuyerName != "" { t.Errorf("got %+v", got) } - if got.GrossValue != 0 || got.StartDate != nil || got.EndDate != nil || got.HasDocuments { + if got.GrossValue != 0 || got.StartDate != nil || got.EndDate != nil || + got.HasDocuments { t.Errorf("unexpected tender fields: %+v", got) } if len(got.Releases) != 1 { @@ -366,18 +398,46 @@ func TestConvert(t *testing.T) { return time.Date(2026, 1, d, 0, 0, 0, 0, time.UTC) } releases := []ocds.Release{ - {OCID: "ocds-b", ID: "1", Date: day(1), Tender: &ocds.Tender{ - Title: "B", - }}, - {OCID: "ocds-a", ID: "2", Date: day(3), Tender: &ocds.Tender{ - Title: "A latest", Lots: []ocds.Lot{{ID: "1"}, {ID: "2"}}, - }}, - {OCID: "ocds-a", ID: "1", Date: day(2), Tender: &ocds.Tender{ - Title: "A first", - }}, - {OCID: "ocds-a", ID: "3", Date: day(1), Tender: &ocds.Tender{ - Title: "A oldest", - }}, + { + OCID: "ocds-b", + ID: "1", + Date: day(1), + Tender: &ocds.Tender{ + Title: "B", + }, + }, + { + OCID: "ocds-a", + ID: "2", + Date: day(3), + Tender: &ocds.Tender{ + Title: "A latest", + Lots: []ocds.Lot{ + { + ID: "1", + }, + { + ID: "2", + }, + }, + }, + }, + { + OCID: "ocds-a", + ID: "1", + Date: day(2), + Tender: &ocds.Tender{ + Title: "A first", + }, + }, + { + OCID: "ocds-a", + ID: "3", + Date: day(1), + Tender: &ocds.Tender{ + Title: "A oldest", + }, + }, } got := Convert(releases) @@ -395,7 +455,9 @@ func TestConvert(t *testing.T) { t.Errorf("ocds-a title = %q, want %q", a.Title, "A latest") } if !a.LatestReleaseDate.Equal(day(3)) { - t.Errorf("ocds-a LatestReleaseDate = %v, want %v", a.LatestReleaseDate, day(3)) + t.Errorf( + "ocds-a LatestReleaseDate = %v, want %v", a.LatestReleaseDate, day(3), + ) } if !a.MultiLot { t.Error("ocds-a MultiLot = false, want true for two lots") @@ -408,7 +470,8 @@ func TestConvert(t *testing.T) { t.Errorf("ocds-a release ids = %v, want %v", ids, want) } - if b.Title != "B" || b.MultiLot || !b.LatestReleaseDate.Equal(day(1)) || len(b.Releases) != 1 { + if b.Title != "B" || b.MultiLot || !b.LatestReleaseDate.Equal(day(1)) || + len(b.Releases) != 1 { t.Errorf("ocds-b = %+v", b) } } @@ -417,12 +480,22 @@ func TestConvert(t *testing.T) { func TestConvertSameDate(t *testing.T) { date := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) got := Convert([]ocds.Release{ - {OCID: "ocds-a", ID: "1", Date: date, Tender: &ocds.Tender{ - Title: "first", - }}, - {OCID: "ocds-a", ID: "2", Date: date, Tender: &ocds.Tender{ - Title: "second", - }}, + { + OCID: "ocds-a", + ID: "1", + Date: date, + Tender: &ocds.Tender{ + Title: "first", + }, + }, + { + OCID: "ocds-a", + ID: "2", + Date: date, + Tender: &ocds.Tender{ + Title: "second", + }, + }, }) if len(got) != 1 || got[0].Title != "second" { t.Fatalf("got %+v, want one opportunity titled %q", got, "second") @@ -446,13 +519,24 @@ func TestConvertSetsSource(t *testing.T) { } for _, o := range Convert([]ocds.Release{ - {OCID: "ocds-a", ID: "1", Tender: &ocds.Tender{ - Title: "A", - }}, - {OCID: "ocds-a", ID: "2", Tender: &ocds.Tender{ - Title: "A amended", - }}, - {OCID: "ocds-b", ID: "1"}, + { + OCID: "ocds-a", + ID: "1", + Tender: &ocds.Tender{ + Title: "A", + }, + }, + { + OCID: "ocds-a", + ID: "2", + Tender: &ocds.Tender{ + Title: "A amended", + }, + }, + { + OCID: "ocds-b", + ID: "1", + }, }) { if o.Source != want { t.Errorf("Convert: %s Source = %q, want %q", o.UUID, o.Source, want) diff --git a/internal/httpx/httpx_test.go b/internal/httpx/httpx_test.go index bbda3ac..ae9e88f 100644 --- a/internal/httpx/httpx_test.go +++ b/internal/httpx/httpx_test.go @@ -20,7 +20,8 @@ func TestWriteProblem(t *testing.T) { if ct := rec.Header().Get("Content-Type"); ct != "application/problem+json" { t.Errorf("Content-Type = %q", ct) } - if nosniff := rec.Header().Get("X-Content-Type-Options"); nosniff != "nosniff" { + nosniff := rec.Header().Get("X-Content-Type-Options") + if nosniff != "nosniff" { t.Errorf("X-Content-Type-Options = %q", nosniff) } var p Problem @@ -42,7 +43,10 @@ func TestWriteProblem(t *testing.T) { // Without a request (or a request ID) the problem has no requestId, and an // empty detail is left out. func TestWriteProblemMinimal(t *testing.T) { - for _, req := range []*http.Request{nil, httptest.NewRequest("GET", "/", nil)} { + for _, req := range []*http.Request{ + nil, + httptest.NewRequest("GET", "/", nil), + } { rec := httptest.NewRecorder() WriteProblem(rec, req, http.StatusTooManyRequests, "") var body map[string]any diff --git a/internal/httpx/problem.go b/internal/httpx/problem.go index b5eb610..5bdaf8b 100644 --- a/internal/httpx/problem.go +++ b/internal/httpx/problem.go @@ -17,7 +17,9 @@ type Problem struct { } // WriteProblem writes a problem+json error response. -func WriteProblem(w http.ResponseWriter, r *http.Request, status int, detail string) { +func WriteProblem( + w http.ResponseWriter, r *http.Request, status int, detail string, +) { problem := Problem{ Type: "about:blank", Title: http.StatusText(status), @@ -27,6 +29,7 @@ func WriteProblem(w http.ResponseWriter, r *http.Request, status int, detail str if r != nil { problem.RequestID = RequestID(r.Context()) } + w.Header().Set("Content-Type", "application/problem+json") w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(status) diff --git a/internal/logging/accesslog.go b/internal/logging/accesslog.go index 204010d..2352118 100644 --- a/internal/logging/accesslog.go +++ b/internal/logging/accesslog.go @@ -19,19 +19,19 @@ type statusRecorder struct { bytes int } -func (r *statusRecorder) WriteHeader(code int) { - if r.status == 0 { - r.status = code +func (s *statusRecorder) WriteHeader(code int) { + if s.status == 0 { + s.status = code } - r.ResponseWriter.WriteHeader(code) + s.ResponseWriter.WriteHeader(code) } -func (r *statusRecorder) Write(data []byte) (int, error) { - if r.status == 0 { - r.status = http.StatusOK +func (s *statusRecorder) Write(data []byte) (int, error) { + if s.status == 0 { + s.status = http.StatusOK } - written, err := r.ResponseWriter.Write(data) - r.bytes += written + written, err := s.ResponseWriter.Write(data) + s.bytes += written return written, err } @@ -42,6 +42,7 @@ func logField(value string) string { if value == "" { return "-" } + var builder strings.Builder for i := 0; i < len(value); i++ { char := value[i] @@ -51,16 +52,17 @@ func logField(value string) string { builder.WriteByte(char) } } + return builder.String() } // clientKey attempts to produce a client host name as a key identifier to split // requests in the log between sources. Could be extended to also check // Forwarded-For or other proxy headers -func clientKey(req *http.Request) string { - host, _, err := net.SplitHostPort(req.RemoteAddr) +func clientKey(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return req.RemoteAddr + return r.RemoteAddr } return host } @@ -68,19 +70,21 @@ func clientKey(req *http.Request) string { // AccessLog wraps next, writing a line to logger for each request in the // Apache/nginx Common Log Format, or the Combined Log Format (which adds the // referer and user agent) if combined is true. -func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { +func AccessLog( + logger *log.Logger, combined bool, next http.Handler, +) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() recorder := &statusRecorder{ ResponseWriter: w, } - next.ServeHTTP(recorder, req) + next.ServeHTTP(recorder, r) if recorder.status == 0 { recorder.status = http.StatusOK } user := "-" - if username, _, ok := req.BasicAuth(); ok && username != "" { + if username, _, ok := r.BasicAuth(); ok && username != "" { user = logField(username) } size := "-" // CLF uses "-" for an empty body @@ -89,16 +93,16 @@ func AccessLog(logger *log.Logger, combined bool, next http.Handler) http.Handle } line := fmt.Sprintf( `%s - %s [%s] "%s" %d %s`, - clientKey(req), + clientKey(r), user, start.Format("02/Jan/2006:15:04:05 -0700"), - logField(req.Method+" "+req.RequestURI+" "+req.Proto), + logField(r.Method+" "+r.RequestURI+" "+r.Proto), recorder.status, size, ) if combined { line += fmt.Sprintf( - ` "%s" "%s"`, logField(req.Referer()), logField(req.UserAgent()), + ` "%s" "%s"`, logField(r.Referer()), logField(r.UserAgent()), ) } logger.Println(line) diff --git a/internal/logging/accesslog_test.go b/internal/logging/accesslog_test.go index b70fb3a..3efeab0 100644 --- a/internal/logging/accesslog_test.go +++ b/internal/logging/accesslog_test.go @@ -65,6 +65,7 @@ func TestStatusRecorder(t *testing.T) { rec := &statusRecorder{ ResponseWriter: inner, } + rec.Write([]byte("hello")) rec.Write([]byte(" world")) if rec.status != http.StatusOK || rec.bytes != 11 { diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go index 44c88cf..0a2974f 100644 --- a/internal/logging/logging_test.go +++ b/internal/logging/logging_test.go @@ -29,7 +29,7 @@ func saveGlobals(t *testing.T) { func TestLevels(t *testing.T) { saveGlobals(t) - slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + slog.SetDefault(slog.New(slog.DiscardHandler)) SetLevel(slog.LevelInfo) if IsDebugEnabled() { @@ -57,6 +57,13 @@ func TestLevels(t *testing.T) { type wrapper struct{ slog.Handler } func TestIsJSONHandler(t *testing.T) { + // The handlers' types are what is tested, so slog.DiscardHandler won't do. + //nolint:sloglint // see above + jsonWithAttrs := slog.NewJSONHandler(io.Discard, nil).WithAttrs( + []slog.Attr{slog.String("k", "v")}, + ) + + //nolint:sloglint // as above tests := []struct { name string h slog.Handler @@ -66,9 +73,7 @@ func TestIsJSONHandler(t *testing.T) { {"text", slog.NewTextHandler(io.Discard, nil), false}, {"tint", tint.NewTextHandler(io.Discard, nil), false}, {"nil", nil, false}, - {"json with attrs", slog.NewJSONHandler(io.Discard, nil).WithAttrs( - []slog.Attr{slog.String("k", "v")}, - ), true}, + {"json with attrs", jsonWithAttrs, true}, // Only slog's own JSON handler is recognised, not one wrapped in another. {"wrapped json", wrapper{ Handler: slog.NewJSONHandler(io.Discard, nil), @@ -108,7 +113,8 @@ func TestLogToFileWritesJSON(t *testing.T) { t.Fatal(err) } s := string(data) - if !strings.Contains(s, `"msg":"hello file"`) || !strings.Contains(s, `"k":"v"`) { + if !strings.Contains(s, `"msg":"hello file"`) || + !strings.Contains(s, `"k":"v"`) { t.Fatalf("log file missing entry: %s", s) } if strings.Contains(s, "hidden") { @@ -118,7 +124,9 @@ func TestLogToFileWritesJSON(t *testing.T) { func TestLogToFileBadPath(t *testing.T) { saveGlobals(t) - if err := LogToFile(filepath.Join(t.TempDir(), "missing", "app.log")); err == nil { + if err := LogToFile( + filepath.Join(t.TempDir(), "missing", "app.log"), + ); err == nil { t.Fatal("expected an error for a path in a missing directory") } } @@ -131,6 +139,7 @@ func TestLogToFileTracksFile(t *testing.T) { if !IsFileLogger() { t.Fatal("IsFileLogger() = false after LogToFile") } + if err := Shutdown(); err != nil { t.Fatal(err) } @@ -155,6 +164,7 @@ func TestLogToTerminal(t *testing.T) { if IsJSONHandler(h) { t.Fatal("terminal handler reported as JSON") } + SetLevel(slog.LevelWarn) if h.Enabled(context.Background(), slog.LevelInfo) { t.Fatal("terminal handler ignores the package level") diff --git a/internal/middleware/ratelimit.go b/internal/middleware/ratelimit.go index 054c4d0..be3b500 100644 --- a/internal/middleware/ratelimit.go +++ b/internal/middleware/ratelimit.go @@ -53,7 +53,9 @@ func RateLimit(limiter Limiter, key KeyFunc) func(http.Handler) http.Handler { seconds = 1 } w.Header().Set("Retry-After", strconv.Itoa(seconds)) - httpx.WriteProblem(w, r, http.StatusTooManyRequests, "rate limit exceeded") + httpx.WriteProblem( + w, r, http.StatusTooManyRequests, "rate limit exceeded", + ) return } next.ServeHTTP(w, r) @@ -85,10 +87,7 @@ type limiterEntry struct { // requests per second with the given burst. A janitor goroutine evicts // entries idle for longer than ttl until ctx is cancelled. func NewKeyedLimiter( - ctx context.Context, - requestsPerSecond float64, - burst int, - ttl time.Duration, + ctx context.Context, requestsPerSecond float64, burst int, ttl time.Duration, ) *KeyedLimiter { limiter := &KeyedLimiter{ limit: rate.Limit(requestsPerSecond), @@ -99,9 +98,11 @@ func NewKeyedLimiter( for i := range limiter.shards { limiter.shards[i].entries = make(map[string]*limiterEntry) } + if ttl > 0 { go limiter.janitor(ctx) } + return limiter } @@ -125,14 +126,15 @@ func (k *KeyedLimiter) Allow(key string) (bool, time.Duration) { limiter := entry.limiter shard.mu.Unlock() - res := limiter.ReserveN(now, 1) - if !res.OK() { + reservation := limiter.ReserveN(now, 1) + if !reservation.OK() { return false, time.Second } - if delay := res.DelayFrom(now); delay > 0 { - res.CancelAt(now) + if delay := reservation.DelayFrom(now); delay > 0 { + reservation.CancelAt(now) return false, delay } + return true, 0 } diff --git a/internal/middleware/ratelimit_test.go b/internal/middleware/ratelimit_test.go index e74c6a0..e7d0c24 100644 --- a/internal/middleware/ratelimit_test.go +++ b/internal/middleware/ratelimit_test.go @@ -8,17 +8,21 @@ import ( func TestRateLimit(t *testing.T) { l := NewKeyedLimiter(t.Context(), 1, 2, 0) - h := RateLimit(l, nil)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - })) + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + }) + h := RateLimit(l, nil)(next) + var last int for i := 0; i < 4; i++ { rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest("GET", "/", nil)) last = rec.Code - if last == http.StatusTooManyRequests && rec.Header().Get("Retry-After") == "" { + if last == http.StatusTooManyRequests && + rec.Header().Get("Retry-After") == "" { t.Fatal("missing Retry-After") } } + if last != http.StatusTooManyRequests { t.Fatalf("want 429, got %d", last) } diff --git a/internal/middleware/real_ip.go b/internal/middleware/real_ip.go index a93e0ce..dfc425d 100644 --- a/internal/middleware/real_ip.go +++ b/internal/middleware/real_ip.go @@ -116,7 +116,8 @@ func RealIP(opts RealIPOptions) func(http.Handler) http.Handler { break } } - if proto := forwardedProto(r.Header); proto == "http" || proto == "https" { + proto := forwardedProto(r.Header) + if proto == "http" || proto == "https" { info.Scheme = proto } } else if opts.StripUntrusted { diff --git a/internal/middleware/real_ip_test.go b/internal/middleware/real_ip_test.go index 8254724..7ac94d5 100644 --- a/internal/middleware/real_ip_test.go +++ b/internal/middleware/real_ip_test.go @@ -9,18 +9,23 @@ import ( // serveRealIP runs a request from peer with the given headers through RealIP // and returns the ClientInfo the next handler sees. -func serveRealIP(t *testing.T, opts RealIPOptions, peer string, h map[string]string) ClientInfo { +func serveRealIP( + t *testing.T, opts RealIPOptions, peer string, h map[string]string, +) ClientInfo { t.Helper() var got ClientInfo next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = Client(r) }) + req := httptest.NewRequest("GET", "/", nil) req.RemoteAddr = peer for k, v := range h { req.Header.Set(k, v) } + RealIP(opts)(next).ServeHTTP(httptest.NewRecorder(), req) + return got } @@ -32,15 +37,19 @@ func TestRealIP(t *testing.T) { Headers: defaultHeaders, } // Spoofed left-most entry is ignored; right-most untrusted wins. - c := serveRealIP(t, opts, "127.0.0.1:1234", map[string]string{ - "X-Forwarded-For": "6.6.6.6, 203.0.113.9, 127.0.0.1", "X-Forwarded-Proto": "https", - }) + headers := map[string]string{ + "X-Forwarded-For": "6.6.6.6, 203.0.113.9, 127.0.0.1", + "X-Forwarded-Proto": "https", + } + c := serveRealIP(t, opts, "127.0.0.1:1234", headers) if c.IP.String() != "203.0.113.9" || c.Scheme != "https" || !c.ViaProxy { t.Fatalf("xff: %+v", c) } - c = serveRealIP(t, opts, "127.0.0.1:1234", map[string]string{ + + headers = map[string]string{ "Forwarded": `for="[2001:db8::7]:4711";proto=https`, - }) + } + c = serveRealIP(t, opts, "127.0.0.1:1234", headers) if c.IP.String() != "2001:db8::7" { t.Fatalf("forwarded: %+v", c) } @@ -51,7 +60,12 @@ func TestUntrustedPeerHeadersIgnored(t *testing.T) { Trusted: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, Headers: defaultHeaders, } - c := serveRealIP(t, opts, "127.0.0.1:1234", map[string]string{"X-Forwarded-For": "203.0.113.9"}) + c := serveRealIP( + t, + opts, + "127.0.0.1:1234", + map[string]string{"X-Forwarded-For": "203.0.113.9"}, + ) if c.IP.String() != "127.0.0.1" || c.ViaProxy { t.Fatalf("untrusted XFF honoured: %+v", c) } diff --git a/internal/mock/findatender/catalog.go b/internal/mock/findatender/catalog.go index 30215e2..6ff32b1 100644 --- a/internal/mock/findatender/catalog.go +++ b/internal/mock/findatender/catalog.go @@ -79,33 +79,19 @@ func parseCatalog(data []byte, name string) (*Catalog, error) { func (c *Catalog) validate() error { var errs []error - for _, pool := range []struct { - name string - n int + pools := []struct { + name string + count int }{ - { - name: "buyers", - n: len(c.Buyers), - }, - { - name: "suppliers", - n: len(c.Suppliers), - }, - { - name: "subjects", - n: len(c.Subjects), - }, - { - name: "methods", - n: len(c.Methods), - }, - { - name: "classifications", - n: len(c.Classifications), - }, - } { - if pool.n == 0 { - errs = append(errs, fmt.Errorf("no %s", pool.name)) + {name: "buyers", count: len(c.Buyers)}, + {name: "suppliers", count: len(c.Suppliers)}, + {name: "subjects", count: len(c.Subjects)}, + {name: "methods", count: len(c.Methods)}, + {name: "classifications", count: len(c.Classifications)}, + } + for _, pool := range pools { + if pool.count == 0 { + errs = append(errs, fmt.Errorf("mock: no %s", pool.name)) } } @@ -136,8 +122,9 @@ func (c *Catalog) validate() error { keys = append(keys[:5], "...") } errs = append(errs, fmt.Errorf( - "%d classification key(s) not found in classifications: %s", - len(missing), strings.Join(keys, ", "), + "mock: %d classification key(s) not found in classifications: %s", + len(missing), + strings.Join(keys, ", "), )) } @@ -164,13 +151,19 @@ func (c *Catalog) subject(rng *rand.Rand) ocds.Tender { subject := clone(c.Subjects[rng.Intn(len(c.Subjects))]) tender := subject.Tender tender.Classification = c.resolveOne(subject.Classification) - tender.AdditionalClassifications = c.resolve(subject.AdditionalClassifications) + tender.AdditionalClassifications = c.resolve( + subject.AdditionalClassifications, + ) + tender.Items = nil for _, subjectItem := range subject.Items { item := subjectItem.Item - item.AdditionalClassifications = c.resolve(subjectItem.AdditionalClassifications) + item.AdditionalClassifications = c.resolve( + subjectItem.AdditionalClassifications, + ) tender.Items = append(tender.Items, item) } + return tender } @@ -195,12 +188,13 @@ func (c *Catalog) resolve(keys []string) []ocds.Classification { return out } -// clone deep-copies a value through JSON, which is all the generated types need. +// clone deep-copies a value through JSON, which is all the generated types +// need. func clone[T any](value T) T { var out T data, err := json.Marshal(value) if err == nil { - json.Unmarshal(data, &out) //nolint:errcheck,gosec // b was just marshalled from a T + json.Unmarshal(data, &out) //nolint:errcheck,gosec // data was just marshalled from a T } return out } diff --git a/internal/mock/findatender/catalog_test.go b/internal/mock/findatender/catalog_test.go index fa0e717..0916931 100644 --- a/internal/mock/findatender/catalog_test.go +++ b/internal/mock/findatender/catalog_test.go @@ -31,8 +31,8 @@ func TestEmbeddedCatalogIsValid(t *testing.T) { if err != nil { t.Fatal(err) } - if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || len(cat.Subjects) == 0 || - len(cat.Methods) == 0 { + if len(cat.Buyers) == 0 || len(cat.Suppliers) == 0 || + len(cat.Subjects) == 0 || len(cat.Methods) == 0 { t.Fatalf( "embedded catalog has an empty pool: %d buyers, %d suppliers, %d subjects, %d methods", len(cat.Buyers), @@ -45,8 +45,8 @@ func TestEmbeddedCatalogIsValid(t *testing.T) { func TestParseCatalog(t *testing.T) { cat := loadTestCatalog(t) - if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || len(cat.Subjects) != 2 || - len(cat.Methods) != 2 { + if len(cat.Buyers) != 2 || len(cat.Suppliers) != 1 || + len(cat.Subjects) != 2 || len(cat.Methods) != 2 { t.Fatalf( "got %d buyers, %d suppliers, %d subjects, %d methods", len(cat.Buyers), @@ -55,18 +55,22 @@ func TestParseCatalog(t *testing.T) { len(cat.Methods), ) } + if d := cat.Buyers[0].Details; d == nil || len(d.Classifications) != 1 || d.Classifications[0].Scheme != "UK_CA_TYPE" { t.Fatalf("buyer details.classifications not read: %+v", d) } + s := cat.Subjects[0] if s.Classification != "CPV-77314000" || len(s.Items) != 1 { t.Fatalf("subject classification key/items not read: %+v", s) } - if ac := s.Items[0].AdditionalClassifications; len(ac) != 1 || ac[0] != "CPV-77310000" { + if ac := s.Items[0].AdditionalClassifications; len(ac) != 1 || + ac[0] != "CPV-77310000" { t.Fatalf("items.additionalClassifications keys not read: %+v", ac) } - if v := s.Lots[0].Value; v == nil || v.AmountGross == nil || *v.AmountGross != 120000 { + if v := s.Lots[0].Value; v == nil || v.AmountGross == nil || + *v.AmountGross != 120000 { t.Fatalf("lot value not read: %+v", v) } } @@ -86,7 +90,8 @@ func TestSubjectResolvesClassifications(t *testing.T) { t.Fatalf("main classification = %+v", cl) } ac := tender.Items[0].AdditionalClassifications - if len(ac) != 1 || ac[0].Description != "Planting and maintenance services of green areas" { + if len(ac) != 1 || + ac[0].Description != "Planting and maintenance services of green areas" { t.Fatalf("item classifications = %+v", ac) } // The catalog itself still holds keys, untouched by picking. diff --git a/internal/mock/findatender/generator.go b/internal/mock/findatender/generator.go index 8c89d7c..74b040b 100644 --- a/internal/mock/findatender/generator.go +++ b/internal/mock/findatender/generator.go @@ -17,61 +17,63 @@ const ocidPrefix = "ocds-mock-" // moves an existing procedure on (planning -> tender -> award) so the same ocid // appears in several releases, as it does on the real service. type Generator struct { - cat *Catalog - store *Store - rnd *rand.Rand - opts options + catalog *Catalog + store *Store + rng *rand.Rand + opts options } // NewGenerator returns a generator drawing from catalog and storing releases in // store. The same seed and store contents give the same releases. It logs to // slog.Default() unless given WithLogger. -func NewGenerator(catalog *Catalog, store *Store, seed int64, opts ...Option) *Generator { +func NewGenerator( + catalog *Catalog, store *Store, seed int64, opts ...Option, +) *Generator { return &Generator{ - cat: catalog, - store: store, - rnd: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data - opts: applyOptions(opts), + catalog: catalog, + store: store, + rng: rand.New(rand.NewSource(seed)), //nolint:gosec // seeded for repeatable mock data + opts: applyOptions(opts), } } -// Make returns a new release dated at (to the second). About a third of the -// time it moves on a procedure still open from the last 60 days; otherwise it -// starts a new one. -func (g *Generator) Make(at time.Time) ocds.Release { - at = at.UTC().Truncate(time.Second) - if g.rnd.Float64() < 0.35 { - if prev, ok := g.store.RandomOpen(g.rnd, at.AddDate(0, 0, -60)); ok { - return g.advance(prev, at) +// Make returns a new release with the given date (to the second). About a third +// of the time it moves on a procedure still open from the last 60 days; +// otherwise it starts a new one. +func (g *Generator) Make(date time.Time) ocds.Release { + date = date.UTC().Truncate(time.Second) + if g.rng.Float64() < 0.35 { + if previous, ok := g.store.RandomOpen(g.rng, date.AddDate(0, 0, -60)); ok { + return g.advance(previous, date) } } - return g.newProcedure(at) + return g.newProcedure(date) } // newProcedure returns the first release of a new procedure: a tender (or, // one time in five, a planning notice) from a random catalog buyer and // subject, using a random procurement method. Values are scaled by a random // factor, and a tender without lots gets a single lot covering it all. -func (g *Generator) newProcedure(at time.Time) ocds.Release { - buyer := g.cat.buyer(g.rnd) +func (g *Generator) newProcedure(date time.Time) ocds.Release { + buyer := g.catalog.buyer(g.rng) ensurePartyID(&buyer) buyer.Roles = []string{"buyer"} - method := g.cat.method(g.rnd) + method := g.catalog.method(g.rng) tag, status := "tender", "active" - if g.rnd.Float64() < 0.2 { + if g.rng.Float64() < 0.2 { tag, status = "planning", "planned" } ocid := g.newOCID() - tender := g.cat.subject(g.rnd) + tender := g.catalog.subject(g.rng) tender.ID = ocds.ID(ocid) tender.ProcurementMethod = method.Method tender.ProcurementMethodDetails = method.Details - factor := 0.5 + g.rnd.Float64()*1.5 + factor := 0.5 + g.rng.Float64()*1.5 if tender.Value == nil { - amount := float64((g.rnd.Intn(5000) + 10) * 1000) + amount := float64((g.rng.Intn(5000) + 10) * 1000) tender.Value = &ocds.Value{ Amount: &amount, Currency: "GBP", @@ -96,14 +98,14 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release { setStatus(&tender, status) if status == "active" { tender.TenderPeriod = &ocds.Period{ - EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))), + EndDate: ptr(date.AddDate(0, 0, 30+g.rng.Intn(30))), } } return ocds.Release{ OCID: ocid, - ID: g.store.NextNoticeID(at.Year()), - Date: at, + ID: g.store.NextNoticeID(date.Year()), + Date: date, Tag: []string{tag}, InitiationType: "tender", Parties: []ocds.Organization{buyer}, @@ -116,13 +118,15 @@ func (g *Generator) newProcedure(at time.Time) ocds.Release { } } -// advance returns the next release in prev's procedure: planning moves to +// advance returns the next release in previous's procedure: planning moves to // tender, and a tender either has its deadline extended (tenderUpdate) or is // awarded to a supplier from the catalog. -func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { - release := clone(prev) - release.ID = g.store.NextNoticeID(at.Year()) - release.Date = at +func (g *Generator) advance( + previous ocds.Release, date time.Time, +) ocds.Release { + release := clone(previous) + release.ID = g.store.NextNoticeID(date.Year()) + release.Date = date release.Awards = nil if release.Tender == nil { release.Tender = &ocds.Tender{ @@ -131,11 +135,11 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { } next := "award" - switch prev.Tag[0] { + switch previous.Tag[0] { case "planning": next = "tender" case "tender": - if g.rnd.Float64() < 0.25 { + if g.rng.Float64() < 0.25 { next = "tenderUpdate" } } @@ -145,12 +149,13 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { case "tender": setStatus(release.Tender, "active") release.Tender.TenderPeriod = &ocds.Period{ - EndDate: ptr(at.AddDate(0, 0, 30+g.rnd.Intn(30))), + EndDate: ptr(date.AddDate(0, 0, 30+g.rng.Intn(30))), } case "tenderUpdate": // A typical update: the deadline is extended by two weeks. - end := at.AddDate(0, 0, 30) - if period := release.Tender.TenderPeriod; period != nil && period.EndDate != nil { + end := date.AddDate(0, 0, 30) + if period := release.Tender.TenderPeriod; period != nil && + period.EndDate != nil { end = *period.EndDate } release.Tender.TenderPeriod = &ocds.Period{ @@ -158,7 +163,7 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { } case "award": setStatus(release.Tender, "complete") - supplier := g.cat.supplier(g.rnd) + supplier := g.catalog.supplier(g.rng) ensurePartyID(&supplier) supplier.Roles = []string{"supplier"} release.Parties = append(release.Parties, supplier) @@ -166,7 +171,7 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { var value *ocds.Value if release.Tender.Value != nil { scaled := clone(*release.Tender.Value) - scaleValue(&scaled, 0.8+g.rnd.Float64()*0.2) + scaleValue(&scaled, 0.8+g.rng.Float64()*0.2) value = &scaled } @@ -180,7 +185,7 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { release.Awards = []ocds.Award{{ ID: "1", Status: "active", - Date: ptr(at), + Date: ptr(date), Value: value, Suppliers: []ocds.OrganizationReference{ { @@ -198,26 +203,26 @@ func (g *Generator) advance(prev ocds.Release, at time.Time) ocds.Release { // newOCID returns a random ocid not already in the store. func (g *Generator) newOCID() string { for { - id := fmt.Sprintf("%s%06x", ocidPrefix, g.rnd.Intn(1<<24)) + id := fmt.Sprintf("%s%06x", ocidPrefix, g.rng.Intn(1<<24)) if !g.store.HasOCID(id) { return id } } } -// gap returns a random duration from min to max inclusive, or min if max is -// not larger. -func (g *Generator) gap(min, max time.Duration) time.Duration { - if max <= min { - return min +// gap returns a random duration from minGap to maxGap inclusive, or minGap if +// maxGap is not larger. +func (g *Generator) gap(minGap, maxGap time.Duration) time.Duration { + if maxGap <= minGap { + return minGap } - return min + time.Duration(g.rnd.Int63n(int64(max-min)+1)) + return minGap + time.Duration(g.rng.Int63n(int64(maxGap-minGap)+1)) } // CatchUp fills the period since the last stored release (or the last backfill // period, whichever is shorter) so restarts leave no gap in the timeline. func (g *Generator) CatchUp( - now time.Time, backfill, min, max time.Duration, + now time.Time, backfill, minGap, maxGap time.Duration, ) (int, error) { start := now.Add(-backfill) if last, ok := g.store.Last(); ok && last.After(start) { @@ -225,7 +230,8 @@ func (g *Generator) CatchUp( } count := 0 - for when := start.Add(g.gap(min, max)); !when.After(now); when = when.Add(g.gap(min, max)) { + when := start.Add(g.gap(minGap, maxGap)) + for ; !when.After(now); when = when.Add(g.gap(minGap, maxGap)) { if err := g.store.AppendNoSync(g.Make(when)); err != nil { return count, err } @@ -235,11 +241,12 @@ func (g *Generator) CatchUp( return count, g.store.Sync() } -// Run publishes a new release after a random gap between min and max until ctx ends. -func (g *Generator) Run(ctx context.Context, min, max time.Duration) { +// Run publishes a new release after a random gap between minGap and maxGap +// until ctx ends. +func (g *Generator) Run(ctx context.Context, minGap, maxGap time.Duration) { logger := g.opts.logger("generator") for { - gap := g.gap(min, max) + gap := g.gap(minGap, maxGap) logger.Info("next release", "in", gap.Round(time.Second)) timer := time.NewTimer(gap) diff --git a/internal/mock/findatender/generator_test.go b/internal/mock/findatender/generator_test.go index 92918af..a9eeca3 100644 --- a/internal/mock/findatender/generator_test.go +++ b/internal/mock/findatender/generator_test.go @@ -2,8 +2,7 @@ package mock import ( "context" - "io" - "log" + "log/slog" "math/rand" "path/filepath" "strings" @@ -13,19 +12,13 @@ import ( "git.avsdev.uk/Go/job-search/internal/mock/findatender/ocds" ) -func quietLog(t *testing.T) { - t.Helper() - w := log.Writer() - log.SetOutput(io.Discard) - t.Cleanup(func() { - log.SetOutput(w) - }) -} +// quiet discards the logging of the Store and Generator it is given to. +var quiet = WithLogger(slog.New(slog.DiscardHandler)) func TestGap(t *testing.T) { //nolint:gosec // G404: seeded math/rand keeps the test deterministic g := &Generator{ - rnd: rand.New(rand.NewSource(1)), + rng: rand.New(rand.NewSource(1)), } if d := g.gap(time.Second, time.Second); d != time.Second { t.Errorf("min == max: %s", d) @@ -34,7 +27,8 @@ func TestGap(t *testing.T) { t.Errorf("max < min: %s, want min", d) } for i := 0; i < 1000; i++ { - if d := g.gap(time.Second, 2*time.Second); d < time.Second || d > 2*time.Second { + if d := g.gap(time.Second, 2*time.Second); d < time.Second || + d > 2*time.Second { t.Fatalf("gap out of range: %s", d) } } @@ -135,7 +129,14 @@ func TestAdvance(t *testing.T) { Amount: ptr(10000.0), Currency: "GBP", }, - Lots: []ocds.Lot{{ID: "1"}, {ID: ""}}, + Lots: []ocds.Lot{ + { + ID: "1", + }, + { + ID: "", + }, + }, TenderPeriod: &ocds.Period{ EndDate: &end, }, @@ -147,7 +148,8 @@ func TestAdvance(t *testing.T) { seen[r.Tag[0]] = true switch r.Tag[0] { case "tenderUpdate": - if got := *r.Tender.TenderPeriod.EndDate; !got.Equal(end.AddDate(0, 0, 14)) { + got := *r.Tender.TenderPeriod.EndDate + if !got.Equal(end.AddDate(0, 0, 14)) { t.Fatalf("updated deadline %s", got) } case "award": @@ -155,13 +157,15 @@ func TestAdvance(t *testing.T) { t.Fatalf("award: status %s, %d awards", r.Tender.Status, len(r.Awards)) } aw := r.Awards[0] - if aw.Value == nil || *aw.Value.Amount < 8000 || *aw.Value.Amount > 10000 { + if aw.Value == nil || *aw.Value.Amount < 8000 || + *aw.Value.Amount > 10000 { t.Fatalf("award value %+v", aw.Value) } if len(aw.RelatedLots) != 1 || aw.RelatedLots[0] != "1" { t.Fatalf("related lots %v", aw.RelatedLots) } - if len(aw.Suppliers) != 1 || len(r.Parties) != 1 || r.Parties[0].Roles[0] != "supplier" { + if len(aw.Suppliers) != 1 || len(r.Parties) != 1 || + r.Parties[0].Roles[0] != "supplier" { t.Fatalf("suppliers %v, parties %v", aw.Suppliers, r.Parties) } default: @@ -188,7 +192,8 @@ func TestAdvance(t *testing.T) { r := g.advance(prev, at) switch r.Tag[0] { case "tenderUpdate": - if got := *r.Tender.TenderPeriod.EndDate; !got.Equal(at.AddDate(0, 0, 44)) { + got := *r.Tender.TenderPeriod.EndDate + if !got.Equal(at.AddDate(0, 0, 44)) { t.Fatalf("default deadline %s", got) } case "award": @@ -200,9 +205,8 @@ func TestAdvance(t *testing.T) { } func TestRunPublishesUntilCancelled(t *testing.T) { - quietLog(t) cat := loadTestCatalog(t) - store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) + store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl"), quiet) if err != nil { t.Fatal(err) } @@ -210,8 +214,9 @@ func TestRunPublishesUntilCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) + generator := NewGenerator(cat, store, 1, quiet) go func() { - NewGenerator(cat, store, 1).Run(ctx, time.Millisecond, 2*time.Millisecond) + generator.Run(ctx, time.Millisecond, 2*time.Millisecond) close(done) }() @@ -234,7 +239,6 @@ func TestRunPublishesUntilCancelled(t *testing.T) { // A failing store is logged and Run carries on until cancelled. func TestRunSurvivesStoreErrors(t *testing.T) { - quietLog(t) cat := loadTestCatalog(t) store, err := OpenStore(filepath.Join(t.TempDir(), "r.jsonl")) if err != nil { @@ -244,7 +248,8 @@ func TestRunSurvivesStoreErrors(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() - NewGenerator(cat, store, 1).Run(ctx, time.Millisecond, time.Millisecond) + generator := NewGenerator(cat, store, 1, quiet) + generator.Run(ctx, time.Millisecond, time.Millisecond) if store.Len() != 0 { t.Errorf("Len = %d, want 0", store.Len()) } diff --git a/internal/mock/findatender/handler.go b/internal/mock/findatender/handler.go index ea14610..ca812fd 100644 --- a/internal/mock/findatender/handler.go +++ b/internal/mock/findatender/handler.go @@ -31,8 +31,10 @@ const ( type server struct { store *Store baseURL string - loc *time.Location // zone for updatedFrom/updatedTo without an offset; nil means UTC - opts options + // location is the zone for updatedFrom/updatedTo without an offset; nil + // means UTC. + location *time.Location + opts options } // NewHandler serves release packages from store. baseURL is the public base @@ -42,16 +44,13 @@ type server struct { // Stored release dates are UTC; comparisons are on absolute instants. It // logs to slog.Default() unless given WithLogger. func NewHandler( - store *Store, - baseURL string, - location *time.Location, - opts ...Option, + store *Store, baseURL string, location *time.Location, opts ...Option, ) http.HandlerFunc { handler := &server{ - store: store, - baseURL: baseURL, - loc: location, - opts: applyOptions(opts), + store: store, + baseURL: baseURL, + location: location, + opts: applyOptions(opts), } return handler.handle } @@ -86,8 +85,8 @@ func badRequest(w http.ResponseWriter, msg string) { } // parseTime reads a query timestamp. Values without an offset (the documented -// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with an -// offset or Z (RFC 3339) mean exactly what they say. +// yyyy-MM-ddTHH:mm:ss, or a plain date) are local time in location; values with +// an offset or Z (RFC 3339) mean exactly what they say. func parseTime(value string, location *time.Location) (time.Time, error) { if location == nil { location = time.UTC @@ -98,7 +97,8 @@ func parseTime(value string, location *time.Location) (time.Time, error) { } for _, layout := range []string{timeLayout, "2006-01-02"} { - if parsed, err := time.ParseInLocation(layout, value, location); err == nil { + parsed, err := time.ParseInLocation(layout, value, location) + if err == nil { return parsed, nil } } @@ -115,22 +115,24 @@ type pageQuery struct { // handle answers a GET for a page of releases as an OCDS release package, // with a links.next when more releases match. -func (s *server) handle(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { +func (s *server) handle(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { w.WriteHeader(http.StatusMethodNotAllowed) return } - query := req.URL.Query() + query := r.URL.Query() page, err := s.parseQuery(query) if err != nil { badRequest(w, err.Error()) return } - releases, next, more := s.store.Page(page.from, page.upper, page.start, page.limit) + releases, next, more := s.store.Page( + page.from, page.upper, page.start, page.limit, + ) releasePackage := ocds.ReleasePackage{ - URI: s.baseURL + req.URL.RequestURI(), + URI: s.baseURL + r.URL.RequestURI(), Version: "1.1", Extensions: ocds.DeclaredExtensions, PublishedDate: ptr(time.Now().UTC().Truncate(time.Second)), @@ -167,13 +169,15 @@ func (s *server) parseQuery(query url.Values) (pageQuery, error) { if value := query.Get("limit"); value != "" { number, err := strconv.Atoi(value) if err != nil || number < 1 || number > maxLimit { - return page, fmt.Errorf("limit must be an integer between 1 and %d", maxLimit) + return page, fmt.Errorf( + "limit must be an integer between 1 and %d", maxLimit, + ) } page.limit = number } if value := query.Get("updatedFrom"); value != "" { - from, err := parseTime(value, s.loc) + from, err := parseTime(value, s.location) if err != nil { return page, fmt.Errorf("updatedFrom %w", err) } @@ -181,15 +185,15 @@ func (s *server) parseQuery(query url.Values) (pageQuery, error) { } if value := query.Get("updatedTo"); value != "" { - to, err := parseTime(value, s.loc) + until, err := parseTime(value, s.location) if err != nil { return page, fmt.Errorf("updatedTo %w", err) } - if query.Get("updatedFrom") != "" && to.Before(page.from) { + if query.Get("updatedFrom") != "" && until.Before(page.from) { return page, errors.New("updatedTo must not be before updatedFrom") } - if to.Before(page.upper) { - page.upper = to + if until.Before(page.upper) { + page.upper = until } } diff --git a/internal/mock/findatender/handler_test.go b/internal/mock/findatender/handler_test.go index 7152191..83c4b45 100644 --- a/internal/mock/findatender/handler_test.go +++ b/internal/mock/findatender/handler_test.go @@ -120,7 +120,8 @@ func TestHandlerRejectsBadRequests(t *testing.T) { t.Errorf("%s: Content-Type %q", q, ct) } var body map[string]string - if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil || body["error"] != want { + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil || + body["error"] != want { t.Errorf("%s: body %s, want error %q", q, rec.Body.String(), want) } } @@ -147,7 +148,12 @@ func TestParseTimeFormats(t *testing.T) { t.Errorf("parseTime(%q) = %s, %v; want %s", v, got, err, want) } } - for _, v := range []string{"", "08/10/2026", "2026-10-08 12:00:00", "2026-10-08T25:00:00"} { + for _, v := range []string{ + "", + "08/10/2026", + "2026-10-08 12:00:00", + "2026-10-08T25:00:00", + } { if _, err := parseTime(v, nil); err == nil { t.Errorf("parseTime(%q): expected error", v) } diff --git a/internal/mock/findatender/ratelimit.go b/internal/mock/findatender/ratelimit.go index 17354c0..119d90d 100644 --- a/internal/mock/findatender/ratelimit.go +++ b/internal/mock/findatender/ratelimit.go @@ -66,7 +66,8 @@ func (rl *RateLimiter) allow(key string) (bool, int) { // Move to whichever fixed window, counted from the anchor, now falls in. index := now.Sub(client.anchor) / rl.window - if windowStart := client.anchor.Add(index * rl.window); !windowStart.Equal(client.windowStart) { + windowStart := client.anchor.Add(index * rl.window) + if !windowStart.Equal(client.windowStart) { client.windowStart = windowStart client.count = 0 } @@ -82,10 +83,12 @@ func (rl *RateLimiter) allow(key string) (bool, int) { if nextHour := hour.Add(time.Hour); nextHour.Before(next) { next = nextHour } + seconds := int((next.Sub(now) + time.Second - 1) / time.Second) // round up if seconds < 1 { seconds = 1 } + return false, seconds } @@ -100,10 +103,10 @@ func (rl *RateLimiter) prune(hour time.Time) { // clientKey produces an identifier for the client based on remote address. // Could be updated to support proxy and forwarded-for headers. -func clientKey(req *http.Request) string { - host, _, err := net.SplitHostPort(req.RemoteAddr) +func clientKey(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return req.RemoteAddr + return r.RemoteAddr } return host } @@ -112,18 +115,20 @@ func clientKey(req *http.Request) string { // to make a request in their current window or returns a "TooManyRequests" // status and the time the client should wait before making another request. func (rl *RateLimiter) Middleware(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - ok, retry := rl.allow(clientKey(req)) + return func(w http.ResponseWriter, r *http.Request) { + ok, retry := rl.allow(clientKey(r)) if !ok { w.Header().Set("Retry-After", strconv.Itoa(retry)) w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusTooManyRequests) fmt.Fprintf( //nolint:errcheck // the client may have gone; nothing to do - w, "Rate limit of %d exceeded. Please retry after %d seconds.", - rl.limit, retry, + w, + "Rate limit of %d exceeded. Please retry after %d seconds.", + rl.limit, + retry, ) return } - next(w, req) + next(w, r) } } diff --git a/internal/mock/findatender/ratelimit_test.go b/internal/mock/findatender/ratelimit_test.go index bf2521d..754f99f 100644 --- a/internal/mock/findatender/ratelimit_test.go +++ b/internal/mock/findatender/ratelimit_test.go @@ -10,7 +10,8 @@ import ( ) func TestRateLimitWindows(t *testing.T) { - now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) // first request of the hour + // The first request of the hour. + now := time.Date(2026, 10, 6, 10, 3, 20, 0, time.UTC) rl := NewRateLimiter(12, 5*time.Minute) rl.now = func() time.Time { return now diff --git a/internal/mock/findatender/store.go b/internal/mock/findatender/store.go index dee16e1..7cfa027 100644 --- a/internal/mock/findatender/store.go +++ b/internal/mock/findatender/store.go @@ -22,17 +22,17 @@ import ( // Store keeps every release in memory and appends each new one to a JSON Lines // file, so releases survive restarts. Releases are kept in date order. type Store struct { - mu sync.RWMutex - f *os.File - releases []ocds.Release - times []time.Time - latest map[string]int // ocid -> index of its most recent release - lastNum map[int]int // year -> highest notice number used + mu sync.RWMutex + file *os.File + releases []ocds.Release + times []time.Time + latest map[string]int // ocid -> index of its most recent release + lastNumber map[int]int // year -> highest notice number used // Procedures not yet awarded, as a slice plus positions so that adding, // removing and picking one at random are all constant time. - open []string - openPos map[string]int + open []string + openPosition map[string]int opts options } @@ -44,11 +44,12 @@ type Store struct { // It logs to slog.Default() unless given WithLogger. func OpenStore(path string, opts ...Option) (*Store, error) { store := &Store{ - latest: map[string]int{}, - lastNum: map[int]int{}, - openPos: map[string]int{}, - opts: applyOptions(opts), + latest: map[string]int{}, + lastNumber: map[int]int{}, + openPosition: map[string]int{}, + opts: applyOptions(opts), } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return nil, err } @@ -57,6 +58,7 @@ func OpenStore(path string, opts ...Option) (*Store, error) { if err != nil { return nil, err } + // The file is written in order, but sort anyway in case it was hand-edited. sort.SliceStable(releases, func(i, other int) bool { return releases[i].Date.Before(releases[other].Date) @@ -69,9 +71,10 @@ func OpenStore(path string, opts ...Option) (*Store, error) { store.index(i, release) } - if store.f, err = openForAppend(path); err != nil { + if store.file, err = openForAppend(path); err != nil { return nil, err } + return store, nil } @@ -106,6 +109,7 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) { if err := scanner.Err(); err != nil { return nil, fmt.Errorf("reading %s: %w", path, err) } + return releases, nil } @@ -113,18 +117,25 @@ func readReleases(logger *slog.Logger, path string) ([]ocds.Release, error) { // mid-write, it ends the partial record so the next one starts on a fresh // line. func openForAppend(path string) (*os.File, error) { - file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600) + file, err := os.OpenFile( + filepath.Clean(path), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o600, + ) if err != nil { return nil, err } + if info, err := file.Stat(); err == nil && info.Size() > 0 { last := make([]byte, 1) - if _, err := file.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' { + _, readErr := file.ReadAt(last, info.Size()-1) + if readErr == nil && last[0] != '\n' { if _, err := file.Write([]byte("\n")); err != nil { - return nil, errors.Join(fmt.Errorf("repairing %s: %w", path, err), file.Close()) + return nil, errors.Join( + fmt.Errorf("repairing %s: %w", path, err), file.Close(), + ) } } } + return file, nil } @@ -133,51 +144,58 @@ func openForAppend(path string) (*os.File, error) { // highest notice number used each year. func (s *Store) index(i int, release ocds.Release) { s.latest[release.OCID] = i + if len(release.Tag) > 0 && release.Tag[0] == "award" { s.closeProcedure(release.OCID) - } else if _, ok := s.openPos[release.OCID]; !ok { - s.openPos[release.OCID] = len(s.open) + } else if _, ok := s.openPosition[release.OCID]; !ok { + s.openPosition[release.OCID] = len(s.open) s.open = append(s.open, release.OCID) } + if numberText, year, ok := strings.Cut(release.ID, "-"); ok { number, numberErr := strconv.Atoi(numberText) yearNumber, yearErr := strconv.Atoi(year) - if numberErr == nil && yearErr == nil && number > s.lastNum[yearNumber] { - s.lastNum[yearNumber] = number + if numberErr == nil && yearErr == nil && number > s.lastNumber[yearNumber] { + s.lastNumber[yearNumber] = number } } } // closeProcedure removes ocid from the open procedures, if it is there. func (s *Store) closeProcedure(ocid string) { - position, ok := s.openPos[ocid] + position, ok := s.openPosition[ocid] if !ok { return } + last := len(s.open) - 1 s.open[position] = s.open[last] - s.openPos[s.open[position]] = position + s.openPosition[s.open[position]] = position s.open = s.open[:last] - delete(s.openPos, ocid) + delete(s.openPosition, ocid) } // Append persists a release and flushes it to disk. -func (s *Store) Append(release ocds.Release) error { return s.append(release, true) } +func (s *Store) Append(release ocds.Release) error { + return s.append(release, true) +} // AppendNoSync persists a release without an fsync; call Sync afterwards. // Used for bulk catch-up, where a sync per record would be very slow. -func (s *Store) AppendNoSync(release ocds.Release) error { return s.append(release, false) } +func (s *Store) AppendNoSync(release ocds.Release) error { + return s.append(release, false) +} // Sync flushes appended releases to disk. Call it after AppendNoSync. func (s *Store) Sync() error { s.mu.Lock() defer s.mu.Unlock() - return s.f.Sync() + return s.file.Sync() } -// append stores release, syncing the file to disk if sync is true. A date earlier -// than the newest stored release (for example after a clock change) is moved -// up so the order stays valid. +// append stores release, syncing the file to disk if sync is true. A date +// earlier than the newest stored release (for example after a clock change) is +// moved up so the order stays valid. func (s *Store) append(release ocds.Release, sync bool) error { s.mu.Lock() defer s.mu.Unlock() @@ -186,27 +204,34 @@ func (s *Store) append(release ocds.Release, sync bool) error { if count := len(s.times); count > 0 && release.Date.Before(s.times[count-1]) { release.Date = s.times[count-1] } + data, err := json.Marshal(release) if err != nil { return err } - if _, err := s.f.Write(append(data, '\n')); err != nil { + if _, err := s.file.Write(append(data, '\n')); err != nil { return err } if sync { - if err := s.f.Sync(); err != nil { + if err := s.file.Sync(); err != nil { return err } } + s.releases = append(s.releases, release) s.times = append(s.times, release.Date) s.index(len(s.releases)-1, release) + return nil } // Page returns up to limit releases dated in [from, upper], starting no earlier -// than position start, plus the position to continue from and whether more exist. -func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, int, bool) { +// than position start, plus the position to continue from and whether more +// exist. +func (s *Store) Page( + from, upper time.Time, + start, limit int, +) ([]ocds.Release, int, bool) { s.mu.RLock() defer s.mu.RUnlock() @@ -216,12 +241,15 @@ func (s *Store) Page(from, upper time.Time, start, limit int) ([]ocds.Release, i if start > i { i = start } + out := make([]ocds.Release, 0, limit) for i < len(s.releases) && len(out) < limit && !s.times[i].After(upper) { out = append(out, s.releases[i]) i++ } + more := i < len(s.releases) && !s.times[i].After(upper) + return out, i, more } @@ -255,13 +283,15 @@ func (s *Store) HasOCID(ocid string) bool { func (s *Store) NextNoticeID(year int) string { s.mu.RLock() defer s.mu.RUnlock() - return fmt.Sprintf("%06d-%d", s.lastNum[year]+1, year) + return fmt.Sprintf("%06d-%d", s.lastNumber[year]+1, year) } // RandomOpen picks a procedure that has not been awarded and whose latest // release is after since. Procedures idle since before then are dropped from // the open set, as if they had been abandoned. -func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) { +func (s *Store) RandomOpen( + rng *rand.Rand, since time.Time, +) (ocds.Release, bool) { s.mu.Lock() defer s.mu.Unlock() for len(s.open) > 0 { @@ -279,5 +309,5 @@ func (s *Store) RandomOpen(rng *rand.Rand, since time.Time) (ocds.Release, bool) func (s *Store) Close() error { s.mu.Lock() defer s.mu.Unlock() - return s.f.Close() + return s.file.Close() } diff --git a/internal/mock/findatender/store_test.go b/internal/mock/findatender/store_test.go index ef15654..bf62d98 100644 --- a/internal/mock/findatender/store_test.go +++ b/internal/mock/findatender/store_test.go @@ -56,7 +56,9 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) { t.Fatalf("after reopen: %d releases, want %d", store2.Len(), len(all)) } gen2 := NewGenerator(cat, store2, 7) - m, _ := gen2.CatchUp(now.Add(time.Hour), 48*time.Hour, 3*time.Minute, 15*time.Minute) + m, _ := gen2.CatchUp( + now.Add(time.Hour), 48*time.Hour, 3*time.Minute, 15*time.Minute, + ) if m < 4 || m > 20 { t.Fatalf("one hour gap produced %d releases", m) } @@ -66,9 +68,9 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) { // The HTTP handler pages through the store without gaps or repeats. srv := &server{ - store: store2, - baseURL: "", - loc: time.UTC, + store: store2, + baseURL: "", + location: time.UTC, } seen := map[string]bool{} next := BasePath + "?updatedFrom=2026-10-07T00:00:00&limit=7" @@ -102,7 +104,9 @@ func TestStorePersistsAndGeneratorProgresses(t *testing.T) { func writeLines(t *testing.T, path string, lines ...string) { t.Helper() - if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o600); err != nil { + if err := os.WriteFile( + path, []byte(strings.Join(lines, "\n")), 0o600, + ); err != nil { t.Fatal(err) } } @@ -122,7 +126,8 @@ func releaseLine(t *testing.T, ocid, id, tag string, at time.Time) string { } // Unreadable lines are skipped, blank lines ignored, a hand-edited file is put -// back in date order, and the index (open procedures, notice numbers) is rebuilt. +// back in date order, and the index (open procedures, notice numbers) is +// rebuilt. func TestOpenStoreRecoversFromMessyFile(t *testing.T) { path := filepath.Join(t.TempDir(), "r.jsonl") t0 := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) @@ -148,7 +153,8 @@ func TestOpenStoreRecoversFromMessyFile(t *testing.T) { for _, r := range all { ids = append(ids, r.ID) } - if got, want := strings.Join(ids, ","), "000003-2026,000005-2026,000007-2026"; got != want { + want := "000003-2026,000005-2026,000007-2026" + if got := strings.Join(ids, ","); got != want { t.Fatalf("releases %s, want %s", got, want) } if got := store.NextNoticeID(2026); got != "000008-2026" { @@ -199,7 +205,11 @@ func TestOpenStoreAfterTruncatedWrite(t *testing.T) { } defer store2.Close() if store2.Len() != 2 || !store2.HasOCID("ocds-c") { - t.Fatalf("after reopen: Len = %d, has ocds-c = %v", store2.Len(), store2.HasOCID("ocds-c")) + t.Fatalf( + "after reopen: Len = %d, has ocds-c = %v", + store2.Len(), + store2.HasOCID("ocds-c"), + ) } } @@ -294,7 +304,8 @@ func TestRandomOpenDropsStaleProcedures(t *testing.T) { //nolint:gosec // G404: seeded math/rand keeps the test deterministic rnd := rand.New(rand.NewSource(1)) for i := 0; i < 10; i++ { - if r, ok := store.RandomOpen(rnd, t0.Add(time.Minute)); !ok || r.OCID != "new" { + if r, ok := store.RandomOpen(rnd, t0.Add(time.Minute)); !ok || + r.OCID != "new" { t.Fatalf("RandomOpen = %s, %v; want new", r.OCID, ok) } } diff --git a/internal/store/README.md b/internal/store/README.md index 003c8db..4d7e797 100644 --- a/internal/store/README.md +++ b/internal/store/README.md @@ -78,6 +78,18 @@ Reads select whole rows (`SELECT *`, `t.*`, `c.*`) which map to a Go struct. sql `ListOpportunities` is assembled by `listOpportunitiesQuery` from fragments in `queries.go`: a base `SELECT`, one `filterOpportunity*` clause per filter that is set, and the ordering and paging tail. Each filter fragment takes exactly one argument, except the `filterOpportunityMark*` fragments, which take none (`MarkUnset` puts `NOT` in front of them). Adding a filter means one fragment and one line in `listOpportunitiesQuery`. The classifications and the marks for a page of opportunities are then loaded with one `IN (...)` query each. +### SQL style + +A soft convention for the SQL in `queries.go`, followed so the queries look alike and read easily. A query that reads better another way can depart from it. + +- A statement starts on the line after the opening backtick. A fragment combined into another query (`filterOpportunity*`) starts straight after it. +- Keywords are uppercase (`SELECT`, `INSERT INTO`, `ON CONFLICT`); table and column names are the lowercase snake_case of the schema. +- A short statement may stay on one line (`SELECT t.* FROM opportunities t WHERE t.uuid = ?`). A longer one has one clause per line, each at the start of the line: `SELECT`, `FROM`, `JOIN`, `WHERE`, `ORDER BY`, `LIMIT`. +- A column list that doesn't fit on one line goes inside its brackets, indented one tab and filled to about 80 characters per line. The `VALUES` list wraps at the same columns, so each line of values matches the line of columns above it. +- `ON CONFLICT … DO UPDATE SET` puts one `column = excluded.column` per line, indented one tab. +- A subquery (`EXISTS (…)`, `IN (…)`) is indented one tab inside its brackets, with the closing bracket at the start of a line. +- Tables are aliased consistently: `t` for `opportunities`, `c` for `classification_codes`, `tc` for `opportunity_classifications`, `m` for `opportunity_marks`. + ## Store API ```go diff --git a/internal/store/migrate.go b/internal/store/migrate.go index 6239689..c5c88c0 100644 --- a/internal/store/migrate.go +++ b/internal/store/migrate.go @@ -17,21 +17,6 @@ import ( //go:embed migrations/*.sql var migrationsFS embed.FS -// ErrNeedsMigration is returned by Open when the database has pending -// migrations. Apply them with the migrate command, or pass -// WithAutoMigrate. -var ErrNeedsMigration = errors.New("store: database needs migrating") - -// ErrSchemaTooNew is returned by Open when the database has migrations -// this build doesn't know about, i.e. it was migrated by a newer build. -var ErrSchemaTooNew = errors.New("store: database schema is newer than this build") - -// ErrWouldEmptyDatabase is returned by StepDown when the migration would drop -// all of the tables and thus empty the database. -var ErrWouldEmptyDatabase = errors.New( - "migration cancelled; database would be emptied by undoing all migrations", -) - // NewMigrator returns a goose Provider over the store's embedded migrations, // for applying, rolling back or inspecting them. Each migration runs in its // own transaction, so a failed migration leaves the database at the previous @@ -96,13 +81,15 @@ func checkNotNewer(ctx context.Context, provider *goose.Provider) error { if err != nil { return err } + if latest := LatestVersion(provider); current > latest { return fmt.Errorf( - "database is at version %d but this build only knows up to %d; use a newer build", + "store: database is at version %d but this build only knows up to %d; use a newer build", current, latest, ) } + return nil } @@ -116,7 +103,7 @@ func prepareSchema(ctx context.Context, db *sql.DB, opts options) error { if opts.autoMigrate { if _, err := provider.Up(ctx); err != nil { - return fmt.Errorf("store: migrating database: %w", err) + return fmt.Errorf("migrating database: %w", err) } } @@ -184,6 +171,7 @@ func StepDown( applied++ } } + all := steps == 0 || steps >= applied if all && !confirmDrop { return results, ErrWouldEmptyDatabase diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index ec5eadf..c7cf115 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -29,10 +29,12 @@ func newMigrator(t *testing.T) (*goose.Provider, *sql.DB) { t.Cleanup(func() { db.Close() }) + m, err := store.NewMigrator(db) if err != nil { t.Fatal(err) } + return m, db } @@ -105,7 +107,8 @@ func TestOpenRejectsNewerSchema(t *testing.T) { "check": nil, "auto-migrate": {store.WithAutoMigrate()}, } { - if _, err := store.Open(ctx, path, opts...); !errors.Is(err, store.ErrSchemaTooNew) { + _, err := store.Open(ctx, path, opts...) + if !errors.Is(err, store.ErrSchemaTooNew) { t.Errorf("%s: err = %v, want ErrSchemaTooNew", name, err) } } @@ -135,10 +138,12 @@ func TestMigrateDownAllAndUpAgain(t *testing.T) { t.Fatalf("down to 0: %v", err) } var tables int - if err := db.QueryRow(` + query := ` SELECT count(*) FROM sqlite_master WHERE type IN ('table', 'index', 'trigger') - AND name NOT LIKE 'sqlite_%' AND name != 'goose_db_version'`).Scan(&tables); err != nil { + AND name NOT LIKE 'sqlite_%' + AND name != 'goose_db_version'` + if err := db.QueryRow(query).Scan(&tables); err != nil { t.Fatal(err) } if tables != 0 { @@ -229,7 +234,10 @@ func TestDownAllRequiresConfirmation(t *testing.T) { } hasErrored := false - for name, steps := range map[string]int{"steps 0": 0, "more steps than applied": 99} { + for name, steps := range map[string]int{ + "steps 0": 0, + "more steps than applied": 99, + } { _, err := store.StepDown(ctx, m, steps, false) if !errors.Is(err, store.ErrWouldEmptyDatabase) { t.Errorf("%s: %s", name, err) @@ -277,20 +285,11 @@ func TestNewerDatabaseIsRefused(t *testing.T) { t.Fatal(err) } - if _, err := store.StepUp( - ctx, - m, - 0, - ); err == nil || + if _, err := store.StepUp(ctx, m, 0); err == nil || !strings.Contains(err.Error(), "newer build") { t.Errorf("up: err = %v, want a newer-build error", err) } - if _, err := store.StepDown( - ctx, - m, - 1, - false, - ); err == nil || + if _, err := store.StepDown(ctx, m, 1, false); err == nil || !strings.Contains(err.Error(), "newer build") { t.Errorf("down: err = %v, want a newer-build error", err) } @@ -307,7 +306,9 @@ func TestFailedMigrationIsRolledBack(t *testing.T) { // Occupy migration 4's table name with a view. Its CREATE TABLE IF NOT // EXISTS then skips, and the CREATE INDEX that follows fails because // views can't be indexed, part-way through the migration. - if _, err := db.Exec(`CREATE VIEW classification_codes AS SELECT 1 AS x`); err != nil { + if _, err := db.Exec( + `CREATE VIEW classification_codes AS SELECT 1 AS x`, + ); err != nil { t.Fatal(err) } @@ -319,8 +320,9 @@ func TestFailedMigrationIsRolledBack(t *testing.T) { t.Errorf("version after failure: %d, want 3", v) } var indexes int - db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name LIKE 'classification_codes_%'`). - Scan(&indexes) + query := `SELECT count(*) FROM sqlite_master + WHERE name LIKE 'classification_codes_%'` + db.QueryRow(query).Scan(&indexes) if indexes != 0 { t.Errorf("failed migration left %d objects behind", indexes) } diff --git a/internal/store/queries.go b/internal/store/queries.go index 8a3d462..6c4c609 100644 --- a/internal/store/queries.go +++ b/internal/store/queries.go @@ -18,23 +18,27 @@ package store // ---------- opportunities ---------- -// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares against. +// selectOpportunityLatestReleaseDate reads the date SaveOpportunity compares +// against. const selectOpportunityLatestReleaseDate = ` SELECT latest_release_date FROM opportunities WHERE uuid = ?` -// upsertOpportunity inserts or updates a opportunity. Named parameters (types.Opportunity). +// upsertOpportunity inserts or updates a opportunity. Named parameters +// (types.Opportunity). // // An UPDATE on conflict, not a REPLACE: REPLACE would delete the row first, // cascading to the opportunity's releases and changing its id. const upsertOpportunity = ` INSERT INTO opportunities ( uuid, uuid_type, title, status, description, buyer_name, buyer_id, - main_category, procurement_method, multi_lot, sme_suitable, vcse_suitable, - start_date, end_date, gross_value, has_documents, latest_release_date + main_category, procurement_method, multi_lot, sme_suitable, + vcse_suitable, start_date, end_date, gross_value, has_documents, + latest_release_date ) VALUES ( :uuid, :uuid_type, :title, :status, :description, :buyer_name, :buyer_id, - :main_category, :procurement_method, :multi_lot, :sme_suitable, :vcse_suitable, - :start_date, :end_date, :gross_value, :has_documents, :latest_release_date + :main_category, :procurement_method, :multi_lot, :sme_suitable, + :vcse_suitable, :start_date, :end_date, :gross_value, :has_documents, + :latest_release_date ) ON CONFLICT (uuid) DO UPDATE SET uuid_type = excluded.uuid_type, @@ -162,7 +166,8 @@ INSERT INTO opportunity_release_tags (uuid, release_id, tag) VALUES (:uuid, :release_id, :tag) ON CONFLICT DO NOTHING` -// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's releases. +// selectReleaseTagsByOpportunity reads the tags of all of a opportunity's +// releases. const selectReleaseTagsByOpportunity = ` SELECT * FROM opportunity_release_tags @@ -180,13 +185,14 @@ ORDER BY release_id, tag` const upsertClassificationCodeReturningID = ` INSERT INTO classification_codes (is_cpv, code, classification) VALUES (:is_cpv, :code, :classification) -ON CONFLICT (is_cpv, code, classification) DO UPDATE SET is_cpv = excluded.is_cpv +ON CONFLICT (is_cpv, code, classification) + DO UPDATE SET is_cpv = excluded.is_cpv RETURNING id` // ---------- opportunity_classifications ---------- -// deleteOpportunityClassifications clears a opportunity's mappings before they are -// rewritten. +// deleteOpportunityClassifications clears a opportunity's mappings before they +// are rewritten. const deleteOpportunityClassifications = ` DELETE FROM opportunity_classifications WHERE opportunity_uuid = ?` @@ -197,8 +203,8 @@ VALUES (?, ?) ON CONFLICT DO NOTHING` // selectClassificationsForOpportunities reads the classifications of several -// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by Rebind. -// c.* must map entirely onto types.ClassificationCode. +// opportunities. "IN (?)" is expanded by sqlx.In, which must be followed by +// Rebind. c.* must map entirely onto types.ClassificationCode. const selectClassificationsForOpportunities = ` SELECT tc.opportunity_uuid, c.* FROM opportunity_classifications tc diff --git a/internal/store/queries_test.go b/internal/store/queries_test.go index 67c711d..e893fa4 100644 --- a/internal/store/queries_test.go +++ b/internal/store/queries_test.go @@ -30,26 +30,50 @@ type queryCheck struct { // missing here. var checks = map[string]queryCheck{ // opportunities - "selectOpportunityLatestReleaseDate": {sql: selectOpportunityLatestReleaseDate}, - "upsertOpportunity": {sql: upsertOpportunity, named: true}, - "selectOpportunityByUUID": {sql: selectOpportunityByUUID}, + "selectOpportunityLatestReleaseDate": { + sql: selectOpportunityLatestReleaseDate, + }, + "upsertOpportunity": {sql: upsertOpportunity, named: true}, + "selectOpportunityByUUID": {sql: selectOpportunityByUUID}, "listOpportunitiesSelect": { sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage, }, "listOpportunitiesOrderAndPage": { sql: listOpportunitiesSelect + listOpportunitiesOrderAndPage, }, - "filterOpportunityStatus": {sql: withFilter(filterOpportunityStatus)}, - "filterOpportunityClassificationCode": {sql: withFilter(filterOpportunityClassificationCode)}, - "filterOpportunityTitleMatch": {sql: withFilter(filterOpportunityTitleMatch)}, - "filterOpportunityReleasedFrom": {sql: withFilter(filterOpportunityReleasedFrom)}, - "filterOpportunityReleasedBefore": {sql: withFilter(filterOpportunityReleasedBefore)}, - "filterOpportunityMarkHidden": {sql: withFilter(filterOpportunityMarkHidden)}, - "filterOpportunityMarkSkipped": {sql: withFilter(filterOpportunityMarkSkipped)}, - "filterOpportunityMarkInterested": {sql: withFilter(filterOpportunityMarkInterested)}, - "filterOpportunityMarkBidSubmitted": {sql: withFilter(filterOpportunityMarkBidSubmitted)}, - "filterOpportunityMarkWon": {sql: withFilter(filterOpportunityMarkWon)}, - "filterOpportunityMarkLost": {sql: withFilter(filterOpportunityMarkLost)}, + "filterOpportunityStatus": { + sql: withFilter(filterOpportunityStatus), + }, + "filterOpportunityClassificationCode": { + sql: withFilter(filterOpportunityClassificationCode), + }, + "filterOpportunityTitleMatch": { + sql: withFilter(filterOpportunityTitleMatch), + }, + "filterOpportunityReleasedFrom": { + sql: withFilter(filterOpportunityReleasedFrom), + }, + "filterOpportunityReleasedBefore": { + sql: withFilter(filterOpportunityReleasedBefore), + }, + "filterOpportunityMarkHidden": { + sql: withFilter(filterOpportunityMarkHidden), + }, + "filterOpportunityMarkSkipped": { + sql: withFilter(filterOpportunityMarkSkipped), + }, + "filterOpportunityMarkInterested": { + sql: withFilter(filterOpportunityMarkInterested), + }, + "filterOpportunityMarkBidSubmitted": { + sql: withFilter(filterOpportunityMarkBidSubmitted), + }, + "filterOpportunityMarkWon": { + sql: withFilter(filterOpportunityMarkWon), + }, + "filterOpportunityMarkLost": { + sql: withFilter(filterOpportunityMarkLost), + }, // opportunity_ocds_releases "insertRelease": {sql: insertRelease, named: true}, @@ -61,12 +85,22 @@ var checks = map[string]queryCheck{ "selectReleaseTagsByOpportunity": {sql: selectReleaseTagsByOpportunity}, // classification_codes - "upsertClassificationCodeReturningID": {sql: upsertClassificationCodeReturningID, named: true}, + "upsertClassificationCodeReturningID": { + sql: upsertClassificationCodeReturningID, + named: true, + }, // opportunity_classifications - "deleteOpportunityClassifications": {sql: deleteOpportunityClassifications}, - "insertOpportunityClassification": {sql: insertOpportunityClassification}, - "selectClassificationsForOpportunities": {sql: selectClassificationsForOpportunities, in: true}, + "deleteOpportunityClassifications": { + sql: deleteOpportunityClassifications, + }, + "insertOpportunityClassification": { + sql: insertOpportunityClassification, + }, + "selectClassificationsForOpportunities": { + sql: selectClassificationsForOpportunities, + in: true, + }, // opportunity_marks "insertOpportunityMarks": {sql: insertOpportunityMarks, named: true}, @@ -75,12 +109,17 @@ var checks = map[string]queryCheck{ } func withFilter(fragment string) string { - return listOpportunitiesSelect + "\nWHERE " + fragment + listOpportunitiesOrderAndPage + return listOpportunitiesSelect + "\nWHERE " + fragment + + listOpportunitiesOrderAndPage } func openTestDB(t *testing.T) *sqlx.DB { t.Helper() - s, err := Open(context.Background(), filepath.Join(t.TempDir(), "queries.db"), WithAutoMigrate()) + s, err := Open( + context.Background(), + filepath.Join(t.TempDir(), "queries.db"), + WithAutoMigrate(), + ) if err != nil { t.Fatalf("Open: %v", err) } @@ -144,11 +183,13 @@ func TestListOpportunitiesQueryAllFilters(t *testing.T) { Limit: 10, Offset: 5, }) + stmt, err := db.Preparex(query) if err != nil { t.Fatalf("prepare:\n%s\n%v", query, err) } defer stmt.Close() + var rows []opportunityRow if err := stmt.Select(&rows, args...); err != nil { t.Errorf("run with %d args:\n%s\n%v", len(args), query, err) @@ -162,6 +203,7 @@ func TestQueriesAllListed(t *testing.T) { if err != nil { t.Fatal(err) } + var declared []string for _, decl := range file.Decls { gen, ok := decl.(*ast.GenDecl) @@ -174,17 +216,21 @@ func TestQueriesAllListed(t *testing.T) { } } } + sort.Strings(declared) if len(declared) == 0 { t.Fatal("found no constants in queries.go") } + for _, name := range declared { if _, ok := checks[name]; !ok { t.Errorf("queries.go constant %s is not listed in checks", name) } } + for name := range checks { - if i := sort.SearchStrings(declared, name); i == len(declared) || declared[i] != name { + if i := sort.SearchStrings(declared, name); i == len(declared) || + declared[i] != name { t.Errorf("checks lists %s, which queries.go does not declare", name) } } @@ -194,20 +240,27 @@ func TestQueriesAllListed(t *testing.T) { // together, for writes that bypass OpportunityMarks.Validate. func TestMarksCheckConstraint(t *testing.T) { db := openTestDB(t) - if _, err := db.Exec(`INSERT INTO opportunities + insert := `INSERT INTO opportunities (uuid, uuid_type, title, status, latest_release_date) - VALUES ('ocds-1', 'ocid', 'T', 'active', '2026-01-01T00:00:00Z')`); err != nil { + VALUES ('ocds-1', 'ocid', 'T', 'active', '2026-01-01T00:00:00Z')` + if _, err := db.Exec(insert); err != nil { t.Fatal(err) } - if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won, lost) - VALUES ('ocds-1', 1, 1)`); err == nil { + + insertWonAndLost := `INSERT INTO opportunity_marks (uuid, won, lost) + VALUES ('ocds-1', 1, 1)` + if _, err := db.Exec(insertWonAndLost); err == nil { t.Error("inserting won and lost together succeeded") } - if _, err := db.Exec(`INSERT INTO opportunity_marks (uuid, won) - VALUES ('ocds-1', 1)`); err != nil { + + insertWon := `INSERT INTO opportunity_marks (uuid, won) + VALUES ('ocds-1', 1)` + if _, err := db.Exec(insertWon); err != nil { t.Fatalf("inserting won: %v", err) } - if _, err := db.Exec(`UPDATE opportunity_marks SET lost = 1 WHERE uuid = 'ocds-1'`); err == nil { + + updateLost := `UPDATE opportunity_marks SET lost = 1 WHERE uuid = 'ocds-1'` + if _, err := db.Exec(updateLost); err == nil { t.Error("updating to won and lost together succeeded") } } diff --git a/internal/store/store.go b/internal/store/store.go index b0e0743..9571c5c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -18,8 +18,25 @@ import ( "git.avsdev.uk/Go/job-search/internal/types" ) -// ErrNotFound is returned when a requested opportunity does not exist. -var ErrNotFound = errors.New("store: not found") +// Errors returned by the store. +var ( + // ErrNotFound is returned when a requested opportunity does not exist. + ErrNotFound = errors.New("store: not found") + // ErrNeedsMigration is returned by Open when the database has pending + // migrations. Apply them with the migrate command, or pass + // WithAutoMigrate. + ErrNeedsMigration = errors.New("store: database needs migrating") + // ErrSchemaTooNew is returned by Open when the database has migrations + // this build doesn't know about, i.e. it was migrated by a newer build. + ErrSchemaTooNew = errors.New( + "store: database schema is newer than this build", + ) + // ErrWouldEmptyDatabase is returned by StepDown when the migration would + // drop all of the tables and thus empty the database. + ErrWouldEmptyDatabase = errors.New( + "store: migration cancelled; database would be emptied by undoing all migrations", + ) +) // DefaultListLimit is used when OpportunityFilter.Limit is zero or negative. const DefaultListLimit = 50 @@ -32,14 +49,15 @@ type Store struct { // SaveMultipleResult reports what SaveOpportunities changed. type SaveMultipleResult struct { - // NewOpportunities is the number of opportunities that were not already stored. + // NewOpportunities is the number of opportunities that were not already + // stored. NewOpportunities int // NewReleases is the number of releases that were not already stored. NewReleases int - // UpdatedOpportunities is incremented for each opportunity row and its classifications - // that are updated (when every release in the save was older than the - // one the stored opportunity was last updated from). It does not include new - // opportunities. + // UpdatedOpportunities is incremented for each opportunity row and its + // classifications that are updated (when every release in the save was older + // than the one the stored opportunity was last updated from). It does not + // include new opportunities. UpdatedOpportunities int // OpportunityResults is a slice of the results of each opportunity saved. OpportunityResults []SaveResult @@ -47,8 +65,8 @@ type SaveMultipleResult struct { // SaveResult reports what SaveOpportunity changed. type SaveResult struct { - // OpportunityUpdated is true if the opportunity row and its classifications were - // written. It is false when every release in the save was older than the + // OpportunityUpdated is true if the opportunity row and its classifications + // were written. It is false when every release in the save was older than the // one the stored opportunity was last updated from. OpportunityUpdated bool // NewReleases is the number of releases that were not already stored. @@ -58,6 +76,7 @@ type SaveResult struct { // MarkFilter filters ListOpportunities on one of an opportunity's marks. type MarkFilter int +// Values of MarkFilter. const ( // MarkAny does not filter. It is the zero value. MarkAny MarkFilter = iota @@ -68,8 +87,8 @@ const ( MarkUnset ) -// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued fields do not -// filter. +// OpportunityFilter selects opportunities for ListOpportunities. Zero-valued +// fields do not filter. type OpportunityFilter struct { Status string // ClassificationCode matches opportunities with a classification of this code @@ -117,14 +136,18 @@ func prepareOpportunity(opportunity *types.Opportunity) error { opportunity.Marks.UUID = opportunity.UUID if err := opportunity.Marks.Validate(); err != nil { - return fmt.Errorf("store: opportunity %s: %w", opportunity.UUID, err) + return fmt.Errorf("opportunity %s: %w", opportunity.UUID, err) } latest := opportunity.LatestReleaseDate for i := range opportunity.Releases { release := &opportunity.Releases[i] if release.ReleaseID == "" { - return fmt.Errorf("store: opportunity %s: release %d has no release id", opportunity.UUID, i) + return fmt.Errorf( + "store: opportunity %s: release %d has no release id", + opportunity.UUID, + i, + ) } release.UUID = opportunity.UUID release.ReleaseDate = release.ReleaseDate.UTC() @@ -138,7 +161,9 @@ func prepareOpportunity(opportunity *types.Opportunity) error { } if latest.IsZero() { - return fmt.Errorf("store: opportunity %s has no release date", opportunity.UUID) + return fmt.Errorf( + "store: opportunity %s has no release date", opportunity.UUID, + ) } opportunity.LatestReleaseDate = latest.UTC() @@ -188,12 +213,15 @@ func Open(ctx context.Context, path string, opts ...Option) (*Store, error) { if err != nil { return nil, err } + if err := sqldb.PingContext(ctx); err != nil { return nil, errors.Join(err, sqldb.Close()) } + if err := prepareSchema(ctx, sqldb, settings); err != nil { return nil, errors.Join(err, sqldb.Close()) } + return &Store{ db: sqlx.NewDb(sqldb, "sqlite3"), }, nil @@ -207,10 +235,10 @@ func (s *Store) Close() error { return s.db.Close() } // reads fail loudly ("missing destination name ...") until it is mapped, // instead of being silently ignored. // -// opportunityRow gives the opportunities table's internal id column a home, so it stays -// out of types.Opportunity. sqlx maps the embedded Opportunity's fields as if declared -// here. The other tables have no internal columns and read straight into -// internal/types. +// opportunityRow gives the opportunities table's internal id column a home, so +// it stays out of types.Opportunity. sqlx maps the embedded Opportunity's +// fields as if declared here. The other tables have no internal columns and +// read straight into internal/types. type opportunityRow struct { ID int64 `db:"id"` types.Opportunity @@ -230,8 +258,7 @@ type opportunityRow struct { // opportunity is normalised in place (see prepareOpportunity), and when it is // written its classifications' IDs are filled in. func (s *Store) SaveOpportunity( - ctx context.Context, - opportunity *types.Opportunity, + ctx context.Context, opportunity *types.Opportunity, ) (SaveResult, error) { if err := prepareOpportunity(opportunity); err != nil { return SaveResult{}, err @@ -243,14 +270,16 @@ func (s *Store) SaveOpportunity( } defer tx.Rollback() //nolint:errcheck // no-op after Commit - res, _, err := saveOpportunity(ctx, tx, opportunity) + result, _, err := saveOpportunity(ctx, tx, opportunity) if err != nil { return SaveResult{}, err } + if err := tx.Commit(); err != nil { return SaveResult{}, err } - return res, nil + + return result, nil } // SaveOpportunities stores each of opportunities and its releases, all in one @@ -282,28 +311,29 @@ func (s *Store) SaveOpportunities( } defer tx.Rollback() //nolint:errcheck // no-op after Commit - allRes := SaveMultipleResult{ + total := SaveMultipleResult{ OpportunityResults: make([]SaveResult, len(opportunities)), } for i, opportunity := range opportunities { - res, isNew, err := saveOpportunity(ctx, tx, opportunity) + result, isNew, err := saveOpportunity(ctx, tx, opportunity) if err != nil { return SaveMultipleResult{}, err } - allRes.OpportunityResults[i] = res - allRes.NewReleases += res.NewReleases + total.OpportunityResults[i] = result + total.NewReleases += result.NewReleases switch { case isNew: - allRes.NewOpportunities++ - case res.OpportunityUpdated: - allRes.UpdatedOpportunities++ + total.NewOpportunities++ + case result.OpportunityUpdated: + total.UpdatedOpportunities++ } } if err := tx.Commit(); err != nil { return SaveMultipleResult{}, err } - return allRes, nil + + return total, nil } // saveOpportunity does the work of SaveOpportunity and SaveOpportunities for @@ -311,28 +341,31 @@ func (s *Store) SaveOpportunities( // not already stored. func saveOpportunity( ctx context.Context, tx *sqlx.Tx, opportunity *types.Opportunity, -) (res SaveResult, isNew bool, err error) { - var stored time.Time - err = tx.GetContext(ctx, &stored, selectOpportunityLatestReleaseDate, opportunity.UUID) +) (SaveResult, bool, error) { + var ( + result SaveResult + isNew bool + stored time.Time + ) + err := tx.GetContext( + ctx, &stored, selectOpportunityLatestReleaseDate, opportunity.UUID, + ) switch { case errors.Is(err, sql.ErrNoRows): isNew = true - res.OpportunityUpdated = true + result.OpportunityUpdated = true case err != nil: return SaveResult{}, false, fmt.Errorf( - "store: reading opportunity %s: %w", - opportunity.UUID, - err, + "reading opportunity %s: %w", opportunity.UUID, err, ) default: - res.OpportunityUpdated = !opportunity.LatestReleaseDate.Before(stored) + result.OpportunityUpdated = !opportunity.LatestReleaseDate.Before(stored) } - if res.OpportunityUpdated { + + if result.OpportunityUpdated { if err := writeOpportunity(ctx, tx, opportunity); err != nil { return SaveResult{}, false, fmt.Errorf( - "store: writing opportunity %s: %w", - opportunity.UUID, - err, + "writing opportunity %s: %w", opportunity.UUID, err, ) } } @@ -343,10 +376,11 @@ func saveOpportunity( return SaveResult{}, false, err } if inserted { - res.NewReleases++ + result.NewReleases++ } } - return res, isNew, nil + + return result, isNew, nil } // insertReleaseWithTags stores release and its tags unless it is already @@ -357,77 +391,96 @@ func insertReleaseWithTags( result, err := tx.NamedExecContext(ctx, insertRelease, release) if err != nil { return false, fmt.Errorf( - "store: inserting release %s/%s: %w", + "inserting release %s/%s: %w", release.UUID, release.ReleaseID, err, ) } + if affected, err := result.RowsAffected(); err != nil || affected == 0 { - return false, err // n == 0: already stored + return false, err // affected == 0: already stored } + for _, tag := range release.Tags { if _, err := tx.NamedExecContext(ctx, insertReleaseTag, tag); err != nil { return false, fmt.Errorf( - "store: inserting tag for %s/%s: %w", release.UUID, release.ReleaseID, err, + "inserting tag for %s/%s: %w", + release.UUID, + release.ReleaseID, + err, ) } } + return true, nil } -// writeOpportunity upserts opportunity's row, gives it a marks row if it has none -// (existing marks are kept), and replaces its classifications, filling in +// writeOpportunity upserts opportunity's row, gives it a marks row if it has +// none (existing marks are kept), and replaces its classifications, filling in // their IDs. func writeOpportunity( - ctx context.Context, - tx *sqlx.Tx, - opportunity *types.Opportunity, + ctx context.Context, tx *sqlx.Tx, opportunity *types.Opportunity, ) (err error) { - if _, err := tx.NamedExecContext(ctx, upsertOpportunity, opportunity); err != nil { - return err - } - if _, err := tx.NamedExecContext(ctx, insertOpportunityMarks, opportunity.Marks); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, deleteOpportunityClassifications, opportunity.UUID); err != nil { + _, err = tx.NamedExecContext(ctx, upsertOpportunity, opportunity) + if err != nil { return err } - statement, err := tx.PrepareNamedContext(ctx, upsertClassificationCodeReturningID) + _, err = tx.NamedExecContext(ctx, insertOpportunityMarks, opportunity.Marks) + if err != nil { + return err + } + + _, err = tx.ExecContext( + ctx, deleteOpportunityClassifications, opportunity.UUID, + ) + if err != nil { + return err + } + + statement, err := tx.PrepareNamedContext( + ctx, upsertClassificationCodeReturningID, + ) if err != nil { return err } defer func() { err = errors.Join(err, statement.Close()) }() + for i := range opportunity.Classifications { classification := &opportunity.Classifications[i] - if err := statement.GetContext(ctx, &classification.ID, classification); err != nil { + + err = statement.GetContext(ctx, &classification.ID, classification) + if err != nil { return err } - if _, err := tx.ExecContext( - ctx, - insertOpportunityClassification, - opportunity.UUID, - classification.ID, - ); err != nil { + + _, err = tx.ExecContext( + ctx, insertOpportunityClassification, opportunity.UUID, classification.ID, + ) + if err != nil { return err } } + return nil } // SetMarks replaces the marks of the opportunity marks.UUID, or returns // ErrNotFound. Marks that fail marks.Validate (e.g. types.ErrWonAndLost) are // not stored. -func (s *Store) SetMarks(ctx context.Context, marks types.OpportunityMarks) error { +func (s *Store) SetMarks( + ctx context.Context, marks types.OpportunityMarks, +) error { if marks.UUID == "" { return errors.New("store: marks have no UUID") } if err := marks.Validate(); err != nil { - return fmt.Errorf("store: opportunity %s: %w", marks.UUID, err) + return fmt.Errorf("opportunity %s: %w", marks.UUID, err) } + _, err := s.db.NamedExecContext(ctx, upsertOpportunityMarks, marks) if isForeignKeyError(err) { return ErrNotFound @@ -439,12 +492,15 @@ func (s *Store) SetMarks(ctx context.Context, marks types.OpportunityMarks) erro // breaks a foreign key, e.g. one referring to a missing opportunity. func isForeignKeyError(err error) bool { var sqliteErr sqlite3.Error - return errors.As(err, &sqliteErr) && sqliteErr.ExtendedCode == sqlite3.ErrConstraintForeignKey + return errors.As(err, &sqliteErr) && + sqliteErr.ExtendedCode == sqlite3.ErrConstraintForeignKey } // GetOpportunity returns the opportunity with its classifications, marks and // all of its releases (with tags), or ErrNotFound. -func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportunity, error) { +func (s *Store) GetOpportunity( + ctx context.Context, uuid string, +) (*types.Opportunity, error) { var row opportunityRow err := s.db.GetContext(ctx, &row, selectOpportunityByUUID, uuid) if errors.Is(err, sql.ErrNoRows) { @@ -453,6 +509,7 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu if err != nil { return nil, err } + opportunities := []types.Opportunity{row.Opportunity} if err := s.loadClassifications(ctx, opportunities); err != nil { return nil, err @@ -460,9 +517,11 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu if err := s.loadMarks(ctx, opportunities); err != nil { return nil, err } + if opportunities[0].Releases, err = s.ListReleases(ctx, uuid); err != nil { return nil, err } + return &opportunities[0], nil } @@ -470,29 +529,32 @@ func (s *Store) GetOpportunity(ctx context.Context, uuid string) (*types.Opportu // classifications and marks but without releases (use ListReleases or // GetOpportunity for those). func (s *Store) ListOpportunities( - ctx context.Context, - filter OpportunityFilter, + ctx context.Context, filter OpportunityFilter, ) ([]types.Opportunity, error) { query, args := listOpportunitiesQuery(filter) var rows []opportunityRow if err := s.db.SelectContext(ctx, &rows, query, args...); err != nil { return nil, err } + opportunities := make([]types.Opportunity, len(rows)) for i, row := range rows { opportunities[i] = row.Opportunity } + if err := s.loadClassifications(ctx, opportunities); err != nil { return nil, err } if err := s.loadMarks(ctx, opportunities); err != nil { return nil, err } + return opportunities, nil } -// listOpportunitiesQuery assembles the ListOpportunities query from the fragments in -// queries.go, with one argument per filter that is set (none for mark filters). +// listOpportunitiesQuery assembles the ListOpportunities query from the +// fragments in queries.go, with one argument per filter that is set (none for +// mark filters). func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { var where []string var args []any @@ -500,6 +562,7 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { where = append(where, clause) args = append(args, argument) } + if filter.Status != "" { add(filterOpportunityStatus, filter.Status) } @@ -515,35 +578,19 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { if !filter.ReleasedBefore.IsZero() { add(filterOpportunityReleasedBefore, filter.ReleasedBefore.UTC()) } - for _, mark := range []struct { + + marks := []struct { filter MarkFilter clause string }{ - { - filter: filter.Hidden, - clause: filterOpportunityMarkHidden, - }, - { - filter: filter.Skipped, - clause: filterOpportunityMarkSkipped, - }, - { - filter: filter.Interested, - clause: filterOpportunityMarkInterested, - }, - { - filter: filter.BidSubmitted, - clause: filterOpportunityMarkBidSubmitted, - }, - { - filter: filter.Won, - clause: filterOpportunityMarkWon, - }, - { - filter: filter.Lost, - clause: filterOpportunityMarkLost, - }, - } { + {filter: filter.Hidden, clause: filterOpportunityMarkHidden}, + {filter: filter.Skipped, clause: filterOpportunityMarkSkipped}, + {filter: filter.Interested, clause: filterOpportunityMarkInterested}, + {filter: filter.BidSubmitted, clause: filterOpportunityMarkBidSubmitted}, + {filter: filter.Won, clause: filterOpportunityMarkWon}, + {filter: filter.Lost, clause: filterOpportunityMarkLost}, + } + for _, mark := range marks { switch mark.filter { case MarkSet: where = append(where, mark.clause) @@ -557,14 +604,19 @@ func listOpportunitiesQuery(filter OpportunityFilter) (string, []any) { query += "\nWHERE " + strings.Join(where, "\n\tAND ") } query += listOpportunitiesOrderAndPage + return query, append(args, filter.limit(), filter.Offset) } -// loadClassifications fills in Classifications for all opportunities with one query. -func (s *Store) loadClassifications(ctx context.Context, opportunities []types.Opportunity) error { +// loadClassifications fills in Classifications for all opportunities with one +// query. +func (s *Store) loadClassifications( + ctx context.Context, opportunities []types.Opportunity, +) error { if len(opportunities) == 0 { return nil } + uuids := make([]string, len(opportunities)) byUUID := make(map[string]*types.Opportunity, len(opportunities)) for i := range opportunities { @@ -576,27 +628,36 @@ func (s *Store) loadClassifications(ctx context.Context, opportunities []types.O if err != nil { return err } + // opportunity_uuid says which opportunity each row belongs to. var rows []struct { OpportunityUUID string `db:"opportunity_uuid"` types.ClassificationCode } - if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil { + err = s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...) + if err != nil { return err } - for _, releaseRow := range rows { - opportunity := byUUID[releaseRow.OpportunityUUID] - opportunity.Classifications = append(opportunity.Classifications, releaseRow.ClassificationCode) + + for _, classificationRow := range rows { + opportunity := byUUID[classificationRow.OpportunityUUID] + opportunity.Classifications = append( + opportunity.Classifications, classificationRow.ClassificationCode, + ) } + return nil } // loadMarks fills in Marks for all opportunities with one query. An // opportunity with no marks row gets zero marks carrying its UUID. -func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity) error { +func (s *Store) loadMarks( + ctx context.Context, opportunities []types.Opportunity, +) error { if len(opportunities) == 0 { return nil } + uuids := make([]string, len(opportunities)) byUUID := make(map[string]*types.Opportunity, len(opportunities)) for i := range opportunities { @@ -611,42 +672,55 @@ func (s *Store) loadMarks(ctx context.Context, opportunities []types.Opportunity if err != nil { return err } + var rows []types.OpportunityMarks - if err := s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...); err != nil { + err = s.db.SelectContext(ctx, &rows, s.db.Rebind(query), args...) + if err != nil { return err } + for _, row := range rows { byUUID[row.UUID].Marks = row } + return nil } // ListReleases returns a opportunity's releases with their tags, oldest first. -func (s *Store) ListReleases(ctx context.Context, uuid string) ([]types.OpportunityRelease, error) { +func (s *Store) ListReleases( + ctx context.Context, uuid string, +) ([]types.OpportunityRelease, error) { releases := []types.OpportunityRelease{} - if err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid); err != nil { + err := s.db.SelectContext(ctx, &releases, selectReleasesByOpportunity, uuid) + if err != nil { return nil, err } + byID := make(map[string]*types.OpportunityRelease, len(releases)) for i := range releases { byID[releases[i].ReleaseID] = &releases[i] } var tags []types.OpportunityReleaseTag - if err := s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid); err != nil { + err = s.db.SelectContext(ctx, &tags, selectReleaseTagsByOpportunity, uuid) + if err != nil { return nil, err } + for _, tag := range tags { if release := byID[tag.ReleaseID]; release != nil { release.Tags = append(release.Tags, tag) } } + return releases, nil } // ReleaseExists reports whether a release is already stored, so a fetcher // can skip converting it. -func (s *Store) ReleaseExists(ctx context.Context, uuid, releaseID string) (bool, error) { +func (s *Store) ReleaseExists( + ctx context.Context, uuid, releaseID string, +) (bool, error) { var exists bool err := s.db.GetContext(ctx, &exists, selectReleaseExists, uuid, releaseID) return exists, err diff --git a/internal/store/store_test.go b/internal/store/store_test.go index c383787..59bca09 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -30,7 +30,11 @@ func day(d int) time.Time { return time.Date(2026, time.March, d, 9, 30, 0, 0, time.UTC) } -func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *types.Opportunity { +func opportunity( + uuid, title string, + releaseDay int, + releaseIDs ...string, +) *types.Opportunity { start, end := day(20), day(28) t := &types.Opportunity{ UUID: uuid, @@ -39,7 +43,7 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type Status: "active", Description: "A description", BuyerName: "Example Council", - BuyerId: "GB-EXAMPLE", + BuyerID: "GB-EXAMPLE", MainCategory: "services", ProcurementMethod: "open", MultiLot: true, @@ -49,8 +53,16 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type GrossValue: 125000.50, HasDocuments: true, Classifications: []types.ClassificationCode{ - {IsCPV: true, Code: "72000000", Classification: "IT services"}, - {IsCPV: true, Code: "03000000", Classification: "Agricultural products"}, + { + IsCPV: true, + Code: "72000000", + Classification: "IT services", + }, + { + IsCPV: true, + Code: "03000000", + Classification: "Agricultural products", + }, }, // No UUID: SaveOpportunity must fill it in. Marks: types.OpportunityMarks{ @@ -58,18 +70,29 @@ func opportunity(uuid, title string, releaseDay int, releaseIDs ...string) *type BidSubmitted: true, }, } + for i, id := range releaseIDs { t.Releases = append(t.Releases, types.OpportunityRelease{ ReleaseID: id, ReleaseDate: day(releaseDay).Add(time.Duration(i) * time.Minute), JSON: `{"id":"` + id + `"}`, - Tags: []types.OpportunityReleaseTag{{Tag: "opportunity"}, {Tag: "planning"}}, + Tags: []types.OpportunityReleaseTag{ + { + Tag: "opportunity", + }, + { + Tag: "planning", + }, + }, }) } + return t } -func mustSave(t *testing.T, s *store.Store, td *types.Opportunity) store.SaveResult { +func mustSave( + t *testing.T, s *store.Store, td *types.Opportunity, +) store.SaveResult { t.Helper() res, err := s.SaveOpportunity(context.Background(), td) if err != nil { @@ -116,24 +139,30 @@ func TestSaveAndGetRoundTrip(t *testing.T) { } got := mustGet(t, s, "ocds-1") - if got.Title != in.Title || got.BuyerName != in.BuyerName || got.BuyerId != in.BuyerId || - got.MainCategory != in.MainCategory || got.ProcurementMethod != in.ProcurementMethod || - got.UUIDType != "ocid" || got.Status != "active" || got.Description != in.Description || - !got.MultiLot || !got.SMESuitable || got.VCSESuitable || !got.HasDocuments || - got.GrossValue != in.GrossValue { + if got.Title != in.Title || got.BuyerName != in.BuyerName || + got.BuyerID != in.BuyerID || got.MainCategory != in.MainCategory || + got.ProcurementMethod != in.ProcurementMethod || got.UUIDType != "ocid" || + got.Status != "active" || got.Description != in.Description || + !got.MultiLot || !got.SMESuitable || got.VCSESuitable || + !got.HasDocuments || got.GrossValue != in.GrossValue { t.Errorf("opportunity fields differ:\n got %+v\nwant %+v", got, in) } - if got.StartDate == nil || !got.StartDate.Equal(day(20)) || got.EndDate == nil || - !got.EndDate.Equal(day(28)) { + if got.StartDate == nil || !got.StartDate.Equal(day(20)) || + got.EndDate == nil || !got.EndDate.Equal(day(28)) { t.Errorf("dates: got %v / %v", got.StartDate, got.EndDate) } if want := day(1).Add(time.Minute); !got.LatestReleaseDate.Equal(want) { t.Errorf("LatestReleaseDate = %v, want %v", got.LatestReleaseDate, want) } - if len(got.Classifications) != 2 || got.Classifications[0].Code != "03000000" || - got.Classifications[1].Code != "72000000" || got.Classifications[0].ID == 0 { - t.Errorf("classifications: got %+v (leading zero must survive)", got.Classifications) + if len(got.Classifications) != 2 || + got.Classifications[0].Code != "03000000" || + got.Classifications[1].Code != "72000000" || + got.Classifications[0].ID == 0 { + t.Errorf( + "classifications: got %+v (leading zero must survive)", + got.Classifications, + ) } wantMarks := types.OpportunityMarks{ @@ -153,9 +182,8 @@ func TestSaveAndGetRoundTrip(t *testing.T) { !r.ReleaseDate.Equal(day(1)) { t.Errorf("release: got %+v", r) } - if len(r.Tags) != 2 || - r.Tags[0].Tag != "opportunity" || r.Tags[1].Tag != "planning" || - r.Tags[0].ReleaseID != "r1" || + if len(r.Tags) != 2 || r.Tags[0].Tag != "opportunity" || + r.Tags[1].Tag != "planning" || r.Tags[0].ReleaseID != "r1" || r.Tags[0].UUID != "ocds-1" { t.Errorf("tags: got %+v", r.Tags) } @@ -178,7 +206,9 @@ func TestReleasesAreWriteOnce(t *testing.T) { again := opportunity("ocds-1", "Title", 1, "r1") again.Releases[0].JSON = `{"changed":true}` - again.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}} + again.Releases[0].Tags = []types.OpportunityReleaseTag{{ + Tag: "award", + }} if res := mustSave(t, s, again); res.NewReleases != 0 { t.Errorf("re-save: NewReleases = %d, want 0", res.NewReleases) } @@ -206,7 +236,9 @@ func TestOlderReleaseDoesNotOverwriteOpportunity(t *testing.T) { t.Errorf("opportunity was overwritten by an older release: %+v", got) } if len(got.Releases) != 2 || got.Releases[0].ReleaseID != "r1" { - t.Errorf("older release should still be stored, oldest first: %+v", got.Releases) + t.Errorf( + "older release should still be stored, oldest first: %+v", got.Releases, + ) } } @@ -220,7 +252,11 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) { newer := opportunity("ocds-1", "Website redesign", 2, "r2") newer.Status = "complete" newer.Classifications = []types.ClassificationCode{ - {IsCPV: true, Code: "72000000", Classification: "IT services (amended)"}, + { + IsCPV: true, + Code: "72000000", + Classification: "IT services (amended)", + }, } if res := mustSave(t, s, newer); !res.OpportunityUpdated { t.Fatalf("newer save not applied: %+v", res) @@ -246,7 +282,12 @@ func TestNewerReleaseUpdatesOpportunityAndSearchIndex(t *testing.T) { t.Fatal(err) } if len(ts) != want { - t.Errorf("search %q after title change: %d results, want %d", title, len(ts), want) + t.Errorf( + "search %q after title change: %d results, want %d", + title, + len(ts), + want, + ) } } } @@ -257,7 +298,8 @@ func TestClassificationRowsAreShared(t *testing.T) { b := opportunity("ocds-b", "B", 1, "r1") mustSave(t, s, a) mustSave(t, s, b) - if a.Classifications[0].ID == 0 || a.Classifications[0].ID != b.Classifications[0].ID { + if a.Classifications[0].ID == 0 || + a.Classifications[0].ID != b.Classifications[0].ID { t.Errorf( "identical classifications should share an ID: %d vs %d", a.Classifications[0].ID, @@ -267,7 +309,11 @@ func TestClassificationRowsAreShared(t *testing.T) { c := opportunity("ocds-c", "C", 1, "r1") c.Classifications = []types.ClassificationCode{ - {IsCPV: false, Code: "72000000", Classification: "IT services"}, + { + IsCPV: false, + Code: "72000000", + Classification: "IT services", + }, } mustSave(t, s, c) if c.Classifications[0].ID == a.Classifications[0].ID { @@ -279,13 +325,17 @@ func TestSameReleaseIDInDifferentOpportunities(t *testing.T) { s := openStore(t) a := opportunity("ocds-a", "A", 1, "001") b := opportunity("ocds-b", "B", 1, "001") - b.Releases[0].Tags = []types.OpportunityReleaseTag{{Tag: "award"}} + b.Releases[0].Tags = []types.OpportunityReleaseTag{{ + Tag: "award", + }} + if res := mustSave(t, s, a); res.NewReleases != 1 { t.Fatal(res) } if res := mustSave(t, s, b); res.NewReleases != 1 { t.Fatalf("release id reused under another OCID must be stored: %+v", res) } + got := mustGet(t, s, "ocds-b").Releases if len(got) != 1 || len(got[0].Tags) != 1 || got[0].Tags[0].Tag != "award" { t.Errorf("tags leaked between opportunities: %+v", got) @@ -300,7 +350,11 @@ func TestListOpportunitiesFilters(t *testing.T) { roads := opportunity("ocds-2", "Road resurfacing works", 2, "r1") roads.Status = "complete" roads.Classifications = []types.ClassificationCode{ - {IsCPV: true, Code: "45233142", Classification: "Road-repair works"}, + { + IsCPV: true, + Code: "45233142", + Classification: "Road-repair works", + }, } hosting := opportunity("ocds-3", "Managed cloud hosting", 3, "r1") for _, td := range []*types.Opportunity{cloud, roads, hosting} { @@ -312,7 +366,11 @@ func TestListOpportunitiesFilters(t *testing.T) { f store.OpportunityFilter want []string }{ - {"all, newest first", store.OpportunityFilter{}, []string{"ocds-3", "ocds-2", "ocds-1"}}, + { + "all, newest first", + store.OpportunityFilter{}, + []string{"ocds-3", "ocds-2", "ocds-1"}, + }, {"status", store.OpportunityFilter{ Status: "complete", }, []string{"ocds-2"}}, @@ -385,9 +443,13 @@ func TestListOpportunitiesFilters(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got) != 1 || len(got[0].Classifications) != 1 || got[0].Releases != nil || - got[0].Marks.UUID != "ocds-2" || !got[0].Marks.Interested { - t.Errorf("list should include classifications and marks and omit releases: %+v", got) + if len(got) != 1 || len(got[0].Classifications) != 1 || + got[0].Releases != nil || got[0].Marks.UUID != "ocds-2" || + !got[0].Marks.Interested { + t.Errorf( + "list should include classifications and marks and omit releases: %+v", + got, + ) } } @@ -402,26 +464,28 @@ func TestReleaseExistsAndListReleases(t *testing.T) { }{{"ocds-1", "r1", true}, {"ocds-1", "r3", false}, {"ocds-2", "r1", false}} { got, err := s.ReleaseExists(ctx, tc.uuid, tc.id) if err != nil || got != tc.want { - t.Errorf("ReleaseExists(%s, %s) = %v, %v; want %v", tc.uuid, tc.id, got, err, tc.want) + t.Errorf( + "ReleaseExists(%s, %s) = %v, %v; want %v", + tc.uuid, + tc.id, + got, + err, + tc.want, + ) } } rs, err := s.ListReleases(ctx, "ocds-1") - if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" || rs[1].ReleaseID != "r2" || - len(rs[1].Tags) != 2 { + if err != nil || len(rs) != 2 || rs[0].ReleaseID != "r1" || + rs[1].ReleaseID != "r2" || len(rs[1].Tags) != 2 { t.Errorf("ListReleases: %+v, %v", rs, err) } } func TestGetOpportunityNotFound(t *testing.T) { s := openStore(t) - if _, err := s.GetOpportunity( - context.Background(), - "missing", - ); !errors.Is( - err, - store.ErrNotFound, - ) { + _, err := s.GetOpportunity(context.Background(), "missing") + if !errors.Is(err, store.ErrNotFound) { t.Errorf("err = %v, want ErrNotFound", err) } } @@ -473,8 +537,12 @@ func TestResaveLoadedOpportunity(t *testing.T) { ids := []int64{loaded.Classifications[0].ID, loaded.Classifications[1].ID} loaded.Title = "Amended" loaded.Releases = append(loaded.Releases, types.OpportunityRelease{ - ReleaseID: "r2", ReleaseDate: day(2), JSON: `{}`, - Tags: []types.OpportunityReleaseTag{{Tag: "opportunityAmendment"}}, + ReleaseID: "r2", + ReleaseDate: day(2), + JSON: `{}`, + Tags: []types.OpportunityReleaseTag{{ + Tag: "opportunityAmendment", + }}, }) res := mustSave(t, s, loaded) if !res.OpportunityUpdated || res.NewReleases != 1 { @@ -482,10 +550,12 @@ func TestResaveLoadedOpportunity(t *testing.T) { } got := mustGet(t, s, "ocds-1") - if got.Title != "Amended" || len(got.Releases) != 2 || !got.LatestReleaseDate.Equal(day(2)) { + if got.Title != "Amended" || len(got.Releases) != 2 || + !got.LatestReleaseDate.Equal(day(2)) { t.Errorf("resave not applied: %+v", got) } - if got.Classifications[0].ID != ids[0] || got.Classifications[1].ID != ids[1] { + if got.Classifications[0].ID != ids[0] || + got.Classifications[1].ID != ids[1] { t.Errorf( "classification IDs changed on resave: %v -> %d, %d", ids, @@ -527,10 +597,7 @@ func TestSetMarksErrors(t *testing.T) { UUID: "missing", Hidden: true, }, - ); !errors.Is( - err, - store.ErrNotFound, - ) { + ); !errors.Is(err, store.ErrNotFound) { t.Errorf("unknown opportunity: err = %v, want ErrNotFound", err) } if err := s.SetMarks(ctx, types.OpportunityMarks{ @@ -565,7 +632,9 @@ func TestSaveKeepsExistingMarks(t *testing.T) { newest := opportunity("ocds-1", "Amended again", 3, "r3") newest.Marks = types.OpportunityMarks{} - if _, err := s.SaveOpportunities(ctx, []*types.Opportunity{newest}); err != nil { + if _, err := s.SaveOpportunities( + ctx, []*types.Opportunity{newest}, + ); err != nil { t.Fatal(err) } if got := mustGet(t, s, "ocds-1").Marks; got != want { @@ -580,9 +649,13 @@ func TestSaveOpportunitiesStoresMarks(t *testing.T) { b.Marks = types.OpportunityMarks{ Skipped: true, } - if _, err := s.SaveOpportunities(context.Background(), []*types.Opportunity{a, b}); err != nil { + + if _, err := s.SaveOpportunities( + context.Background(), []*types.Opportunity{a, b}, + ); err != nil { t.Fatal(err) } + for uuid, want := range map[string]types.OpportunityMarks{ "ocds-a": {UUID: "ocds-a", Interested: true, BidSubmitted: true}, "ocds-b": {UUID: "ocds-b", Skipped: true}, @@ -610,7 +683,10 @@ func TestListOpportunitiesMarkFilters(t *testing.T) { Lost: true, }) mustSetMarks(t, s, types.OpportunityMarks{ - UUID: "ocds-4", Interested: true, BidSubmitted: true, Won: true, + UUID: "ocds-4", + Interested: true, + BidSubmitted: true, + Won: true, }) set, unset := store.MarkSet, store.MarkUnset @@ -619,7 +695,11 @@ func TestListOpportunitiesMarkFilters(t *testing.T) { f store.OpportunityFilter want []string }{ - {"any", store.OpportunityFilter{}, []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}}, + { + "any", + store.OpportunityFilter{}, + []string{"ocds-4", "ocds-3", "ocds-2", "ocds-1"}, + }, {"hidden", store.OpportunityFilter{ Hidden: set, }, []string{"ocds-2"}}, @@ -698,8 +778,8 @@ func TestListOpportunitiesMarkFilters(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost || !got[2].Marks.Hidden || - got[3].Marks.Hidden || !got[3].Marks.Interested { + if len(got) != 4 || !got[0].Marks.Won || !got[1].Marks.Lost || + !got[2].Marks.Hidden || got[3].Marks.Hidden || !got[3].Marks.Interested { t.Errorf("list marks: %+v", got) } } @@ -727,19 +807,20 @@ func TestWonAndLostAreExclusive(t *testing.T) { Won: true, Lost: true, } - if _, err := s.SaveOpportunity(ctx, in); !errors.Is(err, types.ErrWonAndLost) { + _, err := s.SaveOpportunity(ctx, in) + if !errors.Is(err, types.ErrWonAndLost) { t.Errorf("SaveOpportunity(won and lost): err = %v, want ErrWonAndLost", err) } - if _, err := s.SaveOpportunities( - ctx, - []*types.Opportunity{in}, - ); !errors.Is( - err, - types.ErrWonAndLost, - ) { - t.Errorf("SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err) + + _, err = s.SaveOpportunities(ctx, []*types.Opportunity{in}) + if !errors.Is(err, types.ErrWonAndLost) { + t.Errorf( + "SaveOpportunities(won and lost): err = %v, want ErrWonAndLost", err, + ) } - if _, err := s.GetOpportunity(ctx, "ocds-2"); !errors.Is(err, store.ErrNotFound) { + + _, err = s.GetOpportunity(ctx, "ocds-2") + if !errors.Is(err, store.ErrNotFound) { t.Errorf("rejected opportunity was stored: err = %v", err) } diff --git a/internal/types/opportunities.go b/internal/types/opportunities.go index f2585b9..c233890 100644 --- a/internal/types/opportunities.go +++ b/internal/types/opportunities.go @@ -17,7 +17,7 @@ type Opportunity struct { Description string `db:"description" json:"description"` BuyerName string `db:"buyer_name" json:"buyer_name"` Classifications []ClassificationCode `db:"-" json:"classifications"` - BuyerId string `db:"buyer_id" json:"buyer_id"` + BuyerID string `db:"buyer_id" json:"buyer_id"` MainCategory string `db:"main_category" json:"main_category"` ProcurementMethod string `db:"procurement_method" json:"procurement_method"` MultiLot bool `db:"multi_lot" json:"multi_lot"` @@ -53,7 +53,7 @@ type OpportunityReleaseTag struct { // ErrWonAndLost is returned by OpportunityMarks.Validate when both Won and // Lost are set. -var ErrWonAndLost = errors.New("an opportunity cannot be both won and lost") +var ErrWonAndLost = errors.New("types: an opportunity cannot be both won and lost") // OpportunityMarks are the user's own marks on an opportunity, kept apart // from the fetched data so saving a newer release never changes them. The diff --git a/internal/util/bytesize.go b/internal/util/bytesize.go index c42557c..9868245 100644 --- a/internal/util/bytesize.go +++ b/internal/util/bytesize.go @@ -83,15 +83,19 @@ func ParseByteSize(text string) (ByteSize, error) { return ByteSizeNoLimit, nil } if strings.HasPrefix(trimmed, "-") { - return 0, fmt.Errorf("byte size %q is negative; only -1 (no limit) is allowed", text) + return 0, fmt.Errorf( + "util: byte size %q is negative; only -1 (no limit) is allowed", text, + ) } + parsed, err := humanize.ParseBytes(text) if err != nil { return 0, err } if parsed > math.MaxInt64 { - return 0, fmt.Errorf("byte size %q is too large", text) + return 0, fmt.Errorf("util: byte size %q is too large", text) } + return ByteSize(parsed), nil } diff --git a/internal/util/bytesize_test.go b/internal/util/bytesize_test.go index 0861f7e..3557f32 100644 --- a/internal/util/bytesize_test.go +++ b/internal/util/bytesize_test.go @@ -35,7 +35,9 @@ func TestByteSizeNoLimit(t *testing.T) { for _, text := range []string{"-1", "-1B", "-1b", " -1 "} { size, err := ParseByteSize(text) if err != nil || size != ByteSizeNoLimit { - t.Errorf("ParseByteSize(%q) = %d, %v; want ByteSizeNoLimit", text, size, err) + t.Errorf( + "ParseByteSize(%q) = %d, %v; want ByteSizeNoLimit", text, size, err, + ) } } if got := ByteSizeNoLimit.String(); got != "-1" { @@ -44,7 +46,12 @@ func TestByteSizeNoLimit(t *testing.T) { for _, text := range []string{"-2", "-1KB", "-0.5", "-1KiB", "-100"} { size, err := ParseByteSize(text) if err == nil || !strings.Contains(err.Error(), "only -1 (no limit)") { - t.Errorf("ParseByteSize(%q) = %d, %v; want a negative size error", text, size, err) + t.Errorf( + "ParseByteSize(%q) = %d, %v; want a negative size error", + text, + size, + err, + ) } } } diff --git a/internal/util/context.go b/internal/util/context.go new file mode 100644 index 0000000..b966efc --- /dev/null +++ b/internal/util/context.go @@ -0,0 +1,19 @@ +package util + +import ( + "context" + "time" +) + +// WithTimeout returns ctx with the timeout applied, or ctx itself if the +// timeout is 0 or less (no limit). Either way the returned cancel function +// must be called. +func WithTimeout( + ctx context.Context, timeout time.Duration, +) (context.Context, context.CancelFunc) { + if timeout <= 0 { + return ctx, func() { + } + } + return context.WithTimeout(ctx, timeout) +} diff --git a/internal/util/context_test.go b/internal/util/context_test.go new file mode 100644 index 0000000..575935d --- /dev/null +++ b/internal/util/context_test.go @@ -0,0 +1,20 @@ +package util + +import ( + "context" + "testing" + "time" +) + +func TestWithTimeout(t *testing.T) { + ctx, cancel := WithTimeout(context.Background(), 0) + defer cancel() + if _, ok := ctx.Deadline(); ok { + t.Error("timeout 0 set a deadline") + } + ctx, cancel = WithTimeout(context.Background(), time.Minute) + defer cancel() + if _, ok := ctx.Deadline(); !ok { + t.Error("timeout 1m set no deadline") + } +} diff --git a/internal/util/worker.go b/internal/util/worker.go index b7029d9..d0e5608 100644 --- a/internal/util/worker.go +++ b/internal/util/worker.go @@ -11,18 +11,18 @@ import ( // DefaultWorkerShutdownTimeout is used when Worker.ShutdownTimeout is zero. const DefaultWorkerShutdownTimeout = 10 * time.Second -// ErrWorkerNoWork tells the Worker that a run succeeded but found nothing to do. -// Tasks return it, or an error wrapping it, to have the run treated as a +// ErrWorkerNoWork tells the Worker that a run succeeded but found nothing to +// do. Tasks return it, or an error wrapping it, to have the run treated as a // success (checkpoint advances) while being logged distinctly. For example: // // var ErrNoNewReleases = fmt.Errorf("%w: no new releases", ErrWorkerNoWork) -var ErrWorkerNoWork = errors.New("no work") +var ErrWorkerNoWork = errors.New("util: no work") // Task processes everything that has changed since the given checkpoint. // -// Returning nil or an error matching ErrNoWork advances the checkpoint to the -// time this run started. Any other error leaves the checkpoint unchanged, so -// the next run retries from the same point. +// Returning nil or an error matching ErrWorkerNoWork advances the checkpoint to +// the time this run started. Any other error leaves the checkpoint unchanged, +// so the next run retries from the same point. type Task func(ctx context.Context, since time.Time) error // Worker runs a Task periodically until its context is cancelled. @@ -32,7 +32,8 @@ type Worker struct { // Interval is how often Task runs. Must be greater than zero. Interval time.Duration // ShutdownTimeout is how long an in-flight Task may keep running after - // the parent context is cancelled. Zero means DefaultShutdownTimeout. + // the parent context is cancelled. 0 or less means + // DefaultWorkerShutdownTimeout. ShutdownTimeout time.Duration // Task is the work to perform on each tick. Task Task @@ -45,14 +46,16 @@ type Worker struct { // cancelled. It blocks until shutdown has finished. func (w Worker) Run(ctx context.Context, since time.Time) error { if w.Task == nil { - return fmt.Errorf("worker %q: no task configured", w.Name) + return fmt.Errorf("util: worker %q: no task configured", w.Name) } if w.Interval <= 0 { return fmt.Errorf( - "worker %q: interval must be greater than 0, got %s", - w.Name, w.Interval, + "util: worker %q: interval must be greater than 0, got %s", + w.Name, + w.Interval, ) } + timeout := w.ShutdownTimeout if timeout <= 0 { timeout = DefaultWorkerShutdownTimeout @@ -124,7 +127,11 @@ func (w Worker) execute( logger.Info("task completed", "elapsed", elapsed) return start case errors.Is(err, ErrWorkerNoWork): - logger.Info("task completed with nothing to do", "reason", err, "elapsed", elapsed) + logger.Info( + "task completed with nothing to do", + "reason", err, + "elapsed", elapsed, + ) return start case errors.Is(err, context.Canceled): logger.Warn("task cancelled", "elapsed", elapsed) @@ -142,22 +149,22 @@ func (w Worker) execute( // The result channel is buffered, so if ctx wins the race the goroutine can // still deliver its result and exit instead of blocking forever. func AwaitTask[T any](ctx context.Context, task func() (T, error)) (T, error) { - type result struct { - val T - err error + type outcome struct { + value T + err error } - results := make(chan result, 1) + results := make(chan outcome, 1) go func() { value, err := task() - results <- result{ - val: value, - err: err, + results <- outcome{ + value: value, + err: err, } }() select { case result := <-results: - return result.val, result.err + return result.value, result.err case <-ctx.Done(): var zero T return zero, ctx.Err() diff --git a/internal/util/worker_test.go b/internal/util/worker_test.go index 7d8b81d..4029e48 100644 --- a/internal/util/worker_test.go +++ b/internal/util/worker_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "log/slog" "os" "sync" @@ -14,7 +13,7 @@ import ( func TestMain(m *testing.M) { // Keep worker log lines out of the test output. - slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + slog.SetDefault(slog.New(slog.DiscardHandler)) os.Exit(m.Run()) } @@ -53,10 +52,15 @@ func TestWorkerRunCancelledBeforeStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() called := false - w := Worker{Name: "w", Interval: time.Millisecond, Task: func(context.Context, time.Time) error { - called = true - return nil - }} + w := Worker{ + Name: "w", + Interval: time.Millisecond, + Task: func(context.Context, time.Time) error { + called = true + return nil + }, + } + if err := w.Run(ctx, time.Time{}); err != nil { t.Fatal(err) } @@ -91,12 +95,14 @@ func TestWorkerRunCheckpoints(t *testing.T) { Task: func(_ context.Context, since time.Time) error { mu.Lock() defer mu.Unlock() + i := len(seen) seen = append(seen, since) starts = append(starts, time.Now()) if i == len(results)-1 { cancel() } + return results[i] }, } @@ -129,11 +135,20 @@ func TestWorkerRunCheckpoints(t *testing.T) { } } if !seen[1].After(since0) || seen[1].After(starts[1]) { - t.Fatalf("checkpoint after success = %v, want between %v and %v", seen[1], since0, starts[1]) + t.Fatalf( + "checkpoint after success = %v, want between %v and %v", + seen[1], + since0, + starts[1], + ) } // Run 3 returned ErrWorkerNoWork, so run 4 sees a newer checkpoint. if !seen[4].After(seen[3]) { - t.Fatalf("checkpoint did not advance after ErrWorkerNoWork: %v -> %v", seen[3], seen[4]) + t.Fatalf( + "checkpoint did not advance after ErrWorkerNoWork: %v -> %v", + seen[3], + seen[4], + ) } } @@ -143,7 +158,9 @@ func TestWorkerRunGracefulShutdown(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) var workErr error w := Worker{ - Name: "w", Interval: time.Hour, ShutdownTimeout: time.Second, + Name: "w", + Interval: time.Hour, + ShutdownTimeout: time.Second, Task: func(taskCtx context.Context, _ time.Time) error { cancel() time.Sleep(10 * time.Millisecond) @@ -151,6 +168,7 @@ func TestWorkerRunGracefulShutdown(t *testing.T) { return nil }, } + if err := w.Run(ctx, time.Time{}); err != nil { t.Fatal(err) } @@ -163,7 +181,9 @@ func TestWorkerRunGracefulShutdown(t *testing.T) { func TestWorkerRunShutdownTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) w := Worker{ - Name: "w", Interval: time.Hour, ShutdownTimeout: 20 * time.Millisecond, + Name: "w", + Interval: time.Hour, + ShutdownTimeout: 20 * time.Millisecond, Task: func(taskCtx context.Context, _ time.Time) error { cancel() select { @@ -174,6 +194,7 @@ func TestWorkerRunShutdownTimeout(t *testing.T) { } }, } + start := time.Now() if err := w.Run(ctx, time.Time{}); err != nil { t.Fatal(err) @@ -209,7 +230,9 @@ func TestWorkerExecute(t *testing.T) { got := w.execute(context.Background(), logger, since) if tc.advance { if got.Before(before) { - t.Fatalf("checkpoint %v not advanced to run start (>= %v)", got, before) + t.Fatalf( + "checkpoint %v not advanced to run start (>= %v)", got, before, + ) } } else if !got.Equal(since) { t.Fatalf("checkpoint = %v, want unchanged %v", got, since) @@ -261,10 +284,7 @@ func TestAwaitTaskE(t *testing.T) { func() error { return errFail }, - ); !errors.Is( - err, - errFail, - ) { + ); !errors.Is(err, errFail) { t.Fatalf("got %v, want %v", err, errFail) } @@ -278,10 +298,7 @@ func TestAwaitTaskE(t *testing.T) { <-release return nil }, - ); !errors.Is( - err, - context.DeadlineExceeded, - ) { + ); !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("got %v, want context.DeadlineExceeded", err) } } diff --git a/internal/websocket/client.go b/internal/websocket/client.go index f9a475e..5504f3e 100644 --- a/internal/websocket/client.go +++ b/internal/websocket/client.go @@ -12,6 +12,8 @@ import ( "github.com/coder/websocket" "golang.org/x/time/rate" + + "git.avsdev.uk/Go/job-search/internal/util" ) // clientSendBuffer is how many messages are queued for a client before it @@ -26,7 +28,9 @@ const logMessageLimit = 1 << 10 // *websocket.Conn, or a tcpConn for plain TCP clients (see tcp.go). type msgConn interface { Read(ctx context.Context) (websocket.MessageType, []byte, error) - Write(ctx context.Context, messageType websocket.MessageType, payload []byte) error + Write( + ctx context.Context, messageType websocket.MessageType, payload []byte, + ) error Ping(ctx context.Context) error Close(code websocket.StatusCode, reason string) error CloseNow() error @@ -49,7 +53,7 @@ type client struct { // errUnsupportedData is returned by readLoop (and a tcpConn's Write) for a // binary message. -var errUnsupportedData = errors.New("binary frames are not supported") +var errUnsupportedData = errors.New("websocket: binary frames are not supported") // newClient creates a client for conn. cancel cancels the context the client // runs with; the client calls it when it finishes or is closed. The client's @@ -59,10 +63,7 @@ func newClient( ) *client { limiter := rate.NewLimiter(rate.Inf, 0) if rule := hub.conf.MessageRateLimit; rule.Enabled && rule.PerSecond > 0 { - limiter = rate.NewLimiter( - rate.Limit(rule.PerSecond), - max(rule.Burst, 1), - ) + limiter = rate.NewLimiter(rate.Limit(rule.PerSecond), max(rule.Burst, 1)) } client := &client{ @@ -95,7 +96,7 @@ func (c *client) run(ctx context.Context) { c.sendEnvelope(Envelope{ Type: TypeWelcome, - Payload: mustRaw(map[string]any{ + Payload: rawJSON(map[string]any{ "clientId": c.id, }), }) @@ -129,16 +130,20 @@ func (c *client) readLoop(ctx context.Context) error { if err != nil { return err } + c.logMessage(ctx, "message received", data) c.lastSeen.Store(time.Now().UnixNano()) + if messageType != websocket.MessageText { return errUnsupportedData } + // Check if the client is rate limited for early discard if !c.limiter.Allow() { c.sendError("", "rate limited") continue } + // Unpack the envelope and pass to handle to process if valid var envelope Envelope if err := json.Unmarshal(data, &envelope); err != nil { @@ -157,7 +162,7 @@ func (c *client) handle(envelope Envelope) { c.sendEnvelope(Envelope{ Type: TypePong, ID: envelope.ID, - Payload: mustRaw(map[string]any{"time": time.Now().UTC()}), + Payload: rawJSON(map[string]any{"time": time.Now().UTC()}), }) case TypeEcho: @@ -180,7 +185,7 @@ func (c *client) writeLoop(ctx context.Context) { case <-ctx.Done(): return case msg := <-c.send: - writeCtx, cancel := withTimeout(ctx, c.hub.conf.WriteTimeout) + writeCtx, cancel := util.WithTimeout(ctx, c.hub.conf.WriteTimeout) err := c.conn.Write(writeCtx, websocket.MessageText, msg) cancel() if err != nil { @@ -217,7 +222,7 @@ func (c *client) pingLoop(ctx context.Context) { if time.Now().Before(lastSeen.Add(pingInterval)) { continue } - pingCtx, cancel := withTimeout(ctx, pongTimeout) + pingCtx, cancel := util.WithTimeout(ctx, pongTimeout) err := c.conn.Ping(pingCtx) cancel() if err != nil { @@ -283,29 +288,19 @@ func (c *client) logMessage(ctx context.Context, event string, data []byte) { if !logger.Enabled(ctx, slog.LevelDebug) { return } + logged, truncated := data, false if len(logged) > logMessageLimit { logged, truncated = logged[:logMessageLimit], true } + logger.LogAttrs( ctx, slog.LevelDebug, - event, + event, //nolint:sloglint // a constant message from the caller slog.String("client", c.id), slog.Int("bytes", len(data)), slog.String("message", string(logged)), slog.Bool("truncated", truncated), ) } - -// withTimeout returns ctx with the timeout applied, or ctx itself if the -// timeout is 0 or less (no limit). -func withTimeout( - ctx context.Context, timeout time.Duration, -) (context.Context, context.CancelFunc) { - if timeout <= 0 { - return ctx, func() { - } - } - return context.WithTimeout(ctx, timeout) -} diff --git a/internal/websocket/client_test.go b/internal/websocket/client_test.go index 0312708..646d872 100644 --- a/internal/websocket/client_test.go +++ b/internal/websocket/client_test.go @@ -34,7 +34,9 @@ func (f *fakeConn) Read(ctx context.Context) (ws.MessageType, []byte, error) { return 0, nil, ctx.Err() } -func (f *fakeConn) Write(context.Context, ws.MessageType, []byte) error { return nil } +func (f *fakeConn) Write(context.Context, ws.MessageType, []byte) error { + return nil +} func (f *fakeConn) Ping(context.Context) error { return nil } @@ -102,8 +104,10 @@ func TestSendEnvelopeSlowConsumer(t *testing.T) { func TestClientCloseOnce(t *testing.T) { f := newFakeConn() c, _ := newTestClient(f) + c.close(ws.StatusGoingAway, "bye") c.close(ws.StatusNormalClosure, "again") + if code := <-f.closed; code != ws.StatusGoingAway { t.Errorf("closed with %v; want going away", code) } @@ -147,12 +151,17 @@ func TestClientMessageRateLimit(t *testing.T) { }) c.limiter.Allow() if limited := !c.limiter.Allow(); limited != tt.limited { - t.Errorf("rule %+v: second message limited = %v; want %v", tt.rule, limited, tt.limited) + t.Errorf( + "rule %+v: second message limited = %v; want %v", + tt.rule, + limited, + tt.limited, + ) } } } -func TestMustRaw(t *testing.T) { +func TestRawJSON(t *testing.T) { raw := json.RawMessage(`{"a":1}`) tests := []struct { name string @@ -166,25 +175,12 @@ func TestMustRaw(t *testing.T) { }, ""}, } for _, tt := range tests { - if got := string(mustRaw(tt.v)); got != tt.want { - t.Errorf("%s: mustRaw = %q; want %q", tt.name, got, tt.want) + if got := string(rawJSON(tt.v)); got != tt.want { + t.Errorf("%s: rawJSON = %q; want %q", tt.name, got, tt.want) } } } -func TestWithTimeout(t *testing.T) { - ctx, cancel := withTimeout(context.Background(), 0) - defer cancel() - if _, ok := ctx.Deadline(); ok { - t.Error("timeout 0 set a deadline") - } - ctx, cancel = withTimeout(context.Background(), time.Minute) - defer cancel() - if _, ok := ctx.Deadline(); !ok { - t.Error("timeout 1m set no deadline") - } -} - // syncBuffer is a bytes.Buffer safe for the client's goroutines to log to. type syncBuffer struct { mu sync.Mutex @@ -202,6 +198,7 @@ func (b *syncBuffer) entries(t *testing.T) []map[string]any { t.Helper() b.mu.Lock() defer b.mu.Unlock() + var entries []map[string]any for line := range strings.Lines(b.buf.String()) { var e map[string]any @@ -210,6 +207,7 @@ func (b *syncBuffer) entries(t *testing.T) []map[string]any { } entries = append(entries, e) } + return entries } @@ -283,8 +281,9 @@ func TestLogMessageLimit(t *testing.T) { t.Fatalf("logged %d entries; want 1", len(entries)) } e := entries[0] - if e["message"] != long[:logMessageLimit] || e["bytes"] != float64(len(long)) || - e["truncated"] != true || e["client"] != c.id { + if e["message"] != long[:logMessageLimit] || + e["bytes"] != float64(len(long)) || e["truncated"] != true || + e["client"] != c.id { t.Errorf("entry = %v", e) } diff --git a/internal/websocket/http_test.go b/internal/websocket/http_test.go index 14a7f84..57ef039 100644 --- a/internal/websocket/http_test.go +++ b/internal/websocket/http_test.go @@ -38,10 +38,12 @@ func dialWS(t *testing.T, ctx context.Context, url string) *ws.Conn { t.Cleanup(func() { conn.CloseNow() }) + var e Envelope if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypeWelcome { t.Fatalf("want welcome, got %+v, %v", e, err) } + return conn } @@ -66,7 +68,8 @@ func TestHTTPMessages(t *testing.T) { t.Fatal(err) } var e Envelope - if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || e.ID != "1" { + if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || + e.ID != "1" { t.Fatalf("want pong 1, got %+v, %v", e, err) } } @@ -75,6 +78,7 @@ func TestHTTPMessages(t *testing.T) { func TestPushReachesBothTransports(t *testing.T) { h, addr := tcpHub(t, nil) url := wsHub(t, h) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() conn := dialWS(t, ctx, url) @@ -155,7 +159,8 @@ func TestHTTPMessageRateLimit(t *testing.T) { } } var e Envelope - if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || e.ID != "1" { + if err := wsjson.Read(ctx, conn, &e); err != nil || e.Type != TypePong || + e.ID != "1" { t.Fatalf("first: want pong 1, got %+v, %v", e, err) } err := wsjson.Read(ctx, conn, &e) @@ -173,11 +178,13 @@ func TestHTTPNotAnUpgrade(t *testing.T) { h := NewHub(config.JSCServerWebSocket{}, WithLogger(discardLog)) srv := httptest.NewServer(h) defer srv.Close() + res, err := http.Get(srv.URL) if err != nil { t.Fatal(err) } res.Body.Close() + if res.StatusCode < 400 { t.Errorf("status = %d; want an error", res.StatusCode) } @@ -194,6 +201,7 @@ func TestHTTPRefusedAfterShutdown(t *testing.T) { if err := h.Shutdown(context.Background()); err != nil { t.Fatal(err) } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() conn, _, err := ws.Dial(ctx, url, nil) @@ -201,6 +209,7 @@ func TestHTTPRefusedAfterShutdown(t *testing.T) { t.Fatalf("dial: %v", err) } defer conn.CloseNow() + if _, _, err := conn.Read(ctx); ws.CloseStatus(err) != ws.StatusGoingAway { t.Fatalf("read = %v; want close 1001", err) } diff --git a/internal/websocket/hub.go b/internal/websocket/hub.go index 12935a9..085c30f 100644 --- a/internal/websocket/hub.go +++ b/internal/websocket/hub.go @@ -91,7 +91,11 @@ func (h *Hub) Push(msgType string, payload any) { h.mu.RUnlock() for _, client := range slow { - h.logger().Warn("dropping slow client", "client", client.id, "ip", client.ip) + h.logger().Warn( + "dropping slow client", + "client", client.id, + "ip", client.ip, + ) go client.close(ws.StatusPolicyViolation, "slow consumer") } } @@ -166,11 +170,14 @@ func (h *Hub) maxMessageSize() int { func (h *Hub) register(client *client) bool { h.mu.Lock() defer h.mu.Unlock() + if h.closed { return false } + h.clients[client] = struct{}{} h.wg.Add(1) + return true } diff --git a/internal/websocket/hub_test.go b/internal/websocket/hub_test.go index 6366853..e8b5832 100644 --- a/internal/websocket/hub_test.go +++ b/internal/websocket/hub_test.go @@ -21,16 +21,19 @@ func pipeClient(t *testing.T, h *Hub) (net.Conn, chan struct{}) { server.Close() peer.Close() }) + done := make(chan struct{}) go func() { h.ServeTCP(context.Background(), server) close(done) }() + peer.SetReadDeadline(time.Now().Add(time.Second)) if _, err := bufio.NewReader(peer).ReadString('\n'); err != nil { t.Fatalf("welcome: %v", err) } waitCount(t, h, 1) + return peer, done } @@ -100,7 +103,12 @@ func TestMaxMessageSize(t *testing.T) { for _, tt := range tests { h := NewHub(tt.conf, WithLogger(discardLog)) if got := h.maxMessageSize(); got != tt.want { - t.Errorf("maxMessageSize(%d) = %d; want %d", tt.conf.MaxMessageSize, got, tt.want) + t.Errorf( + "maxMessageSize(%d) = %d; want %d", + tt.conf.MaxMessageSize, + got, + tt.want, + ) } } } diff --git a/internal/websocket/messages.go b/internal/websocket/messages.go index d804a26..72d67f1 100644 --- a/internal/websocket/messages.go +++ b/internal/websocket/messages.go @@ -55,15 +55,16 @@ const ( TypeError = "error" ) -// mustRaw returns value as raw JSON: value itself if it is already +// rawJSON returns value as raw JSON: value itself if it is already // json.RawMessage, nil if value is nil or can't be marshalled. -func mustRaw(value any) json.RawMessage { +func rawJSON(value any) json.RawMessage { if value == nil { return nil } if raw, ok := value.(json.RawMessage); ok { return raw } + encoded, err := json.Marshal(value) if err != nil { return nil diff --git a/internal/websocket/tcp.go b/internal/websocket/tcp.go index 47d9d9d..52f6724 100644 --- a/internal/websocket/tcp.go +++ b/internal/websocket/tcp.go @@ -13,6 +13,8 @@ import ( "time" ws "github.com/coder/websocket" + + "git.avsdev.uk/Go/job-search/internal/util" ) // Plain TCP clients join the hub as if they were WebSocket clients. Instead @@ -55,7 +57,8 @@ func (h *Hub) ServeTCP(ctx context.Context, newConn net.Conn) { } ip := "" - if addrPort, err := netip.ParseAddrPort(newConn.RemoteAddr().String()); err == nil { + addrPort, err := netip.ParseAddrPort(newConn.RemoteAddr().String()) + if err == nil { ip = addrPort.Addr().Unmap().String() } @@ -71,10 +74,20 @@ func (h *Hub) ServeTCP(ctx context.Context, newConn net.Conn) { } defer h.unregister(client) - h.logger().Info("client connected", "client", client.id, "ip", ip, "transport", "tcp") + h.logger().Info( + "client connected", + "client", client.id, + "ip", ip, + "transport", "tcp", + ) client.run(ctx) conn.CloseNow() //nolint:errcheck,gosec // finished with; it may already be closed - h.logger().Info("client disconnected", "client", client.id, "ip", ip, "transport", "tcp") + h.logger().Info( + "client disconnected", + "client", client.id, + "ip", ip, + "transport", "tcp", + ) } // ---------- tcpConn ---------- @@ -88,7 +101,7 @@ type tcpConn struct { scan *bufio.Scanner closeTimeout time.Duration // caps Close's message; 0 or less for none - wm sync.Mutex // serialises writes + writeMu sync.Mutex // serialises writes closeOnce sync.Once // records the first close closed chan struct{} // closed by the first Close or CloseNow @@ -153,6 +166,7 @@ func (t *tcpConn) Read(ctx context.Context) (ws.MessageType, []byte, error) { t.Close(ws.StatusMessageTooBig, "message too long") //nolint:errcheck,gosec // dropped anyway return 0, nil, t.closeErr } + return 0, nil, err } @@ -185,12 +199,9 @@ func (t *tcpConn) Close(code ws.StatusCode, reason string) error { return nil } - ctx := context.Background() - if t.closeTimeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, t.closeTimeout) - defer cancel() - } + // No deadline if closeTimeout is 0 or less. + ctx, cancel := util.WithTimeout(context.Background(), t.closeTimeout) + defer cancel() message, err := json.Marshal(Envelope{ Type: TypeError, @@ -203,6 +214,7 @@ func (t *tcpConn) Close(code ws.StatusCode, reason string) error { if err := t.conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { return err } + return nil } @@ -244,8 +256,8 @@ func (t *tcpConn) isClosed() bool { // write writes data whole, giving up at ctx's deadline or when ctx is // cancelled. func (t *tcpConn) write(ctx context.Context, data []byte) error { - t.wm.Lock() - defer t.wm.Unlock() + t.writeMu.Lock() + defer t.writeMu.Unlock() deadline, _ := ctx.Deadline() // zero means none if err := t.conn.SetWriteDeadline(deadline); err != nil { diff --git a/internal/websocket/tcp_test.go b/internal/websocket/tcp_test.go index 46ae1d5..d37fd09 100644 --- a/internal/websocket/tcp_test.go +++ b/internal/websocket/tcp_test.go @@ -18,11 +18,13 @@ import ( "git.avsdev.uk/Go/job-search/internal/config" ) -var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil)) +var discardLog = slog.New(slog.DiscardHandler) // tcpHub starts a hub serving plain TCP clients, via ServeTCP, on a local // port and returns it with the address. It is shut down when the test ends. -func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string) { +func tcpHub( + t *testing.T, mutate func(*config.JSCServerWebSocket), +) (*Hub, string) { t.Helper() conf := config.JSCServerWebSocket{ MaxMessageSize: 1 << 10, @@ -31,11 +33,13 @@ func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string if mutate != nil { mutate(&conf) } + h := NewHub(conf, WithLogger(discardLog)) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } + ctx, cancel := context.WithCancel(context.Background()) go func() { for { @@ -46,6 +50,7 @@ func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string go h.ServeTCP(ctx, conn) } }() + t.Cleanup(func() { ln.Close() sctx, scancel := context.WithTimeout(context.Background(), time.Second) @@ -53,6 +58,7 @@ func tcpHub(t *testing.T, mutate func(*config.JSCServerWebSocket)) (*Hub, string h.Shutdown(sctx) cancel() }) + return h, ln.Addr().String() } @@ -94,10 +100,12 @@ func (c *tcpClient) read() Envelope { if err != nil { c.t.Fatalf("read: %v (got %q)", err, line) } + var env Envelope if err := json.Unmarshal([]byte(line), &env); err != nil { c.t.Fatalf("bad envelope %q: %v", line, err) } + return env } @@ -107,7 +115,8 @@ func (c *tcpClient) read() Envelope { func (c *tcpClient) waitClosed() { c.t.Helper() c.conn.SetReadDeadline(time.Now().Add(time.Second)) - if _, err := io.ReadAll(c.r); err != nil && !errors.Is(err, syscall.ECONNRESET) { + if _, err := io.ReadAll(c.r); err != nil && + !errors.Is(err, syscall.ECONNRESET) { c.t.Fatalf("connection not closed: %v", err) } } @@ -139,7 +148,8 @@ func TestTCPMessages(t *testing.T) { t.Fatalf("want pong 1, got %+v", e) } c.send(`{"type":"echo","id":"2","payload":{"a":1}}` + "\n") - if e := c.read(); e.Type != TypeEcho || e.ID != "2" || string(e.Payload) != `{"a":1}` { + if e := c.read(); e.Type != TypeEcho || e.ID != "2" || + string(e.Payload) != `{"a":1}` { t.Fatalf("want echo 2, got %+v", e) } c.send("not json\n") @@ -153,7 +163,8 @@ func TestTCPMessages(t *testing.T) { // Pushed messages reach TCP clients. h.Push("item.created", map[string]string{"id": "x"}) - if e := c.read(); e.Type != "item.created" || string(e.Payload) != `{"id":"x"}` { + if e := c.read(); e.Type != "item.created" || + string(e.Payload) != `{"id":"x"}` { t.Fatalf("want item.created, got %+v", e) } } @@ -164,10 +175,12 @@ func TestTCPNoWriteTimeout(t *testing.T) { _, addr := tcpHub(t, func(c *config.JSCServerWebSocket) { c.WriteTimeout = 0 }) + c := dialTCP(t, addr) if e := c.read(); e.Type != TypeWelcome { t.Fatalf("want welcome, got %+v", e) } + c.send(`{"type":"ping","id":"1"}` + "\n") if e := c.read(); e.Type != TypePong { t.Fatalf("want pong, got %+v", e) @@ -179,6 +192,7 @@ func TestTCPClientDisconnect(t *testing.T) { c := dialTCP(t, addr) c.read() // welcome waitCount(t, h, 1) + c.conn.Close() waitCount(t, h, 0) } @@ -190,6 +204,7 @@ func TestTCPLineTooLong(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome + c.send(`{"type":"echo","payload":"` + strings.Repeat("x", 100) + `"}` + "\n") c.waitClosed() waitCount(t, h, 0) @@ -214,7 +229,8 @@ func TestTCPShutdown(t *testing.T) { c.waitClosed() late := dialTCP(t, addr) - if e := late.read(); e.Type != TypeError || e.Error != "server shutting down" { + if e := late.read(); e.Type != TypeError || + e.Error != "server shutting down" { t.Fatalf("late client: want shutdown error, got %+v", e) } late.waitClosed() @@ -229,11 +245,13 @@ func TestTCPPing(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome + c.conn.SetReadDeadline(time.Now().Add(time.Second)) line, err := c.r.ReadString('\n') if err != nil || line != "\n" { t.Fatalf("want a blank ping line, got %q, %v", line, err) } + for i := range 5 { c.send(`{"type":"ping"}` + "\n") for { @@ -291,12 +309,14 @@ func TestTCPNoPongTimeout(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome + for range 3 { c.conn.SetReadDeadline(time.Now().Add(time.Second)) if line, err := c.r.ReadString('\n'); err != nil || line != "\n" { t.Fatalf("want a blank ping line, got %q, %v", line, err) } } + waitCount(t, h, 1) } @@ -307,7 +327,9 @@ func TestTCPDefaultMessageSize(t *testing.T) { }) c := dialTCP(t, addr) c.read() // welcome - c.send(`{"type":"echo","id":"1","payload":"` + strings.Repeat("x", 4096) + `"}` + "\n") + + payload := strings.Repeat("x", 4096) + c.send(`{"type":"echo","id":"1","payload":"` + payload + `"}` + "\n") if e := c.read(); e.Type != TypeEcho || e.ID != "1" { t.Fatalf("want echo, got %+v", e) } @@ -333,10 +355,12 @@ func TestTCPConnRead(t *testing.T) { io.WriteString(peer, "\r\n{\"a\":1}\r\n") peer.Close() }() + typ, data, err := tc.Read(context.Background()) if err != nil || typ != ws.MessageText || string(data) != `{"a":1}` { t.Fatalf("Read = %v, %q, %v", typ, data, err) } + // The peer closing is a normal closure. _, _, err = tc.Read(context.Background()) if ws.CloseStatus(err) != ws.StatusNormalClosure { @@ -370,17 +394,21 @@ func TestTCPConnClose(t *testing.T) { line, _ := bufio.NewReader(peer).ReadString('\n') got <- line }() + if err := tc.Close(ws.StatusGoingAway, "bye"); err != nil { t.Fatalf("Close: %v", err) } if line := <-got; line != `{"type":"error","error":"bye"}`+"\n" { t.Errorf("peer got %q", line) } + _, _, err := tc.Read(context.Background()) var ce ws.CloseError - if !errors.As(err, &ce) || ce.Code != ws.StatusGoingAway || ce.Reason != "bye" { + if !errors.As(err, &ce) || ce.Code != ws.StatusGoingAway || + ce.Reason != "bye" { t.Errorf("Read after Close = %v; want going away, bye", err) } + if err := tc.Close(ws.StatusNormalClosure, "again"); err != nil { t.Errorf("second Close: %v", err) } diff --git a/internal/websocket/tcpserver/server.go b/internal/websocket/tcpserver/server.go index 4be4965..36ca994 100644 --- a/internal/websocket/tcpserver/server.go +++ b/internal/websocket/tcpserver/server.go @@ -17,6 +17,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/config" "git.avsdev.uk/Go/job-search/internal/middleware" + "git.avsdev.uk/Go/job-search/internal/util" "git.avsdev.uk/Go/job-search/internal/websocket" ) @@ -64,7 +65,7 @@ func (s *Server) logger() *slog.Logger { if log == nil { log = slog.Default() } - return log.With("component", "tcp") + return log.With("component", "tcp_server") } // Run listens on the configured host and port and serves plain TCP clients @@ -102,19 +103,17 @@ func (s *Server) serve(ctx context.Context, listen net.Listener) error { listen.Close() //nolint:errcheck,gosec // closed already when ctx was cancelled s.logger().Info("shutting down") - shutdownCtx := context.Background() - if s.conf.Timeouts.ShutdownTimeout > 0 { - var cancel context.CancelFunc - shutdownCtx, cancel = context.WithTimeout( - shutdownCtx, s.conf.Timeouts.ShutdownTimeout, - ) - defer cancel() - } + // No deadline if the shutdown timeout is 0 or less. + shutdownCtx, cancel := util.WithTimeout( + context.Background(), s.conf.Timeouts.ShutdownTimeout, + ) + defer cancel() errs := []error{acceptErr} if err := s.hub.Shutdown(shutdownCtx); err != nil { errs = append(errs, fmt.Errorf("websocket shutdown: %w", err)) } + return errors.Join(errs...) } diff --git a/internal/websocket/tcpserver/server_test.go b/internal/websocket/tcpserver/server_test.go index 0c9d076..5adbf5b 100644 --- a/internal/websocket/tcpserver/server_test.go +++ b/internal/websocket/tcpserver/server_test.go @@ -17,7 +17,7 @@ import ( "git.avsdev.uk/Go/job-search/internal/websocket" ) -var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil)) +var discardLog = slog.New(slog.DiscardHandler) // testConf is a server configuration for tests, changed by mutate if it is // not nil. @@ -55,12 +55,14 @@ func startServer( if err != nil { t.Fatal(err) } + ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) done := make(chan error, 1) go func() { done <- s.serve(ctx, ln) }() + return h, ln.Addr().String(), cancel, done } @@ -95,10 +97,12 @@ func (c *tcpClient) read() websocket.Envelope { if err != nil { c.t.Fatalf("read: %v (got %q)", err, line) } + var env websocket.Envelope if err := json.Unmarshal([]byte(line), &env); err != nil { c.t.Fatalf("bad envelope %q: %v", line, err) } + return env } @@ -107,7 +111,8 @@ func (c *tcpClient) read() websocket.Envelope { func (c *tcpClient) waitClosed() { c.t.Helper() c.conn.SetReadDeadline(time.Now().Add(time.Second)) - if _, err := io.ReadAll(c.r); err != nil && !errors.Is(err, syscall.ECONNRESET) { + if _, err := io.ReadAll(c.r); err != nil && + !errors.Is(err, syscall.ECONNRESET) { c.t.Fatalf("connection not closed: %v", err) } } @@ -129,7 +134,8 @@ func TestServerShutdown(t *testing.T) { } cancel() - if e := c.read(); e.Type != websocket.TypeError || e.Error != "server shutting down" { + if e := c.read(); e.Type != websocket.TypeError || + e.Error != "server shutting down" { t.Fatalf("want shutdown error, got %+v", e) } c.waitClosed() @@ -152,7 +158,9 @@ func TestServerShutdown(t *testing.T) { func TestServerConnectionLimit(t *testing.T) { conf := testConf(func(c *config.JSCServer) { c.WebSocket.ConnectionLimit = config.JSCRateLimitRule{ - Enabled: true, PerSecond: 1, Burst: 1, + Enabled: true, + PerSecond: 1, + Burst: 1, } c.RateLimits.IdleTTL = time.Minute }) @@ -164,7 +172,8 @@ func TestServerConnectionLimit(t *testing.T) { t.Fatalf("first: want welcome, got %+v", e) } second := dial(t, addr) - if e := second.read(); e.Type != websocket.TypeError || e.Error != "rate limited" { + if e := second.read(); e.Type != websocket.TypeError || + e.Error != "rate limited" { t.Fatalf("second: want rate limited, got %+v", e) } second.waitClosed() @@ -203,7 +212,8 @@ func TestServerRunListenError(t *testing.T) { s, _ := newServer(testConf(func(c *config.JSCServer) { c.Port = -1 })) - if err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "listen") { + if err := s.Run(context.Background()); err == nil || + !strings.Contains(err.Error(), "listen") { t.Fatalf("Run = %v; want a listen error", err) } } @@ -231,13 +241,17 @@ func TestServerAcceptRetries(t *testing.T) { t.Fatal(err) } defer ln.Close() + fl := &flakyListener{ Listener: ln, fails: 3, } + err = s.accept(context.Background(), context.Background(), fl) if !errors.Is(err, net.ErrClosed) || fl.fails != 0 { - t.Fatalf("accept = %v with %d failures left; want net.ErrClosed", err, fl.fails) + t.Fatalf( + "accept = %v with %d failures left; want net.ErrClosed", err, fl.fails, + ) } } @@ -249,11 +263,13 @@ func TestServerAcceptCancelledInBackOff(t *testing.T) { t.Fatal(err) } defer ln.Close() + ctx, cancel := context.WithCancel(context.Background()) fl := &flakyListener{ Listener: ln, fails: 1 << 30, } + time.AfterFunc(20*time.Millisecond, cancel) if err := s.accept(ctx, ctx, fl); !errors.Is(err, context.Canceled) { t.Fatalf("accept = %v; want context.Canceled", err)